diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000000..ad411d2a3edcf --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @maralcbr @iconidentify @scottjones diff --git a/.github/workflows/kernel-ci-jobs.yml b/.github/workflows/kernel-ci-jobs.yml new file mode 100644 index 0000000000000..acac9281b4987 --- /dev/null +++ b/.github/workflows/kernel-ci-jobs.yml @@ -0,0 +1,488 @@ +# The kernel CI jobs, shared by kernel-ci.yml and kernel-ci-label.yml. +# tools/aurora-ci/README.rst describes each job and how to run it locally. +# +# Runners: everything runs on GitHub-hosted runners, which cost nothing for a +# public repository. The two CPU-heavy jobs (package, kunit) run on the +# repository variable KERNEL_CI_RUNNER when it is set, as a JSON runs-on value +# such as ["namespace-profile-arm64-linux-apple-12x32"]; it defaults to +# GitHub's arm64 runner. + +name: Kernel CI jobs + +on: + workflow_call: + outputs: + key: + description: Build key (kernel tree plus packaging inputs) + value: ${{ jobs.plan.outputs.key }} + package_run: + description: Run that built the packages, when this run reused them + value: ${{ jobs.plan.outputs.package_run }} + tier: + description: light or full + value: ${{ jobs.plan.outputs.tier }} + +permissions: {} + +env: + BUILD_LABEL: ci:build + RECIPE_REPOSITORY: omacom/omarchy-pkgs + PYTHONDONTWRITEBYTECODE: "1" + +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + +jobs: + plan: + name: plan + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read # look up and download earlier runs' artifacts + contents: read # check out the repository + outputs: + tier: ${{ steps.plan.outputs.tier }} + reason: ${{ steps.plan.outputs.reason }} + base_sha: ${{ steps.plan.outputs.base_sha }} + head_sha: ${{ steps.plan.outputs.head_sha }} + key: ${{ steps.lookup.outputs.key }} + kernel_tree: ${{ steps.lookup.outputs.kernel_tree }} + recipe_commit: ${{ steps.lookup.outputs.recipe_commit }} + package_run: ${{ steps.lookup.outputs.package_run }} + verified_run: ${{ steps.lookup.outputs.verified_run }} + base_tree: ${{ steps.lookup.outputs.base_tree }} + baseline_run: ${{ steps.lookup.outputs.baseline_run }} + steps: + - name: Check out the CI tools + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + filter: blob:none + sparse-checkout: tools/aurora-ci + - name: Plan the run + id: plan + run: python3 -I tools/aurora-ci/plan.py --label "$BUILD_LABEL" + # Only trees are needed: the build key is made of tree hashes. + - name: Fetch the packaging inputs + if: steps.plan.outputs.tier == 'full' + run: | + git clone --quiet --depth 1 --filter=blob:none --no-checkout \ + "https://github.com/$RECIPE_REPOSITORY.git" "$RUNNER_TEMP/omarchy-pkgs" + - name: Look up earlier builds of the same inputs + id: lookup + if: steps.plan.outputs.tier == 'full' + env: + GITHUB_TOKEN: ${{ github.token }} + BASE_SHA: ${{ steps.plan.outputs.base_sha }} + FORCE: ${{ steps.plan.outputs.force }} + run: | + python3 -I tools/aurora-ci/lookup-build.py --kernel . \ + --recipe-repo "$RUNNER_TEMP/omarchy-pkgs" --base-sha "$BASE_SHA" --force "$FORCE" + + commits: + name: commits + needs: plan + if: github.event_name == 'pull_request' || github.event_name == 'merge_group' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read # check out the repository + steps: + - name: Check out the tree + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # checkpatch.pl runs scripts/spdxcheck.py, which needs ply. + - name: Install checkpatch's helpers + run: | + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends python3-ply + - name: Check sign-offs, checkpatch and UAPI changes + env: + BASE_SHA: ${{ needs.plan.outputs.base_sha }} + HEAD_SHA: ${{ needs.plan.outputs.head_sha }} + run: | + python3 -I tools/aurora-ci/check-commits.py --base "$BASE_SHA" --head "$HEAD_SHA" \ + --report "$RUNNER_TEMP/commits.json" + + compile: + name: compile + needs: plan + if: github.event_name == 'pull_request' || github.event_name == 'merge_group' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read # check out the repository + env: + BASE_SHA: ${{ needs.plan.outputs.base_sha }} + HEAD_SHA: ${{ needs.plan.outputs.head_sha }} + steps: + - name: Check out the change + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.plan.outputs.head_sha }} + persist-credentials: false + - name: Fetch history back to the target branch + run: | + git fetch --quiet --no-tags --depth=1 origin "$BASE_SHA" "$HEAD_SHA" + for depth in 64 256 1024 4096; do + git merge-base "$BASE_SHA" "$HEAD_SHA" >/dev/null && break + git fetch --quiet --no-tags --depth="$depth" origin "$BASE_SHA" "$HEAD_SHA" + done + - name: Install LLVM 18 + run: | + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends bc bison clang-18 flex lld-18 llvm-18 + # The configuration the package starts from; compile-check.py merges + # the in-tree fragment into it the way the recipe's configure.sh does. + - name: Fetch the shipped kernel configuration + run: | + git clone --quiet --depth 1 --filter=blob:none --no-checkout \ + "https://github.com/$RECIPE_REPOSITORY.git" "$RUNNER_TEMP/omarchy-pkgs" + git -C "$RUNNER_TEMP/omarchy-pkgs" show HEAD:pkgbuilds/linux-aurora/config > "$RUNNER_TEMP/aurora.config" + echo "omarchy-pkgs $(git -C "$RUNNER_TEMP/omarchy-pkgs" rev-parse HEAD)" + - name: Compile what changed, at every commit, with W=1 + run: | + python3 -I tools/aurora-ci/compile-check.py --tree . --base "$BASE_SHA" --head "$HEAD_SHA" \ + --config "$RUNNER_TEMP/aurora.config" --fragment arch/arm64/configs/asahi.config \ + --out "$RUNNER_TEMP/compile" --per-commit --max-commits 20 --llvm=-18 --jobs "$(nproc)" + - name: Keep the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: compile-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/compile/report.json + ${{ runner.temp }}/compile/summary.md + ${{ runner.temp }}/compile/logs + retention-days: 14 + if-no-files-found: ignore + + devicetree: + name: device trees + needs: plan + if: github.event_name == 'pull_request' || github.event_name == 'merge_group' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read # check out the repository + steps: + # The pull request or queue merge commit, and the target branch tip it + # was merged onto (its first parent). + - name: Check out the tree + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 2 + persist-credentials: false + - name: Install dtschema and yamllint + run: | + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends bison flex python3-dev python3-venv + python3 -m venv "$RUNNER_TEMP/dt-venv" + "$RUNNER_TEMP/dt-venv/bin/pip" install --quiet --require-hashes -r tools/aurora-ci/dt-requirements.txt + echo "$RUNNER_TEMP/dt-venv/bin" >> "$GITHUB_PATH" + - name: Check the device trees and bindings this change touches + run: | + python3 -I tools/aurora-ci/dt-check.py --tree . --base HEAD^1 --head HEAD \ + --out "$RUNNER_TEMP/dt-check" --jobs "$(nproc)" + - name: Keep the report + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: devicetree-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/dt-check/report.json + ${{ runner.temp }}/dt-check/summary.md + ${{ runner.temp }}/dt-check/logs + retention-days: 14 + if-no-files-found: ignore + + host-tests: + name: host tests (${{ matrix.group }}) + needs: plan + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read # check out the repository + strategy: + fail-fast: false + matrix: + group: [tools, aurora-sep] + env: + GROUP: ${{ matrix.group }} + steps: + - name: Check out the tree + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install what the suites need + run: python3 -I tools/aurora-ci/host-tests.py --group "$GROUP" --install + - name: Run the ${{ matrix.group }} suites + run: | + python3 -I tools/aurora-ci/host-tests.py --group "$GROUP" --fetch-commits --jobs 4 \ + --output "$RUNNER_TEMP/host-tests" + - name: Keep the suites' output + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: host-tests-${{ matrix.group }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/host-tests + retention-days: 14 + if-no-files-found: ignore + + self-test: + name: CI self-test + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read # check out the repository + env: + ACTIONLINT_VERSION: 1.7.12 + ACTIONLINT_SHA256: 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 + steps: + # The whole tree: the manifest tests check host-tests.json against it. + - name: Check out the tree + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Unit tests + run: python3 -I -m unittest discover --start-directory tools/aurora-ci/tests --verbose + - name: actionlint + run: | + archive="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" + curl -fsSLo "$RUNNER_TEMP/$archive" \ + "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/$archive" + echo "$ACTIONLINT_SHA256 $RUNNER_TEMP/$archive" | sha256sum --check --strict + tar -xzf "$RUNNER_TEMP/$archive" -C "$RUNNER_TEMP" actionlint + "$RUNNER_TEMP/actionlint" -color + - name: zizmor + uses: zizmorcore/zizmor-action@85b2b9fb8e3325e7a23d7d0a9d7ba5ae0f999e3e # v0.7.1 + with: + version: 1.31.1 + inputs: .github/workflows + persona: pedantic + advanced-security: false + annotations: true + + package: + name: package + needs: plan + if: needs.plan.outputs.tier == 'full' + # A reused build only downloads and checks the packages; that needs no + # build machine. + runs-on: ${{ needs.plan.outputs.package_run != '' && 'ubuntu-24.04' || fromJSON(vars.KERNEL_CI_RUNNER || '"ubuntu-24.04-arm"') }} + timeout-minutes: 180 + permissions: + actions: read # look up and download earlier runs' artifacts + contents: read # check out the repository + env: + KEY: ${{ needs.plan.outputs.key }} + REUSE_RUN: ${{ needs.plan.outputs.package_run }} + steps: + - name: Check out the kernel + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: linux + persist-credentials: false + - name: Check out omarchy-pkgs at the commit the build key names + if: env.REUSE_RUN == '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ env.RECIPE_REPOSITORY }} + ref: ${{ needs.plan.outputs.recipe_commit }} + path: omarchy-pkgs + persist-credentials: false + - name: Point the linux-aurora recipe at this commit + if: env.REUSE_RUN == '' + run: | + mkdir -p out/packages + python3 -I linux/tools/aurora-ci/prepare-recipe.py \ + --recipe omarchy-pkgs/pkgbuilds/linux-aurora --kernel linux \ + --commit "$(git -C linux rev-parse HEAD)" --revision "$GITHUB_RUN_NUMBER" \ + --record out/recipe.json + - name: Build with omarchy-pkgs bin/build + if: env.REUSE_RUN == '' + working-directory: omarchy-pkgs + env: + CONTAINER_ENGINE: docker + run: bin/build --mirror edge --arch aarch64 --package linux-aurora + - name: Collect the packages + if: env.REUSE_RUN == '' + run: | + cp omarchy-pkgs/build-output/edge/aarch64/linux-aurora-*.pkg.tar.zst out/packages/ + ls -l out/packages + - name: Download the packages built for this key + if: env.REUSE_RUN != '' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: kernel-package-${{ env.KEY }} + path: out + run-id: ${{ env.REUSE_RUN }} + github-token: ${{ github.token }} + # Not every runner image ships kmod (Namespace's does not). + - name: Install depmod + run: | + if ! command -v depmod >/dev/null; then + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends kmod + fi + - name: Inspect the packages + run: | + python3 -I linux/tools/aurora-ci/inspect-package.py --packages out/packages \ + --output out/metadata.json + - name: Download the target branch's package metadata + if: needs.plan.outputs.baseline_run != '' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: kernel-metadata-${{ needs.plan.outputs.base_tree }} + path: baseline + run-id: ${{ needs.plan.outputs.baseline_run }} + github-token: ${{ github.token }} + - name: Compare with the target branch + if: needs.plan.outputs.baseline_run != '' + run: | + python3 -I linux/tools/aurora-ci/compare-packages.py --base baseline/metadata.json \ + --head out/metadata.json --retired linux/tools/aurora-ci/retired-dtbs.txt \ + --report out/compare.json + - name: Note a missing baseline + if: needs.plan.outputs.baseline_run == '' + run: | + echo "::notice title=No baseline::The target branch has no package metadata yet; its next push build records it." + - name: Upload the packages + if: env.REUSE_RUN == '' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: kernel-package-${{ env.KEY }} + path: out/ + retention-days: 14 + compression-level: 0 + if-no-files-found: error + - name: Upload the metadata for later comparisons + if: env.REUSE_RUN == '' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: kernel-metadata-${{ needs.plan.outputs.kernel_tree }} + path: out/metadata.json + retention-days: 90 + if-no-files-found: error + + kunit: + name: KUnit + needs: plan + if: needs.plan.outputs.tier == 'full' && needs.plan.outputs.verified_run == '' + runs-on: ${{ fromJSON(vars.KERNEL_CI_RUNNER || '"ubuntu-24.04-arm"') }} + timeout-minutes: 90 + permissions: + contents: read # check out the repository + steps: + - name: Check out the tree + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install the KUnit toolchain + run: python3 -I tools/aurora-ci/run-kunit.py --install-toolchain + - name: Build and run the Apple KUnit suites + run: python3 -I tools/aurora-ci/run-kunit.py --output "$RUNNER_TEMP/kunit" + - name: Keep the results and console log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: kunit-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/kunit/kunit-results.json + ${{ runner.temp }}/kunit/kunit-results.xml + ${{ runner.temp }}/kunit/test.log + ${{ runner.temp }}/kunit/.config + retention-days: 14 + if-no-files-found: ignore + + boot: + name: boot + needs: [plan, package] + if: needs.plan.outputs.tier == 'full' && needs.plan.outputs.verified_run == '' + runs-on: ubuntu-24.04-arm + timeout-minutes: 45 + permissions: + actions: read # look up and download earlier runs' artifacts + contents: read # check out the repository + steps: + - name: Check out the CI tools + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + filter: blob:none + sparse-checkout: tools/aurora-ci + - name: Download the packages + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: kernel-package-${{ needs.plan.outputs.key }} + path: out + run-id: ${{ needs.plan.outputs.package_run || github.run_id }} + github-token: ${{ github.token }} + - name: Install QEMU and busybox + run: | + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends qemu-system-arm busybox-static + if [[ -e /dev/kvm ]]; then + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | + sudo tee /etc/udev/rules.d/99-kernel-ci-kvm.rules >/dev/null + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + fi + - name: Unpack the kernel package + run: | + mkdir root + tar --zstd -xf out/packages/linux-aurora-[0-9]*.pkg.tar.zst -C root + - name: Boot it and load every module + run: | + python3 -I tools/aurora-ci/boot-test.py --root root --busybox /bin/busybox \ + --expected tools/aurora-ci/boot-expected.json --out "$RUNNER_TEMP/boot" + - name: Keep the console log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: boot-${{ github.run_attempt }} + path: ${{ runner.temp }}/boot + retention-days: 14 + if-no-files-found: ignore + + result: + name: result + needs: [plan, commits, compile, devicetree, host-tests, self-test, package, kunit, boot] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read # check out the repository + steps: + - name: Check out the CI tools + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + filter: blob:none + sparse-checkout: tools/aurora-ci + - name: Decide the result + id: result + env: + NEEDS_JSON: ${{ toJSON(needs) }} + run: python3 -I tools/aurora-ci/result.py + - name: Record that this build key passed + if: steps.result.outputs.ok == 'true' && needs.plan.outputs.tier == 'full' && needs.plan.outputs.verified_run == '' + env: + KEY: ${{ needs.plan.outputs.key }} + KERNEL_TREE: ${{ needs.plan.outputs.kernel_tree }} + RECIPE_COMMIT: ${{ needs.plan.outputs.recipe_commit }} + run: | + mkdir -p "$RUNNER_TEMP/verified" + python3 -I -c 'import json, os, sys; json.dump({k.lower(): os.environ[k] for k in + ("KEY", "KERNEL_TREE", "RECIPE_COMMIT", "GITHUB_RUN_ID", "GITHUB_SHA")}, sys.stdout, indent=1)' \ + > "$RUNNER_TEMP/verified/verified.json" + - name: Upload the verification record + if: steps.result.outputs.ok == 'true' && needs.plan.outputs.tier == 'full' && needs.plan.outputs.verified_run == '' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: kernel-verified-${{ needs.plan.outputs.key }} + path: ${{ runner.temp }}/verified/verified.json + retention-days: 90 + if-no-files-found: error diff --git a/.github/workflows/kernel-ci-label.yml b/.github/workflows/kernel-ci-label.yml new file mode 100644 index 0000000000000..17c5568e7fa54 --- /dev/null +++ b/.github/workflows/kernel-ci-label.yml @@ -0,0 +1,29 @@ +# A build on request: applying the ci:build label to a pull request runs the +# full kernel CI (package, KUnit, boot) for its current head, before the merge +# queue would. Applying a label needs triage access, so builds stay a +# maintainer's decision; on a fork, re-apply the label after each push. +# The required check is kernel-ci.yml's; these results are for review. + +name: Kernel CI (requested build) + +on: + pull_request: + branches: [integration/aurora] + types: [labeled] + +permissions: {} + +# Other labels must not cancel a build in progress. +concurrency: + group: kernel-ci-label-${{ github.event.pull_request.number }}-${{ github.event.label.name }} + cancel-in-progress: true + +jobs: + build: + name: requested build + if: github.event.label.name == 'ci:build' + # A reusable workflow named this way always comes from this same commit. + uses: ./.github/workflows/kernel-ci-jobs.yml # zizmor: ignore[self-repository] actionlint does not accept $/ yet + permissions: + actions: read # look up and download earlier runs' artifacts + contents: read # check out the repository diff --git a/.github/workflows/kernel-ci.yml b/.github/workflows/kernel-ci.yml new file mode 100644 index 0000000000000..e02c9ee7481a5 --- /dev/null +++ b/.github/workflows/kernel-ci.yml @@ -0,0 +1,71 @@ +# Kernel CI for integration/aurora. How it works, and how to run each part +# locally: tools/aurora-ci/README.rst. +# +# pull request checks only (commit hygiene, host test suites, the CI's +# own tests), on every push; add the ci:build label for a +# build now (kernel-ci-label.yml) +# merge queue everything: the kernel package built with omarchy-pkgs' +# own recipe, KUnit, a QEMU boot that loads every module, +# and a comparison with the target branch's package +# push the merged commit, reusing the queue's build; attests +# the packages +# nightly rebuilds from scratch, so a change in omarchy-pkgs or in +# the Arch Linux ARM packages it builds with shows up +# before the next release + +name: Kernel CI + +on: + pull_request: + branches: [integration/aurora] + types: [opened, synchronize, reopened, ready_for_review] + merge_group: + types: [checks_requested] + push: + branches: [integration/aurora] + schedule: + - cron: "23 9 * * *" + workflow_dispatch: + +permissions: {} + +# A new push to a pull request replaces its run; every other run is kept. +concurrency: + group: kernel-ci-${{ github.event_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + ci: + name: ci + # A reusable workflow named this way always comes from this same commit. + uses: ./.github/workflows/kernel-ci-jobs.yml # zizmor: ignore[self-repository] actionlint does not accept $/ yet + permissions: + actions: read # look up and download earlier runs' artifacts + contents: read # check out the repository + + # Provenance for the packages of every merged commit: who built them, from + # which commit and workflow. Verify with + # gh attestation verify --repo omacom/linux-aurora + attest: + name: attest + needs: ci + if: github.event_name == 'push' && needs.ci.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + actions: read # download the packages from the run that built them + attestations: write # store the provenance attestation + contents: read # read the repository the attestation names + id-token: write # sign the attestation with this workflow's OIDC identity + steps: + - name: Download the packages + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: kernel-package-${{ needs.ci.outputs.key }} + path: out + run-id: ${{ needs.ci.outputs.package_run || github.run_id }} + github-token: ${{ github.token }} + - name: Attest the packages + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: out/packages/*.pkg.tar.zst diff --git a/.gitignore b/.gitignore index 9875120ea7bde..79980e38c9ca6 100644 --- a/.gitignore +++ b/.gitignore @@ -116,6 +116,7 @@ modules.order !.editorconfig !.get_maintainer.ignore !.gitattributes +!/.github/ !.gitignore !.kunitconfig !.mailmap diff --git a/Documentation/ABI/testing/sysfs-class-power b/Documentation/ABI/testing/sysfs-class-power index 32697b926cc8c..a68e36832c250 100644 --- a/Documentation/ABI/testing/sysfs-class-power +++ b/Documentation/ABI/testing/sysfs-class-power @@ -513,6 +513,11 @@ Description: inhibit-charge: Do not charge while AC is attached inhibit-charge-awake: inhibit-charge only when device is awake force-discharge: Force discharge while AC is attached + auto-discharge: auto, but actively discharge down to + the charge control end threshold. + Needs an end threshold below 100; + an end threshold of 100 returns to + auto ===================== ======================================== What: /sys/class/power_supply//technology diff --git a/Documentation/arch/arm64/apple-j714c.rst b/Documentation/arch/arm64/apple-j714c.rst new file mode 100644 index 0000000000000..dfc676aeea337 --- /dev/null +++ b/Documentation/arch/arm64/apple-j714c.rst @@ -0,0 +1,138 @@ +.. SPDX-License-Identifier: GPL-2.0 + +==================================== +Apple J714c initial native Linux boot +==================================== + +J714c is the 14-inch M5 Max MacBook Pro (T6050). This configuration +provides a RAM-only boot path with AICv3, architectural timers, the +framebuffer left by the boot firmware, and the MTP keyboard and trackpad +over DockChannel. The kernel reaches a BusyBox shell on one CPU with the +built-in keyboard working. + +The device tree describes all 18 CPUs in three clusters of six. For this +one-CPU baseline, secondary cores stay stopped in the loader and the kernel +runs with ``maxcpus=1``. The loader removes unavailable CPUs and fills the +release addresses of the remaining nodes. + +MMIO addresses in the T6050 device tree are physical: the Apple Device Tree +``arm-io`` child address plus 0x200000000. The AIC is at 0x280400000, the +Samsung UART at 0x505200000, the watchdog at 0x28836c000, the GPIO block +at 0x29c000000 and the PMGR window at 0x280600000. The AIC uses the +``apple,t8142-aic3`` fallback and the 3-cell interrupt form. + +SMP bring-up +============ + +Not done. On this machine the secondary CPUs' RVBAR is locked to the +loader image that the boot firmware started. Writes to ``cpu-impl-reg`` +from EL2 raise an SError, so the stock chainload path cannot relocate the +secondaries and the loader does not start them. The ``cpus`` nodes carry +``cpu-release-addr`` for the spin-table path once a loader can start them. + +CPU frequency bring-up +====================== + +Not done. ``apple_soc_cpufreq_init`` reports ``failed to get cluster info: +-19``; the tree has no cluster controller nodes for T6050. The CPUs run at +the frequency the boot firmware left. + +Bootloader requirements +======================= + +Use the T6050-capable Aurora m1n1. Before entering Linux, the loader must: + +* Supply usable memory and reserved regions. T6050 DRAM starts at + 0x10000000000; the DTS memory size is only a loader placeholder. +* Fill CPU release addresses and remove the CPUs it has not started. +* Fill the firmware framebuffer's address, size, geometry and format. The + ``framebuffer@0`` node under ``/chosen`` is enabled in the DTS because + the loader's display setup returns early for T6050. +* Program the MTP DMA address filter from the Apple Device Tree + ``dapf-instance-0`` of ``dart-mtp`` into its filter window, register 1 + of that node (0x294810000). Entry 0 is the PMGR window 0x28830c000 that + the MTP firmware reads first; without it the firmware never sends HELLO. + Register 1 of ``dart-aop`` on T6050 is the block shared by every DART at + 0x2890fc000, and writing filter entries there raises an SError, so the + generic DAPF table must not run on this chip. +* Leave the MTP DART in translate mode. In bypass mode the filter rejects + the firmware's writes to the kernel's buffers. +* Leave the console, interrupt controller and display powered. The DTS + has no PMGR power domains; power stays as the firmware left it. + +On this machine the MTP coprocessor is stopped at kernel entry +(``CPU_CONTROL`` 0) and its DART holds no valid translation, so the kernel +starts the firmware itself. There is no live RTKit session to adopt. + +Build +===== + +On an arm64 Linux build host with the usual kernel build dependencies:: + + out="$PWD/../j714c-build" + make ARCH=arm64 O="$out" \ + KCONFIG_ALLCONFIG=arch/arm64/configs/j714c.config allnoconfig + make ARCH=arm64 O="$out" -j"$(nproc)" Image.gz apple/t6050-j714c.dtb + +``j714c.config`` is ``j813.config`` with ``NR_CPUS=18``, ``ARM64_SME``, +``ARM64_LSUI`` and ``ARM64_ERRATUM_4193714``. Add +``CROSS_COMPILE=aarch64-linux-gnu-`` to both commands when cross-compiling +with GCC from another architecture. + +Boot and qualification +====================== + +Use a RAM-only initramfs with a BusyBox shell. Its init mounts proc, sysfs +and devtmpfs and makes no disk mounts. + +The command line is:: + + maxcpus=1 console=tty0 + +Load Image.gz, the DTB and the initramfs through the loader's USB proxy +with the proxyclient ``linux.py``; on T6050 it continues when the loader +refuses to start secondaries. When the loader itself is chainloaded over +the proxy, ``chainload.py`` skips the secondary RVBAR writes on T6050. + +The qualified console is the firmware framebuffer. The Samsung UART node +records its measured resources; no UART console path has been observed +from this laptop. Before the kernel jump the loader's console is readable +on the USB proxy; after it, the built-in display is the only output. + +Verified on 2026-10-23 with kernel ``7.2.10-aurora-t6050-j714c-wip+``: + +* T6050/J714c identity, one online CPU, timer interrupts, the shell + responding to typed commands. +* ``simple-framebuffer`` registered at 0x11fd31e6880, 3024x1890, + ``x2r10g10b10``, with the console on it. +* The MTP firmware ``AppleMTPFirmwareMac-6300.85.6~243`` starting, and + ``dockchannel-hid`` enumerating the keyboard, multi-touch, actuator and + stm interfaces. Keystrokes reach the console. + +The same result was reproduced with the loader built from the Aurora m1n1 +fork and the kernel built from ``j714c.config`` as above. + +These boot tests do not establish SMP, CPU frequency control, PMGR power +domains, suspend, GPU acceleration, USB, storage, networking, audio, SMC +or temperature monitoring. The Samsung UART, the DockChannel UART and the +``gpio0`` block are described but disabled or unverified. + +Built-in input +============== + +MTP provides keyboard and trackpad HID descriptors at runtime over the +DockChannel fifo at 0x294b14000 with data windows from 0x294b28000. The +``keyboard`` alias lets the loader fill ``hid-country-code`` and +``apple,keyboard-layout-id`` from this machine's own boot environment. + +The T6050 MTP takes the same early AP wake as the T8142 MTP in +``rtkit-helper``. The trackpad enumerates as a multi-touch device; its +firmware file is not supplied and the multi-touch node carries no +``firmware-name``. The STM and actuator interfaces report ready without +the AFE and STM reset GPIOs the J514 tree uses; no SMC GPIO is described. + +Native display +============== + +None. The display stays on the framebuffer the boot firmware left, with no +DCP node and no display reserved-memory setup from the loader. diff --git a/Documentation/arch/arm64/index.rst b/Documentation/arch/arm64/index.rst index 128d3de94bb54..4c69a38d7b84f 100644 --- a/Documentation/arch/arm64/index.rst +++ b/Documentation/arch/arm64/index.rst @@ -10,6 +10,7 @@ ARM64 Architecture acpi_object_usage amu apple-j813 + apple-j714c apple-m6-cpu arm-acpi arm-cca diff --git a/Documentation/devicetree/bindings/display/apple,dcp.yaml b/Documentation/devicetree/bindings/display/apple,dcp.yaml index a7c74d948e6e9..46f48ba6479e0 100644 --- a/Documentation/devicetree/bindings/display/apple,dcp.yaml +++ b/Documentation/devicetree/bindings/display/apple,dcp.yaml @@ -25,6 +25,7 @@ properties: - apple,t6020-dcp - apple,t8140-dcp - apple,t8142-dcp + - apple,t6050-dcp - const: apple,dcp - items: - enum: @@ -32,6 +33,8 @@ properties: - apple,t8112-dcpext - apple,t6000-dcpext - apple,t6020-dcpext + - apple,t6041-dcpext + - apple,t6050-dcpext - const: apple,dcpext - items: - const: apple,t8140-dcpext @@ -70,13 +73,32 @@ properties: clocks: minItems: 1 - maxItems: 2 + maxItems: 64 clock-names: minItems: 1 - items: - - const: clock-154 - - const: clock-194 + maxItems: 64 + + apple,clock-ids: + oneOf: + - type: boolean + - $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 64 + description: | + Ordered firmware provider clock identifiers. Callback clock indices + select the corresponding entry in clocks. An explicitly empty table + is allowed when the provider has no clocks. + + apple,firmware-physical-regs: + $ref: /schemas/types.yaml#/definitions/uint64-array + minItems: 1 + maxItems: 10 + description: | + Coprocessor physical base addresses in display register-table order, + including the bandwidth scratch and optional doorbell windows. + The number of entries must match the complete register table. + Omit on identity-mapped systems; reg supplies the physical view. memory-region: minItems: 1 @@ -321,12 +343,64 @@ required: - apple,bw-scratch allOf: + - if: + properties: + compatible: + contains: + enum: ['apple,t6030-dcpext', 'apple,t8132-dcpext'] + required: + - compatible + then: {} + else: + properties: + mux-controls: + maxItems: 8 + mux-control-names: + maxItems: 8 + items: + pattern: '^(dp-xbar|typec[0-9]+)$' + - if: + not: + properties: + compatible: + contains: + enum: ['apple,t8132-dcp', 'apple,t8132-dcpext'] + then: + required: + - apple,bw-scratch + - if: + properties: + compatible: + contains: + enum: ['apple,t6050-dcp', 'apple,t6050-dcpext'] + then: + required: + - apple,clock-ids + - memory-region + - memory-region-names + - apple,firmware-compat + properties: + apple,firmware-compat: + items: + - const: 27 + - const: 0 + - const: 0 + else: + properties: + clocks: + maxItems: 2 + clock-names: + items: + - const: clock-154 + - const: clock-194 + apple,clock-ids: false + apple,firmware-physical-regs: false - if: not: properties: compatible: contains: - enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext', 'apple,t6050-dcp', 'apple,t6050-dcpext'] then: properties: apple,power-lut-data-x: false @@ -338,7 +412,8 @@ allOf: properties: compatible: contains: - enum: ['apple,t8140-dcp', 'apple,t8140-dcpext'] + enum: ['apple,t8140-dcp', 'apple,t8140-dcpext', + 'apple,t6050-dcp', 'apple,t6050-dcpext'] then: properties: resets: false diff --git a/Documentation/devicetree/bindings/dma/apple,sio.yaml b/Documentation/devicetree/bindings/dma/apple,sio.yaml index 0e3780ad9dd79..dbaaa2d3d1a1a 100644 --- a/Documentation/devicetree/bindings/dma/apple,sio.yaml +++ b/Documentation/devicetree/bindings/dma/apple,sio.yaml @@ -22,6 +22,7 @@ properties: items: - enum: - apple,t6000-sio + - apple,t6030-sio - apple,t8103-sio - const: apple,sio diff --git a/Documentation/devicetree/bindings/iommu/apple,dart.yaml b/Documentation/devicetree/bindings/iommu/apple,dart.yaml index 44a0499ada2cb..d36d00da984da 100644 --- a/Documentation/devicetree/bindings/iommu/apple,dart.yaml +++ b/Documentation/devicetree/bindings/iommu/apple,dart.yaml @@ -88,6 +88,13 @@ properties: handoff. Linux retains occupied root slots and excludes their entire IOVA spans from allocation. This does not assert immutable descendants. + apple,manual-availability: + type: boolean + description: > + The DART's registers answer only once its client, a PCIe root complex, + is up. The client's driver makes the DART available after bringing the + root complex up, and the DART probes then. + apple,tunable: $ref: /schemas/types.yaml#/definitions/uint32-matrix items: diff --git a/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml b/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml new file mode 100644 index 0000000000000..6b1a33cac7aa4 --- /dev/null +++ b/Documentation/devicetree/bindings/net/bluetooth/apple,centauri.yaml @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/bluetooth/apple,centauri.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple N1 Centauri Beta Bluetooth PCI function + +maintainers: + - Dj + +description: | + PCI function two is the Beta Bluetooth HCI/ACL transport. It depends on + the Control function in the same PCI slot, which boots the shared N1 + firmware using the protocol profile described on its own DT node. + The factory address is supplied in network byte order for the firmware + address-provisioning command; HCI returns the standard Bluetooth order. + +properties: + compatible: + const: pci106b,1903 + + reg: + maxItems: 1 + description: Standard five-cell PCI function address + + local-bd-address: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 6 + maxItems: 6 + description: Factory Bluetooth public address, least significant byte + first (the standard Bluetooth property a boot loader fills) + + local-mac-address: + $ref: /schemas/types.yaml#/definitions/uint8-array + minItems: 6 + maxItems: 6 + description: Factory Bluetooth public address in network byte order + +oneOf: + - required: [local-bd-address] + - required: [local-mac-address] + +required: + - compatible + - reg + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml b/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml new file mode 100644 index 0000000000000..f73dd9afceca1 --- /dev/null +++ b/Documentation/devicetree/bindings/net/wireless/apple,centauri.yaml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/wireless/apple,centauri.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Apple N1 Centauri Control PCI function + +maintainers: + - Dj + +description: | + PCI function zero owns the shared N1 firmware bootstrap. The boot producer + supplies the personalized FTAB filename and packed firmware platform ID. + The endpoint must be left in its stopped ROM personality with DMA and MSI + quiesced. The host's root-port reset retrains the link after the ROM + bootstrap. This binding does not authorize adopting an existing live + firmware session or releasing another owner's retained DART streams. + +properties: + compatible: + enum: + - pci106b,1900 + - pci106b,1901 + + reg: + maxItems: 1 + description: Standard five-cell PCI function address + + firmware-name: + $ref: /schemas/types.yaml#/definitions/string + description: Personalized FTAB including the required per-device calibration + + apple,centauri-platform-id: + $ref: /schemas/types.yaml#/definitions/uint32 + description: Firmware chip ID in bits 15..0 and board ID in bits 30..16 + + apple,centauri-protocol: + $ref: /schemas/types.yaml#/definitions/uint32 + const: 27 + description: Shared firmware wire protocol profile + +required: + - compatible + - reg + - firmware-name + - apple,centauri-platform-id + - apple,centauri-protocol + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/pci/apple,pcie.yaml b/Documentation/devicetree/bindings/pci/apple,pcie.yaml index df7ef17ae8d2f..3abab75284892 100644 --- a/Documentation/devicetree/bindings/pci/apple,pcie.yaml +++ b/Documentation/devicetree/bindings/pci/apple,pcie.yaml @@ -56,23 +56,46 @@ properties: - apple,t6000-pcie - const: apple,pcie - items: - - const: apple,t8122-pcie + - enum: + - apple,t6041-pcie + - apple,t8122-pcie - const: apple,t6020-pcie - const: apple,t6020-pcie - const: apple,t8140-pcie + - const: apple,t6050-pcie + - const: apple,t8132-pcie - const: apple,t8103-pciec - items: - const: apple,t8112-pciec - const: apple,t8103-pciec - const: apple,t6000-pciec + - items: + - const: apple,t8132-pciec + - const: apple,t8122-pciec + - const: apple,t8103-pciec + + apple,managed-sids: + $ref: /schemas/types.yaml#/definitions/uint32-array + items: + - const: 1 + - const: 2 + description: > + J713 radio ordinary stream ownership profile. Only these inactive + normal streams and their exact-RID entries may be adopted by the OS. + Other streams and RID entries remain inherited. The documented static + SID16 bypass role may be initialized only from a disabled stream with + no valid root; an existing bypass remains unchanged. PIODMA and other + reserved roles are not modified. This does not authorize a controller + reset or takeover of active page tables. Both the radio DART and PCI + host describe the same two ordinary streams. reg: minItems: 3 - maxItems: 10 + maxItems: 14 reg-names: minItems: 3 - maxItems: 10 + maxItems: 14 ranges: minItems: 2 @@ -84,11 +107,30 @@ properties: minItems: 1 maxItems: 4 + interrupt-names: + minItems: 1 + maxItems: 4 + items: + pattern: "^port[0-3]$" + + nonposted-mmio: true + + apple,msi-vector-base: + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 255 + description: First global vector targeted by this host's local MSI map. + msi-parent: true msi-ranges: maxItems: 1 + apple,msi-vector-base: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + First vector of this PCIe-C controller's slice of the MSI window it + shares with the internal PCIe controller. + iommu-map: true iommu-map-mask: true @@ -105,7 +147,11 @@ properties: apple,pciec-kernel-init: type: boolean - description: The kernel initializes the tunneled controller from reset. + description: | + The kernel initializes the tunneled controller when firmware has not + already initialized it. The compatible selects the initialization + sequence. T600x and T8112 use the Intr2AXI bridge and require tunables + for their associated DART before the tunnel children can be populated. apple,tunable-rc: $ref: '#/$defs/tunable' @@ -142,6 +188,19 @@ patternProperties: description: Optional GPIO to power on the device maxItems: 1 + apple,link-trained-by-bootloader: + type: boolean + description: | + Firmware has initialized power, shared registers, PHY and clocks, + deasserted PERST and trained the link. The OS must validate and adopt + this state, with no reset, tunable replay, clock policy changes or + retraining fallback. Firmware must leave endpoint DMA and MSI/MSI-X + disabled and the port RID table without valid entries. Firmware + retains responsibility for power and external reset throughout use. + On T6050, each such root port 00:.0 is admitted with one + endpoint, device 0 function 0, on the secondary bus the firmware + numbered, at Gen3 x1. + interrupt-controller: true apple,tunable: @@ -169,8 +228,62 @@ required: unevaluatedProperties: false allOf: + - if: + properties: + compatible: + contains: + const: apple,t8132-pciec + then: + required: + - apple,msi-vector-base - $ref: /schemas/pci/pci-host-bridge.yaml# - $ref: /schemas/interrupt-controller/msi-controller.yaml# + - if: + properties: + compatible: + contains: + const: apple,t6050-pcie + then: + properties: + reg: + minItems: 5 + maxItems: 14 + reg-names: + description: + config and rc, then port, phy and intr2axi for each + bootloader-trained root port 00:.0. + minItems: 5 + maxItems: 14 + items: + pattern: "^(config|rc|(port|phy|intr2axi)[0-3])$" + interrupts: + maxItems: 4 + interrupt-names: + items: + pattern: "^port[0-3]$" + apple,msi-vector-base: + const: 64 + power-domains: false + required: + - interrupt-names + - apple,msi-vector-base + - iommu-map + patternProperties: + "^pci@": + properties: + reset-gpios: + description: + The PERST# line of a port whose endpoint needs a root-port + reset after the bootloader trained the link. Left as the + bootloader set it until that reset. + pwren-gpios: false + required: + - apple,link-trained-by-bootloader + else: + patternProperties: + "^pci@": + properties: + apple,link-trained-by-bootloader: false - if: properties: compatible: @@ -213,7 +326,7 @@ allOf: properties: reg: minItems: 5 - maxItems: 7 + maxItems: 9 reg-names: oneOf: - items: @@ -230,6 +343,16 @@ allOf: - const: debug - const: ltssm - enum: [intr2axi, oe-fabric] + - items: + - const: config + - const: rc + - const: port0 + - const: fabric + - const: debug + - const: ioa + - const: ltssm + - const: intr2axi + - const: oe-fabric interrupts: maxItems: 1 ranges: @@ -269,26 +392,44 @@ allOf: interrupts: maxItems: 3 else: + if: + properties: + compatible: + not: + contains: + const: apple,t6050-pcie + then: + properties: + reg: + maxItems: 10 + reg-names: + minItems: 3 + items: + - const: config + - const: rc + - const: port0 + - const: port1 + - const: port2 + - const: port3 + - const: phy0 + - const: phy1 + - const: phy2 + - const: phy3 + - if: properties: - reg-names: - minItems: 3 - items: - - const: config - - const: rc - - const: port0 - - const: port1 - - const: port2 - - const: port3 - - const: phy0 - - const: phy1 - - const: phy2 - - const: phy3 + compatible: + not: + contains: + const: apple,t6050-pcie + then: + patternProperties: + "^pci@": + required: + - reset-gpios patternProperties: "^pci@": properties: apple,tunable: false - required: - - reset-gpios examples: - | diff --git a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml index d86c8b68d1b48..7800f0f106f2c 100644 --- a/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml +++ b/Documentation/devicetree/bindings/power/apple,pmgr-pwrstate.yaml @@ -79,6 +79,16 @@ properties: Forces this power domain to always be powered up. type: boolean + apple,no-auto-pm: + description: | + Never enables automatic power management (auto-enable) for this + domain: once powered up it stays at its requested state until + Linux changes it. Required for a coprocessor CPU domain whose + firmware is started cold by Linux and must not be clock or power + gated behind its back, such as the T6030 external display + coprocessors' CPU domains. + type: boolean + apple,firmware-owned: description: The boot loader hands over the coprocessor in this domain running, and its @@ -86,6 +96,17 @@ properties: domain on and never changes its power target, automatic gating or reset. type: boolean + apple,inherited-on: + description: | + The boot loader leaves this domain powered, with the coprocessor it + feeds running (the internal display chain of the T6030, T6031 and + T8122 MacBook Pros: DISP_SYS, DISP_FE and DISP_CPU). The power state + driver treats it as any domain found on; the display handoff requires + the marker before it adds the PMP and sets apple,min-state on the + chain, so the PMP cannot power the running display coprocessor down. + Unlike apple,firmware-owned, the domain's register is still written. + type: boolean + apple,min-state: description: Specifies the minimum power state for auto-PM. diff --git a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst index a5f0f16a9d46f..6ee033a52c9f7 100644 --- a/Documentation/networking/device_drivers/wifi/mt7932-neo.rst +++ b/Documentation/networking/device_drivers/wifi/mt7932-neo.rst @@ -3,9 +3,9 @@ MediaTek MT7932 radio on the MacBook Neo ======================================== -J700 MT7932 Wi-Fi uses cfg80211 and NetworkManager with firmware-managed -WPA2-CCMP authentication. Bluetooth uses a transport on the radio's second -PCIe function. +J700 MT7932 Wi-Fi uses cfg80211 with firmware-managed WPA2-PSK, WPA3-SAE +and Enhanced Open (OWE) authentication on 2.4 and 5 GHz. Bluetooth uses a +transport on the radio's second PCIe function. The Wi-Fi driver carries configuration, scan IE and WPA2 AKM compatibility fixes on top of its original import. Existing source notices are retained. @@ -33,7 +33,7 @@ associated, but never probes or joins an access point there, and scans made while associated skip them. A channel whose restrictions the driver cannot apply (reduced power, no OFDM, PSD limits) is disabled rather than used, and so is a channel the country package does not permit: every power limit of -that channel in ``policy/.bin`` is left undefined. +that channel in the applied country package is left undefined. Use the ordinary cfg80211 regulatory database and applicable country policy. The validated first-admission fallback is kernel country 00 with firmware XZ. @@ -68,6 +68,103 @@ Use a saved NetworkManager WPA2 profile matching the current interface. KDE audio requires BlueZ, PipeWire, its PulseAudio compatibility service, WirePlumber and Bluetooth audio plugins in the logged-in user's session. +Country packages +---------------- + +Every regulatory change selects a country package again, without a reload. +For country CC the driver uses the first of these that exists and is valid: + +1. ``policy/CC.bin``: J7RP version 1 (2.4 and 5 GHz, at most 16 KiB); +2. ``policy/world-XZ.bin``: the world package. + +Country 00 and cfg80211's numeric codes use only the world package. + +A country without a usable package of its own falls back to the world +package and logs one ``REGULATORY_FALLBACK`` line. cfg80211 keeps the +requested country and its channel list; the firmware gets domain XZ and +only the channels both the country and the world package permit. +Installing the country's package takes effect on the next regulatory +change. Only a missing or invalid world package leaves the radio without a +regulatory domain. + +The applied package can be read back, as root, from +``/sys/kernel/debug/ieee80211/phyN/mt7932_policy``: requested country, +firmware domain, source directory and path, J7RP version, size, the first +eight bytes of its SHA-256, the fallback reason and the result of each +lookup. + +Station security +---------------- + +The driver advertises PSK and SAE with CCMP only. Firmware performs SAE +authentication and the four-way handshake and owns PTK, GTK and IGTK; +there is no host key-install interface or external-auth exchange. + +SAE offload accepts nl80211 WPA version 3, the combined version 2/3 mask, +and version 2 for older offload clients. PSK requires the version 2 bit. +The first AKM always selects the firmware arm; a version mask does not +permit security fallback. + +For SAE, userspace supplies SAE authentication, an 8-63 byte password and +required or optional management frame protection (MFP). Firmware uses required +MFP for the SAE arm. On a PSK+SAE transition network, the first AKM supplied +by userspace selects the arm: PSK keeps the existing binary-PMK request and +disables MFP, while SAE uses the password and required MFP. PSK with required +MFP and PSK-SHA256 are unsupported. + +BIP-CMAC-128 is not advertised because WPA2 with required MFP has not been +implemented or qualified. Advertising BIP would also make supplicants request +required MFP on WPA2 networks. iwd requires BIP-CMAC-128 before selecting SAE, +so it remains WPA2-only with this driver, including on transition networks; +it cannot join a pure-SAE network. SAE offload support here targets +NetworkManager with wpa_supplicant, pending the hardware qualification below. + +The standard SAE PWE settings (unspecified, hunting-and-pecking, +hash-to-element and both) are accepted; firmware chooses the exchange without +a host PWE field. Exact PWE policy behavior on 2.4 and 5 GHz still needs +qualification. SAE-PK, SAE-EXT-KEY and additional ciphers are not +advertised. + +J700 hardware testing with NetworkManager and wpa_supplicant on a PSK+SAE +transition network has confirmed, on 2.4 GHz channel 6 and 5 GHz channel 149: + +* SAE with required MFP (``key-mgmt=sae``), including DHCP, DNS and HTTPS; +* PSK with MFP disabled, and a ``wpa-psk`` profile with optional MFP, which + joins the transition network as PSK; +* repeated reconnection through NetworkManager after each change of profile. + +The request layout, transition admission and PMK derivation also have +automated tests. A pure WPA3 network, exact PWE policy behavior, WPA2 +throughput regression comparisons and long-duration reliability still need +qualification. 2.4 GHz throughput is lower than 5 GHz with every security +mode. + +Enhanced Open (OWE) +------------------- + +The station advertises the OWE AKM and ``OWE_OFFLOAD`` on the supported 2.4 +and 5 GHz channels. Use an OWE profile with CCMP, open-system authentication +and required management frame protection, without a PSK or SAE password. +Firmware performs the Diffie-Hellman exchange, generates RSN information and +owns the handshake and key installation. Userspace must honor the offload +feature instead of expecting a host-generated Diffie-Hellman element in the +association response. + +Before the first OWE association of a firmware session, the driver reserves +temporary live peer IDs and performs a typed disconnect and queue cleanup. +An OWE authentication timeout, firmware rejection or firmware disconnect +then permits one retry inside the connect worker. Before retrying, the +driver waits for its typed disconnect and retires queues and the old peer +through the normal ownership fences. A +failed retirement, cancellation or RF failure ends the request. The retry +shares a ten-second association-event deadline and programs the station +identity again before association. No further retry is made by the worker. + +For OWE transition mode, the supplicant selects the encrypted BSS and can use +a directed scan to find its hidden SSID. Pure OWE, transition mode and +first-authentication behavior after a cold boot still require physical +qualification on 2.4 and 5 GHz. + Local firmware inputs --------------------- @@ -85,8 +182,9 @@ Wi-Fi requests the following files under ``mediatek/mt7932/``: * ``config-original.bin``: the bounded J7CF package with the original 64 or 65 configuration records, including all required keys. * ``policy/world-XZ.bin`` and, when available, the matching country package - under ``policy/``: J7RP containers with original-derived modes and power - tables. Relabeling a different country's payload is unsupported. + under ``policy/`` (see `Country packages`_): J7RP containers with + original-derived modes and power tables. Relabeling a different country's + payload is unsupported. Bluetooth requests these files under ``mediatek/``: @@ -114,6 +212,35 @@ Use ``--band bg`` for 2.4 GHz. Require ``WIFI_PHYSICAL_NETWORK_E2E_PASS`` in the JSON artifact printed on completion. The artifact records three traffic samples and response hashes, omitting network names, addresses and credentials. +Receive diagnostics +------------------- + +``ethtool -S `` reports lifetime host counters for malformed, +unsupported and integrity-rejected RX records, wrong peers, unprotected +data on protected connections, retry duplicates and allocation failures. +Take before/after differences for each traffic sample; reconnecting does +not reset these counters. Descriptor rejection counters include management +records and off-channel traffic. The other counters describe the admitted +peer's Ethernet receive path. + +Retry and A-MSDU counters count received MSDU records, including split +A-MSDU subframes, before protection and duplicate filtering. They are not +over-air MPDU counts, BA-window sizes, TX retries or firmware ACK counters. +Duplicate suppression records sequences only after the protection check, +so a rejected unprotected frame cannot suppress a later decrypted retry. + +Use the current RX bitrate from ``iw dev station dump`` while +traffic is running. The driver expires that sample after one second; an +idle or unsupported RX vector can leave it absent. TX bitrate is not +reported because the firmware's TX-vector query cannot attest peer +identity or freshness. Missing TX bitrate is not a zero-rate measurement. + +Dynamic debug messages ``ASSOC_HT_CAPABILITIES``, +``ASSOC_HE_CAPABILITIES`` and ``ASSOC_START`` describe the firmware-generated +station capabilities and channel event. ``POWER_SAVE_SUBMITTED`` records +the requested mode and submission status, without asserting firmware +readback. These messages contain neither credentials nor packet payloads. + Limitations ----------- @@ -132,7 +259,8 @@ Limitations transport. * The Bluetooth software queue limit does not provide HCI backpressure; saturation can drop an accounted frame. -* Arbitrary scan IEs, WPA3/SAE, required MFP, AP/P2P, general country-package - generation, roaming and long-duration reliability are unqualified. +* Arbitrary scan IEs, AP/P2P, general country-package generation, roaming and + long-duration reliability are unqualified. SAE and required MFP qualification + are described under Station security. * SCO/headset microphone, LE Audio/ISO and simultaneous headset audio are unqualified. diff --git a/Documentation/networking/strparser.rst b/Documentation/networking/strparser.rst index 372106b61e651..d0804c88ad0a0 100644 --- a/Documentation/networking/strparser.rst +++ b/Documentation/networking/strparser.rst @@ -77,7 +77,7 @@ Functions strp_done is called to release any resources held by the stream parser instance. This must be called after the stream processor - has been stopped. + has been stopped. The socket lock must not be held. :: diff --git a/MAINTAINERS b/MAINTAINERS index 85f49f6c6cc48..1d10b8f9158ed 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2076,8 +2076,10 @@ F: include/linux/soc/apple/pci-apple-piodma.h F: rust/kernel/iio/common/aop_sensors.rs F: rust/kernel/soc/apple/aop.rs F: sound/soc/apple/t8140_aop_audio.rs +F: tools/asahi/mt7932-*.py F: tools/testing/apple/ F: tools/testing/kunit/configs/apple_firmware.config +F: tools/testing/kunit/configs/mt7932.config F: tools/testing/selftests/drivers/gpu/apple-dcp/ APPLETALK NETWORK LAYER @@ -2629,6 +2631,10 @@ F: Documentation/devicetree/bindings/dma/apple,sio.yaml F: Documentation/devicetree/bindings/sound/apple,* F: Documentation/devicetree/bindings/sound/cirrus,cs42l84.yaml F: Documentation/devicetree/bindings/sound/trivial-codec.yaml +F: drivers/dma/apple-sio-probe-test.c +F: drivers/dma/apple-sio-probe-test.dtso +F: drivers/dma/apple-sio-t6030-test.dtso +F: drivers/dma/apple-sio-test.c F: drivers/dma/apple-sio.c F: sound/soc/apple/* F: sound/soc/codecs/cs42l83-i2c.c @@ -4431,6 +4437,12 @@ M: Ryan Murray S: Maintained F: tools/aurora-display-notifier/ +AURORA KERNEL CI +M: Chris Kearney <303316+iconidentify@users.noreply.github.com> +S: Maintained +F: .github/workflows/kernel-ci*.yml +F: tools/aurora-ci/ + AUTOFDO BUILD M: Rong Xu M: Han Shen diff --git a/Makefile b/Makefile index be413b167b244..4b60c83ab75c1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 7 PATCHLEVEL = 2 -SUBLEVEL = 10 +SUBLEVEL = 11 EXTRAVERSION = NAME = Baby Opossum Posse diff --git a/arch/arm64/boot/dts/apple/Makefile b/arch/arm64/boot/dts/apple/Makefile index d15eeeb9ece3f..24571db177c07 100644 --- a/arch/arm64/boot/dts/apple/Makefile +++ b/arch/arm64/boot/dts/apple/Makefile @@ -114,3 +114,5 @@ dtb-$(CONFIG_ARCH_APPLE) += t8132-j773g.dtb dtb-$(CONFIG_ARCH_APPLE) += t8140-j700.dtb dtb-$(CONFIG_ARCH_APPLE) += t8142-j813.dtb dtb-$(CONFIG_ARCH_APPLE) += t8152-j873g.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6050-j714c.dtb +dtb-$(CONFIG_ARCH_APPLE) += t6050-k114c.dtb diff --git a/arch/arm64/boot/dts/apple/gen-t6050-j714c-pmp-tunables.py b/arch/arm64/boot/dts/apple/gen-t6050-j714c-pmp-tunables.py new file mode 100644 index 0000000000000..0f761be2602b6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/gen-t6050-j714c-pmp-tunables.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ OR MIT +# +# Generate the J714c per-machine PMP tunables dtsi from the Mac's ADT. +# +# Every /arm-io/pmp0/iop-pmp0-nub property (except ADT structural metadata) +# becomes apple,tunable- with the raw ADT bytes, per the naming in +# omacom-m1n1 src/kboot_pmp_t8122.c and the drivers/soc/apple/t6030-j516s-pmp.dtso +# precedent. apple,board-id / apple,dram-vendor-id live on the pmp node already; +# apple,dram-capacity / apple,pio-ranges are not carried (as in the precedent). +# +# Usage: gen-t6050-j714c-pmp-tunables.py +import sys +sys.path.insert(0, "/home/ed/dev/omarchy/forks/omacom-m1n1/proxyclient") +from m1n1.adt import load_adt, build_prop + +adt_path, out_path = sys.argv[1], sys.argv[2] +adt = load_adt(open(adt_path, "rb").read()) +nub = adt["/arm-io/pmp0/iop-pmp0-nub"] +chosen = adt["/chosen"] + +SKIP = {"name", "compatible", "AAPL,phandle", "region-base", "region-size", + "segment-names", "segment-ranges", "firmware-name"} + +def raw(node, name): + v = node._properties[name] + if name in node._types: + t, _ = node._types[name] + return bytes(build_prop(node._path, name, v, t=t)) + return bytes(v) + +def s(node, key): + v = node._properties.get(key) + if isinstance(v, str): + return v + if isinstance(v, (bytes, bytearray)): + return bytes(v).split(b"\0")[0].decode("ascii", "replace") + return str(v) + +def hexb(b): + return " ".join("%02x" % x for x in b) + +out = [] +out.append("// SPDX-License-Identifier: GPL-2.0+ OR MIT") +out.append("/*") +out.append(" * T6050 J714c per-machine PMP tunables -- GENERATED, do not edit by hand.") +out.append(" *") +out.append(" * Every /arm-io/pmp0/iop-pmp0-nub property (bar ADT structural metadata) as") +out.append(" * apple,tunable-, per omacom-m1n1 src/kboot_pmp_t8122.c and the") +out.append(" * drivers/soc/apple/t6030-j516s-pmp.dtso precedent. Board-specific: include") +out.append(" * only from t6050-j714c.dts, never the SoC dtsi.") +out.append(" *") +out.append(" * Source ADT: %s" % adt_path) +out.append(" * chip-id %#x, board-id %#x" % (chosen._properties.get("chip-id", 0), + chosen._properties.get("board-id", 0))) +out.append(" * firmware-version %r" % s(chosen, "firmware-version")) +out.append(" * system-firmware-version %r" % s(chosen, "system-firmware-version")) +out.append(" * iboot %r / %r" % (s(chosen, "iboot-stage-one-tag"), s(chosen, "iboot-stage-two-tag"))) +out.append(" * PMP firmware image %r (uuid %r)" % (s(nub, "firmware-name"), s(nub, "uuid"))) +out.append(" * Regenerate: gen-t6050-j714c-pmp-tunables.py j714c.adt t6050-j714c-pmp-tunables.dtsi") +out.append(" */") +out.append("") +out.append("&pmp {") +n = 0 +for name in nub._properties.keys(): + if name in SKIP: + continue + out.append("\tapple,tunable-%s = [%s];" % (name, hexb(raw(nub, name)))) + n += 1 +out.append("};") + +open(out_path, "w").write("\n".join(out) + "\n") +print("wrote %s: %d tunables (skipped %d structural)" % + (out_path, n, sum(1 for k in nub._properties if k in SKIP))) diff --git a/arch/arm64/boot/dts/apple/t6030-dcp.dtsi b/arch/arm64/boot/dts/apple/t6030-dcp.dtsi index 9dacebbc8de10..461533b3dc382 100644 --- a/arch/arm64/boot/dts/apple/t6030-dcp.dtsi +++ b/arch/arm64/boot/dts/apple/t6030-dcp.dtsi @@ -144,8 +144,14 @@ <&atcphy2 PHY_TYPE_DP>; phy-names = "typec0", "typec1", "typec2"; mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>, - <&atcphy2_xbar 0>; - mux-control-names = "typec0", "typec1", "typec2"; + <&atcphy2_xbar 0>, + <&atcphy0_xbar 1>, <&atcphy0_xbar 2>, + <&atcphy1_xbar 1>, <&atcphy1_xbar 2>, + <&atcphy2_xbar 1>, <&atcphy2_xbar 2>; + mux-control-names = "typec0", "typec1", "typec2", + "typec0-dpin0", "typec0-dpin1", + "typec1-dpin0", "typec1-dpin1", + "typec2-dpin0", "typec2-dpin1"; apple,connector-type = "USB-C"; apple,dcp-index = <1>; apple,typec-dptx-phys = <0 1 2>; @@ -243,8 +249,14 @@ <&atcphy2 PHY_TYPE_DP>; phy-names = "typec0", "typec1", "typec2"; mux-controls = <&atcphy0_xbar 0>, <&atcphy1_xbar 0>, - <&atcphy2_xbar 0>; - mux-control-names = "typec0", "typec1", "typec2"; + <&atcphy2_xbar 0>, + <&atcphy0_xbar 1>, <&atcphy0_xbar 2>, + <&atcphy1_xbar 1>, <&atcphy1_xbar 2>, + <&atcphy2_xbar 1>, <&atcphy2_xbar 2>; + mux-control-names = "typec0", "typec1", "typec2", + "typec0-dpin0", "typec0-dpin1", + "typec1-dpin0", "typec1-dpin1", + "typec2-dpin0", "typec2-dpin1"; apple,connector-type = "USB-C"; apple,dcp-index = <2>; apple,typec-dptx-phys = <0 1 2>; diff --git a/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi index 36050ececce29..6be636c3577be 100644 --- a/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi +++ b/arch/arm64/boot/dts/apple/t6030-pmgr.dtsi @@ -197,7 +197,11 @@ #power-domain-cells = <0>; #reset-cells = <0>; label = "sio_cpu"; - power-domains = <&ps_sio>; + /* + * The SIO firmware sets up the SPI, UART and audio DMA shims + * however few of them its clients use. + */ + power-domains = <&ps_sio &ps_uart_p &ps_spi_p &ps_audio_p>; }; ps_fpwm0: power-controller@1d8 { diff --git a/arch/arm64/boot/dts/apple/t6030.dtsi b/arch/arm64/boot/dts/apple/t6030.dtsi index 557949060da4c..cedc08f69e1d3 100644 --- a/arch/arm64/boot/dts/apple/t6030.dtsi +++ b/arch/arm64/boot/dts/apple/t6030.dtsi @@ -585,6 +585,33 @@ power-domains = <&ps_sio>; }; + sio_mbox: mbox@289c08000 { + compatible = "apple,t6030-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x89c08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_sio_cpu>; + status = "disabled"; + }; + + /* Enabled by the boot loader with its firmware and parameters. */ + sio: sio@289c00000 { + compatible = "apple,t6030-sio", "apple,sio"; + reg = <0x2 0x89c00000 0x0 0x8000>; + dma-channels = <128>; + #dma-cells = <1>; + mboxes = <&sio_mbox>; + iommus = <&sio_dart 0>; + power-domains = <&ps_sio_cpu>; + status = "disabled"; + }; + admac: dma-controller@28b200000 { compatible = "apple,t6030-admac", "apple,t8122-admac"; reg = <0x2 0x8b200000 0x0 0x34000>; diff --git a/arch/arm64/boot/dts/apple/t603x-j514s-j516s-hdmi.dtsi b/arch/arm64/boot/dts/apple/t603x-j514s-j516s-hdmi.dtsi index 0a92185de199d..10314dba091bc 100644 --- a/arch/arm64/boot/dts/apple/t603x-j514s-j516s-hdmi.dtsi +++ b/arch/arm64/boot/dts/apple/t603x-j514s-j516s-hdmi.dtsi @@ -10,8 +10,20 @@ * * The processor stays a hybrid pipeline: while no HDMI display is attached * it can still be lent to a USB-C port. + * + * The HDMI port's audio is the processor's audio service with DPA1, the + * DisplayPort audio interface its port@0 links (no AP-side registers): SIO + * channel 0x66 carries the samples, as on the M1 Pro/Max HDMI port. The + * loader reserves the SIO firmware and enables the SIO with its parameters. */ +/ { + aliases { + /* The boot loader looks the SIO up by this alias. */ + sio = &sio; + }; +}; + &atcphy3 { status = "okay"; @@ -44,8 +56,14 @@ <&atcphy1 PHY_TYPE_DP>, <&atcphy2 PHY_TYPE_DP>; phy-names = "dp-phy", "typec0", "typec1", "typec2"; mux-controls = <&atcphy3_xbar 0>, <&atcphy0_xbar 0>, - <&atcphy1_xbar 0>, <&atcphy2_xbar 0>; - mux-control-names = "dp-xbar", "typec0", "typec1", "typec2"; + <&atcphy1_xbar 0>, <&atcphy2_xbar 0>, + <&atcphy0_xbar 1>, <&atcphy0_xbar 2>, + <&atcphy1_xbar 1>, <&atcphy1_xbar 2>, + <&atcphy2_xbar 1>, <&atcphy2_xbar 2>; + mux-control-names = "dp-xbar", "typec0", "typec1", "typec2", + "typec0-dpin0", "typec0-dpin1", + "typec1-dpin0", "typec1-dpin1", + "typec2-dpin0", "typec2-dpin1"; mux-index = <0>; apple,dptx-phy = <3>; @@ -53,6 +71,13 @@ #address-cells = <1>; #size-cells = <0>; + port@0 { + reg = <0>; + dcpext_audio: endpoint { + remote-endpoint = <&dpaudio1_dcpext>; + }; + }; + port@1 { reg = <1>; dcpext_dpout: endpoint { @@ -62,6 +87,33 @@ }; }; +/* DPA1: the HDMI port's DisplayPort audio, SIO channel 0x66. */ +&soc { + dpaudio1: audio-controller-1 { + compatible = "apple,t6030-dpaudio", "apple,dpaudio"; + dmas = <&sio 0x66>; + dma-names = "tx"; + power-domains = <&ps_dpa1>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dpaudio1_dcpext: endpoint { + remote-endpoint = <&dcpext_audio>; + }; + }; + }; + }; +}; + +/* The SIO's mailbox; the boot loader enables the SIO with its firmware. */ +&sio_mbox { + status = "okay"; +}; + /* * ATC3 has no USB consumer that would hold its common block on, so across * system sleep it would power down and lose the HDMI PHY state. diff --git a/arch/arm64/boot/dts/apple/t6050-base.dtsi b/arch/arm64/boot/dts/apple/t6050-base.dtsi new file mode 100644 index 0000000000000..6901ba12eb3ea --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-base.dtsi @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Nodes common for T6050 family SoCs (M5 Max) + * + * Copyright The Asahi Linux Contributors + */ + +/ { + #address-cells = <2>; + #size-cells = <2>; + + cpus { + /* + * cpu reg is the MPIDR affinity. On T6050 the boot core reads + * MPIDR_EL1 0x80040000: Aff2 is 4, cluster in Aff1, core in Aff0. + * Clusters 1 and 2 follow that pattern; their cores do not start today. + */ + #address-cells = <2>; + #size-cells = <0>; + + cpu-map { + cluster0 { + core0 { cpu = <&cpu_m00>; }; + core1 { cpu = <&cpu_m01>; }; + core2 { cpu = <&cpu_m02>; }; + core3 { cpu = <&cpu_m03>; }; + core4 { cpu = <&cpu_m04>; }; + core5 { cpu = <&cpu_m05>; }; + }; + + cluster1 { + core0 { cpu = <&cpu_m10>; }; + core1 { cpu = <&cpu_m11>; }; + core2 { cpu = <&cpu_m12>; }; + core3 { cpu = <&cpu_m13>; }; + core4 { cpu = <&cpu_m14>; }; + core5 { cpu = <&cpu_m15>; }; + }; + + cluster2 { + core0 { cpu = <&cpu_p00>; }; + core1 { cpu = <&cpu_p01>; }; + core2 { cpu = <&cpu_p02>; }; + core3 { cpu = <&cpu_p03>; }; + core4 { cpu = <&cpu_p04>; }; + core5 { cpu = <&cpu_p05>; }; + }; + }; + + /* + * T6050 (J714c ADT): 18 CPUs, three 6-core clusters. Clusters 0 and 1 + * are cluster-type "M", cluster 2 is cluster-type "P". reg encodes + * cluster << 8 | core. MIDR part 0x68 ("Sotra C" family). + */ + cpu_m00: cpu@0 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40000>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&mcluster0_opp>; + performance-domains = <&cpufreq_m0>; + }; + + cpu_m01: cpu@1 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40001>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&mcluster0_opp>; + performance-domains = <&cpufreq_m0>; + }; + + cpu_m02: cpu@2 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40002>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&mcluster0_opp>; + performance-domains = <&cpufreq_m0>; + }; + + cpu_m03: cpu@3 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40003>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&mcluster0_opp>; + performance-domains = <&cpufreq_m0>; + }; + + cpu_m04: cpu@4 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40004>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&mcluster0_opp>; + performance-domains = <&cpufreq_m0>; + }; + + cpu_m05: cpu@5 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40005>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_0>; + operating-points-v2 = <&mcluster0_opp>; + performance-domains = <&cpufreq_m0>; + }; + + cpu_m10: cpu@100 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40100>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_1>; + operating-points-v2 = <&mcluster1_opp>; + performance-domains = <&cpufreq_m1>; + }; + + cpu_m11: cpu@101 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40101>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_1>; + operating-points-v2 = <&mcluster1_opp>; + performance-domains = <&cpufreq_m1>; + }; + + cpu_m12: cpu@102 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40102>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_1>; + operating-points-v2 = <&mcluster1_opp>; + performance-domains = <&cpufreq_m1>; + }; + + cpu_m13: cpu@103 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40103>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_1>; + operating-points-v2 = <&mcluster1_opp>; + performance-domains = <&cpufreq_m1>; + }; + + cpu_m14: cpu@104 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40104>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_1>; + operating-points-v2 = <&mcluster1_opp>; + performance-domains = <&cpufreq_m1>; + }; + + cpu_m15: cpu@105 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x40105>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_1>; + operating-points-v2 = <&mcluster1_opp>; + performance-domains = <&cpufreq_m1>; + }; + + cpu_p00: cpu@200 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x10200>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_2>; + operating-points-v2 = <&pcluster0_opp>; + performance-domains = <&cpufreq_p0>; + }; + + cpu_p01: cpu@201 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x10201>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_2>; + operating-points-v2 = <&pcluster0_opp>; + performance-domains = <&cpufreq_p0>; + }; + + cpu_p02: cpu@202 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x10202>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_2>; + operating-points-v2 = <&pcluster0_opp>; + performance-domains = <&cpufreq_p0>; + }; + + cpu_p03: cpu@203 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x10203>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_2>; + operating-points-v2 = <&pcluster0_opp>; + performance-domains = <&cpufreq_p0>; + }; + + cpu_p04: cpu@204 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x10204>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_2>; + operating-points-v2 = <&pcluster0_opp>; + performance-domains = <&cpufreq_p0>; + }; + + cpu_p05: cpu@205 { + compatible = "apple,sotra"; + device_type = "cpu"; + reg = <0x0 0x10205>; + enable-method = "spin-table"; + cpu-release-addr = <0 0>; /* to be filled by loader */ + next-level-cache = <&l2_cache_2>; + operating-points-v2 = <&pcluster0_opp>; + performance-domains = <&cpufreq_p0>; + }; + + l2_cache_0: l2-cache-0 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + }; + + l2_cache_1: l2-cache-1 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + }; + + l2_cache_2: l2-cache-2 { + compatible = "cache"; + cache-level = <2>; + cache-unified; + }; + }; + + timer { + compatible = "arm,armv8-timer"; + interrupt-parent = <&aic>; + interrupt-names = "phys", "virt", "hyp-phys", "hyp-virt"; + interrupts = , + , + , + ; + }; + + clkref: clock-ref { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <24000000>; + clock-output-names = "clkref"; + }; + + clk_200m: clock-200m { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <200000000>; + clock-output-names = "clk_200m"; + }; + + /* + * This is a fabulated representation of the input clock + * to NCO since we don't know the true clock tree. + */ + nco_clkref: clock-ref-nco { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-output-names = "nco_ref"; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + }; + + /* + * Cluster P-states from the ADT: cluster 0 (M) voltage-states22-sram, + * cluster 1 (M) voltage-states23-sram, cluster 2 (P) voltage-states5-sram. + * The frequency in kHz per hardware P-state; opp-level is that P-state. + * No clock-latency-ns, not measured. + */ + mcluster0_opp: opp-table-0 { + compatible = "operating-points-v2"; + opp-shared; + opp01 { + opp-hz = /bits/ 64 <1344000000>; + opp-level = <1>; + }; + opp02 { + opp-hz = /bits/ 64 <1644000000>; + opp-level = <2>; + }; + opp03 { + opp-hz = /bits/ 64 <1992000000>; + opp-level = <3>; + }; + opp04 { + opp-hz = /bits/ 64 <2304000000>; + opp-level = <4>; + }; + opp05 { + opp-hz = /bits/ 64 <2652000000>; + opp-level = <5>; + }; + opp06 { + opp-hz = /bits/ 64 <2964000000>; + opp-level = <6>; + }; + opp07 { + opp-hz = /bits/ 64 <3240000000>; + opp-level = <7>; + }; + opp08 { + opp-hz = /bits/ 64 <3504000000>; + opp-level = <8>; + }; + opp09 { + opp-hz = /bits/ 64 <3696000000>; + opp-level = <9>; + }; + opp10 { + opp-hz = /bits/ 64 <3876000000>; + opp-level = <10>; + }; + opp11 { + opp-hz = /bits/ 64 <4044000000>; + opp-level = <11>; + }; + opp12 { + opp-hz = /bits/ 64 <4176000000>; + opp-level = <12>; + }; + opp13 { + opp-hz = /bits/ 64 <4284000000>; + opp-level = <13>; + }; + opp14 { + opp-hz = /bits/ 64 <4308000000>; + opp-level = <14>; + }; + opp15 { + opp-hz = /bits/ 64 <4380000000>; + opp-level = <15>; + }; + }; + + + mcluster1_opp: opp-table-1 { + compatible = "operating-points-v2"; + opp-shared; + opp01 { + opp-hz = /bits/ 64 <1344000000>; + opp-level = <1>; + }; + opp02 { + opp-hz = /bits/ 64 <1644000000>; + opp-level = <2>; + }; + opp03 { + opp-hz = /bits/ 64 <1992000000>; + opp-level = <3>; + }; + opp04 { + opp-hz = /bits/ 64 <2304000000>; + opp-level = <4>; + }; + opp05 { + opp-hz = /bits/ 64 <2652000000>; + opp-level = <5>; + }; + opp06 { + opp-hz = /bits/ 64 <2964000000>; + opp-level = <6>; + }; + opp07 { + opp-hz = /bits/ 64 <3240000000>; + opp-level = <7>; + }; + opp08 { + opp-hz = /bits/ 64 <3504000000>; + opp-level = <8>; + }; + opp09 { + opp-hz = /bits/ 64 <3696000000>; + opp-level = <9>; + }; + opp10 { + opp-hz = /bits/ 64 <3876000000>; + opp-level = <10>; + }; + opp11 { + opp-hz = /bits/ 64 <4044000000>; + opp-level = <11>; + }; + opp12 { + opp-hz = /bits/ 64 <4176000000>; + opp-level = <12>; + }; + opp13 { + opp-hz = /bits/ 64 <4284000000>; + opp-level = <13>; + }; + opp14 { + opp-hz = /bits/ 64 <4308000000>; + opp-level = <14>; + }; + opp15 { + opp-hz = /bits/ 64 <4380000000>; + opp-level = <15>; + }; + }; + + pcluster0_opp: opp-table-2 { + compatible = "operating-points-v2"; + opp-shared; + opp01 { + opp-hz = /bits/ 64 <1308000000>; + opp-level = <1>; + }; + opp02 { + opp-hz = /bits/ 64 <1620000000>; + opp-level = <2>; + }; + opp03 { + opp-hz = /bits/ 64 <1980000000>; + opp-level = <3>; + }; + opp04 { + opp-hz = /bits/ 64 <2292000000>; + opp-level = <4>; + }; + opp05 { + opp-hz = /bits/ 64 <2580000000>; + opp-level = <5>; + }; + opp06 { + opp-hz = /bits/ 64 <2880000000>; + opp-level = <6>; + }; + opp07 { + opp-hz = /bits/ 64 <3180000000>; + opp-level = <7>; + }; + opp08 { + opp-hz = /bits/ 64 <3432000000>; + opp-level = <8>; + }; + opp09 { + opp-hz = /bits/ 64 <3648000000>; + opp-level = <9>; + }; + opp10 { + opp-hz = /bits/ 64 <3828000000>; + opp-level = <10>; + }; + opp11 { + opp-hz = /bits/ 64 <3984000000>; + opp-level = <11>; + }; + opp12 { + opp-hz = /bits/ 64 <4104000000>; + opp-level = <12>; + }; + opp13 { + opp-hz = /bits/ 64 <4188000000>; + opp-level = <13>; + }; + opp14 { + opp-hz = /bits/ 64 <4236000000>; + opp-level = <14>; + }; + opp15 { + opp-hz = /bits/ 64 <4284000000>; + opp-level = <15>; + }; + opp16 { + opp-hz = /bits/ 64 <4308000000>; + opp-level = <16>; + }; + opp17 { + opp-hz = /bits/ 64 <4332000000>; + opp-level = <17>; + }; + opp18 { + opp-hz = /bits/ 64 <4428000000>; + opp-level = <18>; + }; + opp19 { + opp-hz = /bits/ 64 <4512000000>; + opp-level = <19>; + }; + opp20 { + opp-hz = /bits/ 64 <4608000000>; + opp-level = <20>; + }; + }; + +}; diff --git a/arch/arm64/boot/dts/apple/t6050-die0.dtsi b/arch/arm64/boot/dts/apple/t6050-die0.dtsi new file mode 100644 index 0000000000000..9c354ac761792 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-die0.dtsi @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Devices on the Apple T6050 "M5 Max" SoC (single die) + * + * Register windows from the live J714c device tree (dumps/j714c.adt). + * The six pmgr register blocks are declared here; their power domains + * live in t6050-pmgr.dtsi. + * + * Copyright The Asahi Linux Contributors + */ + +&soc { + aic: interrupt-controller@280400000 { + compatible = "apple,t6050-aic3", "apple,t8142-aic3", "apple,t8122-aic3"; + #interrupt-cells = <3>; + interrupt-controller; + + reg = <0x2 0x80400000 0x0 0x1cc000>, + <0x2 0x80440000 0x0 0x4000>; + reg-names = "core", "event"; + }; + + serial0: serial@505200000 { + compatible = "apple,s5l-uart"; + reg = <0x5 0x05200000 0x0 0x4000>; + reg-io-width = <4>; + interrupt-parent = <&aic>; + interrupts = ; + clocks = <&clkref>, <&clkref>; + clock-names = "uart", "clk_uart_baud0"; + }; + + pmgr: pmgr0: power-management@280600000 { + compatible = "apple,t6050-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x2 0x80600000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + pmgr1: power-management@280900000 { + compatible = "apple,t6050-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x2 0x80900000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + pmgr2: power-management@288300000 { + compatible = "apple,t6050-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x2 0x88300000 0 0xc000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + pmgr3: power-management@2300e80000 { + compatible = "apple,t6050-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x23 0xe80000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + pmgr4: power-management@2380600000 { + compatible = "apple,t6050-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x23 0x80600000 0 0xc000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + pmgr5: power-management@2388300000 { + compatible = "apple,t6050-pmgr", "apple,t8103-pmgr", "syscon", "simple-mfd"; + reg = <0x23 0x88300000 0 0x4000>; + #address-cells = <1>; + #size-cells = <1>; + }; + + cpufreq_m0: cpufreq@210e20000 { + compatible = "apple,t6050-cluster-cpufreq", "apple,cluster-cpufreq"; + reg = <0x2 0x10e20000 0x0 0x1000>; + #performance-domain-cells = <0>; + }; + + cpufreq_m1: cpufreq@211e20000 { + compatible = "apple,t6050-cluster-cpufreq", "apple,cluster-cpufreq"; + reg = <0x2 0x11e20000 0x0 0x1000>; + #performance-domain-cells = <0>; + }; + + cpufreq_p0: cpufreq@212e20000 { + compatible = "apple,t6050-cluster-cpufreq", "apple,cluster-cpufreq"; + reg = <0x2 0x12e20000 0x0 0x1000>; + #performance-domain-cells = <0>; + }; + + wdt: watchdog@28836c000 { + compatible = "apple,t6050-wdt", "apple,t8103-wdt"; + reg = <0x2 0x8836c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + }; + + fpwm0: pwm@505040000 { + compatible = "apple,t6050-fpwm", "apple,s5l-fpwm"; + reg = <0x5 0x05040000 0x0 0x4000>; + power-domains = <&ps_fpwm0>; + clocks = <&clkref>; + #pwm-cells = <2>; + status = "disabled"; + }; + + /* + * AP GPIO block. ADT /arm-io/gpio0: 0x9c000000 + 0x44000, 221 pins, + * seven AIC interrupts, no-restore-gpio-pins 5 6 7 112 141. The t8140 + * driver variant: slots in the block fault when read, so the regmap is + * not cache-initialised and stays inside the valid pins minus the + * reserved ranges (the firmware's no-restore pins, as t8140.dtsi). + * Enabled by the board DT together with its first users, the audio + * codec reset, shutdown and interrupt lines. + */ + pinctrl_ap: pinctrl@29c000000 { + compatible = "apple,t6050-pinctrl", "apple,t8140-pinctrl"; + reg = <0x2 0x9c000000 0x0 0x44000>; + power-domains = <&ps_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_ap 0 0 221>; + apple,npins = <221>; + gpio-reserved-ranges = <5 3>, <112 1>, <141 1>; + + interrupt-controller; + #interrupt-cells = <2>; + interrupt-parent = <&aic>; + interrupts = , + , + , + , + , + , + ; + status = "disabled"; + }; + + /* + * The NUB (always-on die) GPIO block. ADT nub-gpio0@881E4000: 61 pins, + * seven AIC interrupts 706 to 718, pmgr nub_gpio. It holds the reset and + * power lines of the touch-screen (kiwi) interface; its interrupts are + * left unused. Enabled by the board DT together with its first user. + */ + pinctrl_nub: pinctrl@2881e4000 { + compatible = "apple,t6050-pinctrl", "apple,t8140-pinctrl"; + reg = <0x2 0x881e4000 0x0 0x4000>; + power-domains = <&ps_nub_gpio>; + + gpio-controller; + #gpio-cells = <2>; + gpio-ranges = <&pinctrl_nub 0 0 61>; + apple,npins = <61>; + status = "disabled"; + }; + + /* + * I2C controllers. ADT /arm-io/i2c1..3: 0x305014000, 0x305018000, + * 0x30501c000 (+ 0x4000 each), AIC interrupts 1376-1378, pmgr I2C1-3. + * SCL/SDA are AP pins 60/61, 62/63 and 64/65; iBoot leaves them muxed + * (it drives the speaker amps for the boot chime), so no pin groups are + * set here. i2c1 and i2c3 carry the speaker amps, i2c2 the jack codec. + */ + i2c1: i2c@505014000 { + compatible = "apple,t6050-i2c", "apple,t8103-i2c"; + reg = <0x5 0x05014000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c1>; + status = "disabled"; + }; + + i2c2: i2c@505018000 { + compatible = "apple,t6050-i2c", "apple,t8103-i2c"; + reg = <0x5 0x05018000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c2>; + status = "disabled"; + }; + + i2c3: i2c@50501c000 { + compatible = "apple,t6050-i2c", "apple,t8103-i2c"; + reg = <0x5 0x0501c000 0x0 0x4000>; + clocks = <&clkref>; + interrupt-parent = <&aic>; + interrupts = ; + #address-cells = <1>; + #size-cells = <0>; + power-domains = <&ps_i2c3>; + status = "disabled"; + }; + + /* + * AOP (always-on processor) and the audio DMA it owns. ADT /arm-io/aop + * (iop,ascwrap-v6, pre-loaded firmware mac17jaop): asc 0x90600000+0x88000, + * mmio 0x90c00000+0x2f0000, mailbox AIC 578-581 (ADT order [579 578 581 + * 580] = send-not-empty, send-empty, recv-not-empty, recv-empty, as mtp); + * no AP power or clock gates. /arm-io/aop-exclave-mailbox + * (iop,secure-rtbuddy-proxy, T8140's "setup port"): 0x2d067c000+0x4000, + * ADT interrupts [585 582], dart-aop mapper 1; reading it hangs the AP, + * and the firmware boots without it, so it is not described. /arm-io/dart-aop: + * 0x90fc0000+0xc000, AIC 616, mappers aop 0, exclave-aop 1, + * admac-base-ns 8, admac-leap-ns 10, admac-leap-s 11. + * /arm-io/admac-base-ns (admac-evolution,t6050): 0x93800000+0x34000, + * AIC 637, 13 tx + 13 rx channels. DMA windows as t8140.dtsi: the ADT + * has the same vm-base-0 (0x10040000000) and vm-base-1 (0x20000000000). + */ + aop_mbox: mailbox@290608000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x90608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "disabled"; + }; + + /* + * The AOP's "setup port": ADT /arm-io/aop-exclave-mailbox + * (iop,secure-rtbuddy-proxy), the ASC's second mailbox 1 GiB above + * it as on T8140, interrupts [585, 582], messages through dart-aop + * stream 1 (mapper-exclave-aop, vm-base-1 0x20000000000). The + * firmware's TRUSTED compartment serves three endpoints on it (0x20, + * 0x23, 0x32) and may only use DRAM in that stream-1 window; its AFK + * endpoints on the main mailbox answer only after this handshake. + */ + aop_setup_mbox: mailbox@2d067c000 { + compatible = "apple,t6050-aop-setup-mailbox", "apple,t8140-aop-setup-mailbox"; + reg = <0x2 0xd067c000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + iommus = <&aop_dart 1>; + apple,dma-range = <0x200 0x0 0x10 0x0>; + #mbox-cells = <0>; + status = "disabled"; + }; + + aop_dart: iommu@290fc0000 { + /* + * Locked by the loader with inherited roots, like mtp_dart: the + * t8142 compatible turns on apple-dart's retained-root path + * (fw_mirror), which the plain t8110 fallback leaves off. + */ + compatible = "apple,t6050-dart", "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x2 0x90fc0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + status = "disabled"; + }; + + aop: aop@290c00000 { + compatible = "apple,t6050-aop"; + reg = <0x2 0x90c00000 0x0 0x2f0000>, + <0x2 0x90600000 0x0 0x88000>; + reg-names = "mmio", "asc"; + mboxes = <&aop_mbox>, <&aop_setup_mbox>; + mbox-names = "mbox", "setup"; + iommus = <&aop_dart 0>; + /* + * The firmware's UNTRUSTED compartment (the AFK side) may only + * use DRAM at IOVA 0x10040078000..0x20000000000 (its cL4 mmio + * list); a host buffer elsewhere makes it panic. Stream 0's + * inherited root holds the firmware's own slot 16; the host's + * buffers go into the next slot, as on T8140. + */ + apple,dma-range = <0x110 0x0 0x10 0x0>; + status = "disabled"; + + /* + * The AOPAudioService streams on the base-ns ADMAC (ADT + * /arm-io/iop-audio-controller): audio-speaker tx-0 and rx-0 + * (6 x 32-bit out, 12 x 16-bit V/I sense back), audio-codec- + * output tx-2, audio-codec-input rx-2. The ADT's tx-N is Linux + * channel 2N and rx-N is 2N+1 (apple-admac hardwires even + * channels to MEM_TO_DEV). No power domains: the audio fabric + * is the firmware's. + */ + aop_audio: audio { + compatible = "apple,t6050-aop-audio"; + dmas = <&admac_base_ns 0>, <&admac_base_ns 1>, + <&admac_base_ns 4>, <&admac_base_ns 5>, + <&admac_leap_s 1>; + /* + * "hpai" is the microphone array: the ADT routes + * audio-hp-mic to leap-s-adma-rx-0, Linux channel 1 of + * the LEAP-S ADMAC (even channels transmit, odd receive). + */ + dma-names = "spkr", "spkr-sense", "cout", "cin", "hpai"; + #sound-dai-cells = <1>; + }; + }; + + admac_base_ns: dma-controller@293800000 { + compatible = "apple,t6050-admac", "apple,t8140-admac"; + reg = <0x2 0x93800000 0x0 0x34000>; + dma-channels = <13>; + interrupts-extended = <&aic AIC_IRQ 637 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 8>; + apple,dma-range = <0x110 0x0 0x0 0x10000000>; + status = "disabled"; + }; + + /* + * The LEAP-S ADMAC through its non-secure proxy aperture: ADT + * /arm-io/admac-hp-mic-ec-proxy at 0x2d3200000 = admac-leap-s + * 0x293200000 + 1 GiB, the T8140 pattern (its admac_leap_s sits at + * 0x353200000 = 0x313200000 + 1 GiB). dart-aop stream 11. The AOP + * firmware owns the LEAP fabric: no resets and no AP power domains; + * the hpai service's pw0 raises it before a channel is requested. + */ + admac_leap_s: dma-controller@2d3200000 { + compatible = "apple,t6050-admac", "apple,t8140-admac"; + reg = <0x2 0xd3200000 0x0 0x34000>; + dma-channels = <9>; + interrupts-extended = <&aic AIC_IRQ 638 IRQ_TYPE_LEVEL_HIGH>, + <0>, <0>, <0>; + #dma-cells = <1>; + iommus = <&aop_dart 11>; + apple,dma-range = <0x110 0x0 0x0 0x10000000>; + status = "disabled"; + }; + + /* + * J714c also has a dockchannel UART (aapl,dock-channels, + * 0x88e28000/0x10004 + 0x88e0c000/0x18, AIC IRQ 360). The in-tree + * apple,dockchannel-uart driver expects three named windows + * (data/config/irq) and its observability from this laptop without + * a debug cable is unverified, so it is left out of the first boot. + * Add it with the window layout resolved when it is needed. + */ + + /* + * MTP coprocessor and dockchannel HID (keyboard, trackpad), on the + * t8142 J813 layout. Addresses from the J714c ADT: mtp 0x294600000, + * dart-mtp 0x294800000 (0xc000), dockchannel-mtp 0x294b14000 (irq) and + * 0x294b28000.. (data windows). The ADT lists no power or clock gates. + */ + mtp: mtp@294600000 { + compatible = "apple,t6050-mtp", "apple,t8142-rtk-helper-asc4", + "apple,mtp", "apple,rtk-helper-asc4"; + /* The second region is firmware SRAM, not the ASC control window. */ + reg = <0x2 0x94600000 0x0 0x4000>, <0x2 0x94c00000 0x0 0x100000>; + reg-names = "asc", "sram"; + mboxes = <&mtp_mbox>; + iommus = <&mtp_dart 0>; + }; + + mtp_mbox: mailbox@294608000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x94608000 0x0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + mtp_dart: iommu@294800000 { + compatible = "apple,t6050-dart", "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x2 0x94800000 0x0 0xc000>; + interrupts = ; + #iommu-cells = <1>; + /* Keep host allocations clear of firmware's low IOVA mappings. */ + apple,dma-range = <0 0x02000000 0 0x10000000>; + }; + + /* + * /arm-io/dart-apcie0: apcie port 0's, which carries the N1 (its mappers Control, Alpha and + * Beta on streams 1, 2 and 3). The port is trained before Linux (pcie-exp1-host.py v13 over + * the proxy, later the resident m1n1); the board file enables it. + */ + dart_apcie0: iommu@410000000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x4 0x10000000 0x0 0x20000>; + interrupts = ; + #iommu-cells = <1>; + /* iBoot leaves its block gated; it is read at probe */ + power-domains = <&ps_apcie_sys_gp>; + /* Its registers answer only once the root complex is up */ + apple,manual-availability; + status = "disabled"; + }; + + mtp_dockchannel: fifo@294b14000 { + compatible = "apple,t6050-dockchannel", "apple,dockchannel"; + reg = <0x2 0x94b14000 0x0 0x1000>; + reg-names = "irq"; + interrupts = ; + ranges = <0 0x2 0x94b28000 0x20000>; + nonposted-mmio; + #address-cells = <1>; + #size-cells = <1>; + interrupt-controller; + #interrupt-cells = <2>; + + mtp_hid: input@8000 { + compatible = "apple,dockchannel-hid"; + reg = <0x8000 0x1000>, <0xc000 0x1000>, + <0x0000 0x1000>, <0x4000 0x1000>; + reg-names = "config", "data", "rmt-config", "rmt-data"; + iommus = <&mtp_dart 0>; + interrupt-parent = <&mtp_dockchannel>; + interrupts = <2 IRQ_TYPE_LEVEL_HIGH>, <3 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "tx", "rx"; + apple,fifo-size = <0x800>; + apple,helper-cpu = <&mtp>; + }; + }; +}; + +/ { + /* + * The internal PCIe host (apcie), as aurora-silicon's t6050-die0.dtsi (j716c/dt) has it: the + * boot stage trains the links of the ports the board uses and keeps power, clocks, the PHYs + * and PERST; Linux owns the routing, MSI and the DARTs. The board names its ports (registers, + * interrupts, iommu-map, bus-range, a node per port). One MSI domain for all ports; the + * J714c's ADT gives pci-bridge0 vectors 0-31 of apcie0's 128 (AIC 1824 + vector). + */ + pcie0: pcie@1cb0000000 { + compatible = "apple,t6050-pcie"; + device_type = "pci"; + /* The GP port's domain; its parents up to ps_apcie_gp follow it. */ + power-domains = <&ps_aphy_gp_aus5>; + nonposted-mmio; + interrupt-parent = <&aic>; + + msi-controller; + msi-parent = <&pcie0>; + msi-ranges = <&aic AIC_IRQ 1824 IRQ_TYPE_EDGE_RISING 32>; + apple,msi-vector-base = <0>; + + iommu-map-mask = <0xffff>; + dma-coherent; + linux,pci-domain = <0>; + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x02000000 0x0 0x80000000 0xb 0x80000000 0x0 0x40000000>, + <0x43000000 0xb 0xc0000000 0xb 0xc0000000 0x0 0x20000000>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-gpu.dtsi b/arch/arm64/boot/dts/apple/t6050-gpu.dtsi new file mode 100644 index 0000000000000..064e5aa9c7721 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-gpu.dtsi @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T6050 "M5 Max" GPU: G17 with two coprocessors, GFX and GFX1. + * Registers and interrupts from the J714c ADT (sgx, gfx-asc, gfx1-asc). + * Disabled until the loader hands the GPU over. + * + * Copyright The Asahi Linux Contributors + */ + +/ { + reserved-memory { + uat_handoff: uat-handoff { + reg = <0 0 0 0>; + }; + + /* Only reserved: the GPU node does not reference it */ + uat_l2_pagetables: uat-l2-pagetables { + reg = <0 0 0 0>; + }; + + uat_pagetables: uat-pagetables { + reg = <0 0 0 0>; + }; + + uat_ttbs: uat-ttbs { + reg = <0 0 0 0>; + }; + }; + + /* + * ADT sgx perf-states, first set: frequency and core voltage per GPU + * performance state, in performance-state order. + */ + gpu_opp: opp-table-gpu { + compatible = "operating-points-v2"; + + opp00 { + opp-hz = /bits/ 64 <0>; + opp-microvolt = <125000>; + }; + opp01 { + opp-hz = /bits/ 64 <338000000>; + opp-microvolt = <630000>; + }; + opp02 { + opp-hz = /bits/ 64 <486000000>; + opp-microvolt = <670000>; + }; + opp03 { + opp-hz = /bits/ 64 <636000000>; + opp-microvolt = <705000>; + }; + opp04 { + opp-hz = /bits/ 64 <796000000>; + opp-microvolt = <745000>; + }; + opp05 { + opp-hz = /bits/ 64 <888000000>; + opp-microvolt = <755000>; + }; + opp06 { + opp-hz = /bits/ 64 <988000000>; + opp-microvolt = <780000>; + }; + opp07 { + opp-hz = /bits/ 64 <1084000000>; + opp-microvolt = <815000>; + }; + opp08 { + opp-hz = /bits/ 64 <1182000000>; + opp-microvolt = <825000>; + }; + opp09 { + opp-hz = /bits/ 64 <1278000000>; + opp-microvolt = <860000>; + }; + opp10 { + opp-hz = /bits/ 64 <1374000000>; + opp-microvolt = <900000>; + }; + opp11 { + opp-hz = /bits/ 64 <1470000000>; + opp-microvolt = <940000>; + }; + opp12 { + opp-hz = /bits/ 64 <1578000000>; + opp-microvolt = <1020000>; + }; + opp13 { + opp-hz = /bits/ 64 <1620000000>; + opp-microvolt = <1055000>; + }; + }; +}; + +&soc { + gpu: gpu@2302600000 { + compatible = "apple,agx-t6050"; + reg = <0x23 0x02600000 0x0 0x88000>, + <0x23 0x00000000 0x0 0x3fdc000>; + reg-names = "asc", "sgx"; + mboxes = <&gfx_mbox>, <&gfx1_mbox>; + power-domains = <&ps_gfx>; + memory-region = <&uat_ttbs>, <&uat_pagetables>, + <&uat_handoff>; + memory-region-names = "ttbs", "pagetables", "handoff"; + operating-points-v2 = <&gpu_opp>; + status = "disabled"; + }; + + /* The ADT lists each ASC's interrupts in swapped pairs (2798, 2797, 2800, 2799). */ + gfx_mbox: mailbox@2302600000 { + compatible = "apple,t6050-ascwrap-v6", "apple,t8140-ascwrap-v6"; + reg = <0x23 0x02600000 0x0 0x88000>, + <0x23 0x02050000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupt-parent = <&aic>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; + + gfx1_mbox: mailbox@2302e00000 { + compatible = "apple,t6050-ascwrap-v6", "apple,t8140-ascwrap-v6"; + reg = <0x23 0x02e00000 0x0 0x88000>, + <0x23 0x02850000 0x0 0x8>; + reg-names = "wrapper", "iop-vbar"; + interrupt-parent = <&aic>; + interrupts = , + ; + interrupt-names = "send-empty", "recv-not-empty"; + apple,firmware-role = "GFX1"; + power-domains = <&ps_gfx>; + #mbox-cells = <0>; + status = "disabled"; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j714c-display.dtsi b/arch/arm64/boot/dts/apple/t6050-j714c-display.dtsi new file mode 100644 index 0000000000000..a8f6c91a80483 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j714c-display.dtsi @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J714c internal display pipeline: the DCP coprocessor, its mailbox, both + * DARTs, the bandwidth scratch window and the firmware memory. + * + * iBoot leaves the DCP running macOS 27 firmware with its memory mapped + * through the DARTs; the loader does not touch it on T6050, so everything + * the driver needs to adopt the running session is written here from this + * machine's Apple Device Tree: register windows (/arm-io bus addresses gain + * 0x2_0000_0000 on the CPU side), the iop-dcp-nub segment table (segment 8 + * starts at the boot framebuffer) and the panel. + */ + +/ { + aliases { + dcp = &dcp; + disp0 = &display; + disp0_piodma = &disp0_piodma; + }; + + /* + * The ADT gives no rate for it: macOS 27.2's DCP reads 810 MHz for + * getClockFrequency(0) ("returned pixelClock = 810000000"). + */ + clk_disp0: clock-disp0 { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <810000000>; + clock-output-names = "clk_disp0"; + }; + + display: display-subsystem { + compatible = "apple,t6050-display-subsystem", "apple,display-subsystem"; + /* disp0_dart must be first since the loader locks it */ + iommus = <&disp0_dart 0>; + power-domains = <&ps_disp_fe>; + memory-region = <&dcp_fw_8>; + memory-region-names = "framebuffer"; + phandle = <&display>; + }; +}; + +&{/reserved-memory} { + /* + * iop-dcp-nub segment-ranges (phys, size): reserved RAM. The DCP's DART is locked on + * the firmware's four-level root, whose slot 16 maps these at their remap DVAs; Linux + * resolves them through those tables and never maps into that slot. + */ + dcp_fw_text: asc-firmware@1000031c000 { + compatible = "apple,asc-mem"; + reg = <0x100 0x0031c000 0x0 0x9a4000>; + no-map; + }; + dcp_fw_data: asc-firmware@11fd21ac000 { + compatible = "apple,asc-mem"; + reg = <0x11f 0xd21ac000 0x0 0xb04000>; + no-map; + }; + dcp_fw_shared: asc-firmware@11fd218c000 { + compatible = "apple,asc-mem"; + reg = <0x11f 0xd218c000 0x0 0x20000>; + no-map; + }; + dcp_fw_shared_rw: asc-firmware@11fd2cb0000 { + compatible = "apple,asc-mem"; + reg = <0x11f 0xd2cb0000 0x0 0x8000>; + no-map; + }; + /* + * The OS log (segment flags 0xa) runs at its physical address, not through + * the DCP's DART: reserved, no DART mapping, as m1n1 marks it on T6030. + */ + dcp_fw_oslog: asc-firmware@10005cd4000 { + compatible = "apple,asc-mem"; + reg = <0x100 0x05cd4000 0x0 0x28000>; + apple,dcp-os-log; + no-map; + }; + dcp_fw_5: asc-firmware@11fd8cd8000 { + compatible = "apple,asc-mem"; + reg = <0x11f 0xd8cd8000 0x0 0xc00000>; + no-map; + }; + dcp_fw_6: asc-firmware@11fd8978000 { + compatible = "apple,asc-mem"; + reg = <0x11f 0xd8978000 0x0 0x360000>; + no-map; + }; + dcp_fw_7: asc-firmware@11fd88b8000 { + compatible = "apple,asc-mem"; + reg = <0x11f 0xd88b8000 0x0 0xc0000>; + no-map; + }; + /* + * ADT /vram: the framebuffer iBoot scans out of (m1n1: FDT: framebuffer@11fd310c000), + * DCP firmware segment 8. The display subsystem takes the boot framebuffer over from it. + */ + dcp_fw_8: framebuffer@11fd310c000 { + compatible = "framebuffer"; + reg = <0x11f 0xd310c000 0x0 0x57ac000>; + no-map; + }; + dcp_fw_9: asc-firmware@11fd3104000 { + compatible = "apple,asc-mem"; + reg = <0x11f 0xd3104000 0x0 0x4000>; + no-map; + }; +}; + +&soc { + /* ADT disp0 reg[4] (0x86700000, 0x114000): scratch at +0x20000, doorbell at +0xa4000. */ + pmgr_dcp: power-management@286720000 { + reg = <0x2 0x86720000 0x0 0x8>, <0x2 0x867a4000 0x0 0x4>; + reg-names = "dcp-bw-scratch", "dcp-bw-doorbell"; + #apple,bw-scratch-cells = <3>; + #apple,bw-doorbell-cells = <2>; + }; + + /* ADT dcp interrupts [1044 1043 1046 1045 ...]: the mailbox's four, ascending. */ + dcp_mbox: mailbox@48ae08000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x8ae08000 0x0 0x4000>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + power-domains = <&ps_disp_cpu>; + }; + + /* ADT dart-disp0: sid 0 (display) and 16 (piodma); Linux's window 1-2 TiB (DCP DMA is 41 bits) */ + disp0_dart: iommu@48a300000 { + compatible = "apple,t6050-dart", "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0x8a300000 0x0 0xc000>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x100 0x0>; + power-domains = <&ps_disp_cpu>; + }; + + /* ADT dart-dcp: sid 23, the same window; the firmware's slot 16 (1 TiB + 64 GiB) is kept */ + dcp_dart: iommu@48a340000 { + compatible = "apple,t6050-dart", "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x4 0x8a340000 0x0 0x20000>; + interrupts = ; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x0 0x100 0x0>; + power-domains = <&ps_disp_cpu>; + }; + + dcp: dcp@48ae00000 { + compatible = "apple,t6050-dcp", "apple,dcp"; + mboxes = <&dcp_mbox>; + mbox-names = "mbox"; + iommus = <&dcp_dart 23>; + reg-names = "coproc", "disp-0", "disp-1", "disp-2", + "disp-3", "disp-4", "iop-vbar"; + reg = <0x4 0x8ae00000 0x0 0x4000>, + <0x4 0x88000000 0x0 0x4000000>, + <0x4 0x8a240000 0x0 0x10000>, + <0x4 0x8a3ac000 0x0 0x4000>, + <0x4 0x8a800000 0x0 0x800000>, + <0x2 0x86700000 0x0 0x114000>, + <0x4 0x8a850000 0x0 0x4000>; + apple,bw-scratch = <&pmgr_dcp 0 5 0>; + apple,bw-doorbell = <&pmgr_dcp 1 6>; + apple,iomfb-surfaces = <0 0 1 0>; + apple,firmware-compat = <27 0 0>; + clocks = <&clk_disp0>; + apple,clock-ids = <0>; + memory-region = <&dcp_fw_8>, <&dcp_fw_text>, <&dcp_fw_data>, <&dcp_fw_shared>, + <&dcp_fw_shared_rw>, <&dcp_fw_oslog>, <&dcp_fw_5>, <&dcp_fw_6>, + <&dcp_fw_7>, <&dcp_fw_9>; + memory-region-names = "framebuffer", "fw-text", "fw-data", "fw-shared", + "fw-shared-data-rw", "fw-os-log", "fw-5", "fw-6", + "fw-7", "fw-9"; + power-domains = <&ps_disp_cpu>; + phandle = <&dcp>; + /* required bus properties for 'piodma' subdevice */ + #address-cells = <2>; + #size-cells = <2>; + + disp0_piodma: piodma { + iommus = <&disp0_dart 16>; + phandle = <&disp0_piodma>; + }; + + /* 14.2", 3024x1964 at 254 ppi; ADT user-accessible-max-nits 1000. */ + panel: panel { + compatible = "apple,panel-mini-led", "apple,panel"; + width-mm = <302>; + height-mm = <196>; + adj-height-mm = <189>; + apple,max-brightness = <1000>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j714c-isp.dtsi b/arch/arm64/boot/dts/apple/t6050-j714c-isp.dtsi new file mode 100644 index 0000000000000..4d3c0dc24cbd1 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j714c-isp.dtsi @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J714c ISP: T6050's ISP17a (ASC version 0x110003), laid out and booted as + * aurora's T6040 (M4 Pro/Max) one, at the same addresses: isp0 0x580000000, + * dart-isp0 0x5828c0000/0x5828e0000/0x582900000 (+0xc000), the mailbox at + * +0x2924000. T8140's mailbox offset, +0x2920000, stalls the bus here. + * From the J714c ADT: IRQs [513 514 515 516 526] (513 carries the mailbox), + * dart-isp0 IRQ 531, remap streams 1 6 7 10 11 -> 0, sensor-type 0x73. + * + * The ADT marks the ISP sub-domains no_ps, but pmgr1's slots run by device id + * 8 bytes apart from ISP_SYS (218, +0x3a0): ISP_CPU 220 +0x3b0, ISP_FE 221 + * +0x3b8, DPRX 225 +0x3d8, ISP_SECURE 226 +0x3e0, ISP_BE 227 +0x3e8, ISP_CLR + * 228 +0x3f0, with the ADT's parents. All reach active from the AP. + * + * iBoot loads the firmware's __TEXT and __DATA (isp0 segment-ranges) into its + * 32 MiB main memory at 0x100_0000_0000; they end at 0x1fec000. The physical + * addresses are this boot firmware's (2026-11-08) and move with it. No "isp" + * alias: the boot loader leaves this node as it is. + * + * The front camera is an IMX958 (sensor 0x958), as on the M4 Pro/Max. + */ + +&{/reserved-memory} { + isp_fw_text: isp-firmware-text@10001224000 { + reg = <0x100 0x01224000 0x0 0xc1c000>; + iommu-addresses = <&isp 0x100 0x0 0x0 0xc1c000>; + no-map; + }; + + isp_fw_data: isp-firmware-data@10002930000 { + reg = <0x100 0x02930000 0x0 0x13d0000>; + iommu-addresses = <&isp 0x100 0xc1c000 0x0 0x13d0000>; + no-map; + }; + + /* the firmware's main memory, the driver backs it above the data */ + isp_heap: isp-heap { + iommu-addresses = <&isp 0x100 0x0 0x0 0x2000000>; + }; +}; + +&pmgr1 { + ps_isp_cpu: power-controller@3b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b0 4>; + label = "isp_cpu"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_isp_sys>; + }; + + ps_isp_fe: power-controller@3b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b8 4>; + label = "isp_fe"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_isp_sys>; + }; + + ps_isp_dprx: power-controller@3d8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d8 4>; + label = "isp_dprx"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_secure: power-controller@3e0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e0 4>; + label = "isp_secure"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_be: power-controller@3e8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3e8 4>; + label = "isp_be"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_isp_fe>; + }; + + ps_isp_clr: power-controller@3f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f0 4>; + label = "isp_clr"; + #power-domain-cells = <0>; + #reset-cells = <0>; + power-domains = <&ps_isp_be>; + }; +}; + +&soc { + isp_dart0: iommu@5828c0000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + /* ADT dart-isp0 reg[0] and reg[5], its address filter */ + reg = <0x5 0x828c0000 0x0 0xc000>, <0x5 0x828d0000 0x0 0x4000>; + reg-names = "dart", "dapf"; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,remap-streams = <1 0>, <6 0>, <7 0>, <10 0>, <11 0>; + }; + + isp_dart1: iommu@5828e0000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x5 0x828e0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,remap-streams = <1 0>, <6 0>, <7 0>, <10 0>, <11 0>; + }; + + isp_dart2: iommu@582900000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x5 0x82900000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>; + apple,remap-streams = <1 0>, <6 0>, <7 0>, <10 0>, <11 0>; + }; + + /* + * As on T6040: "coproc" up to its control block; "mbox", "gpio" and + * "mbox2" are the interrupt words, the firmware handshake words and + * the doorbell of the mailbox at +0x2924000. + */ + isp: isp@580000000 { + compatible = "apple,t6050-isp", "apple,isp"; + iommus = <&isp_dart0 0>, <&isp_dart1 0>, <&isp_dart2 0>; + reg-names = "coproc", "mbox", "gpio", "mbox2", "wdt", "cil"; + reg = <0x5 0x80000000 0x0 0x1604000>, + <0x5 0x82924000 0x0 0x40>, + <0x5 0x829241d0 0x0 0x20>, + <0x5 0x82924464 0x0 0x10>, + <0x5 0xc3b50000 0x0 0x8000>, + <0x2 0xdc000234 0x0 0x4>; + interrupt-parent = <&aic>; + interrupts = ; + power-domains = <&ps_isp_sys>, <&ps_isp_cpu>, <&ps_isp_fe>, + <&ps_isp_dprx>, <&ps_isp_secure>, <&ps_isp_be>, + <&ps_isp_clr>, <&ps_ispsens0>; + memory-region = <&isp_fw_text>, <&isp_fw_data>, <&isp_heap>; + memory-region-names = "text", "data", "heap"; + /* the shared memory window above the firmware's 32 MiB */ + apple,dart-vm-size = /bits/ 64 <0xde600000>; + /* the ADT isp0 sensor-type */ + apple,platform-id = <0x73>; + apple,temporal-filter = <1>; + /* ADT isp0 cam-connections-scheme, as on the M4 Pro/Max J614/J616 */ + apple,cam-connections-scheme = <0x10>; + /* + * The camera light: ADT eic cil-gpio-offset 141 in gpio0 + * (0x2dc000000), cil-gpio-config 0x474002. Active high: driven + * low it is off even while the sensor streams (measured). + * "wdt" is the eic's mmio-wd-isp: frames pass only while it is fed, + * which the driver does only while the light is on. + */ + apple,cil-config = <0x474002>; + apple,cil-active-high; + + /* + * The full IMX958, cropped to the centre: inside the lens circle and + * about the field of view macOS shows. 1920x1080 and 1280x720 scale + * a 2400x1350 window; 2560x1440 is the centre 1:1. 1920x1080 and + * 2560x1440 use firmware configuration 6, the one macOS streams (two + * exposures, 3040x5760, merged to 3040x2880); 1280x720 uses + * configuration 7 (3040x2880). + */ + sensor-presets { + /* 1920x1080 */ + preset0 { + apple,config-index = <6>; + apple,input-size = <3040 2880>; + apple,output-size = <1920 1080>; + apple,crop = <320 765 2400 1350>; + }; + /* 1280x720 */ + preset1 { + apple,config-index = <7>; + apple,input-size = <3040 2880>; + apple,output-size = <1280 720>; + apple,crop = <320 765 2400 1350>; + }; + /* 2560x1440 */ + preset2 { + apple,config-index = <6>; + apple,input-size = <3040 2880>; + apple,output-size = <2560 1440>; + apple,crop = <240 720 2560 1440>; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j714c-pcie-tunables.dtsi b/arch/arm64/boot/dts/apple/t6050-j714c-pcie-tunables.dtsi new file mode 100644 index 0000000000000..4fc5b05ce944f --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j714c-pcie-tunables.dtsi @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * J714c apcie0 and pci-bridge0 tunables, copied unchanged from this machine's ADT + * (2026-11-05) as (offset, 4, mask, value): what m1n1 copies on chips it knows and + * the kernel applies when it trains the N1's port itself. cio3pllcore offsets are + * relative to pcieclkgen (+0x2200, the T8140 layout). + */ +&pcie0 { + apple,tunable-axi2af = + <0x0 4 0x800000ff 0x80000022>, + <0x4 4 0x80ff00ff 0x80220002>, + <0x8 4 0x80ff00ff 0x80220002>, + <0xc 4 0x80ff00ff 0x80220002>, + <0x10 4 0x80ff00ff 0x80220002>, + <0x14 4 0x80ff00ff 0x80220002>, + <0x18 4 0x80ff00ff 0x80220002>, + <0x1c 4 0x800000ff 0x80000019>, + <0x20 4 0x800000ff 0x80000088>, + <0x24 4 0x80ff00ff 0x80880008>, + <0x28 4 0x80ff00ff 0x80880008>, + <0x2c 4 0x80ff00ff 0x80880008>, + <0x30 4 0x80ff00ff 0x80880008>, + <0x34 4 0x800000ff 0x80000019>, + <0x38 4 0x800000ff 0x80000019>, + <0x3c 4 0x800000ff 0x8000001c>, + <0x100 4 0xff 0xff>, + <0x104 4 0x3 0x3>, + <0x108 4 0xffff0007 0x100001>, + <0x300 4 0x3 0x1>, + <0x304 4 0x3f3f0000 0x8080000>, + <0x308 4 0x3ff 0xd>, + <0x30c 4 0x700003ff 0xc>, + <0x310 4 0x80001f 0x800001>, + <0x314 4 0x8080001f 0x80800001>, + <0x318 4 0x80001f 0x800003>, + <0x31c 4 0x8080001f 0x80800003>, + <0x320 4 0x80001f 0x800003>, + <0x324 4 0x8080001f 0x80800003>, + <0x328 4 0x80001f 0x800003>, + <0x32c 4 0x80001f 0x800003>, + <0x330 4 0x80001f 0x800003>, + <0x334 4 0x80001f 0x800003>, + <0x338 4 0x80001f 0x800003>, + <0x33c 4 0x80001f 0x800003>, + <0x340 4 0x80001f 0x800003>, + <0x404 4 0xffff0023 0x1c0021>, + <0x408 4 0x3ff 0xd>, + <0x40c 4 0x700003ff 0xc>, + <0x410 4 0x8000001f 0x80000001>, + <0x414 4 0x8000001f 0x80000001>, + <0x418 4 0x8000001f 0x80000003>, + <0x41c 4 0x8000001f 0x80000003>, + <0x420 4 0x8000001f 0x80000003>, + <0x424 4 0x8000001f 0x80000003>, + <0x428 4 0x8000001f 0x80000003>, + <0x42c 4 0x8000001f 0x80000003>, + <0x430 4 0x8000001f 0x80000003>, + <0x434 4 0x8000001f 0x80000003>, + <0x438 4 0x8000001f 0x80000003>, + <0x43c 4 0x8000001f 0x80000003>, + <0x440 4 0x8000001f 0x80000003>, + <0x948 4 0x7ff07ff 0x200000>, + <0xa48 4 0x7ff07ff 0x100>, + <0xb00 4 0x20000000 0x20000000>, + <0xb48 4 0x7ff07ff 0x800080>, + <0xe00 4 0xffffffff 0x7fffffff>, + <0xe04 4 0x100 0x100>, + <0x4008 4 0x1 0x1>, + <0x400c 4 0x1 0x1>, + <0x4014 4 0x1 0x0>, + <0x4018 4 0x1 0x0>, + <0x401c 4 0x1 0x1>, + <0x10f8 4 0x1 0x1>, + <0x11f8 4 0x1 0x1>, + <0x12f8 4 0x1 0x1>, + <0x13f8 4 0x1 0x1>, + <0x14f8 4 0x1 0x1>, + <0x10fc 4 0x1 0x1>, + <0x11fc 4 0x1 0x1>, + <0x12fc 4 0x1 0x1>, + <0x13fc 4 0x1 0x1>, + <0x14fc 4 0x1 0x1>, + <0x20028 4 0xfc2f 0x0>, + <0x2002c 4 0x7 0x0>, + <0x20030 4 0xff 0x0>, + <0x20034 4 0x3ff 0x0>, + <0x20058 4 0x30b 0x0>; + apple,tunable-cio3pllcore = /* at pcieclkgen + 0x2200 */ + <0x2200 4 0xa0b 0xa01>, + <0x2224 4 0xc00 0x800>, + <0x2228 4 0xf00 0xb00>, + <0x2238 4 0x20 0x0>, + <0x224c 4 0xff 0x95>, + <0x22e8 4 0xe0000 0x20000>, + <0x2300 4 0xffffff 0xb40b4>; + apple,tunable-pcieclkgen = + <0x0 4 0x3e0 0x220>; + apple,tunable-phy = + <0x20000 4 0x4000000 0x0>, + <0x10000 4 0x10000000 0x0>, + <0x14000 4 0x10000000 0x0>, + <0x18000 4 0x10000000 0x0>, + <0x1c000 4 0x10000000 0x0>; + apple,tunable-phy-ip-pll = + <0x180828 4 0xffff 0xb44>, + <0x180838 4 0x2 0x2>, + <0x180c1c 4 0x7e0 0x7c0>, + <0x180e10 4 0x3e0 0x280>, + <0x1820a4 4 0x3c3c000 0x1010000>, + <0x182200 4 0x8 0x8>, + <0x18221c 4 0x7c0 0x500>, + <0x182224 4 0xc00 0x800>, + <0x182228 4 0xf00 0xb00>, + <0x182230 4 0x3f 0x34>, + <0x18224c 4 0xfe7f 0x4412>, + <0x1822b0 4 0x700000 0x300000>, + <0x182338 4 0x1f00 0x1700>, + <0x183000 4 0x301fbf00 0x1e3400>, + <0x183004 4 0x1fffff 0x8cf39>, + <0x183014 4 0xff0000ff 0x40000080>, + <0x183008 4 0x1fffff 0x15930a>, + <0x18300c 4 0x1fffff 0x1a2548>, + <0x183010 4 0x1fffff 0x1baaa8>; + apple,tunable-phy-ip-auspma = + <0x184008 4 0xffffffff 0xb>, + <0x190008 4 0xffffffff 0xb>, + <0x19c008 4 0xffffffff 0xb>, + <0x1a8008 4 0xffffffff 0xb>, + <0x1860c8 4 0xff 0x0>, + <0x1920c8 4 0xff 0x0>, + <0x19e0c8 4 0xff 0x0>, + <0x1aa0c8 4 0xff 0x0>, + <0x1884c0 4 0x43f0 0x4150>, + <0x188574 4 0x7f8000 0x0>, + <0x1944c0 4 0x43f0 0x4150>, + <0x194574 4 0x7f8000 0x0>, + <0x1a04c0 4 0x43f0 0x4150>, + <0x1a0574 4 0x7f8000 0x0>, + <0x1ac4c0 4 0x43f0 0x4150>, + <0x1ac574 4 0x7f8000 0x0>, + <0x189814 4 0x7e00 0xc00>, + <0x189838 4 0xfe00000 0x5e00000>, + <0x1898b4 4 0x3f000000 0xa000000>, + <0x195814 4 0x7e00 0xc00>, + <0x195838 4 0xfe00000 0x5e00000>, + <0x1958b4 4 0x3f000000 0xa000000>, + <0x1a1814 4 0x7e00 0xc00>, + <0x1a1838 4 0xfe00000 0x5e00000>, + <0x1a18b4 4 0x3f000000 0xa000000>, + <0x1ad814 4 0x7e00 0xc00>, + <0x1ad838 4 0xfe00000 0x5e00000>, + <0x1ad8b4 4 0x3f000000 0xa000000>, + <0x18a050 4 0x8000000 0x8000000>, + <0x18a140 4 0xffff 0x7d0>, + <0x18a154 4 0x1f00 0x1c00>, + <0x18a2c0 4 0x1f803f 0x10008>, + <0x18a2c8 4 0x1f803f 0x20011>, + <0x18a2d0 4 0x3f 0x22>, + <0x18a31c 4 0xf0ff0f0 0x7071010>, + <0x18a338 4 0xffff 0xa0a>, + <0x18a33c 4 0x7f0f 0x2408>, + <0x18a340 4 0xf 0x8>, + <0x18a378 4 0xff00 0x600>, + <0x18a37c 4 0xff00 0x600>, + <0x18a380 4 0xff00 0xf00>, + <0x18a384 4 0xff00 0xf00>, + <0x18a3b8 4 0xffffffff 0x85e000>, + <0x18a3c8 4 0xffffffff 0x300>, + <0x196050 4 0x8000000 0x8000000>, + <0x196140 4 0xffff 0x7d0>, + <0x196154 4 0x1f00 0x1c00>, + <0x1962c0 4 0x1f803f 0x10008>, + <0x1962c8 4 0x1f803f 0x20011>, + <0x1962d0 4 0x3f 0x22>, + <0x19631c 4 0xf0ff0f0 0x7071010>, + <0x196338 4 0xffff 0xa0a>, + <0x19633c 4 0x7f0f 0x2408>, + <0x196340 4 0xf 0x8>, + <0x196378 4 0xff00 0x600>, + <0x19637c 4 0xff00 0x600>, + <0x196380 4 0xff00 0xf00>, + <0x196384 4 0xff00 0xf00>, + <0x1963b8 4 0xffffffff 0x85e000>, + <0x1963c8 4 0xffffffff 0x300>, + <0x1a2050 4 0x8000000 0x8000000>, + <0x1a2140 4 0xffff 0x7d0>, + <0x1a2154 4 0x1f00 0x1c00>, + <0x1a22c0 4 0x1f803f 0x10008>, + <0x1a22c8 4 0x1f803f 0x20011>, + <0x1a22d0 4 0x3f 0x22>, + <0x1a231c 4 0xf0ff0f0 0x7071010>, + <0x1a2338 4 0xffff 0xa0a>, + <0x1a233c 4 0x7f0f 0x2408>, + <0x1a2340 4 0xf 0x8>, + <0x1a2378 4 0xff00 0x600>, + <0x1a237c 4 0xff00 0x600>, + <0x1a2380 4 0xff00 0xf00>, + <0x1a2384 4 0xff00 0xf00>, + <0x1a23b8 4 0xffffffff 0x85e000>, + <0x1a23c8 4 0xffffffff 0x300>, + <0x1ae050 4 0x8000000 0x8000000>, + <0x1ae140 4 0xffff 0x7d0>, + <0x1ae154 4 0x1f00 0x1c00>, + <0x1ae2c0 4 0x1f803f 0x10008>, + <0x1ae2c8 4 0x1f803f 0x20011>, + <0x1ae2d0 4 0x3f 0x22>, + <0x1ae31c 4 0xf0ff0f0 0x7071010>, + <0x1ae338 4 0xffff 0xa0a>, + <0x1ae33c 4 0x7f0f 0x2408>, + <0x1ae340 4 0xf 0x8>, + <0x1ae378 4 0xff00 0x600>, + <0x1ae37c 4 0xff00 0x600>, + <0x1ae380 4 0xff00 0xf00>, + <0x1ae384 4 0xff00 0xf00>, + <0x1ae3b8 4 0xffffffff 0x85e000>, + <0x1ae3c8 4 0xffffffff 0x300>, + <0x18c410 4 0x100000 0x0>, + <0x18c438 4 0x1800000 0x0>, + <0x18c4a4 4 0x10 0x10>, + <0x198410 4 0x100000 0x0>, + <0x198438 4 0x1800000 0x0>, + <0x1984a4 4 0x10 0x10>, + <0x1a4410 4 0x100000 0x0>, + <0x1a4438 4 0x1800000 0x0>, + <0x1a44a4 4 0x10 0x10>, + <0x1b0410 4 0x100000 0x0>, + <0x1b0438 4 0x1800000 0x0>, + <0x1b04a4 4 0x10 0x10>; +}; + +&port00 { + apple,apcie-config-tunables = + <0x140 4 0x1 0x1>, + <0x144 4 0xffffff 0x253770>, + <0x8a8 4 0x1 0x1>, + <0x8ac 4 0x1 0x1>; + apple,pcie-rc-tunables = + <0x78 4 0x7000 0x0>, + <0x194 4 0x2 0x2>, + <0xb80 4 0x7f3f3f 0x7f1e1e>, + <0xb88 4 0x7f3f3f 0x7f1e1e>; + apple,pcie-rc-gen3-shadow-tunables = + <0x154 4 0xf0f0f0f 0x5050505>, + <0x890 4 0x3000000 0x0>, + <0x8a8 4 0xffff0f 0x1>; + apple,pcie-rc-gen4-shadow-tunables = + <0x178 4 0xffff 0x5555>, + <0x890 4 0x3000000 0x1000000>, + <0x8a8 4 0xffff0f 0x1>; + apple,pcie-rc-gen5-shadow-tunables = + <0x1ac 4 0xffff 0x5555>, + <0x890 4 0x3000000 0x2000000>, + <0x8a8 4 0xffff0f 0x1>; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j714c-pmp-tunables.dtsi b/arch/arm64/boot/dts/apple/t6050-j714c-pmp-tunables.dtsi new file mode 100644 index 0000000000000..79de0f2d0e631 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j714c-pmp-tunables.dtsi @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T6050 J714c per-machine PMP tunables -- GENERATED, do not edit by hand. + * + * Every /arm-io/pmp0/iop-pmp0-nub property (bar ADT structural metadata) as + * apple,tunable-, per omacom-m1n1 src/kboot_pmp_t8122.c and the + * drivers/soc/apple/t6030-j516s-pmp.dtso precedent. Board-specific: include + * only from t6050-j714c.dts, never the SoC dtsi. + * + * Source ADT: /home/ed/dev/omarchy/j714c/dumps/j714c-mboot-60.65.adt + * chip-id 0x6050, board-id 0xa + * firmware-version 'mBoot-20457.60.65.0.1' + * system-firmware-version 'mBoot-20457.60.65.0.1' + * iboot 'mBoot-20457.60.65.0.1' / 'mBoot-20457.60.65.0.1' + * PMP firmware image 't6050pmp' (uuid '5BDC0E6C-F792-3E25-BEE9-5BB8808E37A3') + * Regenerate: gen-t6050-j714c-pmp-tunables.py j714c.adt t6050-j714c-pmp-tunables.dtsi + */ + +&pmp { + apple,tunable-soc-ni7-bw-threshold = [00 00 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 00 00 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni4-bw-enable = [01 00 00 00]; + apple,tunable-lts-clpc-voltage = [01 00 00 00 d2 03 00 00 00 00 66 00 33 33 66 00 00 00 00 00 50 41 43 43 00 00 00 00]; + apple,tunable-ptd-driver-version = [01 00 00 00]; + apple,tunable-agx-dcs-lat-enable = [00 00 00 00]; + apple,tunable-ave1-af-bw-threshold-a = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni3-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni9-bw-enable = [00 00 00 00]; + apple,tunable-afi-bw-enable = [01 00 00 00]; + apple,tunable-vlpm-threshold = [55 01 00 00]; + apple,tunable-agx-dcs-bw-enable = [01 00 00 00]; + apple,tunable-mc_cmd_ptd_count = [02 00 00 00]; + apple,tunable-soc-ni8-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni3-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-mc_cmd_ack_ptd_count = [01 00 00 00]; + apple,tunable-agx-af-bw-enable = [01 00 00 00]; + apple,tunable-soc-device = [01 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 02 00 00 00 45 4e 4f 4e 01 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4d 43 43 00 00 00 00 03 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 00 04 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 00 05 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 00 06 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 00 07 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 00 08 00 00 00 45 4e 4f 4e 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 00 09 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 36 00 0a 00 00 00 45 4e 4f 4e 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 37 00 0b 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 38 00 0c 00 00 00 45 4e 4f 4e 01 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 39 00 0d 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 30 00 00 00 0e 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 31 00 00 00 0f 00 00 00 45 4e 4f 4e 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 10 00 00 00 5f 58 46 47 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 00 00 00 00 00 11 00 00 00 45 4e 4f 4e 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 12 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 49 53 50 00 00 00 00 00 13 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 49 4e 54 00 14 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 30 15 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 31 16 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 32 17 00 00 00 45 4e 4f 4e 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 49 53 50 45 58 54 33 18 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 30 00 00 00 00 19 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 31 00 00 00 00 1a 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 44 00 00 00 00 00 1b 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 53 52 30 00 00 00 00 1c 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 53 52 31 00 00 00 00 1d 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4a 50 45 47 00 00 00 00 1e 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 43 4f 44 45 43 00 00 1f 00 00 00 45 4e 4f 4e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 52 4f 52 45 53 00 00 20 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 30 00 00 00 00 21 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 31 00 00 00 00 22 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 32 00 00 00 00 23 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 33 00 00 00 00 24 00 00 00 45 4e 4f 4e 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 55 53 53 00 00 00 00 25 00 00 00 45 4e 4f 4e 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 00 00 00 00 00]; + apple,tunable-ans-dma-af-bw-threshold-a = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-mcache-stream = [1e 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2b 00 00 00 03 00 00 00 01 00 00 00 02 00 00 00 02 00 00 00 0e 00 00 00 0e 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 05 00 00 00 01 00 00 00 01 00 00 00 07 00 00 00 07 00 00 00 20 00 00 00 20 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 06 00 00 00 02 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 24 00 00 00 24 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0b 00 00 00 05 00 00 00 01 00 00 00 0b 00 00 00 0b 00 00 00 30 00 00 00 30 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 0e 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0d 00 00 00 38 00 00 00 38 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 11 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3c 00 00 00 3c 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1b 00 00 00 04 00 00 00 01 00 00 00 0e 00 00 00 0e 00 00 00 3d 00 00 00 3d 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 13 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 40 00 00 00 40 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 24 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 41 00 00 00 41 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1c 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 0f 00 00 00 42 00 00 00 42 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0f 00 00 00 04 00 00 00 01 00 00 00 10 00 00 00 10 00 00 00 44 00 00 00 44 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 01 00 00 00 11 00 00 00 11 00 00 00 48 00 00 00 48 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 13 00 00 00 13 00 00 00 50 00 00 00 50 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 01 00 00 00 15 00 00 00 15 00 00 00 58 00 00 00 5a 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 10 00 00 00 04 00 00 00 01 00 00 00 1f 00 00 00 1f 00 00 00 80 00 00 00 bf 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 20 00 00 00 03 00 00 00 02 00 00 00 03 00 00 00 06 00 00 00 10 00 00 00 1f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 01 00 00 00 1f 00 00 00 03 00 00 00 02 00 00 00 1e 00 00 00 1e 00 00 00 7c 00 00 00 7c 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 12 00 00 00 05 00 00 00 02 00 00 00 09 00 00 00 09 00 00 00 28 00 00 00 28 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0a 00 00 00 05 00 00 00 02 00 00 00 0c 00 00 00 0c 00 00 00 34 00 00 00 34 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 15 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5d 00 00 00 5d 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 36 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5e 00 00 00 5e 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 37 00 00 00 07 00 00 00 02 00 00 00 16 00 00 00 16 00 00 00 5f 00 00 00 5f 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 0d 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 08 00 00 00 08 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 0c 00 00 00 0b 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 09 00 00 00 09 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 19 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 1a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 05 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 29 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 06 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2a 00 00 00 08 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 07 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 2c 00 00 00 03 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 64 00 00 00 64 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2d 00 00 00 0c 00 00 00 03 00 00 00 18 00 00 00 18 00 00 00 65 00 00 00 65 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 2f 00 00 00 02 00 00 00 03 00 00 00 1a 00 00 00 1a 00 00 00 6c 00 00 00 6c 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 26 00 00 00 05 00 00 00 03 00 00 00 1c 00 00 00 1c 00 00 00 74 00 00 00 74 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 14 00 00 00 04 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 0a 00 00 00 0a 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 27 00 00 00 04 00 00 00 03 00 00 00 1d 00 00 00 1d 00 00 00 78 00 00 00 78 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 23 00 00 00 00 00 00 00 03 00 00 00 14 00 00 00 14 00 00 00 54 00 00 00 54 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 28 00 00 00 00 00 00 00 03 00 00 00 1b 00 00 00 1b 00 00 00 70 00 00 00 70 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 30 00 00 00 0e 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 61 00 00 00 61 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 31 00 00 00 0f 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 60 00 00 00 60 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 35 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 68 00 00 00 68 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 1d 00 00 00 09 00 00 00 03 00 00 00 19 00 00 00 19 00 00 00 69 00 00 00 6b 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 33 00 00 00 11 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 62 00 00 00 62 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 34 00 00 00 12 00 00 00 03 00 00 00 17 00 00 00 17 00 00 00 63 00 00 00 63 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 16 00 00 00 0b 00 00 00 03 00 00 00 ff ff 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00]; + apple,tunable-agx-af-lat-enable = [00 00 00 00]; + apple,tunable-soc-ni1-bw-threshold-a = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni3-bw-enable = [01 00 00 00]; + apple,tunable-ave1-af-bw-enable = [00 00 00 00]; + apple,tunable-soc-ni8-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc3-af-bw-enable = [01 00 00 00]; + apple,tunable-lts-config = [03 00 00 00 69 00 00 00 c3 1c 15 00 00 00 00 00 0f dc 0f 00 6e 00 00 00 68 00 00 00 00 00 11 00 33 b3 00 00 2f bd 48 01 03 00 00 00 69 00 00 00 c3 1c 15 00 00 00 00 00 0f dc 0f 00 6e 00 00 00 68 00 00 00 00 00 11 00 33 b3 00 00 2f bd 48 01 01 00 00 00 69 00 00 00 bf d9 01 00 00 00 00 00 da fc 0f 00 6e 00 00 00 66 00 00 00 00 00 11 00 33 b3 00 00 f9 fe c3 00 02 00 00 00 3b 01 00 00 a6 79 0a 00 00 00 00 00 2b ee 0f 00 6e 00 00 00 69 00 00 00 00 00 11 00 33 b3 00 00 97 6e 39 00]; + apple,tunable-soc-ni6-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave0-af-bw-threshold-a = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-bwr-catch-up-factor = [00 00 01 00]; + apple,tunable-fast-die-ctrl-ts = [00 00 05 00]; + apple,tunable-soc-ni8-bw-enable = [00 00 00 00]; + apple,tunable-mc_quota_rsp_ptd_count = [08 00 00 00]; + apple,tunable-soc-ni4-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-lts-ctrl-ts = [00 00 05 00]; + apple,tunable-atc1-af-bw-enable = [01 00 00 00]; + apple,tunable-ave0-af-bw-enable = [00 00 00 00]; + apple,tunable-atc2-af-bw-enable = [01 00 00 00]; + apple,tunable-fast-die-ctrl-ts-avg-temp = [00 00 64 00]; + apple,tunable-agx-af-bw-constraint-enable = [01 00 00 00]; + apple,tunable-soc-ni2-bw-enable = [01 00 00 00]; + apple,tunable-agx-afr-bw-enable = [01 00 00 00]; + apple,tunable-lts-acsk-tddb-config = [01 00 00 00 69 00 00 00 10 14 24 00 00 00 00 00 95 c2 0f 00 1a 04 00 00 56 04 00 00 00 00 00 00]; + apple,tunable-soc-ni7-bw-enable = [01 00 00 00]; + apple,tunable-ane-lnk1-af-bw-threshold-a = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 14 2e 21 00 00 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-user-power-managed = [01 00 00 00]; + apple,tunable-soc-ni0-bw-threshold = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc0-af-bw-enable = [01 00 00 00]; + apple,tunable-auss-af-bw-enable = [01 00 00 00]; + apple,tunable-dcs-bw-enable = [01 00 00 00]; + apple,tunable-no-shutdown = [01 00 00 00]; + apple,tunable-soc-ni4-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni5-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ane-dcs-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni9-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni9-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ptd-range = [01 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 4e 55 4c 4c 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 01 00 00 00 10 00 00 00 50 4d 50 2d 53 54 41 54 55 53 00 00 00 00 00 00 03 00 00 00 08 00 00 00 04 00 00 00 00 00 00 00 44 56 46 53 2d 53 54 41 54 45 00 00 00 00 00 00 04 00 00 00 0c 00 00 00 04 00 00 00 00 00 00 00 50 4d 50 54 4f 4f 4c 00 00 00 00 00 00 00 00 00 05 00 00 00 10 00 00 00 08 00 00 00 00 00 00 00 50 4d 53 2d 50 4d 47 52 2d 50 4b 54 00 00 00 00 06 00 00 00 18 00 00 00 20 00 00 00 00 00 00 00 50 4d 53 2d 44 50 45 2d 50 4b 54 00 00 00 00 00 07 00 00 00 38 00 00 00 48 00 00 00 00 00 00 00 50 4d 53 2d 44 45 56 43 2d 50 4b 54 00 00 00 00 08 00 00 00 80 00 00 00 10 00 00 00 00 00 00 00 50 4d 53 2d 50 57 52 53 2d 50 4b 54 00 00 00 00 09 00 00 00 90 00 00 00 50 01 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 4b 54 00 00 00 00 00 0a 00 00 00 e0 01 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 52 45 51 00 00 0b 00 00 00 e8 01 00 00 08 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 50 53 2d 41 43 4b 00 00 0c 00 00 00 f0 01 00 00 30 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 44 56 46 53 00 00 00 00 0d 00 00 00 20 02 00 00 1a 00 00 00 00 00 00 00 53 4f 43 2d 44 45 56 2d 42 57 52 00 00 00 00 00 0e 00 00 00 3a 02 00 00 02 00 00 00 00 00 00 00 44 56 46 53 2d 44 49 45 2d 54 4f 2d 44 49 45 00 0f 00 00 00 3c 02 00 00 0a 00 00 00 00 00 00 00 4d 43 2d 44 52 56 2d 54 4f 2d 43 54 52 4c 00 00 10 00 00 00 58 02 00 00 02 00 00 00 0c 00 00 00 46 41 53 54 2d 44 49 45 2d 43 45 00 00 00 00 00 11 00 00 00 5a 02 00 00 07 00 00 00 0d 00 00 00 53 4d 43 2d 54 4f 2d 50 50 4d 00 00 00 00 00 00 12 00 00 00 61 02 00 00 09 00 00 00 00 00 00 00 4d 43 2d 43 54 52 4c 2d 54 4f 2d 44 52 56 00 00 13 00 00 00 6a 02 00 00 01 00 00 00 0a 00 00 00 4c 54 53 2d 56 4f 4c 54 2d 43 41 50 00 00 00 00 18 00 00 00 70 02 00 00 05 00 00 00 0e 00 00 00 50 50 4d 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 19 00 00 00 75 02 00 00 05 00 00 00 0f 00 00 00 43 4c 50 43 2d 54 4f 2d 53 4d 43 00 00 00 00 00 1a 00 00 00 7a 02 00 00 20 00 00 00 00 00 00 00 47 46 58 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 1b 00 00 00 9a 02 00 00 09 00 00 00 00 00 00 00 46 41 53 54 2d 44 49 45 2d 54 4f 2d 53 4d 43 00 1c 00 00 00 a3 02 00 00 01 00 00 00 00 00 00 00 4d 43 2d 43 54 52 4c 2d 54 4f 2d 53 4d 43 00 00 1d 00 00 00 a4 02 00 00 08 00 00 00 00 00 00 00 41 4e 45 2d 54 4f 2d 53 4d 43 00 00 00 00 00 00 20 00 00 00 b4 02 00 00 01 00 00 00 00 00 00 00 4c 54 53 2d 54 4f 2d 47 46 58 00 00 00 00 00 00 21 00 00 00 b5 02 00 00 01 00 00 00 0b 00 00 00 50 4d 50 2d 54 4f 2d 47 46 58 00 00 00 00 00 00 22 00 00 00 b6 02 00 00 08 00 00 00 00 00 00 00 50 4d 55 2d 54 4f 2d 47 46 58 00 00 00 00 00 00 28 00 00 00 20 03 00 00 14 00 00 00 00 00 00 00 50 4d 53 2d 56 49 52 54 2d 43 4e 54 52 00 00 00]; + apple,tunable-ans-dma-af-bw-threshold = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-mc_quota_req_ptd_count = [08 00 00 00]; + apple,tunable-ans-nand-af-bw-enable = [01 00 00 00]; + apple,tunable-ans-dcs-bw-enable = [01 00 00 00]; + apple,tunable-soc-ni1-bw-threshold = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-device-ps-group = [00 f0 ff ff ff 01 00 00]; + apple,tunable-soc-ni2-bw-threshold-a = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ane-af-bw-enable = [00 00 00 00]; + apple,tunable-soc-ni7-bw-threshold-a = [00 00 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 00 00 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni1-bw-enable = [01 00 00 00]; + apple,tunable-auss-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-sram-index = [01 00 00 00]; + apple,tunable-soc-ni6-bw-enable = [01 00 00 00]; + apple,tunable-coredump-enable = [40 00 00 00]; + apple,tunable-dvfs-domain = [01 00 00 00 07 00 00 00 00 00 00 00 44 43 53 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 0d 00 00 00 21 00 00 00 41 46 52 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 05 00 00 00 00 00 00 00 53 4f 43 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 04 00 00 00 00 00 00 00 44 49 53 50 5f 49 4e 54 00 00 00 00 00 00 00 00 05 00 00 00 04 00 00 00 00 00 00 00 44 49 53 50 5f 45 58 54 00 00 00 00 00 00 00 00]; + apple,tunable-vlpm-timer-limit = [64 00 00 00]; + apple,tunable-fast-die-ctrl-ce-map = [01 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 30 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 31 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 5f 41 4e 45 00 00 00 00 00 00 00 00]; + apple,tunable-soc-ni0-bw-threshold-a = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-auss-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ave0-af-bw-threshold = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni2-bw-threshold = [8f c2 15 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 8f c2 14 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-jpeg-af-bw-threshold-a = [99 99 03 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 99 99 02 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-controller = [01 00 00 00 10 00 00 00 1b 00 00 00 00 00 00 00 00 00 00 00 44 49 45 2d 54 45 4d 50 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 46 49 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 47 58 2d 41 46 52 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 30 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 31 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 32 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 33 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 34 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 35 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 36 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 37 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 38 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 53 4f 43 2d 4e 49 39 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 11 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 4c 4e 4b 30 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 4c 4e 4b 31 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 13 00 00 00 0f 00 00 00 12 00 00 00 00 00 00 00 00 00 00 00 4d 45 4d 2d 43 41 43 48 45 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 4e 41 4e 44 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 15 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 44 4d 41 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 16 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 30 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 17 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 31 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 18 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 32 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 54 43 33 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1a 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 30 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1b 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 56 45 31 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1c 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 55 53 53 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4a 50 45 47 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 41 46 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 1f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 45 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 20 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 4e 53 2d 44 43 53 2d 42 57 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 21 00 00 00 08 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 56 4c 50 4d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 22 00 00 00 20 00 00 00 13 00 00 00 00 00 00 00 00 00 00 00 4c 49 46 45 54 49 4d 45 2d 53 45 52 56 4f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-ave1-af-bw-threshold = [99 99 09 00 00 00 00 00 00 00 00 00 03 00 00 00 66 66 0e 00 00 00 00 00 99 99 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 0d 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni6-bw-threshold = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-nand-af-bw-threshold-a = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-agx-afr-bw-constraint-enable = [01 00 00 00]; + apple,tunable-soc-ni0-bw-enable = [01 00 00 00]; + apple,tunable-sochot-sensors = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 08 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00 0f 00 00 00 10 00 00 00 11 00 00 00 12 00 00 00 13 00 00 00 14 00 00 00 15 00 00 00 16 00 00 00 17 00 00 00 18 00 00 00 19 00 00 00 1a 00 00 00 1b 00 00 00 1c 00 00 00 1d 00 00 00 1e 00 00 00 1f 00 00 00 20 00 00 00 21 00 00 00 22 00 00 00 23 00 00 00 24 00 00 00 25 00 00 00 26 00 00 00 27 00 00 00 28 00 00 00 29 00 00 00 2a 00 00 00 2b 00 00 00 2c 00 00 00 2d 00 00 00 2e 00 00 00 2f 00 00 00 30 00 00 00 31 00 00 00 32 00 00 00 33 00 00 00 34 00 00 00 35 00 00 00 36 00 00 00 37 00 00 00 38 00 00 00 39 00 00 00 3a 00 00 00 3b 00 00 00 3c 00 00 00 3d 00 00 00 3e 00 00 00 3f 00 00 00 40 00 00 00 41 00 00 00 42 00 00 00 43 00 00 00 44 00 00 00 45 00 00 00]; + apple,tunable-soc-ni5-bw-enable = [01 00 00 00]; + apple,tunable-agx-dcs-bw-constraint-enable = [01 00 00 00]; + apple,tunable-temp-sensor = [01 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 30 69 00 00 00 00 00 00 00 00 00 00 02 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 31 69 00 00 00 00 00 00 00 00 00 00 03 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 32 69 00 00 00 00 00 00 00 00 00 00 04 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 35 69 00 00 00 00 00 00 00 00 00 00 05 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 36 69 00 00 00 00 00 00 00 00 00 00 06 00 00 00 08 00 00 00 11 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 80 06 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 61 30 30 37 69 00 00 00 00 00 00 00 00 00 00 07 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 63 69 00 00 00 00 00 00 00 00 00 00 08 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 64 69 00 00 00 00 00 00 00 00 00 00 09 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 30 69 00 00 00 00 00 00 00 00 00 00 0a 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 31 69 00 00 00 00 00 00 00 00 00 00 0b 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 66 69 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 67 69 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 33 69 00 00 00 00 00 00 00 00 00 00 0e 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 34 69 00 00 00 00 00 00 00 00 00 00 0f 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 65 69 00 00 00 00 00 00 00 00 00 00 10 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 09 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 32 69 00 00 00 00 00 00 00 00 00 00 11 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 68 69 00 00 00 00 00 00 00 00 00 00 12 00 00 00 08 00 00 00 0f 00 00 00 00 00 00 00 0b 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 35 69 00 00 00 00 00 00 00 00 00 00 13 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 00 80 01 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6c 69 00 00 00 00 00 00 00 00 00 00 14 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 00 80 01 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6d 69 00 00 00 00 00 00 00 00 00 00 15 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 06 00 00 00 00 00 00 00 66 e6 04 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 6f 69 00 00 00 00 00 00 00 00 00 00 16 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 07 00 00 00 00 00 00 00 66 e6 04 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 70 69 00 00 00 00 00 00 00 00 00 00 17 00 00 00 08 00 00 00 0f 00 00 00 01 00 00 00 0a 00 00 00 00 00 00 00 66 e6 04 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 73 69 00 00 00 00 00 00 00 00 00 00 18 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 39 69 00 00 00 00 00 00 00 00 00 00 19 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 61 69 00 00 00 00 00 00 00 00 00 00 1a 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 36 69 00 00 00 00 00 00 00 00 00 00 1b 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 70 30 30 37 69 00 00 00 00 00 00 00 00 00 00 1c 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 62 69 00 00 00 00 00 00 00 00 00 00 1d 00 00 00 08 00 00 00 0f 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 70 30 30 38 69 00 00 00 00 00 00 00 00 00 00 1e 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 33 69 00 00 00 00 00 00 00 00 00 00 1f 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 34 69 00 00 00 00 00 00 00 00 00 00 20 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 30 69 00 00 00 00 00 00 00 00 00 00 21 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 31 69 00 00 00 00 00 00 00 00 00 00 22 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 36 69 00 00 00 00 00 00 00 00 00 00 23 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 37 69 00 00 00 00 00 00 00 00 00 00 24 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 35 69 00 00 00 00 00 00 00 00 00 00 25 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 32 69 00 00 00 00 00 00 00 00 00 00 26 00 00 00 08 00 00 00 0d 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 38 69 00 00 00 00 00 00 00 00 00 00 27 00 00 00 08 00 00 00 0d 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 6c 69 00 00 00 00 00 00 00 00 00 00 28 00 00 00 08 00 00 00 0d 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 6d 69 00 00 00 00 00 00 00 00 00 00 29 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 39 69 00 00 00 00 00 00 00 00 00 00 2a 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 61 69 00 00 00 00 00 00 00 00 00 00 2b 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 63 69 00 00 00 00 00 00 00 00 00 00 2c 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 64 69 00 00 00 00 00 00 00 00 00 00 2d 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 66 69 00 00 00 00 00 00 00 00 00 00 2e 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 67 69 00 00 00 00 00 00 00 00 00 00 2f 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 62 69 00 00 00 00 00 00 00 00 00 00 30 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 65 69 00 00 00 00 00 00 00 00 00 00 31 00 00 00 08 00 00 00 0d 00 00 00 02 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 30 68 69 00 00 00 00 00 00 00 00 00 00 32 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 33 69 00 00 00 00 00 00 00 00 00 00 33 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 34 69 00 00 00 00 00 00 00 00 00 00 34 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 30 69 00 00 00 00 00 00 00 00 00 00 35 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 31 69 00 00 00 00 00 00 00 00 00 00 36 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 36 69 00 00 00 00 00 00 00 00 00 00 37 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 37 69 00 00 00 00 00 00 00 00 00 00 38 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 35 69 00 00 00 00 00 00 00 00 00 00 39 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 32 69 00 00 00 00 00 00 00 00 00 00 3a 00 00 00 08 00 00 00 0e 00 00 00 00 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 38 69 00 00 00 00 00 00 00 00 00 00 3b 00 00 00 08 00 00 00 0e 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 6c 69 00 00 00 00 00 00 00 00 00 00 3c 00 00 00 08 00 00 00 0e 00 00 00 01 00 00 00 04 00 00 00 00 00 00 00 cc cc 03 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 6d 69 00 00 00 00 00 00 00 00 00 00 3d 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 39 69 00 00 00 00 00 00 00 00 00 00 3e 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 61 69 00 00 00 00 00 00 00 00 00 00 3f 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 02 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 63 69 00 00 00 00 00 00 00 00 00 00 40 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 03 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 64 69 00 00 00 00 00 00 00 00 00 00 41 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 04 00 00 00 00 00 00 00 00 00 0a 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 66 69 00 00 00 00 00 00 00 00 00 00 42 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 05 00 00 00 00 00 00 00 00 00 0e 00 47 a1 03 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 67 69 00 00 00 00 00 00 00 00 00 00 43 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 06 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 62 69 00 00 00 00 00 00 00 00 00 00 44 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 07 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 65 69 00 00 00 00 00 00 00 00 00 00 45 00 00 00 08 00 00 00 0e 00 00 00 02 00 00 00 08 00 00 00 00 00 00 00 00 00 0e 00 47 61 05 00 6e 00 00 00 7d 00 00 00 54 6d 30 31 68 69 00 00 00 00 00 00 00 00 00 00 46 00 00 00 01 00 00 00 10 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 6d 00 00 00 6d 00 00 00 4d 54 52 5f 54 4f 50 5f 41 47 58 00 00 00 00 00]; + apple,tunable-lts-voltage = [01 00 00 00 03 00 00 00 0d 00 00 00 09 00 00 00 4d 41 43 43 30 00 00 00 02 00 00 00 03 00 00 00 0e 00 00 00 0a 00 00 00 4d 41 43 43 31 00 00 00 03 00 00 00 03 00 00 00 0f 00 00 00 02 00 00 00 50 41 43 43 00 00 00 00 04 00 00 00 01 00 00 00 11 00 00 00 00 00 00 00 41 4e 45 00 00 00 00 00 05 00 00 00 03 00 00 00 10 00 00 00 04 00 00 00 41 47 58 00 00 00 00 00]; + apple,tunable-afc-turbo-enable = [01 00 00 00]; + apple,tunable-jpeg-af-bw-enable = [00 00 00 00]; + apple,tunable-soc-ni5-bw-threshold-a = [ae 07 16 00 00 00 00 00 00 00 00 00 03 00 00 00 eb 91 2b 00 00 00 00 00 ae 07 15 00 03 00 00 00 00 00 ff ff 00 00 00 00 eb 91 2a 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-pm-ptd-ranges = [01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 05 00 00 00 06 00 00 00 07 00 00 00 08 00 00 00 28 00 00 00 09 00 00 00 0a 00 00 00 0b 00 00 00 0c 00 00 00 0d 00 00 00 0e 00 00 00]; + apple,tunable-ane-lnk0-af-bw-threshold-a = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 99 19 39 00 00 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 14 2e 21 00 00 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 0a 17 10 00 00 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-agx-afr-lat-enable = [00 00 00 00]; + apple,tunable-ans-nand-af-bw-threshold = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 05 00 05 00 00 00 3d 0a 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 04 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-ans-dma-af-bw-enable = [01 00 00 00]; + apple,tunable-lts-ctrl-loop = [01 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 01 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 30 00 00 00 00 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 03 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 4d 41 43 43 31 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 02 00 00 00 01 00 00 00 00 00 00 00 02 00 00 00 03 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 50 41 43 43 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 03 00 00 00 02 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 47 50 55 00 00 00 00 00 00 00 00 00 00 00 00 00]; + apple,tunable-jpeg-af-bw-threshold = [99 99 03 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 99 99 02 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-chip-variant = [01 00 00 00]; + apple,tunable-agx-afr-bw-dvfs-filter = [fe ff ff ff]; + apple,tunable-agx-dcs-bw-dvfs-filter = [fe ff ff ff]; + apple,tunable-ane-dcs-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 46 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 45 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00]; + apple,tunable-dcs-bw-threshold = [00 c0 4f 00 01 00 00 00 00 00 00 00 00 00 00 00 5c 8f 9f 00 02 00 00 00 00 c0 4e 00 00 00 00 00 eb 11 e8 00 03 00 00 00 5c 8f 9e 00 00 00 00 00 f5 28 5c 01 04 00 00 00 eb 11 e7 00 00 00 00 00 66 26 96 01 05 00 00 00 f5 28 5b 01 00 00 00 00 33 33 d0 01 06 00 00 00 66 26 95 01 00 00 00 00 00 00 ff ff 07 00 00 00 33 33 cf 01 00 00 00 00]; + apple,tunable-afi-bw-threshold-a = [47 21 58 00 00 00 00 00 00 00 00 00 03 00 00 00 cc 4c ae 00 00 00 00 00 47 21 57 00 03 00 00 00 66 66 e8 00 00 00 00 00 cc 4c ad 00 03 00 00 00 66 66 e8 00 00 00 00 00 66 66 e7 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 e7 00 03 00 00 00]; + apple,tunable-ane-af-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-agx-af-bw-threshold-a = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 d6 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 02 00 00 00 00 00 00 d5 01 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 80 5b 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 85 02 00 00 00 00 00 80 5a 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 e1 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 85 02 00 00 00 00 00 00 e0 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 bc 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 bb 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 0e 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5e 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00]; + apple,tunable-agx-af-bw-constraint-a = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 04 00 00 00 01 00 00 00 01 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 04 00 00 00 02 00 00 00 02 00 00 00 04 00 00 00]; + apple,tunable-agx-af-bw-threshold = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 d6 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 85 02 00 00 00 00 00 00 d5 01 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 80 5b 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 85 02 00 00 00 00 00 80 5a 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 e1 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 85 02 00 00 00 00 00 00 e0 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 84 02 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 bc 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 bb 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 0f 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 0e 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a4 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a3 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 5e 01 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 5d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00]; + apple,tunable-agx-afr-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 5c 8f 96 00 00 00 00 00 00 00 00 00 00 00 00 00 66 e6 e9 00 00 00 00 00 5c 8f 95 00 00 00 00 00 d7 e3 48 01 00 00 00 00 66 e6 e8 00 00 00 00 00 f5 28 b8 01 00 00 00 00 d7 e3 47 01 00 00 00 00 99 d9 0a 02 00 00 00 00 f5 28 b7 01 00 00 00 00 d7 23 69 02 00 00 00 00 99 d9 09 02 00 00 00 00 c2 f5 cb 02 00 00 00 00 d7 23 68 02 00 00 00 00 cc 0c 37 03 00 00 00 00 c2 f5 ca 02 00 00 00 00 33 b3 a7 03 00 00 00 00 cc 0c 36 03 00 00 00 00 e1 3a 1f 04 00 00 00 00 33 b3 a6 03 00 00 00 00 d7 a3 9d 04 00 00 00 00 e1 3a 1e 04 00 00 00 00 1e 05 2d 05 00 00 00 00 d7 a3 9c 04 00 00 00 00 00 00 ff ff 00 00 00 00 1e 05 2c 05 00 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 1e 05 d1 00 00 00 00 00 00 00 00 00 06 00 00 00 7a 54 28 01 00 00 00 00 1e 05 d0 00 06 00 00 00 66 66 8f 01 00 00 00 00 7a 54 27 01 06 00 00 00 d7 63 dd 01 00 00 00 00 66 66 8e 01 06 00 00 00 5c 8f 36 02 00 00 00 00 d7 63 dc 01 06 00 00 00 c2 75 94 02 00 00 00 00 5c 8f 35 02 06 00 00 00 ae 87 fa 02 00 00 00 00 c2 75 93 02 06 00 00 00 1e 45 66 03 00 00 00 00 ae 87 f9 02 06 00 00 00 47 e1 d8 03 00 00 00 00 1e 45 65 03 06 00 00 00 47 61 52 04 00 00 00 00 47 e1 d7 03 06 00 00 00 51 38 dc 04 00 00 00 00 47 61 51 04 06 00 00 00 00 00 ff ff 00 00 00 00 51 38 db 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 3d ca 14 01 00 00 00 00 00 00 00 00 06 00 00 00 33 f3 76 01 00 00 00 00 3d ca 13 01 06 00 00 00 28 1c c2 01 00 00 00 00 33 f3 75 01 06 00 00 00 c2 35 18 02 00 00 00 00 28 1c c1 01 06 00 00 00 f5 28 73 02 00 00 00 00 c2 35 17 02 06 00 00 00 51 38 d6 02 00 00 00 00 f5 28 72 02 06 00 00 00 8f 02 3f 03 00 00 00 00 51 38 d5 02 06 00 00 00 85 ab ae 03 00 00 00 00 8f 02 3e 03 06 00 00 00 51 38 25 04 00 00 00 00 85 ab ad 03 06 00 00 00 00 c0 ab 04 00 00 00 00 51 38 24 04 06 00 00 00 00 00 ff ff 00 00 00 00 00 c0 aa 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 66 a6 66 01 00 00 00 00 00 00 00 00 06 00 00 00 85 eb af 01 00 00 00 00 66 a6 65 01 06 00 00 00 51 f8 03 02 00 00 00 00 85 eb ae 01 06 00 00 00 c2 f5 5c 02 00 00 00 00 51 f8 02 02 06 00 00 00 8f 02 be 02 00 00 00 00 c2 f5 5b 02 06 00 00 00 7a d4 24 03 00 00 00 00 8f 02 bd 02 06 00 00 00 ae 87 92 03 00 00 00 00 7a d4 23 03 06 00 00 00 b8 1e 07 04 00 00 00 00 ae 87 91 03 06 00 00 00 14 6e 8b 04 00 00 00 00 b8 1e 06 04 06 00 00 00 00 00 ff ff 00 00 00 00 14 6e 8a 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 85 eb af 01 00 00 00 00 00 00 00 00 06 00 00 00 51 f8 03 02 00 00 00 00 85 eb ae 01 06 00 00 00 c2 f5 5c 02 00 00 00 00 51 f8 02 02 06 00 00 00 8f 02 be 02 00 00 00 00 c2 f5 5b 02 06 00 00 00 7a d4 24 03 00 00 00 00 8f 02 bd 02 06 00 00 00 ae 87 92 03 00 00 00 00 7a d4 23 03 06 00 00 00 b8 1e 07 04 00 00 00 00 ae 87 91 03 06 00 00 00 14 6e 8b 04 00 00 00 00 b8 1e 06 04 06 00 00 00 00 00 ff ff 00 00 00 00 14 6e 8a 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 70 bd ef 01 00 00 00 00 00 00 00 00 06 00 00 00 8f c2 46 02 00 00 00 00 70 bd ee 01 06 00 00 00 5c cf a5 02 00 00 00 00 8f c2 45 02 06 00 00 00 f5 a8 0a 03 00 00 00 00 5c cf a4 02 06 00 00 00 d7 63 76 03 00 00 00 00 f5 a8 09 03 06 00 00 00 8f 02 e9 03 00 00 00 00 d7 63 75 03 06 00 00 00 28 1c 6b 04 00 00 00 00 8f 02 e8 03 06 00 00 00 00 00 ff ff 00 00 00 00 28 1c 6a 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 5c 8f 30 02 00 00 00 00 00 00 00 00 06 00 00 00 99 99 8d 02 00 00 00 00 5c 8f 2f 02 06 00 00 00 70 7d f0 02 00 00 00 00 99 99 8c 02 06 00 00 00 8f 42 5a 03 00 00 00 00 70 7d ef 02 06 00 00 00 f5 e8 ca 03 00 00 00 00 8f 42 59 03 06 00 00 00 3d ca 4a 04 00 00 00 00 f5 e8 c9 03 06 00 00 00 00 00 ff ff 00 00 00 00 3d ca 49 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 d7 63 75 02 00 00 00 00 00 00 00 00 06 00 00 00 5c 4f d6 02 00 00 00 00 d7 63 74 02 06 00 00 00 b8 1e 3e 03 00 00 00 00 5c 4f d5 02 06 00 00 00 cc cc ac 03 00 00 00 00 b8 1e 3d 03 06 00 00 00 e1 7a 2a 04 00 00 00 00 cc cc ab 03 06 00 00 00 00 00 ff ff 00 00 00 00 e1 7a 29 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 d7 23 bc 02 00 00 00 00 00 00 00 00 06 00 00 00 e1 fa 21 03 00 00 00 00 d7 23 bb 02 06 00 00 00 a3 b0 8e 03 00 00 00 00 e1 fa 20 03 06 00 00 00 f5 28 0a 04 00 00 00 00 a3 b0 8d 03 06 00 00 00 00 00 ff ff 00 00 00 00 f5 28 09 04 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a d7 05 03 00 00 00 00 00 00 00 00 06 00 00 00 0a 97 70 03 00 00 00 00 0a d7 04 03 06 00 00 00 0a d7 e9 03 00 00 00 00 0a 97 6f 03 06 00 00 00 00 00 ff ff 00 00 00 00 0a d7 e8 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 eb 51 25 03 00 00 00 00 00 00 00 00 06 00 00 00 cc 0c 99 03 00 00 00 00 eb 51 24 03 06 00 00 00 00 00 ff ff 00 00 00 00 cc 0c 98 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 40 48 03 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 40 47 03 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00]; + apple,tunable-agx-af-bw-dvfs-filter = [fe ff ff ff]; + apple,tunable-afi-bw-threshold = [47 21 58 00 00 00 00 00 00 00 00 00 03 00 00 00 cc 4c ae 00 00 00 00 00 47 21 57 00 03 00 00 00 66 66 e8 00 00 00 00 00 cc 4c ad 00 03 00 00 00 00 00 ff ff 00 00 00 00 66 66 e7 00 03 00 00 00]; + apple,tunable-agx-af-bw-constraint = [00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 00 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 01 00 00 00 01 00 00 00 03 00 00 00 02 00 00 00 02 00 00 00 03 00 00 00 02 00 00 00 02 00 00 00 03 00 00 00 02 00 00 00 02 00 00 00 03 00 00 00 02 00 00 00 02 00 00 00 03 00 00 00]; + apple,tunable-ane-af-bw-threshold-a = [00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00 00 00 05 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0e 00 00 00 00 00 00 00 04 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 0d 00 06 00 00 00 00 00 32 00 00 00 00 00 00 00 31 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 31 00 06 00 00 00]; + apple,tunable-ans-dcs-bw-threshold = [99 19 00 00 05 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 5c 0f 00 00 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 05 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-agx-dcs-bw-threshold = [00 00 ff ff 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 45 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 90 00 00 00 00 00 00 00 44 00 00 00 00 00 00 00 d2 00 00 00 00 00 00 00 8f 00 00 00 00 00 cc cc 47 01 00 00 00 00 00 00 d1 00 00 00 00 00 00 00 ff ff 00 00 00 00 cc cc 46 01 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 00 00 00 00 00 00 2e 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 90 00 00 00 00 00 00 00 2d 00 06 00 00 00 00 00 d2 00 00 00 00 00 00 00 8f 00 06 00 00 00 cc cc 47 01 00 00 00 00 00 00 d1 00 06 00 00 00 00 00 ff ff 00 00 00 00 cc cc 46 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 90 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 be 00 00 00 00 00 00 00 8f 00 06 00 00 00 cc cc 47 01 00 00 00 00 00 00 bd 00 06 00 00 00 00 00 8e 01 00 00 00 00 cc cc 46 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 8d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 90 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 be 00 00 00 00 00 00 00 8f 00 06 00 00 00 cc cc 47 01 00 00 00 00 00 00 bd 00 06 00 00 00 00 00 8e 01 00 00 00 00 cc cc 46 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 8d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 86 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 a0 00 00 00 00 00 00 00 85 00 06 00 00 00 cc cc 47 01 00 00 00 00 00 00 9f 00 06 00 00 00 00 00 8e 01 00 00 00 00 cc cc 46 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 8d 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 09 00 06 00 00 00 51 f8 18 01 00 00 00 00 00 00 13 00 06 00 00 00 00 00 82 01 00 00 00 00 51 f8 17 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 81 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 14 00 00 00 00 00 00 00 09 00 06 00 00 00 51 f8 18 01 00 00 00 00 00 00 13 00 06 00 00 00 00 00 82 01 00 00 00 00 51 f8 17 01 06 00 00 00 00 00 c7 01 00 00 00 00 00 00 81 01 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 c6 01 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 69 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 8e 00 00 00 00 00 00 00 68 00 06 00 00 00 00 00 b9 00 00 00 00 00 00 00 8d 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 b8 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 7a 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 a3 00 00 00 00 00 00 00 79 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 a2 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 8c 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 8b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 8c 00 00 00 00 00 00 00 09 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 8b 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 06 00 00 00]; + apple,tunable-agx-dcs-bw-constraint = [00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 06 00 00 00 01 00 00 00 01 00 00 00 06 00 00 00 01 00 00 00 01 00 00 00 06 00 00 00 01 00 00 00 01 00 00 00 06 00 00 00 02 00 00 00 02 00 00 00 06 00 00 00 03 00 00 00 03 00 00 00 06 00 00 00 04 00 00 00 04 00 00 00 06 00 00 00 04 00 00 00 04 00 00 00 06 00 00 00 05 00 00 00 05 00 00 00 06 00 00 00 05 00 00 00 05 00 00 00 06 00 00 00]; + apple,tunable-atc2-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni6-push-thr = [24 00 00 00]; + apple,tunable-soc-ni1-push-thr = [f5 06 00 00]; + apple,tunable-atc0-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni5-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni0-push-thr = [f5 06 00 00]; + apple,tunable-atc2-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni8-push-thr = [f5 06 00 00]; + apple,tunable-ane-lnk1-af-bw-threshold = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-atc1-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-agx-afr-bw-constraint = [00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0c 00 00 00 01 00 00 00 01 00 00 00 0c 00 00 00 02 00 00 00 02 00 00 00 0c 00 00 00 03 00 00 00 03 00 00 00 0c 00 00 00 04 00 00 00 04 00 00 00 0c 00 00 00 05 00 00 00 05 00 00 00 0c 00 00 00 06 00 00 00 06 00 00 00 0c 00 00 00 07 00 00 00 07 00 00 00 0c 00 00 00 08 00 00 00 08 00 00 00 0c 00 00 00 09 00 00 00 09 00 00 00 0c 00 00 00 0a 00 00 00 0a 00 00 00 0c 00 00 00 0b 00 00 00 0b 00 00 00 0c 00 00 00 0c 00 00 00 0c 00 00 00 0c 00 00 00]; + apple,tunable-soc-ni2-push-thr = [f5 06 00 00]; + apple,tunable-atc3-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni7-push-thr = [f5 06 00 00]; + apple,tunable-ane-lnk0-af-bw-threshold = [00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 0a 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 09 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 03 00 00 00 47 a1 19 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 47 a1 18 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 99 19 3a 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 99 19 39 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 14 2e 22 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 14 2e 21 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 01 00 00 00 00 00 00 00 00 00 06 00 00 00 0a 17 11 00 00 00 00 00 00 00 00 00 06 00 00 00 00 00 ff ff 00 00 00 00 0a 17 10 00 06 00 00 00]; + apple,tunable-atc3-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-atc0-af-bw-threshold = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni9-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni4-push-thr = [f5 06 00 00]; + apple,tunable-atc1-af-bw-threshold-a = [51 b8 09 00 00 00 00 00 00 00 00 00 03 00 00 00 00 00 ff ff 00 00 00 00 51 b8 08 00 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00 00 00 ff ff 00 00 00 00 00 00 fe ff 03 00 00 00]; + apple,tunable-soc-ni3-push-thr = [f5 06 00 00]; + apple,tunable-soc-ni3-bw-a-enable = [01 00 00 00]; + apple,tunable-agx-afr-lat-constraint-enable = [01 00 00 00]; + apple,tunable-soc-ni4-bw-a-enable = [01 00 00 00]; + apple,tunable-ane-af-bw-a-enable = [01 00 00 00]; + apple,tunable-ave1-af-bw-a-enable = [00 00 00 00]; + apple,tunable-soc-ni5-bw-a-enable = [01 00 00 00]; + apple,tunable-atc2-af-bw-a-enable = [01 00 00 00]; + apple,tunable-ans-nand-af-bw-a-enable = [01 00 00 00]; + apple,tunable-ave0-af-bw-a-enable = [00 00 00 00]; + apple,tunable-soc-ni6-bw-a-enable = [01 00 00 00]; + apple,tunable-atc1-af-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni7-bw-a-enable = [01 00 00 00]; + apple,tunable-atc0-af-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni8-bw-a-enable = [00 00 00 00]; + apple,tunable-ans-dma-af-bw-a-enable = [01 00 00 00]; + apple,tunable-agx-af-bw-constraint-a-enable = [01 00 00 00]; + apple,tunable-soc-ni0-bw-a-enable = [01 00 00 00]; + apple,tunable-agx-af-bw-a-enable = [00 00 00 00]; + apple,tunable-jpeg-af-bw-a-enable = [00 00 00 00]; + apple,tunable-soc-ni1-bw-a-enable = [01 00 00 00]; + apple,tunable-afi-bw-a-enable = [01 00 00 00]; + apple,tunable-atc3-af-bw-a-enable = [01 00 00 00]; + apple,tunable-agx-dcs-lat-constraint-enable = [01 00 00 00]; + apple,tunable-auss-af-bw-a-enable = [01 00 00 00]; + apple,tunable-soc-ni2-bw-a-enable = [01 00 00 00]; + apple,tunable-agx-af-lat-constraint-enable = [01 00 00 00]; + apple,tunable-soc-ni9-bw-a-enable = [00 00 00 00]; + apple,tunable-disable-vlpm-feature = [00 00 00 00]; + apple,tunable-uuid = [35 42 44 43 30 45 36 43 2d 46 37 39 32 2d 33 45 32 35 2d 42 45 45 39 2d 35 42 42 38 38 30 38 45 33 37 41 33 00]; + apple,tunable-pre-loaded = [01 00 00 00]; + apple,tunable-agx-dvfs-state-voltages = [7d 00 00 00 76 02 00 00 9e 02 00 00 c1 02 00 00 e9 02 00 00 f3 02 00 00 0c 03 00 00 2f 03 00 00 39 03 00 00 5c 03 00 00 84 03 00 00 ac 03 00 00 fc 03 00 00 1f 04 00 00 7d 00 00 00 7d 00 00 00]; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j714c-usb.dtsi b/arch/arm64/boot/dts/apple/t6050-j714c-usb.dtsi new file mode 100644 index 0000000000000..358865bacd93b --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j714c-usb.dtsi @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * USB-C port 0 of the MacBook Pro 14" (M5 Max, J714c), after the M5 Air + * description (t8142-j813-usb.dtsi). Addresses from the ADT: usb-drd0, + * dart-usb0, atc-phy0, nub-spmi-a0/hpm0, arm-io base 0x200000000. + * Ports 1 and 2 (usb-drd1/2, hpm1 on nub-spmi-a0, hpm2 on nub-spmi-a1) + * follow once port 0 works. + * + * NOT INCLUDED YET: the T6050 Type-C PHY is a new generation. Its ADT node + * has 44 register windows and a tunable set (AUS40CMN, CIO4PLL, + * LN0/1_RX_CFG_TX_OF_RXCLK, ...) that no loader table or kernel PHY + * generation describes; with the T8142 description the PHY probe fails on + * the missing tunables and the controller never binds. Needs a T6050 + * generation in drivers/phy/apple/atc.c and a tunable table in the loader. + */ + +#include +#include + +/ { + aliases { + atcphy0 = &atcphy0; + usb0 = &dwc3_0; + }; +}; + +&soc { + nub_spmi_a0: spmi@28a188000 { + compatible = "apple,t6050-spmi", "apple,t8142-spmi"; + reg = <0x2 0x8a188000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a0>; + + hpm0: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + + typec0: connector { + compatible = "usb-c-connector"; + label = "USB-C port 0"; + power-role = "dual"; + data-role = "host"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + typec0_hs: endpoint { + remote-endpoint = <&dwc3_0_hs>; + }; + }; + port@1 { + reg = <1>; + typec0_ss: endpoint { + remote-endpoint = <&atcphy0_typec>; + }; + }; + }; + }; + }; + }; + + dwc3_0: usb@2a2280000 { + compatible = "apple,t6050-dwc3", "apple,t8142-dwc3", "apple,t8103-dwc3"; + reg = <0x2 0xa2280000 0x0 0xcd00>, <0x2 0xa228cd00 0x0 0x4b00>; + reg-names = "dwc3-core", "dwc3-apple"; + interrupts = ; + dr_mode = "host"; + usb-role-switch; + role-switch-default-mode = "host"; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + resets = <&atcphy0>; + phys = <&atcphy0 PHY_TYPE_USB2>, <&atcphy0 PHY_TYPE_USB3>; + phy-names = "usb2-phy", "usb3-phy"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + dwc3_0_hs: endpoint { + remote-endpoint = <&typec0_hs>; + }; + }; + port@1 { + reg = <1>; + dwc3_0_ss: endpoint { + remote-endpoint = <&atcphy0_usb>; + }; + }; + }; + }; + + dwc3_0_dart_0: iommu@2a2f00000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xa2f00000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_0_dart_1: iommu@2a2f80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xa2f80000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + atcphy0: phy@2a3000000 { + compatible = "apple,t6050-atcphy", "apple,t8142-atcphy"; + reg = <0x2 0xa3000000 0x0 0x4c000>, + <0x2 0xa3050000 0x0 0x8000>, + <0x2 0xa2800000 0x0 0x8000>, + <0x2 0xa2a90000 0x0 0x4000>, + <0x2 0xa2a84000 0x0 0x4000>; + reg-names = "core", "lpdptx", "axi2af", "usb2phy", "pipehandler"; + #phy-cells = <1>; + #reset-cells = <0>; + orientation-switch; + mode-switch; + power-domains = <&ps_atc0_usb>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + port@0 { + reg = <0>; + atcphy0_typec: endpoint { + remote-endpoint = <&typec0_ss>; + }; + }; + port@1 { + reg = <1>; + atcphy0_usb: endpoint { + remote-endpoint = <&dwc3_0_ss>; + }; + }; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j714c-usb2.dtsi b/arch/arm64/boot/dts/apple/t6050-j714c-usb2.dtsi new file mode 100644 index 0000000000000..9e615ed141401 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j714c-usb2.dtsi @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * USB 2.0 host on USB-C ports 0, 1 and 2 of the MacBook Pro 14" (M5 Max, J714c). + * + * The full T6050 Type-C PHY has no Linux driver yet (see t6050-j714c-usb.dtsi). + * Each port's apple,t6050-usb2-phy puts its usb2phy, pipe handler and dwc3 + * core into USB 2.0 host mode (the loader's usb_t6050_host_prep sequence), + * and the generic xHCI platform driver drives the xHCI registers after + * phy_init(). PHY windows: atc-phyN reg 0 (usb2phy) and usb-drdN reg 3 + * (pipe handler). Addresses from + * the ADT: usb-drd0 0x2a2280000, dart-usb0 0x2a2f00000/0x2a2f80000, + * usb-drd1 0x2aa280000, dart-usb1 0x2aaf00000/0x2aaf80000, + * usb-drd2 0x382280000, dart-usb2 0x382f00000/0x382f80000. + * Port 1 is the proxy cable. + */ + +#include + +/ { + aliases { + usb0 = &dwc3_0; + usb1 = &dwc3_1; + usb2 = &dwc3_2; + }; +}; + +&soc { + /* + * The Type-C controllers, polled: ports 0 and 1 (ADT hpm0 left-back, + * hpm1 left-front) on nub-spmi-a0, port 2 (hpm2, right) on nub-spmi-a1. + */ + nub_spmi_a0: spmi@28a188000 { + compatible = "apple,t6050-spmi", "apple,t8142-spmi"; + reg = <0x2 0x8a188000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a0>; + + hpm0: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + /* ADT target-type "J714c": first four characters reversed */ + apple,unlock-key = "417J"; + + connector { + compatible = "usb-c-connector"; + label = "USB-C port 0"; + power-role = "dual"; + data-role = "host"; + + /* attach and detach reach the USB 2.0 PHY */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec0_hs: endpoint { + remote-endpoint = <&usb2phy_0_role>; + }; + }; + }; + }; + }; + + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + /* ADT target-type "J714c": first four characters reversed */ + apple,unlock-key = "417J"; + + connector { + compatible = "usb-c-connector"; + label = "USB-C port 1"; + power-role = "dual"; + data-role = "host"; + }; + }; + }; + + nub_spmi_a1: spmi@28a1a8000 { + compatible = "apple,t6050-spmi", "apple,t8142-spmi"; + reg = <0x2 0x8a1a8000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a1>; + + hpm2: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + /* ADT target-type "J714c": first four characters reversed */ + apple,unlock-key = "417J"; + + connector { + compatible = "usb-c-connector"; + label = "USB-C port 2"; + power-role = "dual"; + data-role = "host"; + + /* attach and detach reach the USB 2.0 PHY */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec2_hs: endpoint { + remote-endpoint = <&usb2phy_2_role>; + }; + }; + }; + }; + }; + }; + + usb2phy_0: phy@2a2a90000 { + compatible = "apple,t6050-usb2-phy"; + reg = <0x2 0xa2a90000 0x0 0x4000>, <0x2 0xa2a84000 0x0 0x4000>, + <0x2 0xa2280000 0x0 0x11800>; + reg-names = "usb2phy", "pipehandler", "core"; + #phy-cells = <0>; + power-domains = <&ps_atc0_usb>; + usb-role-switch; + + port { + usb2phy_0_role: endpoint { + remote-endpoint = <&typec0_hs>; + }; + }; + }; + + dwc3_0: usb@2a2280000 { + compatible = "generic-xhci"; + /* Keeps the PHY set up and the domain on across s2idle. */ + wakeup-source; + reg = <0x2 0xa2280000 0x0 0x8000>; + interrupts = ; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + phys = <&usb2phy_0>; + phy-names = "usb2-phy"; + }; + + dwc3_0_dart_0: iommu@2a2f00000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xa2f00000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_0_dart_1: iommu@2a2f80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xa2f80000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + /* + * Port 1 also gets SuperSpeed: the Type-C PHY windows (atc-phy1 reg 2, + * 38, 39, 40, 41, 42, 43; usb-drd1 reg 1, 5, 6, 8), the PMGR power + * states of the CIO domains and of the PHY CPU, and the ADT tunables, + * offsets relative to the window of the same name. + */ + usb2phy_1: phy@2aaa90000 { + compatible = "apple,t6050-usb2-phy"; + reg = <0x2 0xaaa90000 0x0 0x4000>, <0x2 0xaaa84000 0x0 0x4000>, + <0x2 0xaa280000 0x0 0x11800>, <0x2 0xaaa80000 0x0 0x4000>, + <0x2 0xaa200000 0x0 0x4000>, <0x2 0xaa980000 0x0 0x4000>, + <0x2 0xaa080000 0x0 0x4000>, <0x2 0xab000000 0x0 0x30000>, + <0x2 0xab140000 0x0 0x1c000>, <0x2 0xab200000 0x0 0x4000>, + <0x2 0xab244000 0x0 0x4000>, <0x2 0xa8000000 0x0 0x8000>, + <0x2 0xa8020000 0x0 0x4000>, <0x2 0xab250000 0x0 0x4000>, + <0x2 0x80600000 0x0 0x1000>, <0x2 0x88300000 0x0 0x1000>; + reg-names = "usb2phy", "pipehandler", "core", "pctl", + "drd-cfg", "ausbc-debug", "bulk-fabric", "atc", + "atc-mxwrap", "atc-power", "atc-fabric", "atc-axi2af", + "atc-axi2af-lioa", "atc-common-cfg", "pmgr", "pmgr-mxwrap"; + #phy-cells = <0>; + power-domains = <&ps_atc1_usb>; + apple,typec-controller = <&hpm1>; + + apple,tunable-gbl = + <0xc100 0x8 0x8>, + <0xc164 0x2 0x0>, + <0xc104 0x1ef00 0x2f00>, + <0xc11c 0x100 0x0>, + <0xc12c 0xa07ff 0x800c8>, + <0xc200 0x2000100 0x2000100>, + <0xc604 0x1000 0x0>, + <0xc628 0xff0000ff 0x6000006>, + <0xc62c 0xff0000ff 0x8000008>, + <0xc680 0xffffffff 0x6f000006>, + <0xc684 0xffffffff 0x60000>; + apple,tunable-cfg = + <0x20 0xff 0x1b>; + apple,tunable-link = + <0xd010 0xff000000 0xff000000>, + <0xd020 0x7ff00000 0x4e900000>, + <0xd024 0x1f8000 0xb8000>, + <0xd060 0x3ffff 0x600>; + apple,tunable-ausbc-debug = + <0xc 0x1f 0x0>; + apple,tunable-bulk-fabric = + <0x8 0x8000 0x0>, + <0x14 0x8000 0x0>, + <0x20 0x8000 0x0>; + apple,tunable-pipehandler = + <0x44 0x4 0x4>; + apple,tunable-cio = + <0xcd20 0xff0000 0x1e0000>; + apple,tunable-atc-common-cfg = + <0x8 0x1 0x1>; + apple,tunable-atc-fabric = + <0x4 0x3f003f 0x200020>, + <0xc 0xf1 0xa1>, + <0x10 0x801f 0x4>, + <0x18 0x3f003f 0x200020>, + <0x20 0xf1 0xa1>, + <0x24 0x801f 0x4>, + <0x2c 0xff00ff 0x960096>, + <0x34 0xf1 0x71>, + <0x38 0x801f 0x4>, + <0x40 0x3f 0x20>, + <0x48 0x801f 0x4>, + <0x50 0x7f007f 0x450045>, + <0x58 0x801f 0x4>, + <0x60 0x3f 0x20>, + <0x68 0x801f 0x4>, + <0x70 0x1 0x1>, + <0x74 0x801f 0x4>, + <0x7c 0x7f007f 0x400040>, + <0x80 0xf1 0x11>, + <0x84 0x1 0x1>, + <0x88 0x1 0x1>, + <0x8c 0x801f 0x4>, + <0x94 0x3f003f 0x240024>, + <0x98 0xf1 0x11>, + <0x9c 0x801f 0x4>, + <0xa4 0x1ff 0x1e0>, + <0xa8 0x801f 0x4>, + <0xb0 0x3f 0x20>, + <0xb8 0x801f 0x4>, + <0xc0 0x3f 0x20>, + <0xc8 0x801f 0x4>, + <0xd0 0x1f001f 0x100010>, + <0xd8 0x1 0x1>, + <0xdc 0x1 0x1>, + <0xe0 0x801f 0x4>, + <0xe8 0xf 0x8>, + <0xec 0x801f 0x4>, + <0xf4 0xffffffff 0x1010101>, + <0xf8 0x801f 0x4>, + <0x100 0x1 0x1>, + <0x104 0x1 0x1>, + <0x108 0x801f 0x4>, + <0x110 0x801f 0x4>, + <0x118 0x801f 0x4>, + <0x120 0x1 0x1>, + <0x124 0x1 0x1>, + <0x128 0x801f 0x4>, + <0x130 0xffffffff 0x1010101>, + <0x134 0x801f 0x4>, + <0x13c 0x1 0x1>, + <0x140 0x1 0x1>, + <0x144 0x801f 0x4>, + <0x14c 0x1 0x1>, + <0x150 0x1 0x1>, + <0x154 0x801f 0x4>, + <0x15c 0x801f 0x4>, + <0x164 0xffffffff 0x1010101>, + <0x168 0xffff 0x101>, + <0x16c 0x801f 0x4>, + <0x174 0x1 0x1>, + <0x178 0x1 0x1>, + <0x17c 0x801f 0x4>, + <0x184 0x1 0x1>, + <0x188 0x1 0x1>, + <0x18c 0x801f 0x4>; + apple,tunable-atc-axi2af = + <0x0 0x800000ff 0x80000023>, + <0x4 0x80ff00ff 0x80220002>, + <0x8 0x80ff00ff 0x80220002>, + <0xc 0x80ff00ff 0x80220006>, + <0x10 0x80ff00ff 0x80220002>, + <0x14 0x80ff00ff 0x80200004>, + <0x18 0x80ff00ff 0x80220002>, + <0x1c 0x80ff00ff 0x80220002>, + <0x20 0x80ff00ff 0x80220002>, + <0x24 0x800000ff 0x8000001a>, + <0x28 0x800000ff 0x80000046>, + <0x2c 0x80ff00ff 0x80460004>, + <0x30 0x80ff00ff 0x80460004>, + <0x34 0x80ff00ff 0x80420008>, + <0x38 0x80ff00ff 0x80460004>, + <0x3c 0x80ff00ff 0x80460004>, + <0x40 0x800000ff 0x8000001a>, + <0x44 0x800000ff 0x8000001a>, + <0x48 0x800000ff 0x8000001d>, + <0x100 0xff 0xff>, + <0x104 0x3 0x3>, + <0x300 0x3 0x1>, + <0x304 0x3f3f0000 0x4040000>, + <0x308 0x3ff 0xd>, + <0x30c 0x700003ff 0xc>, + <0x310 0x80001f 0x800001>, + <0x314 0x8080001f 0x80800001>, + <0x318 0x80001f 0x800001>, + <0x31c 0x8080001f 0x80800001>, + <0x320 0x80001f 0x800003>, + <0x324 0x8080001f 0x80800003>, + <0x328 0x80001f 0x800003>, + <0x32c 0x8080001f 0x80800003>, + <0x330 0x80001f 0x800003>, + <0x334 0x8080001f 0x80800003>, + <0x338 0x80001f 0x800003>, + <0x33c 0x80001f 0x800003>, + <0x340 0x80001f 0x800003>, + <0x344 0x80001f 0x800003>, + <0x348 0x80001f 0x800003>, + <0x34c 0x80001f 0x800003>, + <0x350 0x80001f 0x800003>, + <0x404 0xffff0023 0x1d0021>, + <0x408 0x3ff 0xd>, + <0x40c 0x700003ff 0xc>, + <0x410 0x8000001f 0x80000001>, + <0x414 0x8000001f 0x80000001>, + <0x418 0x8000001f 0x80000001>, + <0x41c 0x8000001f 0x80000001>, + <0x420 0x8000001f 0x80000003>, + <0x424 0x8000001f 0x80000003>, + <0x428 0x8000001f 0x80000003>, + <0x42c 0x8000001f 0x80000003>, + <0x430 0x8000001f 0x80000003>, + <0x434 0x8000001f 0x80000003>, + <0x438 0x8000001f 0x80000003>, + <0x43c 0x8000001f 0x80000003>, + <0x440 0x8000001f 0x80000003>, + <0x444 0x8000001f 0x80000003>, + <0x448 0x8000001f 0x80000003>, + <0x44c 0x8000001f 0x80000003>, + <0x450 0x8000001f 0x80000003>, + <0x948 0x7ff07ff 0x280000>, + <0xa48 0x7ff07ff 0x400040>, + <0xb48 0x7ff07ff 0xb8010e>, + <0xc48 0x7ff07ff 0x400020>, + <0xd48 0x7ff07ff 0x8a008a>, + <0x1000 0xffffffff 0x7fffffff>, + <0x1004 0x100 0x100>, + <0x4008 0x1 0x1>, + <0x400c 0x1 0x1>, + <0x4014 0x1 0x1>, + <0x4018 0x1 0x0>, + <0x401c 0x1 0x1>, + <0x108 0xffff0007 0x100001>; + apple,tunable-atc-pll = + <0xa04 0x7c 0x50>, + <0x3224 0xc00 0x800>, + <0x3230 0x1f 0xf>, + <0x3240 0x7d8007f 0x4000029>, + <0x32b0 0xb800000 0x1800000>, + <0x32e8 0x60001 0x40001>, + <0x3338 0xf800000 0x7800000>; + apple,tunable-atc-axi2af-lioa = + <0x28 0xfc2f 0x0>, + <0x2c 0x7 0x0>, + <0x30 0xff 0x0>, + <0x34 0x3ff 0x0>, + <0x58 0x30b 0x0>; + apple,tunable-atc-lanes = + <0xb698 0x7e00000 0x4400000>, + <0xb69c 0x7e00000 0x3800000>, + <0xb6a0 0x7e00000 0x3800000>, + <0xd094 0x80000 0x80000>, + <0xd098 0x80000 0x80000>, + <0xd0a4 0x800000 0x800000>, + <0xd0a8 0x200000 0x200000>, + <0x1b698 0x7e00000 0x4400000>, + <0x1b69c 0x7e00000 0x3800000>, + <0x1b6a0 0x7e00000 0x3800000>, + <0x1d094 0x80000 0x80000>, + <0x1d098 0x80000 0x80000>, + <0x1d0a4 0x800000 0x800000>, + <0x1d0a8 0x200000 0x200000>; + }; + + dwc3_1: usb@2aa280000 { + compatible = "generic-xhci"; + /* Keeps the PHY set up and the domain on across s2idle. */ + wakeup-source; + reg = <0x2 0xaa280000 0x0 0x8000>; + interrupts = ; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + phys = <&usb2phy_1>; + phy-names = "usb2-phy"; + }; + + dwc3_1_dart_0: iommu@2aaf00000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xaaf00000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_1_dart_1: iommu@2aaf80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xaaf80000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + usb2phy_2: phy@382a90000 { + compatible = "apple,t6050-usb2-phy"; + reg = <0x3 0x82a90000 0x0 0x4000>, <0x3 0x82a84000 0x0 0x4000>, + <0x3 0x82280000 0x0 0x11800>; + reg-names = "usb2phy", "pipehandler", "core"; + #phy-cells = <0>; + power-domains = <&ps_atc2_usb>; + usb-role-switch; + + port { + usb2phy_2_role: endpoint { + remote-endpoint = <&typec2_hs>; + }; + }; + }; + + dwc3_2: usb@382280000 { + compatible = "generic-xhci"; + /* Keeps the PHY set up and the domain on across s2idle. */ + wakeup-source; + reg = <0x3 0x82280000 0x0 0x8000>; + interrupts = ; + iommus = <&dwc3_2_dart_0 0>, <&dwc3_2_dart_1 1>; + power-domains = <&ps_atc2_usb>; + phys = <&usb2phy_2>; + phy-names = "usb2-phy"; + }; + + dwc3_2_dart_0: iommu@382f00000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x3 0x82f00000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc2_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_2_dart_1: iommu@382f80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x3 0x82f80000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc2_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-j714c.dts b/arch/arm64/boot/dts/apple/t6050-j714c.dts new file mode 100644 index 0000000000000..c471b37c8a080 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-j714c.dts @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Pro (14-inch, M5 Max, 18 CPU cores) + * + * target-type: J714c + * + * Initial RAM boot resources measured on J714c/T6050. + * The loader supplies usable memory, CPU release addresses and + * framebuffer geometry. Peripheral power domains come from + * t6050-pmgr.dtsi; the framebuffer holds the display domain. + * + * The loader's command line MUST include maxcpus=1 for the first boot: + * spin-table bring-up of the secondaries is blocked by the T6050 + * firmware-parking problem under investigation (dumps/HANDOFF.md), + * and arm64 spin-table waits unboundedly for each CPU to check in. + * + * Copyright The Asahi Linux Contributors + */ + +/dts-v1/; + +#include +#include +#include +#include + +#include "t6050.dtsi" +#include "t6050-smc.dtsi" +#include "t6050-pmp.dtsi" +#include "t6050-j714c-pmp-tunables.dtsi" +#include "t6050-j714c-display.dtsi" +#include "t6050-j714c-isp.dtsi" + +/ { + compatible = "apple,j714c", "apple,t6050", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M5 Max, 18 CPU cores)"; + + #address-cells = <2>; + #size-cells = <2>; + interrupt-parent = <&aic>; + + aliases { + gpu = &gpu; + keyboard = &keyboard; + serial0 = &serial0; + /* kboot fills the N1's factory addresses from /chosen through these */ + wifi0 = &n1_alpha; + bluetooth0 = &n1_beta; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0:1500000n8"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* Filled by the loader. */ + status = "okay"; /* m1n1 only enables it on chips it knows. */ + power-domains = <&ps_disp_cpu>; + apple,internal-panel; /* simpledrm reports it as eDP */ + }; + }; + + memory@10000000000 { + device_type = "memory"; + reg = <0x100 0 0 0>; /* Filled by the loader. */ + }; + + led-controller { + compatible = "pwm-leds"; + led-0 { + pwms = <&fpwm0 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&fpwm0 { + status = "okay"; +}; + +&mtp { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + /* + * ADT multi-touch function-afe-reset = smc-pmu pKW8('pcIO', 0x10000, + * 0x11c0000), SMC pcIO output 28; level 1 asserts the reset. Left to + * the MTP firmware, the AFE sometimes never answers ("No HINT_L"). + */ + apple,afe-reset-gpios = <&smc_gpio 28 GPIO_ACTIVE_HIGH>; + + mtp_mt: multi-touch { + firmware-name = "apple/tpmtfw-j714c.bin"; + apple,power-method = <2>; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; +#include "t6050-j714c-usb2.dtsi" + +/* Enable the GPU and its mailboxes; the loader disables them again if the ADT + * perf/region data does not validate (dt_set_gpu_g17). */ +&gfx_mbox { + status = "okay"; +}; + +&gfx1_mbox { + status = "okay"; +}; + +&gpu { + status = "okay"; +}; + +#include "hwmon-common.dtsi" +#include "hwmon-fan-dual.dtsi" +#include "hwmon-laptop.dtsi" + +/* + * The T6050 SMC spells the fan mode keys F0md/F1md; F0Md/F1Md from + * hwmon-fan*.dtsi do not exist (as on J714s in omarchy-m5). + */ +&smc_hwmon { + fan-F0Ac { + apple,fan-mode = "F0md"; + }; + + fan-F1Ac { + apple,fan-mode = "F1md"; + }; +}; + +/* + * Audio: the amplifiers and the jack codec on I2C; the I2S side is the + * AOP's (below). + * + * ADT /arm-io/i2c1: audio-speaker 0x38 (the primary amp, holds + * speaker-config), audio-speaker-left-woofer-2 0x39, audio-speaker-left- + * tweeter 0x3a. /arm-io/i2c3: audio-speaker-right-woofer-1 0x3b, + * audio-speaker-right-woofer-2 0x3c, audio-speaker-right-tweeter 0x3d. All + * six are sn012776. /arm-io/i2c2: audio-codec-output 0x4b, cs42l84, the + * headphone jack. + * + * Lines, all on gpio0 (pinctrl_ap): amps share shutdown GPIO 107 + * (function-reset GPIO(0x6b)) and interrupt 106; codec reset GPIO 139 + * (function-reset GPIO(0x8b)), interrupt 134. Speaker-id GPIOs 136/137 are + * unused. Polarities follow the in-tree J516 and the J716s reference: + * active-high shutdown and reset, level-low interrupts; the ADT flag words + * (0x10001 amps, 0x1 codec) are not decoded. + * + * Sense slots: speaker-config row n is the amp the ADT names speakerN + * (audio-speaker, then speaker1..speaker5 = LW1 RW1 LW2 RW2 LT RT); amp n + * carries I in slot 4n and V in slot 4n+2, the tas2764/J516 convention + * (the ADT labels the two the other way round). + * + * T6050 has no AP-side MCA: the I2S ports and their power domains belong + * to the AOP firmware (ADT /arm-io/iop-audio-controller, AOPAudioService, + * streams on admac-base-ns), and t8140_aop_audio's T6050 variant provides + * the CPU DAIs for the sound node below, as on T8140. The amps leave + * hardware shutdown only from the ASoC component probe, so the shared + * shutdown line stays low until the card binds. + */ +&pinctrl_ap { + status = "okay"; +}; + +&i2c1 { + status = "okay"; + + speaker_left_woof1: codec@38 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x38>; + shutdown-gpios = <&pinctrl_ap 107 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 1"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <0>; + ti,vmon-slot-no = <2>; + }; + + speaker_left_woof2: codec@39 { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x39>; + shutdown-gpios = <&pinctrl_ap 107 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Woofer 2"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <8>; + ti,vmon-slot-no = <10>; + }; + + speaker_left_tweet: codec@3a { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3a>; + shutdown-gpios = <&pinctrl_ap 107 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Left Tweeter"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <16>; + ti,vmon-slot-no = <18>; + }; +}; + +&i2c2 { + status = "okay"; + + jack_codec: codec@4b { + compatible = "cirrus,cs42l84"; + reg = <0x4b>; + reset-gpios = <&pinctrl_ap 139 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + interrupts-extended = <&pinctrl_ap 134 IRQ_TYPE_LEVEL_LOW>; + sound-name-prefix = "Jack"; + }; +}; + +&i2c3 { + status = "okay"; + + speaker_right_woof1: codec@3b { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3b>; + shutdown-gpios = <&pinctrl_ap 107 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 1"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <4>; + ti,vmon-slot-no = <6>; + }; + + speaker_right_woof2: codec@3c { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3c>; + shutdown-gpios = <&pinctrl_ap 107 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Woofer 2"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <12>; + ti,vmon-slot-no = <14>; + }; + + speaker_right_tweet: codec@3d { + compatible = "ti,sn012776", "ti,tas2764"; + reg = <0x3d>; + shutdown-gpios = <&pinctrl_ap 107 GPIO_ACTIVE_HIGH>; + #sound-dai-cells = <0>; + sound-name-prefix = "Right Tweeter"; + interrupts-extended = <&pinctrl_ap 106 IRQ_TYPE_LEVEL_LOW>; + ti,imon-slot-no = <20>; + ti,vmon-slot-no = <22>; + }; +}; + +/* + * AOP: the co-processor, its mailbox, the base-ns ADMAC and the audio + * streams. The sound card: the jack on the AOP's cout/cin streams, the six + * amps on its spkr stream (ADT order, amp-tx-zd-config rows 0 and 1 as the + * idle masks), speakers off by default in the j714 macaudio config. + */ +&aop_dart { + status = "okay"; +}; + +&aop_mbox { + status = "okay"; +}; + +&aop_setup_mbox { + status = "okay"; +}; + +&aop { + status = "okay"; +}; + +&aop_audio { + apple,chassis-name = "J714"; + apple,machine-kind = "MacBook Pro"; +}; + +&admac_base_ns { + status = "okay"; +}; + +&admac_leap_s { + status = "okay"; +}; + +/ { + sound: sound { + compatible = "apple,j714-macaudio", "apple,macaudio"; + model = "MacBook Pro J714"; + #address-cells = <1>; + #size-cells = <0>; + + dai-link@0 { + reg = <0>; + link-name = "Headphone Jack"; + + cpu { + sound-dai = <&aop_audio 0>; + }; + codec { + sound-dai = <&jack_codec>; + }; + }; + + dai-link@1 { + reg = <1>; + link-name = "Speakers"; + dai-tdm-idle-mode-0 = "zero"; + dai-tdm-idle-mode-1 = "zero"; + dai-tdm-slot-tx-idle-mask-0 = <0xf0f0f0>; + dai-tdm-slot-tx-idle-mask-1 = <0x0f0f0f>; + + cpu { + sound-dai = <&aop_audio 4>; + }; + codec { + sound-dai = <&speaker_left_woof1>, + <&speaker_right_woof1>, + <&speaker_left_woof2>, + <&speaker_right_woof2>, + <&speaker_left_tweet>, + <&speaker_right_tweet>; + }; + }; + }; +}; + +/* + * iBoot leaves the N1 port's gates in automatic power management with the + * target on and the block gated (APCIE_SYS_GP and APHY_GP_AUS5 actual 0, + * APCIE_PHY_SW actual 4). The kernel trains the port, so these gates stay + * at the requested state. + */ +&ps_apcie_sys_gp { + apple,no-auto-pm; +}; + +&ps_apcie_phy_sw { + apple,no-auto-pm; +}; + +&ps_aphy_gp_aus5 { + apple,no-auto-pm; +}; + +/* + * The N1 (Wi-Fi and Bluetooth) on apcie0 port 0, as aurora-silicon's t6050-j716c.dts names it; + * the J714c has the same port, registers and pins (ADT 2026-11-05). iBoot leaves the port + * untrained and the N1 off (SMC GPIO gP13 low, PERST# GPIO 80 low); the kernel trains it. + * Platform id = (board-id 0x0a << 16) | 0x6050 by aurora's J716c rule; the FTAB comes from the + * macOS install (not shipped by aurora). + */ +&pcie0 { + reg = <0x1c 0xb0000000 0x0 0x10000000>, + <0x4 0x14000000 0x0 0x4000>, + <0x4 0x10028000 0x0 0x8000>, + <0x4 0x17010000 0x0 0x4000>, + <0x4 0x10024000 0x0 0x4000>, + <0x4 0x17020000 0x0 0x4000>, + <0x4 0x17004000 0x0 0x4000>, + <0x4 0x16000000 0x0 0x24000>, /* ADT reg[5] */ + <0x4 0x15044000 0x0 0x4000>, /* ADT reg[6] */ + <0x4 0x17000000 0x0 0x24000>, /* ADT reg[2] */ + <0x4 0x17800000 0x0 0x1b4000>; /* ADT reg[4] */ + /* phy-shared: ADT reg[3]; phy-common: ADT reg[2] + 0x4000; the rest: tunable windows */ + reg-names = "config", "rc", "port0", "phy0", "intr2axi0", "phy-shared", "phy-common", + "axi2af", "pcieclkgen", "phy-window", "phy-ip"; + interrupts = ; + interrupt-names = "port0"; + + /* The N1's three functions are on DART streams 1-3 */ + iommu-map = <0x100 &dart_apcie0 1 3>; + bus-range = <0 4>; + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0 0 0 0>; + bus-range = <1 1>; + apple,link-trained-by-bootloader; + /* PERST#, for the kernel's train and the root-port reset after the N1's ROM stage */ + reset-gpios = <&pinctrl_ap 80 GPIO_ACTIVE_LOW>; + /* The N1's power/reset: ADT function-reset = pKW4('gP13', ...), SMC GPIO pin 0x13 */ + pwren-gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; + status = "okay"; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + + n1_control: control@0,0 { + compatible = "pci106b,1900"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + firmware-name = "apple/centauri/j714c-ftab.bin"; + apple,centauri-platform-id = <0x000a6050>; + apple,centauri-protocol = <27>; + }; + + n1_alpha: wifi@0,1 { + compatible = "pci106b,1902"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + local-mac-address = [00 00 00 00 00 00]; + }; + + n1_beta: bluetooth@0,2 { + compatible = "pci106b,1903"; + reg = <0x10200 0x0 0x0 0x0 0x0>; + local-bd-address = [00 00 00 00 00 00]; + }; + }; +}; + +&dart_apcie0 { + status = "okay"; +}; + +#include "t6050-j714c-pcie-tunables.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6050-k114c-pcie-tunables.dtsi b/arch/arm64/boot/dts/apple/t6050-k114c-pcie-tunables.dtsi new file mode 100644 index 0000000000000..f84b8366f1665 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-k114c-pcie-tunables.dtsi @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * K114c apcie0 tunables that differ from the J714c's. t6050-k114c.dts includes + * t6050-j714c-pcie-tunables.dtsi first and this file after it, so the two + * properties below replace the J714c values; axi2af, pcieclkgen, phy and + * phy-ip-auspma are entry-for-entry identical on this machine (K114c ADT + * of 2026-11-10, ioreg gather). + * + * cio3pllcore: the ADT stores the offsets relative to the cio3pllcore window + * at pcieclkgen + 0x2200; the driver applies the property on the pcieclkgen + * window ("apple,tunable-cio3pllcore" tuned against "pcieclkgen"), so 0x2200 + * is added, the same re-basing the J714c file uses. The offsets then equal + * the J714c's; one value differs (offset 0x224c: 0x94 here, 0x95 there). + * phy-ip-pll: three values differ from the J714c (the PLL trim). + * Entries are copied unchanged from the ADT as (offset, 4, mask, value). + */ +&pcie0 { + apple,tunable-cio3pllcore = + <0x2200 4 0xa0b 0xa01>, + <0x2224 4 0xc00 0x800>, + <0x2228 4 0xf00 0xb00>, + <0x2238 4 0x20 0x0>, + <0x224c 4 0xff 0x94>, + <0x22e8 4 0xe0000 0x20000>, + <0x2300 4 0xffffff 0xb40b4>; + + apple,tunable-phy-ip-pll = + <0x180828 4 0xffff 0xb44>, + <0x180838 4 0x2 0x2>, + <0x180c1c 4 0x7e0 0x7c0>, + <0x180e10 4 0x3e0 0x1e0>, + <0x1820a4 4 0x3c3c000 0x1010000>, + <0x182200 4 0x8 0x8>, + <0x18221c 4 0x7c0 0x500>, + <0x182224 4 0xc00 0x800>, + <0x182228 4 0xf00 0xb00>, + <0x182230 4 0x3f 0x32>, + <0x18224c 4 0xfe7f 0x4412>, + <0x1822b0 4 0x700000 0x300000>, + <0x182338 4 0x1f00 0x1700>, + <0x183000 4 0x301fbf00 0x1e3400>, + <0x183004 4 0x1fffff 0x8d0e3>, + <0x183014 4 0xff0000ff 0x40000080>, + <0x183008 4 0x1fffff 0x15930a>, + <0x18300c 4 0x1fffff 0x1a2548>, + <0x183010 4 0x1fffff 0x1baaa8>; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-k114c-usb2.dtsi b/arch/arm64/boot/dts/apple/t6050-k114c-usb2.dtsi new file mode 100644 index 0000000000000..2f09b3922bec6 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-k114c-usb2.dtsi @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * USB 2.0 host on USB-C ports 0, 1 and 2 of the MacBook Pro 14" (M5 Max, + * OLED, K114c). + * + * The full T6050 Type-C PHY has no Linux driver yet (t6050-j714c-usb.dtsi on + * the J714c branch sketches it). Each port's apple,t6050-usb2-phy puts its + * usb2phy, pipe handler and dwc3 core into USB 2.0 host mode (the loader's + * usb_t6050_host_prep sequence), and the generic xHCI platform driver drives + * the xHCI registers after phy_init(). PHY windows: atc-phyN reg 0 (usb2phy) + * and usb-drdN reg 3 (pipe handler). Addresses, interrupts and windows are + * verified against this K114c's ADT (ioreg gather, 2026-11-10): usb-drd0 + * 0x2a2280000, dart-usb0 0x2a2f00000/0x2a2f80000, usb-drd1 0x2aa280000, + * dart-usb1 0x2aaf00000/0x2aaf80000, usb-drd2 0x382280000, dart-usb2 + * 0x382f00000/0x382f80000 - all identical to the J714c's. Port 1 is the + * proxy cable, as on the J714c. + * + * The Type-C PHY tunables are chip hardware: copied from the J714c file + * unchanged, after the tunable groups the ADT carries under its own names + * (tunable_ATC0AXI2AF, tunable_ATC0AXI2AF_LIOA, tunable_ATC_COMMON_CFG, + * tunable_ATC_FABRIC) matched it entry for entry on this K114c. + * + * The unlock key is the target type with its first four characters reversed: + * K114c -> "411K". + */ + +#include + +/ { + aliases { + usb0 = &dwc3_0; + usb1 = &dwc3_1; + usb2 = &dwc3_2; + }; +}; + +&soc { + /* + * The Type-C controllers, polled: ports 0 and 1 (ADT hpm0 left-back, + * hpm1 left-front) on nub-spmi-a0, port 2 (hpm2, right) on nub-spmi-a1. + */ + nub_spmi_a0: spmi@28a188000 { + compatible = "apple,t6050-spmi", "apple,t8142-spmi"; + reg = <0x2 0x8a188000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a0>; + + hpm0: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + /* ADT target-type "K114c": first four characters reversed */ + apple,unlock-key = "411K"; + + connector { + compatible = "usb-c-connector"; + label = "USB-C port 0"; + power-role = "dual"; + data-role = "host"; + + /* attach and detach reach the USB 2.0 PHY */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec0_hs: endpoint { + remote-endpoint = <&usb2phy_0_role>; + }; + }; + }; + }; + }; + + hpm1: usb-pd@a { + compatible = "apple,sn201202x"; + reg = <0xa SPMI_USID>; + /* ADT target-type "K114c": first four characters reversed */ + apple,unlock-key = "411K"; + + connector { + compatible = "usb-c-connector"; + label = "USB-C port 1"; + power-role = "dual"; + data-role = "host"; + }; + }; + }; + + nub_spmi_a1: spmi@28a1a8000 { + compatible = "apple,t6050-spmi", "apple,t8142-spmi"; + reg = <0x2 0x8a1a8000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + power-domains = <&ps_nub_spmi_a1>; + + hpm2: usb-pd@c { + compatible = "apple,sn201202x"; + reg = <0xc SPMI_USID>; + /* ADT target-type "K114c": first four characters reversed */ + apple,unlock-key = "411K"; + + connector { + compatible = "usb-c-connector"; + label = "USB-C port 2"; + power-role = "dual"; + data-role = "host"; + + /* attach and detach reach the USB 2.0 PHY */ + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + typec2_hs: endpoint { + remote-endpoint = <&usb2phy_2_role>; + }; + }; + }; + }; + }; + }; + + usb2phy_0: phy@2a2a90000 { + compatible = "apple,t6050-usb2-phy"; + reg = <0x2 0xa2a90000 0x0 0x4000>, <0x2 0xa2a84000 0x0 0x4000>, + <0x2 0xa2280000 0x0 0x11800>; + reg-names = "usb2phy", "pipehandler", "core"; + #phy-cells = <0>; + power-domains = <&ps_atc0_usb>; + usb-role-switch; + + port { + usb2phy_0_role: endpoint { + remote-endpoint = <&typec0_hs>; + }; + }; + }; + + dwc3_0: usb@2a2280000 { + compatible = "generic-xhci"; + /* Keeps the PHY set up and the domain on across s2idle. */ + wakeup-source; + reg = <0x2 0xa2280000 0x0 0x8000>; + interrupts = ; + iommus = <&dwc3_0_dart_0 0>, <&dwc3_0_dart_1 1>; + power-domains = <&ps_atc0_usb>; + phys = <&usb2phy_0>; + phy-names = "usb2-phy"; + }; + + dwc3_0_dart_0: iommu@2a2f00000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xa2f00000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_0_dart_1: iommu@2a2f80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xa2f80000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc0_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + /* + * Port 1 also gets SuperSpeed: the Type-C PHY windows (atc-phy1 reg 2, + * 38, 39, 40, 41, 42, 43; usb-drd1 reg 1, 5, 6, 8), the PMGR power + * states of the CIO domains and of the PHY CPU, and the ADT tunables, + * offsets relative to the window of the same name. + */ + usb2phy_1: phy@2aaa90000 { + compatible = "apple,t6050-usb2-phy"; + reg = <0x2 0xaaa90000 0x0 0x4000>, <0x2 0xaaa84000 0x0 0x4000>, + <0x2 0xaa280000 0x0 0x11800>, <0x2 0xaaa80000 0x0 0x4000>, + <0x2 0xaa200000 0x0 0x4000>, <0x2 0xaa980000 0x0 0x4000>, + <0x2 0xaa080000 0x0 0x4000>, <0x2 0xab000000 0x0 0x30000>, + <0x2 0xab140000 0x0 0x1c000>, <0x2 0xab200000 0x0 0x4000>, + <0x2 0xab244000 0x0 0x4000>, <0x2 0xa8000000 0x0 0x8000>, + <0x2 0xa8020000 0x0 0x4000>, <0x2 0xab250000 0x0 0x4000>, + <0x2 0x80600000 0x0 0x1000>, <0x2 0x88300000 0x0 0x1000>; + reg-names = "usb2phy", "pipehandler", "core", "pctl", + "drd-cfg", "ausbc-debug", "bulk-fabric", "atc", + "atc-mxwrap", "atc-power", "atc-fabric", "atc-axi2af", + "atc-axi2af-lioa", "atc-common-cfg", "pmgr", "pmgr-mxwrap"; + #phy-cells = <0>; + power-domains = <&ps_atc1_usb>; + apple,typec-controller = <&hpm1>; + + apple,tunable-gbl = + <0xc100 0x8 0x8>, + <0xc164 0x2 0x0>, + <0xc104 0x1ef00 0x2f00>, + <0xc11c 0x100 0x0>, + <0xc12c 0xa07ff 0x800c8>, + <0xc200 0x2000100 0x2000100>, + <0xc604 0x1000 0x0>, + <0xc628 0xff0000ff 0x6000006>, + <0xc62c 0xff0000ff 0x8000008>, + <0xc680 0xffffffff 0x6f000006>, + <0xc684 0xffffffff 0x60000>; + apple,tunable-cfg = + <0x20 0xff 0x1b>; + apple,tunable-link = + <0xd010 0xff000000 0xff000000>, + <0xd020 0x7ff00000 0x4e900000>, + <0xd024 0x1f8000 0xb8000>, + <0xd060 0x3ffff 0x600>; + apple,tunable-ausbc-debug = + <0xc 0x1f 0x0>; + apple,tunable-bulk-fabric = + <0x8 0x8000 0x0>, + <0x14 0x8000 0x0>, + <0x20 0x8000 0x0>; + apple,tunable-pipehandler = + <0x44 0x4 0x4>; + apple,tunable-cio = + <0xcd20 0xff0000 0x1e0000>; + apple,tunable-atc-common-cfg = + <0x8 0x1 0x1>; + apple,tunable-atc-fabric = + <0x4 0x3f003f 0x200020>, + <0xc 0xf1 0xa1>, + <0x10 0x801f 0x4>, + <0x18 0x3f003f 0x200020>, + <0x20 0xf1 0xa1>, + <0x24 0x801f 0x4>, + <0x2c 0xff00ff 0x960096>, + <0x34 0xf1 0x71>, + <0x38 0x801f 0x4>, + <0x40 0x3f 0x20>, + <0x48 0x801f 0x4>, + <0x50 0x7f007f 0x450045>, + <0x58 0x801f 0x4>, + <0x60 0x3f 0x20>, + <0x68 0x801f 0x4>, + <0x70 0x1 0x1>, + <0x74 0x801f 0x4>, + <0x7c 0x7f007f 0x400040>, + <0x80 0xf1 0x11>, + <0x84 0x1 0x1>, + <0x88 0x1 0x1>, + <0x8c 0x801f 0x4>, + <0x94 0x3f003f 0x240024>, + <0x98 0xf1 0x11>, + <0x9c 0x801f 0x4>, + <0xa4 0x1ff 0x1e0>, + <0xa8 0x801f 0x4>, + <0xb0 0x3f 0x20>, + <0xb8 0x801f 0x4>, + <0xc0 0x3f 0x20>, + <0xc8 0x801f 0x4>, + <0xd0 0x1f001f 0x100010>, + <0xd8 0x1 0x1>, + <0xdc 0x1 0x1>, + <0xe0 0x801f 0x4>, + <0xe8 0xf 0x8>, + <0xec 0x801f 0x4>, + <0xf4 0xffffffff 0x1010101>, + <0xf8 0x801f 0x4>, + <0x100 0x1 0x1>, + <0x104 0x1 0x1>, + <0x108 0x801f 0x4>, + <0x110 0x801f 0x4>, + <0x118 0x801f 0x4>, + <0x120 0x1 0x1>, + <0x124 0x1 0x1>, + <0x128 0x801f 0x4>, + <0x130 0xffffffff 0x1010101>, + <0x134 0x801f 0x4>, + <0x13c 0x1 0x1>, + <0x140 0x1 0x1>, + <0x144 0x801f 0x4>, + <0x14c 0x1 0x1>, + <0x150 0x1 0x1>, + <0x154 0x801f 0x4>, + <0x15c 0x801f 0x4>, + <0x164 0xffffffff 0x1010101>, + <0x168 0xffff 0x101>, + <0x16c 0x801f 0x4>, + <0x174 0x1 0x1>, + <0x178 0x1 0x1>, + <0x17c 0x801f 0x4>, + <0x184 0x1 0x1>, + <0x188 0x1 0x1>, + <0x18c 0x801f 0x4>; + apple,tunable-atc-axi2af = + <0x0 0x800000ff 0x80000023>, + <0x4 0x80ff00ff 0x80220002>, + <0x8 0x80ff00ff 0x80220002>, + <0xc 0x80ff00ff 0x80220006>, + <0x10 0x80ff00ff 0x80220002>, + <0x14 0x80ff00ff 0x80200004>, + <0x18 0x80ff00ff 0x80220002>, + <0x1c 0x80ff00ff 0x80220002>, + <0x20 0x80ff00ff 0x80220002>, + <0x24 0x800000ff 0x8000001a>, + <0x28 0x800000ff 0x80000046>, + <0x2c 0x80ff00ff 0x80460004>, + <0x30 0x80ff00ff 0x80460004>, + <0x34 0x80ff00ff 0x80420008>, + <0x38 0x80ff00ff 0x80460004>, + <0x3c 0x80ff00ff 0x80460004>, + <0x40 0x800000ff 0x8000001a>, + <0x44 0x800000ff 0x8000001a>, + <0x48 0x800000ff 0x8000001d>, + <0x100 0xff 0xff>, + <0x104 0x3 0x3>, + <0x300 0x3 0x1>, + <0x304 0x3f3f0000 0x4040000>, + <0x308 0x3ff 0xd>, + <0x30c 0x700003ff 0xc>, + <0x310 0x80001f 0x800001>, + <0x314 0x8080001f 0x80800001>, + <0x318 0x80001f 0x800001>, + <0x31c 0x8080001f 0x80800001>, + <0x320 0x80001f 0x800003>, + <0x324 0x8080001f 0x80800003>, + <0x328 0x80001f 0x800003>, + <0x32c 0x8080001f 0x80800003>, + <0x330 0x80001f 0x800003>, + <0x334 0x8080001f 0x80800003>, + <0x338 0x80001f 0x800003>, + <0x33c 0x80001f 0x800003>, + <0x340 0x80001f 0x800003>, + <0x344 0x80001f 0x800003>, + <0x348 0x80001f 0x800003>, + <0x34c 0x80001f 0x800003>, + <0x350 0x80001f 0x800003>, + <0x404 0xffff0023 0x1d0021>, + <0x408 0x3ff 0xd>, + <0x40c 0x700003ff 0xc>, + <0x410 0x8000001f 0x80000001>, + <0x414 0x8000001f 0x80000001>, + <0x418 0x8000001f 0x80000001>, + <0x41c 0x8000001f 0x80000001>, + <0x420 0x8000001f 0x80000003>, + <0x424 0x8000001f 0x80000003>, + <0x428 0x8000001f 0x80000003>, + <0x42c 0x8000001f 0x80000003>, + <0x430 0x8000001f 0x80000003>, + <0x434 0x8000001f 0x80000003>, + <0x438 0x8000001f 0x80000003>, + <0x43c 0x8000001f 0x80000003>, + <0x440 0x8000001f 0x80000003>, + <0x444 0x8000001f 0x80000003>, + <0x448 0x8000001f 0x80000003>, + <0x44c 0x8000001f 0x80000003>, + <0x450 0x8000001f 0x80000003>, + <0x948 0x7ff07ff 0x280000>, + <0xa48 0x7ff07ff 0x400040>, + <0xb48 0x7ff07ff 0xb8010e>, + <0xc48 0x7ff07ff 0x400020>, + <0xd48 0x7ff07ff 0x8a008a>, + <0x1000 0xffffffff 0x7fffffff>, + <0x1004 0x100 0x100>, + <0x4008 0x1 0x1>, + <0x400c 0x1 0x1>, + <0x4014 0x1 0x1>, + <0x4018 0x1 0x0>, + <0x401c 0x1 0x1>, + <0x108 0xffff0007 0x100001>; + apple,tunable-atc-pll = + <0xa04 0x7c 0x50>, + <0x3224 0xc00 0x800>, + <0x3230 0x1f 0xf>, + <0x3240 0x7d8007f 0x4000029>, + <0x32b0 0xb800000 0x1800000>, + <0x32e8 0x60001 0x40001>, + <0x3338 0xf800000 0x7800000>; + apple,tunable-atc-axi2af-lioa = + <0x28 0xfc2f 0x0>, + <0x2c 0x7 0x0>, + <0x30 0xff 0x0>, + <0x34 0x3ff 0x0>, + <0x58 0x30b 0x0>; + apple,tunable-atc-lanes = + <0xb698 0x7e00000 0x4400000>, + <0xb69c 0x7e00000 0x3800000>, + <0xb6a0 0x7e00000 0x3800000>, + <0xd094 0x80000 0x80000>, + <0xd098 0x80000 0x80000>, + <0xd0a4 0x800000 0x800000>, + <0xd0a8 0x200000 0x200000>, + <0x1b698 0x7e00000 0x4400000>, + <0x1b69c 0x7e00000 0x3800000>, + <0x1b6a0 0x7e00000 0x3800000>, + <0x1d094 0x80000 0x80000>, + <0x1d098 0x80000 0x80000>, + <0x1d0a4 0x800000 0x800000>, + <0x1d0a8 0x200000 0x200000>; + }; + + dwc3_1: usb@2aa280000 { + compatible = "generic-xhci"; + /* Keeps the PHY set up and the domain on across s2idle. */ + wakeup-source; + reg = <0x2 0xaa280000 0x0 0x8000>; + interrupts = ; + iommus = <&dwc3_1_dart_0 0>, <&dwc3_1_dart_1 1>; + power-domains = <&ps_atc1_usb>; + phys = <&usb2phy_1>; + phy-names = "usb2-phy"; + }; + + dwc3_1_dart_0: iommu@2aaf00000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xaaf00000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_1_dart_1: iommu@2aaf80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x2 0xaaf80000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc1_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + usb2phy_2: phy@382a90000 { + compatible = "apple,t6050-usb2-phy"; + reg = <0x3 0x82a90000 0x0 0x4000>, <0x3 0x82a84000 0x0 0x4000>, + <0x3 0x82280000 0x0 0x11800>; + reg-names = "usb2phy", "pipehandler", "core"; + #phy-cells = <0>; + power-domains = <&ps_atc2_usb>; + usb-role-switch; + + port { + usb2phy_2_role: endpoint { + remote-endpoint = <&typec2_hs>; + }; + }; + }; + + dwc3_2: usb@382280000 { + compatible = "generic-xhci"; + /* Keeps the PHY set up and the domain on across s2idle. */ + wakeup-source; + reg = <0x3 0x82280000 0x0 0x8000>; + interrupts = ; + iommus = <&dwc3_2_dart_0 0>, <&dwc3_2_dart_1 1>; + power-domains = <&ps_atc2_usb>; + phys = <&usb2phy_2>; + phy-names = "usb2-phy"; + }; + + dwc3_2_dart_0: iommu@382f00000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x3 0x82f00000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc2_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; + + dwc3_2_dart_1: iommu@382f80000 { + compatible = "apple,t6050-dart", "apple,t8110-dart"; + reg = <0x3 0x82f80000 0x0 0xc000>; + interrupts = ; + power-domains = <&ps_atc2_usb>; + #iommu-cells = <1>; + apple,dma-range = <0x100 0x4000 0 0xffff0000>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-k114c.dts b/arch/arm64/boot/dts/apple/t6050-k114c.dts new file mode 100644 index 0000000000000..c4519409283a0 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-k114c.dts @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple MacBook Pro (14-inch, M5 Max, 18 CPU cores, OLED) + * + * target-type: K114c + * + * Initial RAM boot resources measured on the K114c (ADT ioreg gather of + * 2026-11-10). The loader supplies usable memory, CPU release addresses and + * framebuffer geometry. Peripheral power domains come from t6050-pmgr.dtsi; + * the framebuffer holds the display domain. + * + * Compared with Edi Moldovan's J714c this board file leaves out, with + * reasons: + * - the DCP display pipeline (his t6050-j714c-display.dtsi): the console + * runs on the simple framebuffer the loader fills, and the K114c's OLED + * panel and DCP firmware segment table have not been measured, so there + * is no K114c display dtsi to include; + * - the GPU (gfx/gfx1 mbox, gpu): not needed to boot; Edi's loader-side + * ADT validation (dt_set_gpu_g17) is not exercised here either; + * - the PMP (t6050-pmp.dtsi and his pmp-tunables): not needed to boot, and + * this K114c's ADT has no pmp device node at all; + * - the camera and ISP (his t6050-j714c-isp.dtsi): no K114c camera values + * measured; + * - audio (I2C amps and jack codec, the AOP, the sound card): out of scope + * for this bring-up and the K114c ADT amplifier layout is unmeasured. + */ + +/dts-v1/; + +#include +#include +#include +#include + +#include "t6050.dtsi" +#include "t6050-smc.dtsi" + +/ { + compatible = "apple,k114c", "apple,t6050", "apple,arm-platform"; + model = "Apple MacBook Pro (14-inch, M5 Max, 18 CPU cores, OLED)"; + + #address-cells = <2>; + #size-cells = <2>; + interrupt-parent = <&aic>; + + aliases { + keyboard = &keyboard; + serial0 = &serial0; + /* kboot fills the N1's factory addresses from /chosen through these */ + wifi0 = &n1_alpha; + bluetooth0 = &n1_beta; + }; + + chosen { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + stdout-path = "serial0:1500000n8"; + + framebuffer0: framebuffer@0 { + compatible = "apple,simple-framebuffer", "simple-framebuffer"; + reg = <0 0 0 0>; /* Filled by the loader. */ + status = "okay"; /* m1n1 only enables it on chips it knows. */ + power-domains = <&ps_disp_cpu>; + apple,internal-panel; /* simpledrm reports it as eDP */ + }; + }; + + memory@10000000000 { + device_type = "memory"; + reg = <0x100 0 0 0>; /* Filled by the loader. */ + }; + + led-controller { + compatible = "pwm-leds"; + led-0 { + pwms = <&fpwm0 0 40000>; + label = "kbd_backlight"; + function = LED_FUNCTION_KBD_BACKLIGHT; + color = ; + max-brightness = <255>; + default-state = "keep"; + }; + }; +}; + +&fpwm0 { + status = "okay"; +}; + +&mtp { + status = "okay"; +}; + +/* The AP GPIO controller: the N1's PERST# is GPIO 80 on it, so pcie-apple defers until it binds. */ +&pinctrl_ap { + status = "okay"; +}; + +&mtp_mbox { + status = "okay"; +}; + +&mtp_dart { + status = "okay"; +}; + +&mtp_dockchannel { + status = "okay"; +}; + +&mtp_hid { + /* + * ADT multi-touch function-afe-reset = smc-pmu pKW8('pcIO', 0x10000, + * 0x11c0000), SMC pcIO output 28; level 1 asserts the reset. Left to + * the MTP firmware, the AFE sometimes never answers ("No HINT_L"). + */ + apple,afe-reset-gpios = <&smc_gpio 28 GPIO_ACTIVE_HIGH>; + + mtp_mt: multi-touch { + firmware-name = "apple/tpmtfw-k114c.bin"; + apple,power-method = <2>; + }; + + kiwi { + /* The k114c touchscreen half: digitizer TouchScreen (usage + * 0x0d/0x04), rides the same dockchannel MTP as multi-touch + * (macOS shows Transport FIFO, LocationID 172). The load path is + * the generic CMD_SEND_FIRMWARE handshake, not a separate CBOR + * bootloader: the blob is HIDF like the multi-touch one, and the + * loader rewrites the interface byte at iface_offset. The image + * is installed at /lib/firmware/apple/tpkiwifw-k114c.bin. + * + * ADT kiwi functions (all on OIPG, the AP pinctrl): reset gpio + * 25 (01 00 01 00), power gpio 26 (01 01 00 00), + * clock_24MHz_enable gpio 22 (01 01 00 02), display_sync gpio 11 + * (01 01 00 02). Polarity assumed active high; the boot logs + * confirm. The clock-source pmgr gate (ADT phandle 0x1ed CRST) + * has no measured pmgr offset yet; the kernelcache pull decides + * whether kiwi also needs its power domain tied here. */ + firmware-name = "apple/tpkiwifw-k114c.bin"; + apple,power-method = <2>; + apple,reset-gpios = <&pinctrl_ap 25 GPIO_ACTIVE_HIGH>; + apple,power-gpios = <&pinctrl_ap 26 GPIO_ACTIVE_HIGH>; + apple,clock_24MHz_enable-gpios = <&pinctrl_ap 22 GPIO_ACTIVE_HIGH>; + apple,display_sync-gpios = <&pinctrl_ap 11 GPIO_ACTIVE_HIGH>; + }; + + keyboard: keyboard { + hid-country-code = <0>; + apple,keyboard-layout-id = <0>; + }; + + stm { + }; + + actuator { + }; + + tp_accel { + }; +}; + +#include "hwmon-common.dtsi" +#include "hwmon-fan-dual.dtsi" +#include "hwmon-laptop.dtsi" + +/* + * The T6050 SMC spells the fan mode keys F0md/F1md; F0Md/F1Md from + * hwmon-fan*.dtsi do not exist (as on the J714c). + */ +&smc_hwmon { + fan-F0Ac { + apple,fan-mode = "F0md"; + }; + + fan-F1Ac { + apple,fan-mode = "F1md"; + }; +}; + + +#include "t6050-k114c-usb2.dtsi" +/* + * iBoot leaves the N1 port's gates in automatic power management with the + * target on and the block gated (APCIE_SYS_GP and APHY_GP_AUS5 actual 0, + * APCIE_PHY_SW actual 4). The kernel trains the port, so these gates stay + * at the requested state. + */ +&ps_apcie_sys_gp { + apple,no-auto-pm; +}; + +&ps_apcie_phy_sw { + apple,no-auto-pm; +}; + +&ps_aphy_gp_aus5 { + apple,no-auto-pm; +}; + +/* + * The N1 (Wi-Fi and Bluetooth) on apcie0 port 0, the same port, registers + * and pins as the J714c (K114c ADT 2026-11-10: pci-bridge0 function-perst = + * GPIO 80, function-reset = pKW4('gP13', ...), so SMC GPIO 19; dart-apcie0 + * AIC 1524; apcie0 port0 AIC 1523; msi-vector-offset 1824 with pci-bridge0 + * owning 32 vectors; root board-id 0x04). iBoot leaves the port untrained + * and the N1 off (SMC GPIO gP13 low, PERST# GPIO 80 low); the kernel trains + * it (apple_pcie_t6050_cold_train, entered when adopt finds the link down + * and the port names pwren-gpios; apple,link-trained-by-bootloader is the + * adopt path's port-validity flag, not a claim about who trained it). + * Platform id = (board-id 0x04 << 16) | 0x6050 by the J714c/J716c rule; the + * FTAB comes from the macOS install and is not shipped by anyone. + */ +&pcie0 { + reg = <0x1c 0xb0000000 0x0 0x10000000>, /* ADT reg[0] */ + <0x4 0x14000000 0x0 0x4000>, /* ADT reg[1] */ + <0x4 0x10028000 0x0 0x8000>, /* ADT reg[16] */ + <0x4 0x17010000 0x0 0x4000>, /* ADT reg[18] */ + <0x4 0x10024000 0x0 0x4000>, /* ADT reg[19] */ + <0x4 0x17020000 0x0 0x4000>, /* ADT reg[3] */ + <0x4 0x17004000 0x0 0x4000>, /* phy-common: ADT reg[2] + 0x4000 */ + <0x4 0x16000000 0x0 0x1000000>, /* ADT reg[5]: K114c size */ + <0x4 0x15044000 0x0 0x4000>, /* ADT reg[6] */ + <0x4 0x17000000 0x0 0x800000>, /* ADT reg[2]: K114c size */ + <0x4 0x17800000 0x0 0x1b4000>; /* ADT reg[4] */ + reg-names = "config", "rc", "port0", "phy0", "intr2axi0", "phy-shared", "phy-common", + "axi2af", "pcieclkgen", "phy-window", "phy-ip"; + interrupts = ; + interrupt-names = "port0"; + + /* The N1's three functions are on DART streams 1-3 */ + iommu-map = <0x100 &dart_apcie0 1 3>; + bus-range = <0 8>; /* ADT bus-range 0x0,0x8 (the J714c's says 0-4) */ + + port00: pci@0,0 { + device_type = "pci"; + reg = <0x0 0 0 0 0>; + bus-range = <1 1>; + apple,link-trained-by-bootloader; + /* PERST#, for the kernel's train and the root-port reset after the N1's ROM stage */ + reset-gpios = <&pinctrl_ap 80 GPIO_ACTIVE_LOW>; + /* The N1's power/reset: ADT function-reset = pKW4('gP13', ...), SMC GPIO pin 0x13 */ + pwren-gpios = <&smc_gpio 19 GPIO_ACTIVE_HIGH>; + status = "okay"; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + + interrupt-controller; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 7>; + interrupt-map = <0 0 0 1 &port00 0 0 0 0>, + <0 0 0 2 &port00 0 0 0 1>, + <0 0 0 3 &port00 0 0 0 2>, + <0 0 0 4 &port00 0 0 0 3>; + + n1_control: control@0,0 { + compatible = "pci106b,1900"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + firmware-name = "apple/centauri/k114c-ftab.bin"; + apple,centauri-platform-id = <0x00046050>; + apple,centauri-protocol = <27>; + }; + + n1_alpha: wifi@0,1 { + compatible = "pci106b,1902"; + reg = <0x10100 0x0 0x0 0x0 0x0>; + local-mac-address = [00 00 00 00 00 00]; + }; + + n1_beta: bluetooth@0,2 { + compatible = "pci106b,1903"; + reg = <0x10200 0x0 0x0 0x0 0x0>; + local-bd-address = [00 00 00 00 00 00]; + }; + }; +}; + +&dart_apcie0 { + status = "okay"; +}; + +#include "t6050-j714c-pcie-tunables.dtsi" +#include "t6050-k114c-pcie-tunables.dtsi" diff --git a/arch/arm64/boot/dts/apple/t6050-nvme.dtsi b/arch/arm64/boot/dts/apple/t6050-nvme.dtsi new file mode 100644 index 0000000000000..472c3aedf09b3 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-nvme.dtsi @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * NVMe related devices for the Apple T6050 "M5 Max" SoC + * + * Register windows from the J714c ADT (/arm-io/ans and /arm-io/sart-ans); + * node layout and the M4-generation driver variant follow t8140.dtsi. + * + * Copyright The Asahi Linux Contributors + */ + +&soc { + ans_mbox: mailbox@419608000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x4 0x19608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + power-domains = <&ps_ans>; + #mbox-cells = <0>; + }; + + sart: iommu@41dc50000 { + compatible = "apple,t6050-sart"; /* SARTv4, its own register layout */ + reg = <0x4 0x1dc50000 0x0 0xc000>; + power-domains = <&ps_ans>; + }; + + nvme: nvme@45dcc0000 { + compatible = "apple,t6050-nvme-ans2", "apple,t8140-nvme-ans2"; + /* + * The ADT sizes the controller window at 0x10000; the linear + * submission queue registers sit above that, as on T8140. + */ + reg = <0x4 0x5dcc0000 0x0 0x30000>, + <0x4 0x19600000 0x0 0x4000>, + <0x4 0x1dcc0000 0x0 0x60010>; + reg-names = "nvme", "ans", "nvmmu"; + interrupt-parent = <&aic>; + interrupts = ; + mboxes = <&ans_mbox>; + apple,sart = <&sart>; + /* + * The port-0 lane PHY must stay on for the controller: with it + * powered down as unused, enable never reaches CSTS.RDY. + */ + power-domains = <&ps_ans>, <&ps_apcie_sys_st0>, + <&ps_apcie_phy_sw>, <&ps_aphy_st0_aus5>; + power-domain-names = "ans", "apcie0", "apcie_phy", "aphy_st0"; + resets = <&ps_ans>; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-pmgr.dtsi b/arch/arm64/boot/dts/apple/t6050-pmgr.dtsi new file mode 100644 index 0000000000000..8a7cf0a24ed8d --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-pmgr.dtsi @@ -0,0 +1,2728 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * PMGR Power domains for Apple T6050 "M5 Max" SoC + * + * Generated from the J714c ADT with pmgr_adt2dt.py, then edited by hand. + * Regenerating loses the following: + * - the CPU core and cluster domains are left out, as in every other + * pmgr file; + * - parent references to register-less (no_ps) ADT domains point at the + * nearest real ancestor or are dropped; + * - apple,always-on is removed from the 25 domains the boot firmware + * leaves off (b2b fabric, nub SPMI, afr) and added to the core set + * aic, disp_sys, disp_fe, uart_p, spi_p, ans as in t8132-pmgr.dtsi; + * - apple,firmware-owned marks disp_cpu, dispext0-3_cpu (writing them + * raises an SError) and smc_i2cm0-5 (enabling auto-PM on one or more + * of them resets the machine a few seconds into the boot). + * + * Copyright The Asahi Linux Contributors + */ + +&pmgr0 { + + ps_aic: power-controller@108 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + label = "aic"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_pmc: power-controller@110 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + label = "pmc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_pms_c1ppt: power-controller@118 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + label = "pms_c1ppt"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_soc_dpe: power-controller@120 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + label = "soc_dpe"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_pmp: power-controller@128 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + label = "pmp"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_pms_sram: power-controller@130 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + label = "pms_sram"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_pmgr_soc_ocla: power-controller@138 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + label = "pmgr_soc_ocla"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2b_ctrl_nogate: power-controller@140 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + label = "b2b_ctrl_nogate"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab0_acc_0: power-controller@148 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + label = "fab0_acc_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab0_acc_1: power-controller@150 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + label = "fab0_acc_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab2_acc: power-controller@158 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + label = "fab2_acc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab1_acc: power-controller@160 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + label = "fab1_acc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_b2bc_afc_0: power-controller@168 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + label = "b2bc_afc_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_b2bc_afc_0_dfx: power-controller@170 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + label = "b2bc_afc_0_dfx"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bc_afc_0>; + #reset-cells = <0>; + }; + + ps_b2bc_afc_1: power-controller@178 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + label = "b2bc_afc_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_b2bc_afc_1_dfx: power-controller@180 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + label = "b2bc_afc_1_dfx"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bc_afc_1>; + #reset-cells = <0>; + }; + + ps_fab4_soc: power-controller@1e8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e8 4>; + label = "fab4_soc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab5_soc: power-controller@1f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + label = "fab5_soc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_dispext0_sys: power-controller@200 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x200 4>; + label = "dispext0_sys"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc0_common: power-controller@208 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x208 4>; + label = "atc0_common"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc1_common: power-controller@210 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x210 4>; + label = "atc1_common"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_dispext2_sys: power-controller@220 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x220 4>; + label = "dispext2_sys"; + #power-domain-cells = <0>; + power-domains = <&ps_fab4_soc>; + #reset-cells = <0>; + }; + + ps_dispext3_sys: power-controller@228 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x228 4>; + label = "dispext3_sys"; + #power-domain-cells = <0>; + power-domains = <&ps_fab4_soc>; + #reset-cells = <0>; + }; + + ps_dispext1_sys: power-controller@230 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + label = "dispext1_sys"; + #power-domain-cells = <0>; + power-domains = <&ps_fab5_soc>; + #reset-cells = <0>; + }; + + ps_atc2_common: power-controller@238 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + label = "atc2_common"; + #power-domain-cells = <0>; + power-domains = <&ps_fab5_soc>; + #reset-cells = <0>; + }; + + ps_atc3_common: power-controller@240 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + label = "atc3_common"; + #power-domain-cells = <0>; + power-domains = <&ps_fab5_soc>; + #reset-cells = <0>; + apple,firmware-owned; + }; + + ps_auss: power-controller@248 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x248 4>; + label = "auss"; + #power-domain-cells = <0>; + power-domains = <&ps_fab5_soc>; + #reset-cells = <0>; + }; + + ps_gpio: power-controller@250 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x250 4>; + label = "gpio"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_msg: power-controller@258 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + label = "msg"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_ap_tmm: power-controller@260 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x260 4>; + label = "ap_tmm"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_dispext0_fe: power-controller@268 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x268 4>; + label = "dispext0_fe"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_sys>; + #reset-cells = <0>; + }; + + ps_dispext0_adma: power-controller@270 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x270 4>; + label = "dispext0_adma"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_sys>; + #reset-cells = <0>; + }; + + ps_dispext0_cpu: power-controller@278 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x278 4>; + label = "dispext0_cpu"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext0_fe>; + #reset-cells = <0>; + /* The display coprocessor owns this state; writing it faults (SError). */ + apple,firmware-owned; + }; + +/* + * atc0-3_cio and atc3_common are left to the firmware for now: the kernel's + * boot-time power-on of these domains (2026-10-26, J714c) breaks the USB 2.0 + * host on the parked Type-C PHY (the xHCI reset never completes). USB4 needs + * them driven in sequence with the PHY, by the PHY driver, later. + */ + ps_atc0_cio: power-controller@280 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + label = "atc0_cio"; + #power-domain-cells = <0>; + power-domains = <&ps_atc0_common>; + #reset-cells = <0>; + apple,firmware-owned; + }; + + ps_atc0_pcie: power-controller@288 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + label = "atc0_pcie"; + #power-domain-cells = <0>; + power-domains = <&ps_atc0_common>; + #reset-cells = <0>; + }; + + ps_atc1_cio: power-controller@290 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + label = "atc1_cio"; + #power-domain-cells = <0>; + power-domains = <&ps_atc1_common>; + #reset-cells = <0>; + apple,firmware-owned; + }; + + ps_atc1_pcie: power-controller@298 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + label = "atc1_pcie"; + #power-domain-cells = <0>; + power-domains = <&ps_atc1_common>; + #reset-cells = <0>; + }; + + ps_dispext2_fe: power-controller@2a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + label = "dispext2_fe"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext2_sys>; + #reset-cells = <0>; + }; + + ps_dispext2_adma: power-controller@2a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + label = "dispext2_adma"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext2_sys>; + #reset-cells = <0>; + }; + + ps_dispext2_cpu: power-controller@2b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + label = "dispext2_cpu"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext2_fe>; + #reset-cells = <0>; + /* The display coprocessor owns this state; writing it faults (SError). */ + apple,firmware-owned; + }; + + ps_dispext3_fe: power-controller@2b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + label = "dispext3_fe"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext3_sys>; + #reset-cells = <0>; + }; + + ps_dispext3_adma: power-controller@2c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + label = "dispext3_adma"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext3_sys>; + #reset-cells = <0>; + }; + + ps_dispext3_cpu: power-controller@2c8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + label = "dispext3_cpu"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext3_fe>; + #reset-cells = <0>; + /* The display coprocessor owns this state; writing it faults (SError). */ + apple,firmware-owned; + }; + + ps_dispext1_fe: power-controller@2d0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + label = "dispext1_fe"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_sys>; + #reset-cells = <0>; + }; + + ps_dispext1_adma: power-controller@2d8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + label = "dispext1_adma"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_sys>; + #reset-cells = <0>; + }; + + ps_dispext1_cpu: power-controller@2e0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + label = "dispext1_cpu"; + #power-domain-cells = <0>; + power-domains = <&ps_dispext1_fe>; + #reset-cells = <0>; + /* The display coprocessor owns this state; writing it faults (SError). */ + apple,firmware-owned; + }; + + ps_atc2_cio: power-controller@2e8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + label = "atc2_cio"; + #power-domain-cells = <0>; + power-domains = <&ps_atc2_common>; + #reset-cells = <0>; + apple,firmware-owned; + }; + + ps_atc2_pcie: power-controller@2f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + label = "atc2_pcie"; + #power-domain-cells = <0>; + power-domains = <&ps_atc2_common>; + #reset-cells = <0>; + }; + + ps_atc3_cio: power-controller@2f8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + label = "atc3_cio"; + #power-domain-cells = <0>; + power-domains = <&ps_atc3_common>; + #reset-cells = <0>; + apple,firmware-owned; + }; + + ps_atc3_pcie: power-controller@300 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + label = "atc3_pcie"; + #power-domain-cells = <0>; + power-domains = <&ps_atc3_common>; + #reset-cells = <0>; + }; + + ps_atc0_cio_pcie: power-controller@338 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + label = "atc0_cio_pcie"; + #power-domain-cells = <0>; + power-domains = <&ps_atc0_cio>; + #reset-cells = <0>; + }; + + ps_atc0_cio_usb: power-controller@340 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + label = "atc0_cio_usb"; + #power-domain-cells = <0>; + power-domains = <&ps_atc0_cio>; + #reset-cells = <0>; + }; + + ps_atc0_cio_auss: power-controller@348 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + label = "atc0_cio_auss"; + #power-domain-cells = <0>; + power-domains = <&ps_atc0_cio>, <&ps_auss>; + #reset-cells = <0>; + }; + + ps_atc1_cio_pcie: power-controller@350 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + label = "atc1_cio_pcie"; + #power-domain-cells = <0>; + power-domains = <&ps_atc1_cio>; + #reset-cells = <0>; + }; + + ps_atc1_cio_usb: power-controller@358 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x358 4>; + label = "atc1_cio_usb"; + #power-domain-cells = <0>; + power-domains = <&ps_atc1_cio>; + #reset-cells = <0>; + }; + + ps_atc1_cio_auss: power-controller@360 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + label = "atc1_cio_auss"; + #power-domain-cells = <0>; + power-domains = <&ps_atc1_cio>, <&ps_auss>; + #reset-cells = <0>; + }; + + ps_atc2_cio_pcie: power-controller@3f8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3f8 4>; + label = "atc2_cio_pcie"; + #power-domain-cells = <0>; + power-domains = <&ps_atc2_cio>; + #reset-cells = <0>; + }; + + ps_atc2_cio_usb: power-controller@400 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x400 4>; + label = "atc2_cio_usb"; + #power-domain-cells = <0>; + power-domains = <&ps_atc2_cio>; + #reset-cells = <0>; + }; + + ps_atc2_cio_auss: power-controller@408 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + label = "atc2_cio_auss"; + #power-domain-cells = <0>; + power-domains = <&ps_atc2_cio>, <&ps_auss>; + #reset-cells = <0>; + }; + + ps_atc3_cio_pcie: power-controller@410 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x410 4>; + label = "atc3_cio_pcie"; + #power-domain-cells = <0>; + power-domains = <&ps_atc3_cio>; + #reset-cells = <0>; + }; + + ps_atc3_cio_usb: power-controller@418 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x418 4>; + label = "atc3_cio_usb"; + #power-domain-cells = <0>; + power-domains = <&ps_atc3_cio>; + #reset-cells = <0>; + }; + + ps_atc3_cio_auss: power-controller@420 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x420 4>; + label = "atc3_cio_auss"; + #power-domain-cells = <0>; + power-domains = <&ps_atc3_cio>, <&ps_auss>; + #reset-cells = <0>; + }; + + ps_fppt: power-controller@450 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x450 4>; + label = "fppt"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bc_dft: power-controller@458 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x458 4>; + label = "b2bc_dft"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bc_cntl_dfx: power-controller@460 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x460 4>; + label = "b2bc_cntl_dfx"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bc_dbg_0: power-controller@468 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x468 4>; + label = "b2bc_dbg_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bc_dbg_1: power-controller@470 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x470 4>; + label = "b2bc_dbg_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bc_dbg_2: power-controller@478 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x478 4>; + label = "b2bc_dbg_2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; +}; + +&pmgr1 { + + ps_ispsens0: power-controller@100 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + label = "ispsens0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_global_aux_hpd_0: power-controller@108 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + label = "global_aux_hpd_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_global_aux_hpd_1: power-controller@110 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + label = "global_aux_hpd_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_apcie_gp: power-controller@118 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + label = "apcie_gp"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_apcie_st0: power-controller@128 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + label = "apcie_st0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_apcie_st1: power-controller@130 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + label = "apcie_st1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_fab6_soc: power-controller@138 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + label = "fab6_soc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_ans: power-controller@140 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + label = "ans"; + #power-domain-cells = <0>; + power-domains = <&ps_fab6_soc>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_apcie_sys_gp: power-controller@148 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + label = "apcie_sys_gp"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_gp>, <&ps_fab6_soc>; + #reset-cells = <0>; + }; + + ps_apcie_sys_st0: power-controller@150 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + label = "apcie_sys_st0"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_st0>, <&ps_ans>; + #reset-cells = <0>; + }; + + ps_apcie_sys_st1: power-controller@158 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + label = "apcie_sys_st1"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_st1>, <&ps_ans>; + #reset-cells = <0>; + }; + + ps_apcie_sys_gp_oc: power-controller@160 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + label = "apcie_sys_gp_oc"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_sys_gp>; + #reset-cells = <0>; + }; + + ps_apcie_sys_st0_o: power-controller@168 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + label = "apcie_sys_st0_o"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_sys_st0>; + #reset-cells = <0>; + }; + + ps_apcie_sys_st1_o: power-controller@170 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + label = "apcie_sys_st1_o"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_sys_st1>; + #reset-cells = <0>; + }; + + ps_fab7_soc: power-controller@178 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + label = "fab7_soc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_disp_sys: power-controller@180 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + label = "disp_sys"; + #power-domain-cells = <0>; + power-domains = <&ps_fab7_soc>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_scodec: power-controller@190 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + label = "scodec"; + #power-domain-cells = <0>; + power-domains = <&ps_fab7_soc>; + #reset-cells = <0>; + }; + + ps_disp_fe: power-controller@198 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + label = "disp_fe"; + #power-domain-cells = <0>; + power-domains = <&ps_disp_sys>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_disp_adma: power-controller@1a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a0 4>; + label = "disp_adma"; + #power-domain-cells = <0>; + power-domains = <&ps_disp_sys>; + #reset-cells = <0>; + }; + + ps_disp_cpu: power-controller@1a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a8 4>; + label = "disp_cpu"; + #power-domain-cells = <0>; + power-domains = <&ps_disp_fe>; + #reset-cells = <0>; + /* The display coprocessor owns this state; writing it faults (SError). */ + apple,firmware-owned; + }; + + ps_apcie_phy_sw: power-controller@1b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b0 4>; + label = "apcie_phy_sw"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_sys_gp>, <&ps_fab6_soc>; + #reset-cells = <0>; + }; + + ps_scodec_streaming: power-controller@1c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c0 4>; + label = "scodec_streaming"; + #power-domain-cells = <0>; + power-domains = <&ps_scodec>; + #reset-cells = <0>; + }; + + ps_aphy_st0_aus5_a: power-controller@240 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + label = "aphy_st0_aus5_a"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_phy_sw>; + #reset-cells = <0>; + }; + + ps_aphy_st1_aus5_a: power-controller@248 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x248 4>; + label = "aphy_st1_aus5_a"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_phy_sw>; + #reset-cells = <0>; + }; + + ps_aphy_gp_aus5_a: power-controller@258 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x258 4>; + label = "aphy_gp_aus5_a"; + #power-domain-cells = <0>; + power-domains = <&ps_apcie_phy_sw>; + #reset-cells = <0>; + }; + + ps_aphy_st0_aus5: power-controller@268 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x268 4>; + label = "aphy_st0_aus5"; + #power-domain-cells = <0>; + power-domains = <&ps_aphy_st0_aus5_a>; + #reset-cells = <0>; + }; + + ps_aphy_st1_aus5: power-controller@270 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x270 4>; + label = "aphy_st1_aus5"; + #power-domain-cells = <0>; + power-domains = <&ps_aphy_st1_aus5_a>; + #reset-cells = <0>; + }; + + ps_aphy_gp_aus5: power-controller@280 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x280 4>; + label = "aphy_gp_aus5"; + #power-domain-cells = <0>; + power-domains = <&ps_aphy_gp_aus5_a>; + #reset-cells = <0>; + }; + + ps_fab8_soc: power-controller@288 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + label = "fab8_soc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_sio: power-controller@290 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x290 4>; + label = "sio"; + #power-domain-cells = <0>; + power-domains = <&ps_fab8_soc>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_sep: power-controller@298 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + label = "sep"; + #power-domain-cells = <0>; + power-domains = <&ps_fab8_soc>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_sio_cpu: power-controller@2a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + label = "sio_cpu"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fpwm0: power-controller@2a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + label = "fpwm0"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_fpwm2: power-controller@2b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + label = "fpwm2"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_i2c0: power-controller@2c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + label = "i2c0"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_i2c1: power-controller@2c8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c8 4>; + label = "i2c1"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_i2c2: power-controller@2d0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d0 4>; + label = "i2c2"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_i2c3: power-controller@2d8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2d8 4>; + label = "i2c3"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_i2c4: power-controller@2e0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e0 4>; + label = "i2c4"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_i2c5: power-controller@2e8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2e8 4>; + label = "i2c5"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_i2c6: power-controller@2f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f0 4>; + label = "i2c6"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_i2c7: power-controller@2f8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2f8 4>; + label = "i2c7"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_i2c8: power-controller@300 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x300 4>; + label = "i2c8"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_spi_p: power-controller@308 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + label = "spi_p"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_uart_p: power-controller@310 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x310 4>; + label = "uart_p"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_aes: power-controller@318 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + label = "aes"; + #power-domain-cells = <0>; + power-domains = <&ps_sio>; + #reset-cells = <0>; + }; + + ps_spi0: power-controller@320 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + label = "spi0"; + #power-domain-cells = <0>; + power-domains = <&ps_spi_p>; + #reset-cells = <0>; + }; + + ps_spi1: power-controller@328 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x328 4>; + label = "spi1"; + #power-domain-cells = <0>; + power-domains = <&ps_spi_p>; + #reset-cells = <0>; + }; + + ps_spi2: power-controller@330 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + label = "spi2"; + #power-domain-cells = <0>; + power-domains = <&ps_spi_p>; + #reset-cells = <0>; + }; + + ps_spi3: power-controller@338 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + label = "spi3"; + #power-domain-cells = <0>; + power-domains = <&ps_spi_p>; + #reset-cells = <0>; + }; + + ps_spi4: power-controller@340 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + label = "spi4"; + #power-domain-cells = <0>; + power-domains = <&ps_spi_p>; + #reset-cells = <0>; + }; + + ps_spi5: power-controller@348 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x348 4>; + label = "spi5"; + #power-domain-cells = <0>; + power-domains = <&ps_spi_p>; + #reset-cells = <0>; + }; + + ps_qspi: power-controller@350 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x350 4>; + label = "qspi"; + #power-domain-cells = <0>; + power-domains = <&ps_spi_p>; + #reset-cells = <0>; + }; + + ps_uart_n: power-controller@358 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x358 4>; + label = "uart_n"; + #power-domain-cells = <0>; + power-domains = <&ps_uart_p>; + #reset-cells = <0>; + }; + + ps_uart0: power-controller@360 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x360 4>; + label = "uart0"; + #power-domain-cells = <0>; + power-domains = <&ps_uart_p>; + #reset-cells = <0>; + }; + + ps_uart1: power-controller@368 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x368 4>; + label = "uart1"; + #power-domain-cells = <0>; + power-domains = <&ps_uart_p>; + #reset-cells = <0>; + }; + + ps_uart2: power-controller@370 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x370 4>; + label = "uart2"; + #power-domain-cells = <0>; + power-domains = <&ps_uart_p>; + #reset-cells = <0>; + }; + + ps_uart3: power-controller@378 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x378 4>; + label = "uart3"; + #power-domain-cells = <0>; + power-domains = <&ps_uart_p>; + #reset-cells = <0>; + }; + + ps_uart4: power-controller@380 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x380 4>; + label = "uart4"; + #power-domain-cells = <0>; + power-domains = <&ps_uart_p>; + #reset-cells = <0>; + }; + + ps_uart5: power-controller@388 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x388 4>; + label = "uart5"; + #power-domain-cells = <0>; + power-domains = <&ps_uart_p>; + #reset-cells = <0>; + }; + + ps_uart6: power-controller@390 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x390 4>; + label = "uart6"; + #power-domain-cells = <0>; + power-domains = <&ps_uart_p>; + #reset-cells = <0>; + }; + + ps_fab9_soc: power-controller@398 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + label = "fab9_soc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_isp_sys: power-controller@3a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + label = "isp_sys"; + #power-domain-cells = <0>; + power-domains = <&ps_fab9_soc>; + #reset-cells = <0>; + }; + + ps_ane_sys: power-controller@3a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a8 4>; + label = "ane_sys"; + #power-domain-cells = <0>; + power-domains = <&ps_fab8_soc>, <&ps_fab9_soc>; + #reset-cells = <0>; + }; + + ps_ane_mpm: power-controller@3c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + label = "ane_mpm"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + #reset-cells = <0>; + }; + + ps_ane_cpu: power-controller@3c8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c8 4>; + label = "ane_cpu"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + #reset-cells = <0>; + }; + + ps_ane_tdbase: power-controller@3d0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3d0 4>; + label = "ane_tdbase"; + #power-domain-cells = <0>; + power-domains = <&ps_ane_sys>; + #reset-cells = <0>; + }; +}; + +&pmgr2 { + + ps_debug_gated: power-controller@58 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + label = "debug_gated"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_spmi_a0: power-controller@60 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x60 4>; + label = "nub_spmi_a0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_spmi_a1: power-controller@68 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + label = "nub_spmi_a1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_spmi_a2: power-controller@70 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x70 4>; + label = "nub_spmi_a2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_nub_dock_channel: power-controller@80 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x80 4>; + label = "nub_dock_channel"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_gpio: power-controller@88 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x88 4>; + label = "nub_gpio"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_atc0_common_dp_0: power-controller@98 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x98 4>; + label = "atc0_common_dp_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc0_common_dp_1: power-controller@a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa0 4>; + label = "atc0_common_dp_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc0_common_dp_2: power-controller@a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xa8 4>; + label = "atc0_common_dp_2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc0_common_dp_3: power-controller@b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb0 4>; + label = "atc0_common_dp_3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc1_common_dp_0: power-controller@b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xb8 4>; + label = "atc1_common_dp_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc1_common_dp_1: power-controller@c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc0 4>; + label = "atc1_common_dp_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc1_common_dp_2: power-controller@c8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xc8 4>; + label = "atc1_common_dp_2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc1_common_dp_3: power-controller@d0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd0 4>; + label = "atc1_common_dp_3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc2_common_dp_0: power-controller@d8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xd8 4>; + label = "atc2_common_dp_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc2_common_dp_1: power-controller@e0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe0 4>; + label = "atc2_common_dp_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc2_common_dp_2: power-controller@e8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xe8 4>; + label = "atc2_common_dp_2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc2_common_dp_3: power-controller@f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xf0 4>; + label = "atc2_common_dp_3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc3_common_dp_0: power-controller@f8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0xf8 4>; + label = "atc3_common_dp_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc3_common_dp_1: power-controller@100 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + label = "atc3_common_dp_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc3_common_dp_2: power-controller@108 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + label = "atc3_common_dp_2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc3_common_dp_3: power-controller@110 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + label = "atc3_common_dp_3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_nub_spmi_ahb: power-controller@118 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + label = "nub_spmi_ahb"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_spmi0: power-controller@120 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + label = "nub_spmi0"; + #power-domain-cells = <0>; + power-domains = <&ps_nub_spmi_ahb>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_spmi1: power-controller@128 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + label = "nub_spmi1"; + #power-domain-cells = <0>; + power-domains = <&ps_nub_spmi_ahb>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_spmi2: power-controller@130 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + label = "nub_spmi2"; + #power-domain-cells = <0>; + power-domains = <&ps_nub_spmi_ahb>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_spmi3: power-controller@138 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + label = "nub_spmi3"; + #power-domain-cells = <0>; + power-domains = <&ps_nub_spmi_ahb>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_spmi4: power-controller@140 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + label = "nub_spmi4"; + #power-domain-cells = <0>; + power-domains = <&ps_nub_spmi_ahb>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_atc0_usb_aon: power-controller@148 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + label = "atc0_usb_aon"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc0_usb: power-controller@150 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x150 4>; + label = "atc0_usb"; + #power-domain-cells = <0>; + power-domains = <&ps_atc0_common>, <&ps_atc0_usb_aon>; + #reset-cells = <0>; + }; + + ps_atc0_phymxwrap: power-controller@158 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x158 4>; + label = "atc0_phymxwrap"; + #power-domain-cells = <0>; + power-domains = <&ps_atc0_common>, <&ps_atc0_usb_aon>; + #reset-cells = <0>; + }; + + ps_atc1_usb_aon: power-controller@160 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x160 4>; + label = "atc1_usb_aon"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc1_usb: power-controller@168 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x168 4>; + label = "atc1_usb"; + #power-domain-cells = <0>; + power-domains = <&ps_atc1_common>, <&ps_atc1_usb_aon>; + #reset-cells = <0>; + }; + + ps_atc1_phymxwrap: power-controller@170 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x170 4>; + label = "atc1_phymxwrap"; + #power-domain-cells = <0>; + power-domains = <&ps_atc1_common>, <&ps_atc1_usb_aon>; + #reset-cells = <0>; + }; + + ps_atc2_usb_aon: power-controller@178 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x178 4>; + label = "atc2_usb_aon"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc2_usb: power-controller@180 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x180 4>; + label = "atc2_usb"; + #power-domain-cells = <0>; + power-domains = <&ps_atc2_common>, <&ps_atc2_usb_aon>; + #reset-cells = <0>; + }; + + ps_atc2_phymxwrap: power-controller@188 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + label = "atc2_phymxwrap"; + #power-domain-cells = <0>; + power-domains = <&ps_atc2_common>, <&ps_atc2_usb_aon>; + #reset-cells = <0>; + }; + + ps_atc3_usb_aon: power-controller@190 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x190 4>; + label = "atc3_usb_aon"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_atc3_usb: power-controller@198 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + label = "atc3_usb"; + #power-domain-cells = <0>; + power-domains = <&ps_atc3_common>, <&ps_atc3_usb_aon>; + #reset-cells = <0>; + }; + + ps_atc3_phymxwrap: power-controller@1a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a0 4>; + label = "atc3_phymxwrap"; + #power-domain-cells = <0>; + power-domains = <&ps_atc3_common>, <&ps_atc3_usb_aon>; + #reset-cells = <0>; + }; + + ps_smc_i2cm0: power-controller@8028 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8028 4>; + label = "smc_i2cm0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + /* The SMC owns its I2C masters; enabling auto-PM on them resets the machine. */ + apple,firmware-owned; + }; + + ps_smc_i2cm1: power-controller@8030 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8030 4>; + label = "smc_i2cm1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + /* The SMC owns its I2C masters; enabling auto-PM on them resets the machine. */ + apple,firmware-owned; + }; + + ps_smc_i2cm2: power-controller@8038 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8038 4>; + label = "smc_i2cm2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + /* The SMC owns its I2C masters; enabling auto-PM on them resets the machine. */ + apple,firmware-owned; + }; + + ps_smc_i2cm3: power-controller@8040 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8040 4>; + label = "smc_i2cm3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + /* The SMC owns its I2C masters; enabling auto-PM on them resets the machine. */ + apple,firmware-owned; + }; + + ps_smc_i2cm4: power-controller@8048 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8048 4>; + label = "smc_i2cm4"; + #power-domain-cells = <0>; + #reset-cells = <0>; + /* The SMC owns its I2C masters; enabling auto-PM on them resets the machine. */ + apple,firmware-owned; + }; + + ps_smc_i2cm5: power-controller@8050 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8050 4>; + label = "smc_i2cm5"; + #power-domain-cells = <0>; + #reset-cells = <0>; + /* The SMC owns its I2C masters; enabling auto-PM on them resets the machine. */ + apple,firmware-owned; + }; + + ps_smc_i2cm6: power-controller@8058 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8058 4>; + label = "smc_i2cm6"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; +}; + +&pmgr3 { + + ps_gpx: power-controller@0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x0 4>; + label = "gpx"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_afr: power-controller@100 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x100 4>; + label = "afr"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_cs: power-controller@108 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + label = "cs"; + #power-domain-cells = <0>; + power-domains = <&ps_afr>; + #reset-cells = <0>; + }; + + ps_gfx: power-controller@110 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + label = "gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_afr>, <&ps_gpx>; + #reset-cells = <0>; + }; +}; + +&pmgr4 { + + ps_aic_gfx: power-controller@108 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x108 4>; + label = "aic_gfx"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_pms_gfx_c1ppt: power-controller@110 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x110 4>; + label = "pms_gfx_c1ppt"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_soc_dpe_gfx: power-controller@118 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x118 4>; + label = "soc_dpe_gfx"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_pmgr_soc_ocla_g: power-controller@120 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x120 4>; + label = "pmgr_soc_ocla_g"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2b_ctrl_gfx: power-controller@128 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x128 4>; + label = "b2b_ctrl_gfx"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bg_afc_0: power-controller@130 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x130 4>; + label = "b2bg_afc_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_b2bg_afc_0_dfx: power-controller@138 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x138 4>; + label = "b2bg_afc_0_dfx"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bg_afc_0>; + #reset-cells = <0>; + }; + + ps_b2bg_afc_1: power-controller@140 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x140 4>; + label = "b2bg_afc_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_b2bg_afc_1_dfx: power-controller@148 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x148 4>; + label = "b2bg_afc_1_dfx"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bg_afc_1>; + #reset-cells = <0>; + }; + + ps_b2bg_afi_1_dfx: power-controller@188 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x188 4>; + label = "b2bg_afi_1_dfx"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_fppt_gfx: power-controller@198 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x198 4>; + label = "fppt_gfx"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_fab0_soc: power-controller@1a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1a8 4>; + label = "fab0_soc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab1_soc: power-controller@1b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1b0 4>; + label = "fab1_soc"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_venc0_sys: power-controller@1c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c0 4>; + label = "venc0_sys"; + #power-domain-cells = <0>; + power-domains = <&ps_fab0_soc>; + #reset-cells = <0>; + }; + + ps_msr0: power-controller@1c8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1c8 4>; + label = "msr0"; + #power-domain-cells = <0>; + power-domains = <&ps_fab0_soc>; + #reset-cells = <0>; + }; + + ps_jpg: power-controller@1d0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d0 4>; + label = "jpg"; + #power-domain-cells = <0>; + power-domains = <&ps_fab0_soc>; + #reset-cells = <0>; + }; + + ps_avd_sys: power-controller@1d8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1d8 4>; + label = "avd_sys"; + #power-domain-cells = <0>; + power-domains = <&ps_fab1_soc>; + #reset-cells = <0>; + }; + + ps_prores: power-controller@1e0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1e0 4>; + label = "prores"; + #power-domain-cells = <0>; + power-domains = <&ps_fab1_soc>; + #reset-cells = <0>; + }; + + ps_venc1_sys: power-controller@1f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f0 4>; + label = "venc1_sys"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_msr1: power-controller@1f8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x1f8 4>; + label = "msr1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_msr0_ase_core: power-controller@200 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x200 4>; + label = "msr0_ase_core"; + #power-domain-cells = <0>; + power-domains = <&ps_msr0>; + #reset-cells = <0>; + }; + + ps_msr1_ase_core: power-controller@208 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x208 4>; + label = "msr1_ase_core"; + #power-domain-cells = <0>; + power-domains = <&ps_msr1>; + #reset-cells = <0>; + }; + + ps_amcc0: power-controller@218 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x218 4>; + label = "amcc0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc1: power-controller@220 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x220 4>; + label = "amcc1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc2: power-controller@228 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x228 4>; + label = "amcc2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc3: power-controller@230 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x230 4>; + label = "amcc3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_0: power-controller@238 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x238 4>; + label = "fab_afr_amcc_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_1: power-controller@240 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x240 4>; + label = "fab_afr_amcc_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afc_amcc_0: power-controller@288 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x288 4>; + label = "fab_afc_amcc_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc4: power-controller@298 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x298 4>; + label = "amcc4"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc5: power-controller@2a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a0 4>; + label = "amcc5"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc6: power-controller@2a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2a8 4>; + label = "amcc6"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc7: power-controller@2b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b0 4>; + label = "amcc7"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_2: power-controller@2b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2b8 4>; + label = "fab_afr_amcc_2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_3: power-controller@2c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x2c0 4>; + label = "fab_afr_amcc_3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afc_amcc_1: power-controller@308 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x308 4>; + label = "fab_afc_amcc_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc8: power-controller@318 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x318 4>; + label = "amcc8"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc9: power-controller@320 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x320 4>; + label = "amcc9"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc10: power-controller@328 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x328 4>; + label = "amcc10"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc11: power-controller@330 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x330 4>; + label = "amcc11"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_4: power-controller@338 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x338 4>; + label = "fab_afr_amcc_4"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_5: power-controller@340 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x340 4>; + label = "fab_afr_amcc_5"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afc_amcc_2: power-controller@388 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x388 4>; + label = "fab_afc_amcc_2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc12: power-controller@398 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x398 4>; + label = "amcc12"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc13: power-controller@3a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a0 4>; + label = "amcc13"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc14: power-controller@3a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3a8 4>; + label = "amcc14"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc15: power-controller@3b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b0 4>; + label = "amcc15"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_6: power-controller@3b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3b8 4>; + label = "fab_afr_amcc_6"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_7: power-controller@3c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x3c0 4>; + label = "fab_afr_amcc_7"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afc_amcc_3: power-controller@408 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x408 4>; + label = "fab_afc_amcc_3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc16: power-controller@418 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x418 4>; + label = "amcc16"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc17: power-controller@420 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x420 4>; + label = "amcc17"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc18: power-controller@428 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x428 4>; + label = "amcc18"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc19: power-controller@430 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x430 4>; + label = "amcc19"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_8: power-controller@438 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x438 4>; + label = "fab_afr_amcc_8"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_9: power-controller@440 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x440 4>; + label = "fab_afr_amcc_9"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afc_amcc_4: power-controller@488 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x488 4>; + label = "fab_afc_amcc_4"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc20: power-controller@498 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x498 4>; + label = "amcc20"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc21: power-controller@4a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a0 4>; + label = "amcc21"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc22: power-controller@4a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4a8 4>; + label = "amcc22"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc23: power-controller@4b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b0 4>; + label = "amcc23"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_10: power-controller@4b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4b8 4>; + label = "fab_afr_amcc_10"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_11: power-controller@4c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x4c0 4>; + label = "fab_afr_amcc_11"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afc_amcc_5: power-controller@508 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x508 4>; + label = "fab_afc_amcc_5"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc24: power-controller@518 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x518 4>; + label = "amcc24"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc25: power-controller@520 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x520 4>; + label = "amcc25"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc26: power-controller@528 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x528 4>; + label = "amcc26"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc27: power-controller@530 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x530 4>; + label = "amcc27"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_12: power-controller@538 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x538 4>; + label = "fab_afr_amcc_12"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_13: power-controller@540 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x540 4>; + label = "fab_afr_amcc_13"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afc_amcc_6: power-controller@588 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x588 4>; + label = "fab_afc_amcc_6"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc28: power-controller@598 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x598 4>; + label = "amcc28"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc29: power-controller@5a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a0 4>; + label = "amcc29"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc30: power-controller@5a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5a8 4>; + label = "amcc30"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_amcc31: power-controller@5b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5b0 4>; + label = "amcc31"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_14: power-controller@5b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5b8 4>; + label = "fab_afr_amcc_14"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afr_amcc_15: power-controller@5c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x5c0 4>; + label = "fab_afr_amcc_15"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_afc_amcc_7: power-controller@610 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x610 4>; + label = "fab_afc_amcc_7"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_fab_b2bd_afc_0: power-controller@648 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x648 4>; + label = "fab_b2bd_afc_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_fab_b2bd_afc_1: power-controller@650 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x650 4>; + label = "fab_b2bd_afc_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_afc_0: power-controller@658 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x658 4>; + label = "b2bd_afc_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_afc_0_dfx: power-controller@660 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x660 4>; + label = "b2bd_afc_0_dfx"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bd_afc_0>; + #reset-cells = <0>; + }; + + ps_b2bd_afc_1: power-controller@668 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x668 4>; + label = "b2bd_afc_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_afc_1_dfx: power-controller@670 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x670 4>; + label = "b2bd_afc_1_dfx"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bd_afc_1>; + #reset-cells = <0>; + }; + + ps_b2bd_afr_0: power-controller@678 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x678 4>; + label = "b2bd_afr_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_afr_1: power-controller@680 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x680 4>; + label = "b2bd_afr_1"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bd_afr_0>; + #reset-cells = <0>; + }; + + ps_b2bd_afr_dfx_0: power-controller@688 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x688 4>; + label = "b2bd_afr_dfx_0"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bd_afr_0>; + #reset-cells = <0>; + }; + + ps_b2bd_afr_dfx_1: power-controller@690 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x690 4>; + label = "b2bd_afr_dfx_1"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bd_afr_1>; + #reset-cells = <0>; + }; + + ps_b2bd_afr_3: power-controller@698 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x698 4>; + label = "b2bd_afr_3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_afr_2: power-controller@6a0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6a0 4>; + label = "b2bd_afr_2"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bd_afr_3>; + #reset-cells = <0>; + }; + + ps_b2bd_afr_dfx_3: power-controller@6a8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6a8 4>; + label = "b2bd_afr_dfx_3"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bd_afr_3>; + #reset-cells = <0>; + }; + + ps_b2bd_afr_dfx_2: power-controller@6b0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6b0 4>; + label = "b2bd_afr_dfx_2"; + #power-domain-cells = <0>; + power-domains = <&ps_b2bd_afr_2>; + #reset-cells = <0>; + }; + + ps_b2bg_dft: power-controller@6b8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6b8 4>; + label = "b2bg_dft"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bg_cntl_dfx: power-controller@6c0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6c0 4>; + label = "b2bg_cntl_dfx"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bg_dbg_0: power-controller@6c8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6c8 4>; + label = "b2bg_dbg_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bg_dbg_1: power-controller@6d0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6d0 4>; + label = "b2bg_dbg_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bg_dbg_2: power-controller@6d8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6d8 4>; + label = "b2bg_dbg_2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_cntl_dfx_0: power-controller@6e0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6e0 4>; + label = "b2bd_cntl_dfx_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_cntl_dfx_1: power-controller@6e8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6e8 4>; + label = "b2bd_cntl_dfx_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_dbg_0: power-controller@6f0 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6f0 4>; + label = "b2bd_dbg_0"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_dbg_1: power-controller@6f8 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x6f8 4>; + label = "b2bd_dbg_1"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_dbg_2: power-controller@700 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x700 4>; + label = "b2bd_dbg_2"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_dbg_3: power-controller@708 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x708 4>; + label = "b2bd_dbg_3"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_b2bd_dbg_4: power-controller@710 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x710 4>; + label = "b2bd_dbg_4"; + #power-domain-cells = <0>; + #reset-cells = <0>; + }; + + ps_venc0_dma: power-controller@8000 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8000 4>; + label = "venc0_dma"; + #power-domain-cells = <0>; + power-domains = <&ps_venc0_sys>; + #reset-cells = <0>; + }; + + ps_venc0_pipe4: power-controller@8008 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8008 4>; + label = "venc0_pipe4"; + #power-domain-cells = <0>; + power-domains = <&ps_venc0_dma>; + #reset-cells = <0>; + }; + + ps_venc0_pipe5: power-controller@8010 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8010 4>; + label = "venc0_pipe5"; + #power-domain-cells = <0>; + power-domains = <&ps_venc0_dma>; + #reset-cells = <0>; + }; + + ps_venc0_me0: power-controller@8018 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8018 4>; + label = "venc0_me0"; + #power-domain-cells = <0>; + power-domains = <&ps_venc0_dma>; + #reset-cells = <0>; + }; + + ps_venc0_me1: power-controller@8020 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8020 4>; + label = "venc0_me1"; + #power-domain-cells = <0>; + power-domains = <&ps_venc0_me0>; + #reset-cells = <0>; + }; + + ps_venc1_dma: power-controller@8028 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8028 4>; + label = "venc1_dma"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_sys>; + #reset-cells = <0>; + }; + + ps_venc1_pipe4: power-controller@8030 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8030 4>; + label = "venc1_pipe4"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_dma>; + #reset-cells = <0>; + }; + + ps_venc1_pipe5: power-controller@8038 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8038 4>; + label = "venc1_pipe5"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_dma>; + #reset-cells = <0>; + }; + + ps_venc1_me0: power-controller@8040 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8040 4>; + label = "venc1_me0"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_dma>; + #reset-cells = <0>; + }; + + ps_venc1_me1: power-controller@8048 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x8048 4>; + label = "venc1_me1"; + #power-domain-cells = <0>; + power-domains = <&ps_venc1_me0>; + #reset-cells = <0>; + }; +}; + +&pmgr5 { + + ps_debug_gfx_gated: power-controller@58 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x58 4>; + label = "debug_gfx_gated"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; + + ps_nub_gfx_gpio: power-controller@68 { + compatible = "apple,t6050-pmgr-pwrstate", "apple,t8103-pmgr-pwrstate"; + reg = <0x68 4>; + label = "nub_gfx_gpio"; + #power-domain-cells = <0>; + #reset-cells = <0>; + apple,always-on; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-pmp.dtsi b/arch/arm64/boot/dts/apple/t6050-pmp.dtsi new file mode 100644 index 0000000000000..087429d768c90 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-pmp.dtsi @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T6050 power management processor (PMP) and its DART, mailbox and report. + * + * The ADT marks the PMGR gfx state notify_pmp (id 16): the GPU votes through + * the PTD request for soc-device 0x10 ("GFX_", acknowledged), the pmp-gfx + * report entry below, not through T6030's apple_pmp_set_device_power(). + * + * Addresses from the J714c ADT (read offline from j714c.adt): + * /arm-io/pmp0 (iop,ascwrap-v6): reg[0] asc 0x284e00000+0x88000 (CPU_CONTROL at +0x44, + * live-verified), reg[2] firmware/SRAM 0x284500000+0x100000 (= iop-pmp0-nub region-base), + * reg[3] PTD update 0x284250000; mailbox AIC [397 396 399 398]. + * /arm-io/dart-pmp0 (dart,t8110): 0x2841a0000+0xc000, DAPF 0x2841b0000, AIC 371. + * Conventions mirror the AOP/MTP ascwrap-v6 coprocessors in t6050-die0.dtsi. + * + * The pmp node stays status="disabled": pmp-report.c (starts_pmp on T6050) takes the + * start over only from a disabled node. Its always-on entries seed SoC-device power + * requests in the PTD (DISPINT, ANS); the start then checks the halted PMP CPU, the + * SRAM image UUID and boot arguments, and the seeded PTD, and only then enables this + * node for pmp.rs to run the firmware. The start is queued by the first + * apple_pmp_report_wait_ready() caller. Preconditions: the m1n1 dart-pmp0 DAPF entry + * must be resident, or pmp.rs faults on the closed ASC window; the per-mac + * apple,tunable-* (tunable-uuid, tunable-soc-device, tunable-ptd-range) come from the + * ADT iop-pmp0-nub (t6050-j714c-pmp-tunables.dtsi). + */ + +&soc { + pmp_mbox: mailbox@284e08000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x84e08000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + status = "okay"; + }; + + pmp_dart: iommu@2841a0000 { + compatible = "apple,t6050-dart", "apple,t8142-dart", "apple,t8110-dart"; + reg = <0x2 0x841a0000 0x0 0xc000>; + #iommu-cells = <1>; + interrupt-parent = <&aic>; + interrupts = ; + /* Full dart-pmp0 window at 1 TiB (ADT vm-base/vm-size). The firmware's + * __ETEXT at the window base is re-created as an IOMMU_RESV_DIRECT region + * (see &pmp_fw); the iommu core reserves that in the allocator, so rtkit + * buffers never land on it. Pairs with the 42-bit DMA mask. */ + apple,dma-range = /bits/ 64 <0x10000000000 0x1000000000>; + status = "okay"; + }; + + pmp: pmp@284500000 { + compatible = "apple,t6050-pmp-v2", "apple,t6000-pmp-v2"; + /* "pmp" = firmware/SRAM region; "asc" = ascwrap control (CPU_CONTROL at +0x44) */ + reg = <0x2 0x84500000 0x0 0x100000>, + <0x2 0x84e00000 0x0 0x4000>; + reg-names = "pmp", "asc"; + mboxes = <&pmp_mbox>; + mbox-names = "mbox"; + iommus = <&pmp_dart 0>; + memory-region = <&pmp_fw>; + apple,board-id = <0xa>; + apple,dram-vendor-id = <0xff>; + /* ADT /arm-io/pmgr pmc and pmc-pmgr, iop-pmp0-nub soc-chip-variant */ + apple,pmc = <1>; + apple,pmc-pmgr = <0x3f>; + apple,soc-chip-variant = <1>; + /* Asked for by the firmware but not in the ADT nub: values from macOS's + * ApplePMPv2 (ioreg). macOS has no pmc-max-bwr-latency-us either. */ + apple,tunable-ane-af-bw-dvfs-filter = [ff ff ff ff]; + apple,tunable-ane-dcs-bw-dvfs-filter = [ff ff ff ff]; + apple,tunable-ane-lnk0-af-bw-dvfs-filter = [ff ff ff ff]; + apple,tunable-ane-lnk0-af-bw-enable = [01 00 00 00]; + apple,tunable-ane-lnk1-af-bw-dvfs-filter = [ff ff ff ff]; + apple,tunable-ane-lnk1-af-bw-enable = [01 00 00 00]; + apple,tunable-tuning-version = [00 00 00 00]; + apple,tunable-fast-die-ctrl-loop = [01 00 00 00 00 00 00 00 00 00 00 00 8f 02 00 00 + 00 00 01 00 00 00 6e 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 01 00 + 16 00 00 00 01 00 00 00 02 00 00 00 03 00 00 00 + 04 00 00 00 05 00 00 00 06 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 61 6e 65 30 00 00 00 00 00 00 00 00 + 00 00 00 00 02 00 00 00 00 00 00 00 01 00 00 00 + 3d 0a 00 00 00 00 04 00 00 00 6e 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 01 00 16 00 00 00 20 00 00 00 21 00 00 00 + 25 00 00 00 1e 00 00 00 1f 00 00 00 24 00 00 00 + 22 00 00 00 23 00 00 00 26 00 00 00 29 00 00 00 + 2a 00 00 00 2f 00 00 00 2b 00 00 00 2c 00 00 00 + 30 00 00 00 2d 00 00 00 2e 00 00 00 31 00 00 00 + 27 00 00 00 28 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 6d 61 63 63 30 00 00 00 + 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 + 02 00 00 00 41 00 00 00 00 00 04 00 00 00 6e 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 01 00 16 00 00 00 09 00 00 00 + 0a 00 00 00 10 00 00 00 0d 00 00 00 0e 00 00 00 + 12 00 00 00 1a 00 00 00 1b 00 00 00 1d 00 00 00 + 18 00 00 00 19 00 00 00 1c 00 00 00 07 00 00 00 + 08 00 00 00 0f 00 00 00 0b 00 00 00 0c 00 00 00 + 11 00 00 00 13 00 00 00 14 00 00 00 15 00 00 00 + 16 00 00 00 17 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 70 61 63 63 + 00 00 00 00 00 00 00 00 00 00 00 00 04 00 00 00 + 00 00 00 00 03 00 00 00 3d 0a 00 00 00 00 04 00 + 00 00 6e 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 01 00 16 00 00 00 + 34 00 00 00 35 00 00 00 39 00 00 00 32 00 00 00 + 33 00 00 00 38 00 00 00 36 00 00 00 37 00 00 00 + 3a 00 00 00 3d 00 00 00 3e 00 00 00 43 00 00 00 + 3f 00 00 00 40 00 00 00 44 00 00 00 41 00 00 00 + 42 00 00 00 45 00 00 00 3b 00 00 00 3c 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 6d 61 63 63 31 00 00 00 00 00 00 00 00 00 00 00]; + /* pmp-report enables this node once the PTD requests are seeded. */ + status = "disabled"; + }; + + pmp_report: pmp-report@284240000 { + compatible = "apple,t6050-pmp-v2-report"; + /* PTD read aperture + update page (ADT pmp0 reg[3] - 0x10000) */ + reg = <0x2 0x84240000 0x0 0x14000>; + power-domains = <&ps_pms_sram>; + #address-cells = <1>; + #size-cells = <0>; + apple,pmp = <&pmp>; + status = "okay"; + + /* reg = soc-device record index (ADT id - 1); DISPINT is acknowledged at handover. */ + pmp_report_dispint: report@12 { + compatible = "apple,t6050-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x12>; + label = "pmp-dispint"; + #power-domain-cells = <0>; + power-domains = <&ps_disp_cpu>; + apple,always-on; + }; + + pmp_report_ans: report@24 { + compatible = "apple,t6050-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0x24>; + label = "pmp-ans"; + #power-domain-cells = <0>; + power-domains = <&ps_ans>; + apple,always-on; + /* Its soc-device record has no acknowledge flag (as on T6031). */ + apple,no-ack; + }; + + /* The GPU's vote, behind the gfx power state. */ + pmp_report_gfx: report@f { + compatible = "apple,t6050-pmp-v2-report-entry", + "apple,t6000-pmp-v2-report-entry"; + reg = <0xf>; + label = "pmp-gfx"; + #power-domain-cells = <0>; + power-domains = <&ps_gfx>; + }; + }; +}; + +&gpu { + power-domains = <&pmp_report_gfx>; +}; + +/ { + aliases { + pmp0 = &pmp_dart; + }; + + reserved-memory { + #address-cells = <2>; + #size-cells = <2>; + ranges; + + /* The PMP firmware's __ETEXT lives in DRAM at phys 0x10001eb4000, which + * iBoot maps at IOVA 0x10000000000. The kernel's fresh dart-pmp0 page + * table drops that mapping (iommu_iova_to_phys reads 0x0), so the + * firmware cannot reach its own code to finish powering on. Re-create it + * as an IOMMU_RESV_DIRECT mapping on the pmp device and keep Linux off + * the DRAM page (no-map); phys is unreserved RAM otherwise. */ + pmp_fw: pmp-fw@10001eb4000 { + reg = <0x100 0x01eb4000 0x0 0x12000>; + no-map; + iommu-addresses = <&pmp 0x100 0x0 0x0 0x12000>; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050-smc.dtsi b/arch/arm64/boot/dts/apple/t6050-smc.dtsi new file mode 100644 index 0000000000000..3c9e65022b4b7 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050-smc.dtsi @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * T6050 SMC (System Management Controller) + the SPMI/PMIC NVMEM cells its + * reboot and RTC children need. From a research pass (ADT + Asahi drivers); + * the stage-0 proxy probe (scripts/probes/smc-proxy-probe.py) confirmed the + * mailbox, the shmem window and the driver keys (docs/research/ + * smc-keys-j714s.json). Brought up in stages. + * + * Source of every value: runtime ADT build/adt-runtime-j714s.bin + * (see PLAN.md for the property each number comes from). Structure copied + * from t6030.dtsi:937-998 / t6031-die0.dtsi:107-169 / t8122.dtsi:775-889. + * + * Board files add hwmon sensors (hwmon-*.dtsi) on top of &smc_hwmon. The + * T6050 SMC has no F0Md/F1Md: its fan mode keys are F0md/F1md (ui8, + * writable), so every T6050 board overrides apple,fan-mode after including + * hwmon-fan*.dtsi (see the board files). + */ + +#include + +&soc { + /* + * ADT /arm-io/nub-spmi0: compatible "aapl,spmi", gen = 4, + * reg[0] = 0x8a014000 (-> 0x28a014000), ADT size 0x4000. The full + * ADT size is used: the gen 4 FIFO registers below sit at 0x200-0x220, + * outside the 0x100 window t6031 maps for its gen 1 layout (a 0x100 + * resource only worked because ioremap maps a whole 16K page). + * interrupts = [256, 726, 281, ...]: 256 is the controller-internal + * SPMI_IRQ_NOTIFY hwirq (identical in all five SPMI nodes), 726 is the + * AIC line (same index convention as wdt: ADT interrupts[0] = 721). + * This is a gen 4 controller (ADT "gen" = 4): STATUS/CMD/REPLY sit at + * 0x200/0x210/0x220 and RX-empty is bit 30 (m1n1 src/spmi.c). The + * gen 1 layout (t8103) does not apply, so there is no fallback + * compatible. Linux runs gen 4 polled (patches/linux/0008): its IRQ + * registers are undocumented, so the interrupt below stays unused. + * + * No power-domains: the ADT node has no power-gates, and ps_nub_spmi0 + * (pmgr 0x120, boot value 0x1400024f = auto-PM + clock-gated) is + * apple,always-on in t6050-pmgr.dtsi. t6031's equivalent node also has + * none. Referencing it would only add a genpd dependency. + */ + nub_spmi0: spmi@28a014000 { + compatible = "apple,t6050-spmi", "apple,t8142-spmi"; + reg = <0x2 0x8a014000 0x0 0x4000>; + #address-cells = <2>; + #size-cells = <0>; + /* Polled: this tree's gen 4 SPMI driver has no interrupt support. */ + + /* + * ADT /arm-io/nub-spmi0/spmi-abbeyL1: compatible + * "pmu,spmi","pmu,abbey", reg[0] = 0xe -> USID 14, is-primary. + * (spmi-mosquitoM0 = USID 9, btm = USID 0xb on the same bus.) + * + * Cell offsets are the Asahi "chapel" layout (t6031-die0.dtsi: + * 47-90); the ADT abbey properties confirm the ones it lists: + * info-pm_setting = 0x2001 + * info-clock_offset = {0x2100, 6} (info-rtc_scrpad 0x2100) + * info-fault_shadow = {0x867b, 0x10} + * info-scrpad_socd = {0x8b00, 0x1800} + * info-leg_scrpad = 0xf800 -> boot_stage 0xf801 etc. are + * base+1/+2/+3/+0xf as on chapel/stowe + * [INFERENCE: the +N offsets are not in the ADT] + */ + pmic1: pmic@e { + compatible = "apple,abbey-pmic", "apple,spmi-nvmem"; + reg = <0xe SPMI_USID>; + + nvmem-layout { + compatible = "fixed-layout"; + #address-cells = <1>; + #size-cells = <1>; + + pm_setting: pm-setting@2001 { + reg = <0x2001 0x1>; + }; + + rtc_offset: rtc-offset@2100 { + reg = <0x2100 0x6>; + }; + + fault_shadow: fault-shadow@867b { + reg = <0x867b 0x10>; + }; + + socd: socd@8b00 { + reg = <0x8b00 0x1800>; + }; + + boot_stage: boot-stage@f801 { + reg = <0xf801 0x1>; + }; + + boot_error_count: boot-error-count@f802,0 { + reg = <0xf802 0x1>; + bits = <0 4>; + }; + + panic_count: panic-count@f802,4 { + reg = <0xf802 0x1>; + bits = <4 4>; + }; + + boot_error_stage: boot-error-stage@f803 { + reg = <0xf803 0x1>; + }; + + shutdown_flag: shutdown-flag@f80f,3 { + reg = <0xf80f 0x1>; + bits = <3 1>; + }; + }; + }; + }; + + /* + * ADT /arm-io/smc: compatible "iop,ascwrap-v6" (same as ans/mtp), + * reg[0] = 0x8c600000 (ASC wrapper, size 0x88000), mailbox at + * reg[0] + 0x8000 exactly like ans_mbox/mtp_mbox (m1n1 src/asc.c), + * interrupts = [900, 899, 902, 901] = [n+1, n, n+3, n+2] with n = 899, + * so send-empty = 899 ... recv-not-empty = 902 (same reordering as + * ANS 1348,1347,1350,1349 -> 1347..1350 and MTP 677,676,679,678). + * No power-gates / clock-gates in the ADT node -> no power-domains. + */ + smc_mbox: mbox@28c608000 { + compatible = "apple,t6050-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x2 0x8c608000 0x0 0x4000>; + interrupt-parent = <&aic>; + interrupts = , + , + , + ; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; + + /* + * "smc" reg: ASC wrapper (driver does not map it; only index 1 is + * ioremapped). "sram" reg: ADT iop-smc-nub region-base = 0x28de00000, + * region-size = 0x140000 (Asahi uses 0x100000 on M1-M4; the T6050 + * firmware region is 0x140000, and the driver rejects any shmem + * buffer outside this window, drivers/mfd/macsmc.c:320). + * smc_gpio is deliberately defined here although nothing consumes it yet. + */ + smc: smc@28c600000 { + compatible = "apple,t6050-smc", "apple,t8103-smc"; + reg = <0x2 0x8c600000 0x0 0x4000>, + <0x2 0x8de00000 0x0 0x140000>; + reg-names = "smc", "sram"; + mboxes = <&smc_mbox>; + + smc_gpio: gpio { + compatible = "apple,smc-gpio"; + gpio-controller; + #gpio-cells = <2>; + }; + + smc_hwmon: hwmon { + compatible = "apple,smc-hwmon"; + }; + + /* + * Registers SYS_OFF_PRIO_HIGH (192) restart + power-off + * handlers; the apple,t6050-wdt restart handler is priority + * 128 (drivers/watchdog/apple_wdt.c:193), so once this probes, + * `reboot` goes through SMC 'phra' instead of the wdt. The + * PMIC scratchpad cells it writes match t6031 (M3 Max). + */ + smc_reboot: reboot { + compatible = "apple,smc-reboot"; + nvmem-cells = <&shutdown_flag>, <&boot_stage>, + <&boot_error_count>, <&panic_count>; + nvmem-cell-names = "shutdown_flag", "boot_stage", + "boot_error_count", "panic_count"; + }; + + smc_rtc: rtc { + compatible = "apple,smc-rtc"; + nvmem-cells = <&rtc_offset>; + nvmem-cell-names = "rtc_offset"; + }; + }; +}; diff --git a/arch/arm64/boot/dts/apple/t6050.dtsi b/arch/arm64/boot/dts/apple/t6050.dtsi new file mode 100644 index 0000000000000..60c50d7d82f29 --- /dev/null +++ b/arch/arm64/boot/dts/apple/t6050.dtsi @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0+ OR MIT +/* + * Apple T6050 "M5 Max" SoC + * + * Copyright The Asahi Linux Contributors + */ + +#include +#include + +#include "t6050-base.dtsi" + +/ { + compatible = "apple,t6050", "apple,arm-platform"; + + #address-cells = <2>; + #size-cells = <2>; + + soc: soc { + compatible = "simple-bus"; + #address-cells = <2>; + #size-cells = <2>; + + ranges; + nonposted-mmio; + /* Required to get >32-bit DMA via DARTs */ + dma-ranges = <0 0 0 0 0xffffffff 0xffffc000>; + }; +}; + +/* T6050 is a single die: no multi-die templating. */ +#include "t6050-die0.dtsi" +#include "t6050-pmgr.dtsi" +#include "t6050-nvme.dtsi" +#include "t6050-gpu.dtsi" diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi index 1d73abffa6b72..6ead557a8cf19 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi @@ -1783,7 +1783,7 @@ }; dspi2_miso_mosi_pins: iic5-spi3-pins { - pinctrl-single,bits = <0x3 (0x2 << 9) (0x7 << 9)>; + pinctrl-single,bits = <0x0 (0x3 << 9) (0x7 << 9)>; }; i2c5_pins: iic6-i2c-pins { @@ -1803,7 +1803,7 @@ }; gpio1_31_28_pins: xspi1-data74-gpio-pins { - pinctrl-single,bits = <0x0 0x1 (0x7 << 15)>; + pinctrl-single,bits = <0x0 (0x1 << 15) (0x7 << 15)>; }; fspi_data30_pins: xspi1-data30-pins { @@ -1811,7 +1811,7 @@ }; gpio1_27_24_pins: xspi1-data30-gpio-pins { - pinctrl-single,bits = <0x0 0x1 (0x7 << 18)>; + pinctrl-single,bits = <0x0 (0x1 << 18) (0x7 << 18)>; }; fspi_dqs_sck_cs10_pins: xspi1-base-pins { @@ -1819,7 +1819,7 @@ }; gpio1_23_20_pins: xspi1-base-gpio-pins { - pinctrl-single,bits = <0x0 0x1 (0x7 << 21)>; + pinctrl-single,bits = <0x0 (0x1 << 21) (0x7 << 21)>; }; esdhc0_cmd_data30_clk_vsel_pins: sdhc1-base-sdhc-vsel-pins { @@ -1868,12 +1868,19 @@ }; /* RCWSR14 */ + /* + * IIC1_PMUX is a single bit at RCWSR14[10], but bit-per-mux + * uses the mask for the field to clear before applying the + * value. + * So bits 11 and 12, which are reserved, are written as 0 + * on every mux switch, it is harmless. + */ i2c0_pins: iic1-i2c-pins { - pinctrl-single,bits = <0x8 0x0 (0x1 << 10)>; + pinctrl-single,bits = <0x8 0x0 (0x7 << 10)>; }; gpio0_3_2_pins: iic1-gpio-pins { - pinctrl-single,bits = <0x8 (0x1 << 10) (0x1 << 10)>; + pinctrl-single,bits = <0x8 (0x1 << 10) (0x7 << 10)>; }; }; diff --git a/arch/arm64/configs/asahi.config b/arch/arm64/configs/asahi.config index 436a53dd6f864..98f9a98ab9028 100644 --- a/arch/arm64/configs/asahi.config +++ b/arch/arm64/configs/asahi.config @@ -1,3 +1,7 @@ +# Shared Apple Silicon hardware defaults for M1 and later, including Neo. +# Merge over ARM64 defconfig or a distribution base config, then run +# olddefconfig and verify that the requested settings survive resolution. +# Device-specific boot, debug and distribution policy stays in the base. CONFIG_RUST=y CONFIG_ARM64_ACTLR_STATE=y CONFIG_ARCH_APPLE=y @@ -26,6 +30,7 @@ CONFIG_TOUCHSCREEN_APPLE_Z2=m CONFIG_INPUT_MACSMC_INPUT=m CONFIG_I2C_APPLE=m CONFIG_SPI_APPLE=m +CONFIG_SPI_APPLE_QSPI=m CONFIG_SPMI_APPLE=m CONFIG_PINCTRL_APPLE_GPIO=m CONFIG_GPIO_MACSMC=m @@ -33,7 +38,9 @@ CONFIG_POWER_RESET_MACSMC=m CONFIG_MACSMC_POWER=m CONFIG_SENSORS_MACSMC_HWMON=m CONFIG_SENSORS_APPLE_PMP_J613=m -CONFIG_APPLE_WATCHDOG=m +CONFIG_THERMAL_NETLINK=y +CONFIG_NVME_HWMON=y +CONFIG_APPLE_WATCHDOG=y CONFIG_VIDEO_APPLE_AVD=m CONFIG_VIDEO_APPLE_ISP=m CONFIG_DRM=y @@ -44,12 +51,14 @@ CONFIG_DRM_APPLE_AUDIO=y CONFIG_DRM_ACCEL=y CONFIG_DRM_ACCEL_ANE=m CONFIG_DRM_SIMPLEDRM=y +CONFIG_SYSFB_SIMPLEFB=y CONFIG_SND_SOC_APPLE_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m CONFIG_SND_SOC_APPLE_MCA=m CONFIG_SND_SOC_APPLE_MACAUDIO=m CONFIG_SND_SOC_CS42L83=m CONFIG_SND_SOC_CS42L84=m +CONFIG_SND_SOC_MAX98357A=m CONFIG_SND_SOC_TAS2764=m CONFIG_SND_SOC_TAS2770=m CONFIG_HID_APPLE=m @@ -64,12 +73,13 @@ CONFIG_USB_DWC3_APPLE=m CONFIG_TYPEC=m CONFIG_USB4=m CONFIG_USB4_APPLE_SOC=m -CONFIG_RESET_APPLE_CIO=y CONFIG_USB_XHCI_PCI_ASMEDIA=y CONFIG_TYPEC_SN201202X=m CONFIG_RTC_DRV_MACSMC=m +CONFIG_RTC_DRV_APPLE_PMU=m CONFIG_APPLE_ADMAC=m CONFIG_APPLE_SIO=m +CONFIG_APPLE_SIO_T6030=y CONFIG_MFD_MACSMC=m CONFIG_COMMON_CLK_APPLE_NCO=m CONFIG_APPLE_DART=y @@ -83,6 +93,8 @@ CONFIG_APPLE_RTKIT_HELPER=m CONFIG_APPLE_SART=m CONFIG_RUST_APPLE_RTKIT=y CONFIG_APPLE_AOP=m +CONFIG_HW_RANDOM=y +CONFIG_KEYS=y CONFIG_TRUSTED_KEYS=y # CONFIG_TRUSTED_KEYS_TPM is not set # CONFIG_TRUSTED_KEYS_TEE is not set @@ -101,8 +113,13 @@ CONFIG_APPLE_M1_CPU_PMU=y CONFIG_NVMEM_APPLE_EFUSES=m CONFIG_NVMEM_APPLE_SPMI=m CONFIG_MUX_APPLE_DPXBAR=m +CONFIG_TYPEC_TPS6598X=m +CONFIG_RESET_APPLE_CIO=y CONFIG_INFINIBAND=m CONFIG_INFINIBAND_USER_ACCESS=m CONFIG_INFINIBAND_USER_MAD=m CONFIG_RDMA_RXE=m CONFIG_RDMA_SIW=m +CONFIG_DMABUF_HEAPS=y +CONFIG_DMABUF_HEAPS_SYSTEM=y +CONFIG_UDMABUF=y diff --git a/arch/arm64/configs/aurora_j700_defconfig b/arch/arm64/configs/aurora_j700_defconfig deleted file mode 100644 index 246b5294e1565..0000000000000 --- a/arch/arm64/configs/aurora_j700_defconfig +++ /dev/null @@ -1,2841 +0,0 @@ -CONFIG_WERROR=y -CONFIG_LOCALVERSION="-aurora" -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_SYSVIPC=y -CONFIG_POSIX_MQUEUE=y -CONFIG_AUDIT=y -CONFIG_NO_HZ_IDLE=y -CONFIG_HIGH_RES_TIMERS=y -CONFIG_BPF_SYSCALL=y -CONFIG_BPF_JIT=y -CONFIG_PREEMPT=y -CONFIG_IRQ_TIME_ACCOUNTING=y -CONFIG_BSD_PROCESS_ACCT=y -CONFIG_BSD_PROCESS_ACCT_V3=y -CONFIG_TASKSTATS=y -CONFIG_TASK_XACCT=y -CONFIG_TASK_IO_ACCOUNTING=y -CONFIG_IKCONFIG=y -CONFIG_IKCONFIG_PROC=y -CONFIG_UCLAMP_TASK=y -CONFIG_NUMA_BALANCING=y -CONFIG_MEMCG=y -CONFIG_BLK_CGROUP=y -CONFIG_UCLAMP_TASK_GROUP=y -CONFIG_CGROUP_PIDS=y -CONFIG_CGROUP_FREEZER=y -CONFIG_CGROUP_HUGETLB=y -CONFIG_CPUSETS=y -CONFIG_CGROUP_DEVICE=y -CONFIG_CGROUP_CPUACCT=y -CONFIG_CGROUP_PERF=y -CONFIG_CGROUP_BPF=y -CONFIG_CGROUP_MISC=y -CONFIG_USER_NS=y -CONFIG_CHECKPOINT_RESTORE=y -CONFIG_SCHED_AUTOGROUP=y -CONFIG_BLK_DEV_INITRD=y -CONFIG_KALLSYMS_ALL=y -CONFIG_PROFILING=y -CONFIG_RUST=y -CONFIG_ARCH_ACTIONS=y -CONFIG_ARCH_AIROHA=y -CONFIG_ARCH_SUNXI=y -CONFIG_ARCH_ALPINE=y -CONFIG_ARCH_APPLE=y -CONFIG_ARCH_ARTPEC=y -CONFIG_ARCH_AXIADO=y -CONFIG_ARCH_BCM=y -CONFIG_ARCH_BCM2835=y -CONFIG_ARCH_BCM_IPROC=y -CONFIG_ARCH_BCMBCA=y -CONFIG_ARCH_BRCMSTB=y -CONFIG_ARCH_BERLIN=y -CONFIG_ARCH_BLAIZE=y -CONFIG_ARCH_BST=y -CONFIG_ARCH_CIX=y -CONFIG_ARCH_EXYNOS=y -CONFIG_ARCH_K3=y -CONFIG_ARCH_LG1K=y -CONFIG_ARCH_HISI=y -CONFIG_ARCH_KEEMBAY=y -CONFIG_ARCH_MEDIATEK=y -CONFIG_ARCH_MESON=y -CONFIG_ARCH_MICROCHIP=y -CONFIG_ARCH_SPARX5=y -CONFIG_ARCH_MVEBU=y -CONFIG_ARCH_NXP=y -CONFIG_ARCH_LAYERSCAPE=y -CONFIG_ARCH_MXC=y -CONFIG_ARCH_S32=y -CONFIG_ARCH_MA35=y -CONFIG_ARCH_NPCM=y -CONFIG_ARCH_QCOM=y -CONFIG_ARCH_REALTEK=y -CONFIG_ARCH_RENESAS=y -CONFIG_ARCH_ROCKCHIP=y -CONFIG_ARCH_SEATTLE=y -CONFIG_ARCH_INTEL_SOCFPGA=y -CONFIG_ARCH_SOPHGO=y -CONFIG_ARCH_STM32=y -CONFIG_ARCH_SYNQUACER=y -CONFIG_ARCH_TEGRA=y -CONFIG_ARCH_TESLA_FSD=y -CONFIG_ARCH_SPRD=y -CONFIG_ARCH_THUNDER=y -CONFIG_ARCH_THUNDER2=y -CONFIG_ARCH_UNIPHIER=y -CONFIG_ARCH_VEXPRESS=y -CONFIG_ARCH_VISCONTI=y -CONFIG_ARCH_XGENE=y -CONFIG_ARCH_ZYNQMP=y -CONFIG_ARM64_16K_PAGES=y -CONFIG_ARM64_VA_BITS_48=y -CONFIG_NUMA=y -CONFIG_XEN=y -CONFIG_ARM64_MEMORY_MODEL_CONTROL=y -CONFIG_HIBERNATION=y -CONFIG_PM_DEBUG=y -CONFIG_PM_ADVANCED_DEBUG=y -CONFIG_WQ_POWER_EFFICIENT_DEFAULT=y -CONFIG_ENERGY_MODEL=y -CONFIG_ARM_PSCI_CPUIDLE=y -CONFIG_CPU_FREQ=y -CONFIG_CPU_FREQ_STAT=y -CONFIG_CPU_FREQ_GOV_POWERSAVE=m -CONFIG_CPU_FREQ_GOV_USERSPACE=y -CONFIG_CPU_FREQ_GOV_ONDEMAND=y -CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m -CONFIG_CPUFREQ_DT=y -CONFIG_ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM=m -CONFIG_ARM_APPLE_SOC_CPUFREQ=y -CONFIG_ARM_ARMADA_37XX_CPUFREQ=y -CONFIG_ARM_SCPI_CPUFREQ=y -CONFIG_ARM_IMX_CPUFREQ_DT=m -CONFIG_ARM_MEDIATEK_CPUFREQ=y -CONFIG_ARM_QCOM_CPUFREQ_NVMEM=y -CONFIG_ARM_QCOM_CPUFREQ_HW=y -CONFIG_ARM_RASPBERRYPI_CPUFREQ=m -CONFIG_ARM_SCMI_CPUFREQ=y -CONFIG_ARM_TEGRA186_CPUFREQ=y -CONFIG_QORIQ_CPUFREQ=y -CONFIG_ACPI_CPPC_CPUFREQ=m -CONFIG_ACPI=y -CONFIG_ACPI_HOTPLUG_MEMORY=y -CONFIG_ACPI_HMAT=y -CONFIG_ACPI_APEI=y -CONFIG_ACPI_APEI_GHES=y -CONFIG_ACPI_APEI_PCIEAER=y -CONFIG_ACPI_APEI_MEMORY_FAILURE=y -CONFIG_ACPI_APEI_EINJ=y -CONFIG_VIRTUALIZATION=y -CONFIG_KVM=y -CONFIG_COMPAT_32BIT_TIME=y -# CONFIG_RELR is not set -CONFIG_MODULES=y -CONFIG_MODULE_UNLOAD=y -CONFIG_PARTITION_ADVANCED=y -CONFIG_AIX_PARTITION=y -CONFIG_OSF_PARTITION=y -CONFIG_MAC_PARTITION=y -CONFIG_BSD_DISKLABEL=y -CONFIG_MINIX_SUBPARTITION=y -CONFIG_SOLARIS_X86_PARTITION=y -CONFIG_UNIXWARE_DISKLABEL=y -CONFIG_LDM_PARTITION=y -CONFIG_SGI_PARTITION=y -CONFIG_SUN_PARTITION=y -# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set -CONFIG_BINFMT_MISC=m -# CONFIG_COMPAT_BRK is not set -CONFIG_MEMORY_HOTPLUG=y -CONFIG_MEMORY_HOTREMOVE=y -CONFIG_KSM=y -CONFIG_MEMORY_FAILURE=y -CONFIG_TRANSPARENT_HUGEPAGE=y -CONFIG_PAGE_BLOCK_MAX_ORDER=10 -CONFIG_USERFAULTFD=y -CONFIG_NET=y -CONFIG_PACKET=y -CONFIG_PACKET_DIAG=m -CONFIG_UNIX_DIAG=m -CONFIG_TLS=m -CONFIG_XFRM_USER=m -CONFIG_XFRM_INTERFACE=m -CONFIG_XFRM_STATISTICS=y -CONFIG_NET_KEY=m -CONFIG_NET_KEY_MIGRATE=y -CONFIG_IP_MULTICAST=y -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_PNP=y -CONFIG_IP_PNP_DHCP=y -CONFIG_IP_PNP_BOOTP=y -CONFIG_NET_IPIP=m -CONFIG_NET_IPGRE_DEMUX=m -CONFIG_NET_IPGRE=m -CONFIG_NET_IPGRE_BROADCAST=y -CONFIG_NET_IPVTI=m -CONFIG_NET_FOU_IP_TUNNELS=y -CONFIG_INET_AH=m -CONFIG_INET_ESP=m -CONFIG_INET_ESP_OFFLOAD=m -CONFIG_INET_ESPINTCP=y -CONFIG_INET_IPCOMP=m -CONFIG_INET_UDP_DIAG=m -CONFIG_INET_RAW_DIAG=m -CONFIG_INET_DIAG_DESTROY=y -CONFIG_TCP_CONG_ADVANCED=y -CONFIG_TCP_CONG_HSTCP=m -CONFIG_TCP_CONG_HYBLA=m -CONFIG_TCP_CONG_NV=m -CONFIG_TCP_CONG_SCALABLE=m -CONFIG_TCP_CONG_LP=m -CONFIG_TCP_CONG_VENO=m -CONFIG_TCP_CONG_YEAH=m -CONFIG_TCP_CONG_ILLINOIS=m -CONFIG_TCP_CONG_DCTCP=m -CONFIG_TCP_CONG_CDG=m -CONFIG_TCP_CONG_BBR=m -CONFIG_INET6_AH=m -CONFIG_INET6_ESP=m -CONFIG_INET6_ESP_OFFLOAD=m -CONFIG_INET6_ESPINTCP=y -CONFIG_INET6_IPCOMP=m -CONFIG_IPV6_VTI=m -CONFIG_IPV6_GRE=m -CONFIG_IPV6_MULTIPLE_TABLES=y -CONFIG_NETFILTER=y -CONFIG_BRIDGE_NETFILTER=m -CONFIG_NETFILTER_NETLINK_HOOK=m -CONFIG_NF_CONNTRACK=m -CONFIG_NF_CONNTRACK_ZONES=y -CONFIG_NF_CONNTRACK_PROCFS=y -CONFIG_NF_CONNTRACK_EVENTS=y -CONFIG_NF_CONNTRACK_TIMEOUT=y -CONFIG_NF_CONNTRACK_TIMESTAMP=y -CONFIG_NF_CONNTRACK_AMANDA=m -CONFIG_NF_CONNTRACK_FTP=m -CONFIG_NF_CONNTRACK_H323=m -CONFIG_NF_CONNTRACK_IRC=m -CONFIG_NF_CONNTRACK_NETBIOS_NS=m -CONFIG_NF_CONNTRACK_SNMP=m -CONFIG_NF_CONNTRACK_PPTP=m -CONFIG_NF_CONNTRACK_SANE=m -CONFIG_NF_CONNTRACK_SIP=m -CONFIG_NF_CONNTRACK_TFTP=m -CONFIG_NF_CT_NETLINK=m -CONFIG_NF_CT_NETLINK_TIMEOUT=m -CONFIG_NF_CT_NETLINK_HELPER=m -CONFIG_NETFILTER_NETLINK_GLUE_CT=y -CONFIG_NF_TABLES=m -CONFIG_NF_TABLES_INET=y -CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_NUMGEN=m -CONFIG_NFT_CT=m -CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_CONNLIMIT=m -CONFIG_NFT_LOG=m -CONFIG_NFT_LIMIT=m -CONFIG_NFT_MASQ=m -CONFIG_NFT_REDIR=m -CONFIG_NFT_NAT=m -CONFIG_NFT_TUNNEL=m -CONFIG_NFT_QUEUE=m -CONFIG_NFT_QUOTA=m -CONFIG_NFT_REJECT=m -CONFIG_NFT_COMPAT=m -CONFIG_NFT_HASH=m -CONFIG_NFT_FIB_INET=m -CONFIG_NFT_XFRM=m -CONFIG_NFT_SOCKET=m -CONFIG_NFT_OSF=m -CONFIG_NFT_TPROXY=m -CONFIG_NFT_DUP_NETDEV=m -CONFIG_NFT_FWD_NETDEV=m -CONFIG_NFT_FIB_NETDEV=m -CONFIG_NFT_REJECT_NETDEV=m -CONFIG_NF_FLOW_TABLE_INET=m -CONFIG_NF_FLOW_TABLE=m -CONFIG_NF_FLOW_TABLE_PROCFS=y -CONFIG_NETFILTER_XTABLES_LEGACY=y -CONFIG_NETFILTER_XT_SET=m -CONFIG_NETFILTER_XT_TARGET_AUDIT=m -CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m -CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m -CONFIG_NETFILTER_XT_TARGET_CONNMARK=m -CONFIG_NETFILTER_XT_TARGET_DSCP=m -CONFIG_NETFILTER_XT_TARGET_HMARK=m -CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m -CONFIG_NETFILTER_XT_TARGET_LED=m -CONFIG_NETFILTER_XT_TARGET_LOG=m -CONFIG_NETFILTER_XT_TARGET_MARK=m -CONFIG_NETFILTER_XT_TARGET_NFLOG=m -CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m -CONFIG_NETFILTER_XT_TARGET_NOTRACK=m -CONFIG_NETFILTER_XT_TARGET_TEE=m -CONFIG_NETFILTER_XT_TARGET_TPROXY=m -CONFIG_NETFILTER_XT_TARGET_TRACE=m -CONFIG_NETFILTER_XT_TARGET_TCPMSS=m -CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m -CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m -CONFIG_NETFILTER_XT_MATCH_BPF=m -CONFIG_NETFILTER_XT_MATCH_CGROUP=m -CONFIG_NETFILTER_XT_MATCH_CLUSTER=m -CONFIG_NETFILTER_XT_MATCH_COMMENT=m -CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m -CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m -CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m -CONFIG_NETFILTER_XT_MATCH_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m -CONFIG_NETFILTER_XT_MATCH_CPU=m -CONFIG_NETFILTER_XT_MATCH_DCCP=m -CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m -CONFIG_NETFILTER_XT_MATCH_DSCP=m -CONFIG_NETFILTER_XT_MATCH_ESP=m -CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m -CONFIG_NETFILTER_XT_MATCH_HELPER=m -CONFIG_NETFILTER_XT_MATCH_IPCOMP=m -CONFIG_NETFILTER_XT_MATCH_IPRANGE=m -CONFIG_NETFILTER_XT_MATCH_IPVS=m -CONFIG_NETFILTER_XT_MATCH_LENGTH=m -CONFIG_NETFILTER_XT_MATCH_LIMIT=m -CONFIG_NETFILTER_XT_MATCH_MAC=m -CONFIG_NETFILTER_XT_MATCH_MARK=m -CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m -CONFIG_NETFILTER_XT_MATCH_NFACCT=m -CONFIG_NETFILTER_XT_MATCH_OSF=m -CONFIG_NETFILTER_XT_MATCH_OWNER=m -CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m -CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m -CONFIG_NETFILTER_XT_MATCH_QUOTA=m -CONFIG_NETFILTER_XT_MATCH_RATEEST=m -CONFIG_NETFILTER_XT_MATCH_REALM=m -CONFIG_NETFILTER_XT_MATCH_RECENT=m -CONFIG_NETFILTER_XT_MATCH_SCTP=m -CONFIG_NETFILTER_XT_MATCH_SOCKET=m -CONFIG_NETFILTER_XT_MATCH_STATE=m -CONFIG_NETFILTER_XT_MATCH_STATISTIC=m -CONFIG_NETFILTER_XT_MATCH_STRING=m -CONFIG_NETFILTER_XT_MATCH_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_TIME=m -CONFIG_NETFILTER_XT_MATCH_U32=m -CONFIG_IP_SET=m -CONFIG_IP_SET_BITMAP_IP=m -CONFIG_IP_SET_BITMAP_IPMAC=m -CONFIG_IP_SET_BITMAP_PORT=m -CONFIG_IP_SET_HASH_IP=m -CONFIG_IP_SET_HASH_IPMARK=m -CONFIG_IP_SET_HASH_IPPORT=m -CONFIG_IP_SET_HASH_IPPORTIP=m -CONFIG_IP_SET_HASH_IPPORTNET=m -CONFIG_IP_SET_HASH_IPMAC=m -CONFIG_IP_SET_HASH_MAC=m -CONFIG_IP_SET_HASH_NETPORTNET=m -CONFIG_IP_SET_HASH_NET=m -CONFIG_IP_SET_HASH_NETNET=m -CONFIG_IP_SET_HASH_NETPORT=m -CONFIG_IP_SET_HASH_NETIFACE=m -CONFIG_IP_SET_LIST_SET=m -CONFIG_IP_VS=m -CONFIG_NFT_DUP_IPV4=m -CONFIG_NFT_FIB_IPV4=m -CONFIG_NF_TABLES_ARP=y -CONFIG_NF_LOG_ARP=m -CONFIG_NF_LOG_IPV4=m -CONFIG_IP_NF_IPTABLES=m -CONFIG_IP_NF_MATCH_AH=m -CONFIG_IP_NF_MATCH_ECN=m -CONFIG_IP_NF_MATCH_RPFILTER=m -CONFIG_IP_NF_MATCH_TTL=m -CONFIG_IP_NF_TARGET_REJECT=m -CONFIG_IP_NF_NAT=m -CONFIG_IP_NF_TARGET_MASQUERADE=m -CONFIG_IP_NF_TARGET_NETMAP=m -CONFIG_IP_NF_TARGET_REDIRECT=m -CONFIG_IP_NF_TARGET_ECN=m -CONFIG_IP_NF_TARGET_TTL=m -CONFIG_IP_NF_RAW=m -CONFIG_IP_NF_ARPFILTER=m -CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_NFT_DUP_IPV6=m -CONFIG_NFT_FIB_IPV6=m -CONFIG_IP6_NF_IPTABLES=m -CONFIG_IP6_NF_MATCH_AH=m -CONFIG_IP6_NF_MATCH_EUI64=m -CONFIG_IP6_NF_MATCH_FRAG=m -CONFIG_IP6_NF_MATCH_OPTS=m -CONFIG_IP6_NF_MATCH_HL=m -CONFIG_IP6_NF_MATCH_IPV6HEADER=m -CONFIG_IP6_NF_MATCH_MH=m -CONFIG_IP6_NF_MATCH_RPFILTER=m -CONFIG_IP6_NF_MATCH_RT=m -CONFIG_IP6_NF_MATCH_SRH=m -CONFIG_IP6_NF_TARGET_HL=m -CONFIG_IP6_NF_TARGET_REJECT=m -CONFIG_IP6_NF_RAW=m -CONFIG_IP6_NF_NAT=m -CONFIG_IP6_NF_TARGET_MASQUERADE=m -CONFIG_IP6_NF_TARGET_NPT=m -CONFIG_NF_TABLES_BRIDGE=m -CONFIG_NFT_BRIDGE_META=m -CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_CONNTRACK_BRIDGE=m -CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m -CONFIG_BRIDGE_NF_EBTABLES=m -CONFIG_BRIDGE_EBT_BROUTE=m -CONFIG_BRIDGE_EBT_T_FILTER=m -CONFIG_BRIDGE_EBT_T_NAT=m -CONFIG_BRIDGE_EBT_802_3=m -CONFIG_BRIDGE_EBT_AMONG=m -CONFIG_BRIDGE_EBT_ARP=m -CONFIG_BRIDGE_EBT_IP=m -CONFIG_BRIDGE_EBT_IP6=m -CONFIG_BRIDGE_EBT_LIMIT=m -CONFIG_BRIDGE_EBT_MARK=m -CONFIG_BRIDGE_EBT_PKTTYPE=m -CONFIG_BRIDGE_EBT_STP=m -CONFIG_BRIDGE_EBT_VLAN=m -CONFIG_BRIDGE_EBT_ARPREPLY=m -CONFIG_BRIDGE_EBT_DNAT=m -CONFIG_BRIDGE_EBT_MARK_T=m -CONFIG_BRIDGE_EBT_REDIRECT=m -CONFIG_BRIDGE_EBT_SNAT=m -CONFIG_BRIDGE_EBT_LOG=m -CONFIG_BRIDGE_EBT_NFLOG=m -CONFIG_L2TP=m -CONFIG_L2TP_V3=y -CONFIG_L2TP_IP=m -CONFIG_L2TP_ETH=m -CONFIG_BRIDGE=m -CONFIG_BRIDGE_VLAN_FILTERING=y -CONFIG_NET_DSA=m -CONFIG_VLAN_8021Q=m -CONFIG_VLAN_8021Q_GVRP=y -CONFIG_VLAN_8021Q_MVRP=y -CONFIG_NET_SCHED=y -CONFIG_NET_SCH_HTB=m -CONFIG_NET_SCH_HFSC=m -CONFIG_NET_SCH_PRIO=m -CONFIG_NET_SCH_MULTIQ=m -CONFIG_NET_SCH_RED=m -CONFIG_NET_SCH_SFB=m -CONFIG_NET_SCH_SFQ=m -CONFIG_NET_SCH_TEQL=m -CONFIG_NET_SCH_TBF=m -CONFIG_NET_SCH_CBS=m -CONFIG_NET_SCH_ETF=m -CONFIG_NET_SCH_TAPRIO=m -CONFIG_NET_SCH_GRED=m -CONFIG_NET_SCH_NETEM=m -CONFIG_NET_SCH_DRR=m -CONFIG_NET_SCH_MQPRIO=m -CONFIG_NET_SCH_CHOKE=m -CONFIG_NET_SCH_QFQ=m -CONFIG_NET_SCH_CODEL=m -CONFIG_NET_SCH_FQ_CODEL=m -CONFIG_NET_SCH_CAKE=m -CONFIG_NET_SCH_FQ=m -CONFIG_NET_SCH_HHF=m -CONFIG_NET_SCH_PIE=m -CONFIG_NET_SCH_FQ_PIE=m -CONFIG_NET_SCH_INGRESS=m -CONFIG_NET_SCH_PLUG=m -CONFIG_NET_SCH_ETS=m -CONFIG_NET_SCH_DUALPI2=m -CONFIG_NET_CLS_BASIC=m -CONFIG_NET_CLS_ROUTE4=m -CONFIG_NET_CLS_FW=m -CONFIG_NET_CLS_U32=m -CONFIG_CLS_U32_PERF=y -CONFIG_CLS_U32_MARK=y -CONFIG_NET_CLS_FLOW=m -CONFIG_NET_CLS_BPF=m -CONFIG_NET_CLS_FLOWER=m -CONFIG_NET_CLS_MATCHALL=m -CONFIG_NET_EMATCH=y -CONFIG_NET_EMATCH_CMP=m -CONFIG_NET_EMATCH_NBYTE=m -CONFIG_NET_EMATCH_U32=m -CONFIG_NET_EMATCH_META=m -CONFIG_NET_EMATCH_TEXT=m -CONFIG_NET_EMATCH_IPSET=m -CONFIG_NET_EMATCH_IPT=m -CONFIG_NET_CLS_ACT=y -CONFIG_NET_ACT_POLICE=m -CONFIG_NET_ACT_GACT=m -CONFIG_GACT_PROB=y -CONFIG_NET_ACT_MIRRED=m -CONFIG_NET_ACT_SAMPLE=m -CONFIG_NET_ACT_NAT=m -CONFIG_NET_ACT_PEDIT=m -CONFIG_NET_ACT_SIMP=m -CONFIG_NET_ACT_SKBEDIT=m -CONFIG_NET_ACT_CSUM=m -CONFIG_NET_ACT_MPLS=m -CONFIG_NET_ACT_VLAN=m -CONFIG_NET_ACT_BPF=m -CONFIG_NET_ACT_CONNMARK=m -CONFIG_NET_ACT_CTINFO=m -CONFIG_NET_ACT_SKBMOD=m -CONFIG_NET_ACT_TUNNEL_KEY=m -CONFIG_NET_ACT_CT=m -CONFIG_NET_ACT_GATE=m -CONFIG_VSOCKETS=m -CONFIG_NETLINK_DIAG=m -CONFIG_HSR=m -CONFIG_QRTR_SMD=m -CONFIG_QRTR_TUN=m -CONFIG_CAN=m -CONFIG_CAN_J1939=m -CONFIG_CAN_ISOTP=m -CONFIG_BT=m -CONFIG_BT_RFCOMM=m -CONFIG_BT_RFCOMM_TTY=y -CONFIG_BT_BNEP=m -CONFIG_BT_BNEP_MC_FILTER=y -CONFIG_BT_BNEP_PROTO_FILTER=y -CONFIG_BT_HIDP=m -CONFIG_BT_LEDS=y -CONFIG_BT_MSFTEXT=y -CONFIG_BT_AOSPEXT=y -CONFIG_BT_BRCMEXT=y -# CONFIG_BT_DEBUGFS is not set -CONFIG_BT_MTK7932_PCIE=m -CONFIG_BT_HCIBTUSB=m -CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y -CONFIG_BT_HCIBTUSB_MTK=y -CONFIG_BT_HCIUART=m -CONFIG_BT_HCIUART_LL=y -CONFIG_BT_HCIUART_BCM=y -CONFIG_BT_HCIUART_QCA=y -CONFIG_BT_HCIUART_MRVL=y -CONFIG_BT_HCIBCM203X=m -CONFIG_BT_HCIBCM4377=m -CONFIG_BT_HCIBPA10X=m -CONFIG_BT_HCIBFUSB=m -CONFIG_BT_HCIVHCI=m -CONFIG_BT_MRVL=m -CONFIG_BT_MRVL_SDIO=m -CONFIG_BT_ATH3K=m -CONFIG_BT_QCOMSMD=m -CONFIG_BT_NXPUART=m -CONFIG_CFG80211=m -CONFIG_MAC80211=m -CONFIG_RFKILL=y -CONFIG_RFKILL_GPIO=m -CONFIG_NET_9P=y -CONFIG_NET_9P_VIRTIO=y -CONFIG_NFC=m -CONFIG_NFC_NCI=m -CONFIG_NFC_NXP_NCI=m -CONFIG_NFC_NXP_NCI_I2C=m -CONFIG_NFC_S3FWRN5_I2C=m -CONFIG_PCI=y -CONFIG_PCIEPORTBUS=y -CONFIG_PCIEAER=y -CONFIG_PCI_IOV=y -CONFIG_PCI_PASID=y -CONFIG_HOTPLUG_PCI=y -CONFIG_HOTPLUG_PCI_ACPI=y -CONFIG_PCI_AARDVARK=y -CONFIG_PCIE_ALTERA=y -CONFIG_PCIE_ALTERA_MSI=y -CONFIG_PCIE_APPLE=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y -CONFIG_PCIE_BRCMSTB=m -CONFIG_PCI_HOST_THUNDER_PEM=y -CONFIG_PCI_HOST_THUNDER_ECAM=y -CONFIG_PCI_HOST_GENERIC=y -CONFIG_PCIE_MEDIATEK_GEN3=m -CONFIG_PCI_TEGRA=y -CONFIG_PCIE_RCAR_HOST=y -CONFIG_PCIE_RCAR_EP=y -CONFIG_PCIE_RENESAS_RZG3S_HOST=y -CONFIG_PCIE_ROCKCHIP_HOST=m -CONFIG_PCI_XGENE=y -CONFIG_PCIE_XILINX=y -CONFIG_PCIE_XILINX_DMA_PL=y -CONFIG_PCIE_XILINX_NWL=y -CONFIG_PCIE_XILINX_CPM=y -CONFIG_PCI_J721E_HOST=m -CONFIG_PCI_IMX6_HOST=y -CONFIG_PCI_LAYERSCAPE=y -CONFIG_PCI_HISI=y -CONFIG_PCIE_KIRIN=y -CONFIG_PCIE_HISI_STB=y -CONFIG_PCIE_ARMADA_8K=y -CONFIG_PCIE_TEGRA194_HOST=m -CONFIG_PCIE_TEGRA194_EP=m -CONFIG_PCIE_QCOM=y -CONFIG_PCIE_RCAR_GEN4_HOST=m -CONFIG_PCIE_RCAR_GEN4_EP=m -CONFIG_PCIE_ROCKCHIP_DW_HOST=y -CONFIG_PCIE_VISCONTI_HOST=y -CONFIG_PCIE_LAYERSCAPE_GEN4=y -CONFIG_PCI_ENDPOINT=y -CONFIG_PCI_ENDPOINT_CONFIGFS=y -CONFIG_PCI_EPF_TEST=m -CONFIG_DEVTMPFS=y -CONFIG_DEVTMPFS_MOUNT=y -CONFIG_FW_LOADER_USER_HELPER=y -CONFIG_FW_LOADER_COMPRESS=y -CONFIG_FW_LOADER_COMPRESS_ZSTD=y -CONFIG_HISILICON_LPC=y -CONFIG_IMX_AIPSTZ=m -CONFIG_TEGRA_ACONNECT=m -CONFIG_MHI_BUS_PCI_GENERIC=m -CONFIG_ARM_SCPI_PROTOCOL=y -CONFIG_INTEL_STRATIX10_SERVICE=y -CONFIG_INTEL_STRATIX10_RSU=m -CONFIG_MTK_ADSP_IPC=m -CONFIG_SYSFB_SIMPLEFB=y -CONFIG_GOOGLE_FIRMWARE=y -CONFIG_GOOGLE_CBMEM=m -CONFIG_GOOGLE_COREBOOT_TABLE=m -CONFIG_EFI_CAPSULE_LOADER=y -CONFIG_IMX_SCU=y -CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE=y -CONFIG_QCOM_QSEECOM=y -CONFIG_QCOM_QSEECOM_UEFISECAPP=y -CONFIG_EXYNOS_ACPM_PROTOCOL=m -CONFIG_TEGRA_BPMP=y -CONFIG_ZYNQMP_FIRMWARE_DEBUG=y -CONFIG_GNSS=m -CONFIG_GNSS_MTK_SERIAL=m -CONFIG_GNSS_USB=m -CONFIG_MTD=y -CONFIG_MTD_BLOCK=y -CONFIG_MTD_CFI=y -CONFIG_MTD_CFI_ADV_OPTIONS=y -CONFIG_MTD_CFI_INTELEXT=y -CONFIG_MTD_CFI_AMDSTD=y -CONFIG_MTD_CFI_STAA=y -CONFIG_MTD_PHYSMAP=y -CONFIG_MTD_PHYSMAP_OF=y -CONFIG_MTD_DATAFLASH=y -CONFIG_MTD_SST25L=y -CONFIG_MTD_RAW_NAND=y -CONFIG_MTD_NAND_DENALI_DT=y -CONFIG_MTD_NAND_MARVELL=y -CONFIG_MTD_NAND_BRCMNAND=m -CONFIG_MTD_NAND_FSL_IFC=y -CONFIG_MTD_NAND_QCOM=y -CONFIG_MTD_SPI_NAND=m -CONFIG_MTD_SPI_NOR=y -CONFIG_MTD_UBI=m -CONFIG_MTD_HYPERBUS=m -CONFIG_HBMC_AM654=m -CONFIG_ZRAM=y -CONFIG_BLK_DEV_LOOP=y -CONFIG_BLK_DEV_NBD=m -CONFIG_VIRTIO_BLK=y -CONFIG_BLK_DEV_NVME=y -CONFIG_NVME_HWMON=y -CONFIG_NVME_APPLE=y -CONFIG_QCOM_COINCELL=m -CONFIG_QCOM_FASTRPC=m -CONFIG_SRAM=y -CONFIG_PCI_ENDPOINT_TEST=m -CONFIG_XILINX_SDFEC=m -CONFIG_NTSYNC=m -CONFIG_EEPROM_AT24=m -CONFIG_EEPROM_AT25=m -CONFIG_MISC_RTSX_USB=m -CONFIG_UACCE=m -CONFIG_MISC_RP1=m -# CONFIG_SCSI_PROC_FS is not set -CONFIG_BLK_DEV_SD=y -CONFIG_BLK_DEV_SR=m -CONFIG_CHR_DEV_SG=m -CONFIG_SCSI_SAS_ATA=y -CONFIG_SCSI_HISI_SAS=y -CONFIG_SCSI_HISI_SAS_PCI=y -CONFIG_MEGARAID_SAS=y -CONFIG_SCSI_MPT3SAS=m -CONFIG_ATA=y -CONFIG_SATA_AHCI=y -CONFIG_SATA_AHCI_PLATFORM=y -CONFIG_AHCI_BRCM=m -CONFIG_AHCI_DWC=m -CONFIG_AHCI_CEVA=y -CONFIG_AHCI_MVEBU=y -CONFIG_AHCI_XGENE=y -CONFIG_AHCI_QORIQ=y -CONFIG_SATA_SIL24=y -CONFIG_SATA_RCAR=y -CONFIG_PATA_OF_PLATFORM=y -CONFIG_MD=y -CONFIG_MD_LINEAR=m -CONFIG_BLK_DEV_DM=m -CONFIG_DM_CRYPT=m -CONFIG_DM_SNAPSHOT=m -CONFIG_DM_THIN_PROVISIONING=m -CONFIG_DM_CACHE=m -CONFIG_DM_WRITECACHE=m -CONFIG_DM_MIRROR=m -CONFIG_DM_RAID=m -CONFIG_DM_ZERO=m -CONFIG_DM_UEVENT=y -CONFIG_DM_VERITY=m -CONFIG_DM_VERITY_FEC=y -CONFIG_DM_INTEGRITY=m -CONFIG_NETDEVICES=y -CONFIG_BONDING=m -CONFIG_DUMMY=m -CONFIG_WIREGUARD=m -CONFIG_OVPN=m -CONFIG_IFB=m -CONFIG_NET_TEAM=m -CONFIG_NET_TEAM_MODE_BROADCAST=m -CONFIG_NET_TEAM_MODE_ROUNDROBIN=m -CONFIG_NET_TEAM_MODE_RANDOM=m -CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m -CONFIG_NET_TEAM_MODE_LOADBALANCE=m -CONFIG_MACVLAN=m -CONFIG_MACVTAP=m -CONFIG_VXLAN=m -CONFIG_GENEVE=m -CONFIG_TUN=y -CONFIG_VETH=m -CONFIG_VIRTIO_NET=y -CONFIG_MHI_NET=m -CONFIG_NET_DSA_BCM_SF2=m -CONFIG_NET_DSA_MSCC_FELIX=m -CONFIG_ENA_ETHERNET=m -CONFIG_AMD_XGBE=y -CONFIG_NET_XGENE=y -CONFIG_ATL1C=m -CONFIG_BCMGENET=m -CONFIG_BNX2X=m -CONFIG_SYSTEMPORT=m -CONFIG_MACB=y -CONFIG_THUNDER_NIC_PF=y -CONFIG_FEC=y -CONFIG_FSL_FMAN=y -CONFIG_FSL_DPAA_ETH=y -CONFIG_FSL_DPAA2_ETH=y -CONFIG_FSL_ENETC=y -CONFIG_NXP_ENETC4=m -CONFIG_FSL_ENETC_VF=y -CONFIG_FSL_ENETC_QOS=y -CONFIG_NXP_NETC_BLK_CTRL=m -CONFIG_HIX5HD2_GMAC=y -CONFIG_HNS_DSAF=y -CONFIG_HNS_ENET=y -CONFIG_HNS3=y -CONFIG_HNS3_HCLGE=y -CONFIG_HNS3_ENET=y -CONFIG_E1000=y -CONFIG_E1000E=y -CONFIG_IGB=m -CONFIG_IGBVF=y -CONFIG_IGC=m -CONFIG_MVNETA=y -CONFIG_MVPP2=y -CONFIG_SKY2=y -CONFIG_NET_VENDOR_MEDIATEK=y -CONFIG_NET_MEDIATEK_STAR_EMAC=m -CONFIG_MLX4_EN=m -CONFIG_MLX5_CORE=m -CONFIG_MLX5_CORE_EN=y -CONFIG_QCOM_EMAC=m -CONFIG_RMNET=m -CONFIG_R8169=m -CONFIG_SH_ETH=y -CONFIG_RAVB=y -CONFIG_RENESAS_ETHER_SWITCH=y -CONFIG_RTSN=y -CONFIG_SMC91X=y -CONFIG_SMSC911X=y -CONFIG_SNI_AVE=y -CONFIG_SNI_NETSEC=y -CONFIG_STMMAC_ETH=m -CONFIG_DWMAC_MEDIATEK=m -CONFIG_DWMAC_TEGRA=m -CONFIG_TI_K3_AM65_CPSW_NUSS=y -CONFIG_TI_ICSSG_PRUETH=m -CONFIG_XILINX_AXI_EMAC=m -CONFIG_QCOM_IPA=m -CONFIG_MESON_GXL_PHY=m -CONFIG_AQUANTIA_PHY=y -CONFIG_BCM54140_PHY=m -CONFIG_MARVELL_PHY=m -CONFIG_MARVELL_10G_PHY=y -CONFIG_MARVELL_88Q2XXX_PHY=y -CONFIG_MICREL_PHY=y -CONFIG_MICROSEMI_PHY=y -CONFIG_AT803X_PHY=y -CONFIG_QCA808X_PHY=m -CONFIG_REALTEK_PHY=y -CONFIG_ROCKCHIP_PHY=y -CONFIG_DP83867_PHY=y -CONFIG_DP83869_PHY=m -CONFIG_DP83TD510_PHY=y -CONFIG_DP83TG720_PHY=m -CONFIG_VITESSE_PHY=y -CONFIG_XILINX_GMII2RGMII=m -CONFIG_CAN_FLEXCAN=m -CONFIG_CAN_XILINXCAN=m -CONFIG_CAN_M_CAN=m -CONFIG_CAN_M_CAN_PLATFORM=m -CONFIG_CAN_RCAR=m -CONFIG_CAN_RCAR_CANFD=m -CONFIG_CAN_MCP251XFD=m -CONFIG_CAN_8DEV_USB=m -CONFIG_CAN_EMS_USB=m -CONFIG_CAN_ESD_USB=m -CONFIG_CAN_F81604=m -CONFIG_CAN_GS_USB=m -CONFIG_CAN_KVASER_USB=m -CONFIG_CAN_MCBA_USB=m -CONFIG_CAN_PEAK_USB=m -CONFIG_MDIO_GPIO=y -CONFIG_MDIO_BUS_MUX_MULTIPLEXER=y -CONFIG_MDIO_BUS_MUX_MMIOREG=y -CONFIG_PPP=m -CONFIG_PPP_BSDCOMP=m -CONFIG_PPP_DEFLATE=m -CONFIG_PPP_FILTER=y -CONFIG_PPP_MPPE=m -CONFIG_PPP_MULTILINK=y -CONFIG_PPPOE=m -CONFIG_PPTP=m -CONFIG_PPPOL2TP=m -CONFIG_PPP_ASYNC=m -CONFIG_PPP_SYNC_TTY=m -CONFIG_USB_CATC=m -CONFIG_USB_KAWETH=m -CONFIG_USB_PEGASUS=m -CONFIG_USB_RTL8150=m -CONFIG_USB_RTL8152=y -CONFIG_USB_LAN78XX=m -CONFIG_USB_USBNET=y -CONFIG_USB_NET_CDC_EEM=m -CONFIG_USB_NET_HUAWEI_CDC_NCM=m -CONFIG_USB_NET_CDC_MBIM=m -CONFIG_USB_NET_DM9601=m -CONFIG_USB_NET_SR9700=m -CONFIG_USB_NET_SR9800=m -CONFIG_USB_NET_SMSC75XX=m -CONFIG_USB_NET_SMSC95XX=m -CONFIG_USB_NET_GL620A=m -CONFIG_USB_NET_NET1080=m -CONFIG_USB_NET_PLUSB=m -CONFIG_USB_NET_MCS7830=m -CONFIG_USB_NET_RNDIS_HOST=m -CONFIG_USB_NET_CDC_SUBSET=m -CONFIG_USB_ALI_M5632=y -CONFIG_USB_AN2720=y -CONFIG_USB_EPSON2888=y -CONFIG_USB_KC2190=y -CONFIG_USB_NET_ZAURUS=m -CONFIG_USB_NET_CX82310_ETH=m -CONFIG_USB_NET_KALMIA=m -CONFIG_USB_NET_QMI_WWAN=m -CONFIG_USB_HSO=m -CONFIG_USB_NET_INT51X1=m -CONFIG_USB_IPHETH=m -CONFIG_USB_SIERRA_NET=m -CONFIG_USB_VL600=m -CONFIG_USB_NET_CH9200=m -CONFIG_USB_NET_AQC111=m -CONFIG_ATH9K_HTC=m -CONFIG_CARL9170=m -CONFIG_ATH6KL=m -CONFIG_ATH6KL_USB=m -CONFIG_AR5523=m -CONFIG_ATH10K=m -CONFIG_ATH10K_PCI=m -CONFIG_ATH10K_SDIO=m -CONFIG_ATH10K_USB=m -CONFIG_ATH10K_SNOC=m -CONFIG_WCN36XX=m -CONFIG_ATH11K=m -CONFIG_ATH11K_AHB=m -CONFIG_ATH11K_PCI=m -CONFIG_ATH12K=m -CONFIG_BRCMFMAC=m -CONFIG_BRCMFMAC_USB=y -CONFIG_BRCMFMAC_PCIE=y -CONFIG_IWLWIFI=m -CONFIG_IWLDVM=m -CONFIG_IWLMVM=m -CONFIG_LIBERTAS_THINFIRM=m -CONFIG_LIBERTAS_THINFIRM_USB=m -CONFIG_MWIFIEX=m -CONFIG_MWIFIEX_SDIO=m -CONFIG_MWIFIEX_PCIE=m -CONFIG_MWIFIEX_USB=m -CONFIG_MT7601U=m -CONFIG_MT76x0U=m -CONFIG_MT76x2U=m -CONFIG_MT7663U=m -CONFIG_MT7921E=m -CONFIG_MT7921U=m -CONFIG_MT7925U=m -CONFIG_MT7932_FULLMAC=m -CONFIG_RT2X00=m -CONFIG_RT2500USB=m -CONFIG_RT73USB=m -CONFIG_RT2800USB=m -CONFIG_RT2800USB_RT3573=y -CONFIG_RT2800USB_RT53XX=y -CONFIG_RT2800USB_RT55XX=y -CONFIG_RT2800USB_UNKNOWN=y -CONFIG_RTL8187=m -CONFIG_RTL8192DU=m -CONFIG_RTL8XXXU=m -CONFIG_RTW88=m -CONFIG_RTW88_8822BU=m -CONFIG_RTW88_8822CU=m -CONFIG_RTW88_8723DU=m -CONFIG_RTW88_8821CU=m -CONFIG_RTW88_8821AU=m -CONFIG_RTW88_8812AU=m -CONFIG_RTW88_8814AU=m -CONFIG_RTW89=m -CONFIG_RTW89_8851BU=m -CONFIG_RTW89_8852AU=m -CONFIG_RTW89_8852BU=m -CONFIG_RTW89_8852CU=m -CONFIG_RSI_91X=m -CONFIG_WL18XX=m -CONFIG_WLCORE_SDIO=m -CONFIG_ZD1211RW=m -CONFIG_WWAN=m -CONFIG_MHI_WWAN_CTRL=m -CONFIG_MHI_WWAN_MBIM=m -CONFIG_INPUT_JOYDEV=m -CONFIG_INPUT_EVDEV=y -CONFIG_KEYBOARD_ADC=m -CONFIG_KEYBOARD_GPIO=y -CONFIG_KEYBOARD_GPIO_POLLED=m -CONFIG_KEYBOARD_SNVS_PWRKEY=m -CONFIG_KEYBOARD_IMX_SC_KEY=m -CONFIG_KEYBOARD_CROS_EC=y -CONFIG_KEYBOARD_MTK_PMIC=m -CONFIG_MOUSE_ELAN_I2C=m -CONFIG_MOUSE_SYNAPTICS_USB=m -CONFIG_INPUT_JOYSTICK=y -CONFIG_JOYSTICK_IFORCE=m -CONFIG_JOYSTICK_IFORCE_USB=m -CONFIG_JOYSTICK_XPAD=m -CONFIG_JOYSTICK_XPAD_FF=y -CONFIG_JOYSTICK_XPAD_LEDS=y -CONFIG_JOYSTICK_PXRC=m -CONFIG_INPUT_TABLET=y -CONFIG_TABLET_USB_ACECAD=m -CONFIG_TABLET_USB_AIPTEK=m -CONFIG_TABLET_USB_HANWANG=m -CONFIG_TABLET_USB_KBTAB=m -CONFIG_TABLET_USB_PEGASUS=m -CONFIG_INPUT_TOUCHSCREEN=y -CONFIG_TOUCHSCREEN_APPLE_Z2=m -CONFIG_TOUCHSCREEN_ATMEL_MXT=m -CONFIG_TOUCHSCREEN_GOODIX=m -CONFIG_TOUCHSCREEN_GOODIX_BERLIN_SPI=m -CONFIG_TOUCHSCREEN_ILI210X=m -CONFIG_TOUCHSCREEN_ELAN=m -CONFIG_TOUCHSCREEN_EDT_FT5X06=m -CONFIG_TOUCHSCREEN_USB_COMPOSITE=m -CONFIG_TOUCHSCREEN_HIMAX_HX83112B=m -CONFIG_INPUT_MISC=y -CONFIG_INPUT_AW86927=m -CONFIG_INPUT_BBNSM_PWRKEY=m -CONFIG_INPUT_PM8941_PWRKEY=y -CONFIG_INPUT_PM8XXX_VIBRATOR=m -CONFIG_INPUT_TPS65219_PWRBUTTON=m -CONFIG_INPUT_TPS6594_PWRBUTTON=m -CONFIG_INPUT_UINPUT=m -CONFIG_INPUT_PWM_BEEPER=m -CONFIG_INPUT_PWM_VIBRA=m -CONFIG_INPUT_RK805_PWRKEY=m -CONFIG_INPUT_DA9063_ONKEY=m -CONFIG_INPUT_HISI_POWERKEY=y -CONFIG_INPUT_MACSMC_INPUT=m -CONFIG_RMI4_F1A=y -CONFIG_RMI4_F21=y -CONFIG_RMI4_F34=y -CONFIG_RMI4_F55=y -# CONFIG_SERIO_SERPORT is not set -CONFIG_SERIO_AMBAKMI=y -CONFIG_LEGACY_PTY_COUNT=16 -CONFIG_SERIAL_8250=y -CONFIG_SERIAL_8250_CONSOLE=y -CONFIG_SERIAL_8250_NR_UARTS=8 -CONFIG_SERIAL_8250_EXTENDED=y -CONFIG_SERIAL_8250_SHARE_IRQ=y -CONFIG_SERIAL_8250_BCM2835AUX=y -CONFIG_SERIAL_8250_DW=y -CONFIG_SERIAL_8250_EM=y -CONFIG_SERIAL_8250_OMAP=y -CONFIG_SERIAL_8250_MT6577=y -CONFIG_SERIAL_8250_UNIPHIER=y -CONFIG_SERIAL_OF_PLATFORM=y -CONFIG_SERIAL_AMBA_PL011=y -CONFIG_SERIAL_AMBA_PL011_CONSOLE=y -CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y -CONFIG_SERIAL_APPLE_DOCKCHANNEL=y -CONFIG_SERIAL_MESON=y -CONFIG_SERIAL_MESON_CONSOLE=y -CONFIG_SERIAL_SAMSUNG=y -CONFIG_SERIAL_SAMSUNG_CONSOLE=y -CONFIG_SERIAL_TEGRA=y -CONFIG_SERIAL_TEGRA_TCU=y -CONFIG_SERIAL_IMX=y -CONFIG_SERIAL_IMX_CONSOLE=y -CONFIG_SERIAL_SH_SCI=y -CONFIG_SERIAL_RSCI=y -CONFIG_SERIAL_MSM=y -CONFIG_SERIAL_MSM_CONSOLE=y -CONFIG_SERIAL_QCOM_GENI=y -CONFIG_SERIAL_QCOM_GENI_CONSOLE=y -CONFIG_SERIAL_XILINX_PS_UART=y -CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y -CONFIG_SERIAL_FSL_LPUART=y -CONFIG_SERIAL_FSL_LPUART_CONSOLE=y -CONFIG_SERIAL_FSL_LINFLEXUART=y -CONFIG_SERIAL_FSL_LINFLEXUART_CONSOLE=y -CONFIG_SERIAL_STM32=y -CONFIG_SERIAL_STM32_CONSOLE=y -CONFIG_SERIAL_MVEBU_UART=y -CONFIG_SERIAL_OWL=y -CONFIG_SERIAL_DEV_BUS=y -CONFIG_VIRTIO_CONSOLE=y -CONFIG_IPMI_HANDLER=m -CONFIG_IPMI_DEVICE_INTERFACE=m -CONFIG_IPMI_SI=m -CONFIG_HW_RANDOM=y -CONFIG_HW_RANDOM_VIRTIO=y -CONFIG_TCG_TPM=y -CONFIG_TCG_TIS=m -CONFIG_TCG_TIS_SPI=m -CONFIG_TCG_TIS_SPI_CR50=y -CONFIG_TCG_TIS_I2C_CR50=m -CONFIG_TCG_TIS_I2C_INFINEON=y -CONFIG_XILLYUSB=m -CONFIG_I2C_CHARDEV=y -CONFIG_I2C_MUX=y -CONFIG_I2C_MUX_PCA954x=y -CONFIG_I2C_MUX_PINCTRL=m -CONFIG_I2C_BCM2835=m -CONFIG_I2C_CADENCE=m -CONFIG_I2C_DESIGNWARE_CORE=y -CONFIG_I2C_GPIO=m -CONFIG_I2C_IMX=y -CONFIG_I2C_IMX_LPI2C=y -CONFIG_I2C_MESON=y -CONFIG_I2C_MT65XX=y -CONFIG_I2C_MV64XXX=y -CONFIG_I2C_OMAP=y -CONFIG_I2C_OWL=y -CONFIG_I2C_APPLE=y -CONFIG_I2C_PXA=y -CONFIG_I2C_QCOM_CCI=m -CONFIG_I2C_QCOM_GENI=m -CONFIG_I2C_QUP=y -CONFIG_I2C_RIIC=y -CONFIG_I2C_RK3X=y -CONFIG_I2C_RZV2M=m -CONFIG_I2C_S3C2410=y -CONFIG_I2C_SH_MOBILE=y -CONFIG_I2C_TEGRA=y -CONFIG_I2C_UNIPHIER_F=y -CONFIG_I2C_XILINX=m -CONFIG_I2C_RCAR=y -CONFIG_I2C_DIOLAN_U2C=m -CONFIG_I2C_DLN2=m -CONFIG_I2C_CP2615=m -CONFIG_I2C_TINY_USB=m -CONFIG_I2C_CROS_EC_TUNNEL=y -CONFIG_SPI=y -CONFIG_SPI_APPLE=m -CONFIG_SPI_APPLE_QSPI=m -CONFIG_SPI_ARMADA_3700=y -CONFIG_SPI_BCM2835=m -CONFIG_SPI_BCM2835AUX=m -CONFIG_SPI_CADENCE_QUADSPI=y -CONFIG_SPI_CH341=m -CONFIG_SPI_DESIGNWARE=m -CONFIG_SPI_DW_DMA=y -CONFIG_SPI_DW_MMIO=m -CONFIG_SPI_DLN2=m -CONFIG_SPI_FSL_LPSPI=y -CONFIG_SPI_FSL_QUADSPI=y -CONFIG_SPI_NXP_FLEXSPI=y -CONFIG_SPI_IMX=m -CONFIG_SPI_FSL_DSPI=y -CONFIG_SPI_MESON_SPICC=m -CONFIG_SPI_MESON_SPIFC=m -CONFIG_SPI_MT65XX=y -CONFIG_SPI_MTK_NOR=m -CONFIG_SPI_OMAP24XX=m -CONFIG_SPI_ORION=y -CONFIG_SPI_PL022=y -CONFIG_SPI_ROCKCHIP=y -CONFIG_SPI_ROCKCHIP_SFC=m -CONFIG_SPI_RPCIF=m -CONFIG_SPI_RSPI=m -CONFIG_SPI_RZV2H_RSPI=m -CONFIG_SPI_RZV2M_CSI=m -CONFIG_SPI_QCOM_QSPI=m -CONFIG_SPI_QUP=y -CONFIG_SPI_QCOM_GENI=m -CONFIG_SPI_S3C64XX=y -CONFIG_SPI_SH_MSIOF=m -CONFIG_SPI_STM32_OSPI=m -CONFIG_SPI_SUN6I=y -CONFIG_SPI_TEGRA210_QUAD=m -CONFIG_SPI_TEGRA114=m -CONFIG_SPI_XILINX=m -CONFIG_SPI_ZYNQMP_GQSPI=m -CONFIG_SPI_SPIDEV=m -CONFIG_SPMI=y -CONFIG_SPMI_APPLE=y -CONFIG_SPMI_MTK_PMIF=m -CONFIG_PINCTRL_APPLE_GPIO=y -CONFIG_PINCTRL_DA9062=m -CONFIG_PINCTRL_MAX77620=y -CONFIG_PINCTRL_RK805=m -CONFIG_PINCTRL_SINGLE=y -CONFIG_PINCTRL_SX150X=m -CONFIG_PINCTRL_OWL=y -CONFIG_PINCTRL_S700=y -CONFIG_PINCTRL_S900=y -CONFIG_PINCTRL_BRCMSTB=y -CONFIG_PINCTRL_BCM2712=y -CONFIG_PINCTRL_IMX_SCMI=y -CONFIG_PINCTRL_IMX91=y -# CONFIG_PINCTRL_HAWI is not set -# CONFIG_PINCTRL_SM7150 is not set -CONFIG_PINCTRL_QCOM_SPMI_PMIC=y -CONFIG_PINCTRL_LPASS_LPI=m -CONFIG_PINCTRL_MILOS_LPASS_LPI=m -CONFIG_PINCTRL_SC7280_LPASS_LPI=m -CONFIG_PINCTRL_SM6115_LPASS_LPI=m -CONFIG_PINCTRL_SM8250_LPASS_LPI=m -CONFIG_PINCTRL_SM8450_LPASS_LPI=m -CONFIG_PINCTRL_SC8280XP_LPASS_LPI=m -CONFIG_PINCTRL_SM8550_LPASS_LPI=m -CONFIG_PINCTRL_SM8650_LPASS_LPI=m -CONFIG_PINCTRL_SOPHGO_SG2000=y -CONFIG_GPIO_ALTERA=m -CONFIG_GPIO_DAVINCI=y -CONFIG_GPIO_DWAPB=y -CONFIG_GPIO_MB86S7X=y -CONFIG_GPIO_MPC8XXX=y -CONFIG_GPIO_MXC=y -CONFIG_GPIO_PL061=y -CONFIG_GPIO_RCAR=y -CONFIG_GPIO_SYSCON=y -CONFIG_GPIO_UNIPHIER=y -CONFIG_GPIO_VF610=y -CONFIG_GPIO_VISCONTI=y -CONFIG_GPIO_WCD934X=m -CONFIG_GPIO_XGENE=y -CONFIG_GPIO_XGENE_SB=y -CONFIG_GPIO_XILINX=m -CONFIG_GPIO_ZYNQ=m -CONFIG_GPIO_MAX732X=y -CONFIG_GPIO_PCA953X=y -CONFIG_GPIO_PCA953X_IRQ=y -CONFIG_GPIO_PCF857X=m -CONFIG_GPIO_TPIC2810=m -CONFIG_GPIO_ADP5585=m -CONFIG_GPIO_BD9571MWV=m -CONFIG_GPIO_DLN2=m -CONFIG_GPIO_MACSMC=y -CONFIG_GPIO_MAX77620=y -CONFIG_GPIO_SL28CPLD=m -CONFIG_GPIO_MPSSE=m -CONFIG_GPIO_AGGREGATOR=m -CONFIG_W1=m -CONFIG_W1_MASTER_DS2490=m -CONFIG_W1_SLAVE_THERM=m -CONFIG_W1_SLAVE_SMEM=m -CONFIG_W1_SLAVE_DS2405=m -CONFIG_W1_SLAVE_DS2408=m -CONFIG_W1_SLAVE_DS2413=m -CONFIG_W1_SLAVE_DS2406=m -CONFIG_W1_SLAVE_DS2423=m -CONFIG_W1_SLAVE_DS2805=m -CONFIG_W1_SLAVE_DS2430=m -CONFIG_W1_SLAVE_DS2431=m -CONFIG_W1_SLAVE_DS2433=m -CONFIG_W1_SLAVE_DS2433_CRC=y -CONFIG_W1_SLAVE_DS2438=m -CONFIG_W1_SLAVE_DS2780=m -CONFIG_W1_SLAVE_DS2781=m -CONFIG_W1_SLAVE_DS28E04=m -CONFIG_POWER_RESET_GPIO=y -CONFIG_POWER_RESET_GPIO_RESTART=y -CONFIG_POWER_RESET_MACSMC=y -CONFIG_POWER_RESET_MSM=y -CONFIG_POWER_RESET_QCOM_PON=m -CONFIG_POWER_RESET_TORADEX_EC=m -CONFIG_POWER_RESET_XGENE=y -CONFIG_POWER_RESET_SYSCON=y -CONFIG_POWER_RESET_SYSCON_POWEROFF=y -CONFIG_SYSCON_REBOOT_MODE=y -CONFIG_NVMEM_REBOOT_MODE=m -CONFIG_POWER_SEQUENCING_PCIE_M2=m -CONFIG_BATTERY_QCOM_BATTMGR=m -CONFIG_BATTERY_SBS=m -CONFIG_BATTERY_BQ27XXX=y -CONFIG_BATTERY_MAX17042=m -CONFIG_CHARGER_MT6360=m -CONFIG_CHARGER_BQ25890=m -CONFIG_CHARGER_BQ25980=m -CONFIG_CHARGER_RK817=m -CONFIG_MACSMC_POWER=m -CONFIG_SENSORS_AQUACOMPUTER_D5NEXT=m -CONFIG_SENSORS_ASUS_ROG_RYUJIN=m -CONFIG_SENSORS_ARM_SCMI=y -CONFIG_SENSORS_ARM_SCPI=y -CONFIG_SENSORS_CORSAIR_CPRO=m -CONFIG_SENSORS_CORSAIR_PSU=m -CONFIG_SENSORS_GIGABYTE_WATERFORCE=m -CONFIG_SENSORS_GPIO_FAN=m -CONFIG_SENSORS_JC42=m -CONFIG_SENSORS_POWERZ=m -CONFIG_SENSORS_MACSMC_HWMON=y -CONFIG_SENSORS_LM75=m -CONFIG_SENSORS_LM90=m -CONFIG_SENSORS_NZXT_KRAKEN2=m -CONFIG_SENSORS_NZXT_KRAKEN3=m -CONFIG_SENSORS_NZXT_SMART2=m -CONFIG_SENSORS_PWM_FAN=m -CONFIG_SENSORS_RASPBERRYPI_HWMON=m -CONFIG_SENSORS_SL28CPLD=m -CONFIG_SENSORS_AMC6821=m -CONFIG_SENSORS_INA2XX=m -CONFIG_SENSORS_INA3221=m -CONFIG_SENSORS_TMP102=m -CONFIG_THERMAL_NETLINK=y -CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR=y -CONFIG_THERMAL_GOV_STEP_WISE=y -CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y -CONFIG_CPU_THERMAL=y -CONFIG_DEVFREQ_THERMAL=y -CONFIG_THERMAL_EMULATION=y -CONFIG_IMX_SC_THERMAL=m -CONFIG_IMX8MM_THERMAL=m -CONFIG_K3_THERMAL=m -CONFIG_QORIQ_THERMAL=m -CONFIG_SUN8I_THERMAL=y -CONFIG_ROCKCHIP_THERMAL=m -CONFIG_ARMADA_THERMAL=y -CONFIG_MTK_THERMAL=m -CONFIG_MTK_LVTS_THERMAL=m -CONFIG_BCM2711_THERMAL=m -CONFIG_BCM2835_THERMAL=m -CONFIG_BRCMSTB_THERMAL=m -CONFIG_RCAR_THERMAL=y -CONFIG_RCAR_GEN3_THERMAL=y -CONFIG_RZG2L_THERMAL=y -CONFIG_RZG3E_THERMAL=y -CONFIG_RZG3S_THERMAL=m -CONFIG_TEGRA_SOCTHERM=m -CONFIG_TEGRA_BPMP_THERMAL=m -CONFIG_GENERIC_ADC_THERMAL=m -CONFIG_QCOM_TSENS=y -CONFIG_QCOM_SPMI_ADC_TM5=m -CONFIG_QCOM_SPMI_TEMP_ALARM=m -CONFIG_QCOM_LMH=m -CONFIG_UNIPHIER_THERMAL=y -CONFIG_KHADAS_MCU_FAN_THERMAL=m -CONFIG_WATCHDOG=y -CONFIG_XILINX_WATCHDOG=m -CONFIG_XILINX_WINDOW_WATCHDOG=m -CONFIG_SL28CPLD_WATCHDOG=m -CONFIG_ARM_SP805_WATCHDOG=y -CONFIG_ARM_SBSA_WATCHDOG=y -CONFIG_S3C2410_WATCHDOG=y -CONFIG_DW_WATCHDOG=y -CONFIG_K3_RTI_WATCHDOG=m -CONFIG_SUNXI_WATCHDOG=m -CONFIG_NPCM7XX_WATCHDOG=y -CONFIG_IMX2_WDT=y -CONFIG_IMX_SC_WDT=m -CONFIG_IMX7ULP_WDT=m -CONFIG_QCOM_WDT=m -CONFIG_MESON_GXBB_WATCHDOG=m -CONFIG_MESON_WATCHDOG=m -CONFIG_ARM_SMC_WATCHDOG=y -CONFIG_RENESAS_WDT=y -CONFIG_RENESAS_RZG2LWDT=y -CONFIG_RENESAS_RZV2HWDT=y -CONFIG_UNIPHIER_WATCHDOG=y -CONFIG_PM8916_WATCHDOG=m -CONFIG_APPLE_WATCHDOG=y -CONFIG_BCM2835_WDT=y -CONFIG_BCM7038_WDT=m -CONFIG_USBPCWATCHDOG=m -CONFIG_MFD_ADP5585=m -CONFIG_MFD_ALTERA_SYSMGR=y -CONFIG_MFD_BD9571MWV=y -CONFIG_MFD_AXP20X_I2C=y -CONFIG_MFD_AXP20X_RSB=y -CONFIG_MFD_MACSMC=y -CONFIG_MFD_DA9062=m -CONFIG_MFD_DLN2=m -CONFIG_MFD_EXYNOS_LPASS=m -CONFIG_MFD_HI6421_PMIC=y -CONFIG_MFD_HI655X_PMIC=y -CONFIG_MFD_MAX77620=y -CONFIG_MFD_MAX77759=m -CONFIG_MFD_MT6360=y -CONFIG_MFD_MT6397=y -CONFIG_MFD_SPMI_PMIC=y -CONFIG_MFD_RK8XX_I2C=y -CONFIG_MFD_RK8XX_SPI=y -CONFIG_MFD_SEC_ACPM=m -CONFIG_MFD_SEC_I2C=y -CONFIG_MFD_SL28CPLD=y -CONFIG_RZ_MTU3=y -CONFIG_MFD_TI_AM335X_TSCADC=m -CONFIG_MFD_TI_LP873X=m -CONFIG_MFD_TPS65219=y -CONFIG_MFD_TPS6594_I2C=m -CONFIG_MFD_ROHM_BD718XX=y -CONFIG_MFD_STM32_LPTIMER=m -CONFIG_MFD_STM32_TIMERS=m -CONFIG_MFD_WCD934X=m -CONFIG_MFD_KHADAS_MCU=m -CONFIG_MFD_QCOM_PM8008=m -CONFIG_REGULATOR_FIXED_VOLTAGE=y -CONFIG_REGULATOR_AXP20X=y -CONFIG_REGULATOR_BD718XX=y -CONFIG_REGULATOR_BD9571MWV=y -CONFIG_REGULATOR_CROS_EC=y -CONFIG_REGULATOR_DA9211=m -CONFIG_REGULATOR_FAN53555=y -CONFIG_REGULATOR_GPIO=y -CONFIG_REGULATOR_HI6421V530=y -CONFIG_REGULATOR_HI655X=y -CONFIG_REGULATOR_LP873X=m -CONFIG_REGULATOR_MAX77620=y -CONFIG_REGULATOR_MAX8973=y -CONFIG_REGULATOR_MAX20411=m -CONFIG_REGULATOR_MP8859=y -CONFIG_REGULATOR_MT6315=m -CONFIG_REGULATOR_MT6357=y -CONFIG_REGULATOR_MT6358=y -CONFIG_REGULATOR_MT6359=y -CONFIG_REGULATOR_MT6360=y -CONFIG_REGULATOR_MT6397=y -CONFIG_REGULATOR_PCA9450=y -CONFIG_REGULATOR_PF8X00=y -CONFIG_REGULATOR_PFUZE100=y -CONFIG_REGULATOR_PWM=y -CONFIG_REGULATOR_QCOM_PM8008=m -CONFIG_REGULATOR_QCOM_REFGEN=m -CONFIG_REGULATOR_QCOM_RPMH=y -CONFIG_REGULATOR_QCOM_SMD_RPM=y -CONFIG_REGULATOR_QCOM_SPMI=y -CONFIG_REGULATOR_QCOM_USB_VBUS=m -CONFIG_REGULATOR_RAA215300=y -CONFIG_REGULATOR_RK808=y -CONFIG_REGULATOR_S2MPS11=y -CONFIG_REGULATOR_TPS65132=m -CONFIG_REGULATOR_TPS65219=y -CONFIG_REGULATOR_VCTRL=m -CONFIG_RC_CORE=m -CONFIG_RC_DECODERS=y -CONFIG_RC_DEVICES=y -CONFIG_IR_GPIO_CIR=m -CONFIG_IR_MESON=m -CONFIG_IR_SUNXI=m -CONFIG_MEDIA_CEC_SUPPORT=y -CONFIG_CEC_MESON_G12A_AO=m -CONFIG_MEDIA_SUPPORT=m -CONFIG_MEDIA_CAMERA_SUPPORT=y -CONFIG_MEDIA_ANALOG_TV_SUPPORT=y -CONFIG_MEDIA_DIGITAL_TV_SUPPORT=y -CONFIG_MEDIA_SDR_SUPPORT=y -CONFIG_MEDIA_PLATFORM_SUPPORT=y -# CONFIG_DVB_NET is not set -CONFIG_MEDIA_USB_SUPPORT=y -CONFIG_USB_GSPCA=m -CONFIG_USB_GSPCA_BENQ=m -CONFIG_USB_GSPCA_CONEX=m -CONFIG_USB_GSPCA_CPIA1=m -CONFIG_USB_GSPCA_DTCS033=m -CONFIG_USB_GSPCA_ETOMS=m -CONFIG_USB_GSPCA_FINEPIX=m -CONFIG_USB_GSPCA_JEILINJ=m -CONFIG_USB_GSPCA_JL2005BCD=m -CONFIG_USB_GSPCA_KINECT=m -CONFIG_USB_GSPCA_KONICA=m -CONFIG_USB_GSPCA_MARS=m -CONFIG_USB_GSPCA_MR97310A=m -CONFIG_USB_GSPCA_NW80X=m -CONFIG_USB_GSPCA_OV519=m -CONFIG_USB_GSPCA_OV534=m -CONFIG_USB_GSPCA_OV534_9=m -CONFIG_USB_GSPCA_PAC207=m -CONFIG_USB_GSPCA_PAC7302=m -CONFIG_USB_GSPCA_PAC7311=m -CONFIG_USB_GSPCA_SE401=m -CONFIG_USB_GSPCA_SN9C2028=m -CONFIG_USB_GSPCA_SN9C20X=m -CONFIG_USB_GSPCA_SONIXB=m -CONFIG_USB_GSPCA_SONIXJ=m -CONFIG_USB_GSPCA_SPCA1528=m -CONFIG_USB_GSPCA_SPCA500=m -CONFIG_USB_GSPCA_SPCA501=m -CONFIG_USB_GSPCA_SPCA505=m -CONFIG_USB_GSPCA_SPCA506=m -CONFIG_USB_GSPCA_SPCA508=m -CONFIG_USB_GSPCA_SPCA561=m -CONFIG_USB_GSPCA_SQ905=m -CONFIG_USB_GSPCA_SQ905C=m -CONFIG_USB_GSPCA_SQ930X=m -CONFIG_USB_GSPCA_STK014=m -CONFIG_USB_GSPCA_STK1135=m -CONFIG_USB_GSPCA_STV0680=m -CONFIG_USB_GSPCA_SUNPLUS=m -CONFIG_USB_GSPCA_T613=m -CONFIG_USB_GSPCA_TOPRO=m -CONFIG_USB_GSPCA_TOUPTEK=m -CONFIG_USB_GSPCA_TV8532=m -CONFIG_USB_GSPCA_VC032X=m -CONFIG_USB_GSPCA_VICAM=m -CONFIG_USB_GSPCA_XIRLINK_CIT=m -CONFIG_USB_GSPCA_ZC3XX=m -CONFIG_USB_GL860=m -CONFIG_USB_M5602=m -CONFIG_USB_STV06XX=m -CONFIG_USB_PWC=m -CONFIG_USB_VIDEO_CLASS=m -CONFIG_V4L_PLATFORM_DRIVERS=y -CONFIG_SDR_PLATFORM_DRIVERS=y -CONFIG_V4L_MEM2MEM_DRIVERS=y -CONFIG_VIDEO_AMPHION_VPU=m -CONFIG_VIDEO_APPLE_ISP=m -CONFIG_VIDEO_APPLE_AVD=m -CONFIG_VIDEO_CADENCE_CSI2RX=m -CONFIG_VIDEO_WAVE_VPU=m -CONFIG_VIDEO_E5010_JPEG_ENC=m -CONFIG_VIDEO_MEDIATEK_JPEG=m -CONFIG_VIDEO_MEDIATEK_VCODEC=m -CONFIG_VIDEO_MEDIATEK_MDP3=m -CONFIG_VIDEO_IMX7_CSI=m -CONFIG_VIDEO_IMX_MIPI_CSIS=m -CONFIG_VIDEO_IMX8_ISI=m -CONFIG_VIDEO_IMX8_ISI_M2M=y -CONFIG_VIDEO_IMX8_JPEG=m -CONFIG_VIDEO_QCOM_CAMSS=m -CONFIG_VIDEO_QCOM_IRIS=m -CONFIG_VIDEO_QCOM_VENUS=m -CONFIG_VIDEO_RCAR_CSI2=m -CONFIG_VIDEO_RCAR_ISP=m -CONFIG_VIDEO_RCAR_VIN=m -CONFIG_VIDEO_RZG2L_CSI2=m -CONFIG_VIDEO_RZG2L_CRU=m -CONFIG_VIDEO_RENESAS_FCP=m -CONFIG_VIDEO_RENESAS_FDP1=m -CONFIG_VIDEO_RENESAS_VSP1=m -CONFIG_VIDEO_RCAR_DRIF=m -CONFIG_VIDEO_ROCKCHIP_RGA=m -CONFIG_VIDEO_ROCKCHIP_CIF=m -CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC=m -CONFIG_VIDEO_SAMSUNG_S5P_JPEG=m -CONFIG_VIDEO_SAMSUNG_S5P_MFC=m -CONFIG_VIDEO_SUN6I_CSI=m -CONFIG_VIDEO_SYNOPSYS_HDMIRX=m -CONFIG_VIDEO_SYNOPSYS_HDMIRX_LOAD_DEFAULT_EDID=y -CONFIG_VIDEO_TI_J721E_CSI2RX=m -CONFIG_VIDEO_HANTRO=m -CONFIG_VIDEO_IMX219=m -CONFIG_VIDEO_IMX412=m -CONFIG_VIDEO_OV5640=m -CONFIG_VIDEO_OV5645=m -CONFIG_VIDEO_S5KJN1=m -CONFIG_DRM=y -CONFIG_DRM_ADP=m -CONFIG_DRM_APPLE=m -CONFIG_DRM_HDLCD=m -CONFIG_DRM_MALI_DISPLAY=m -CONFIG_DRM_KOMEDA=m -CONFIG_DRM_ASAHI=m -CONFIG_DRM_FSL_LDB=m -CONFIG_DRM_I2C_NXP_TDA998X=m -CONFIG_DRM_ITE_IT6263=m -CONFIG_DRM_LONTIUM_LT8912B=m -CONFIG_DRM_LONTIUM_LT9611=m -CONFIG_DRM_LONTIUM_LT9611UXC=m -CONFIG_DRM_LONTIUM_LT8713SX=m -CONFIG_DRM_ITE_IT66121=m -CONFIG_DRM_NWL_MIPI_DSI=m -CONFIG_DRM_PARADE_PS8640=m -CONFIG_DRM_SII902X=m -CONFIG_DRM_SIMPLE_BRIDGE=m -CONFIG_DRM_THINE_THC63LVD1024=m -CONFIG_DRM_TOSHIBA_TC358767=m -CONFIG_DRM_TOSHIBA_TC358768=m -CONFIG_DRM_TI_TFP410=m -CONFIG_DRM_TI_SN65DSI83=m -CONFIG_DRM_TI_SN65DSI86=m -CONFIG_DRM_ANALOGIX_ANX7625=m -CONFIG_DRM_I2C_ADV7511=m -CONFIG_DRM_I2C_ADV7511_AUDIO=y -CONFIG_DRM_CDNS_DSI=m -CONFIG_DRM_CDNS_MHDP8546=m -CONFIG_DRM_IMX8MP_DW_HDMI_BRIDGE=m -CONFIG_DRM_DW_HDMI_AHB_AUDIO=m -CONFIG_DRM_DW_HDMI_CEC=m -CONFIG_DRM_DW_HDMI_QP_CEC=y -CONFIG_DRM_ETNAVIV=m -CONFIG_DRM_EXYNOS=m -CONFIG_DRM_EXYNOS5433_DECON=y -CONFIG_DRM_EXYNOS7_DECON=y -CONFIG_DRM_EXYNOS_DSI=y -# CONFIG_DRM_EXYNOS_DP is not set -CONFIG_DRM_EXYNOS_HDMI=y -CONFIG_DRM_EXYNOS_MIC=y -CONFIG_DRM_GUD=m -CONFIG_DRM_HISI_HIBMC=m -CONFIG_DRM_HISI_KIRIN=m -CONFIG_DRM_POWERVR=m -CONFIG_DRM_IMX_DCSS=m -CONFIG_DRM_LIMA=m -CONFIG_DRM_MEDIATEK=m -CONFIG_DRM_MEDIATEK_DP=m -CONFIG_DRM_MEDIATEK_HDMI=m -CONFIG_DRM_MEDIATEK_HDMI_V2=m -CONFIG_DRM_MESON=m -CONFIG_DRM_MSM=m -CONFIG_DRM_MXSFB=m -CONFIG_DRM_IMX_LCDIF=m -CONFIG_DRM_NOUVEAU=m -CONFIG_DRM_PANEL_BOE_TV101WUM_NL6=m -CONFIG_DRM_PANEL_LVDS=m -CONFIG_DRM_PANEL_HIMAX_HX8279=m -CONFIG_DRM_PANEL_HIMAX_HX83112A=m -CONFIG_DRM_PANEL_HIMAX_HX83112B=m -CONFIG_DRM_PANEL_ILITEK_ILI9882T=m -CONFIG_DRM_PANEL_KHADAS_TS050=m -CONFIG_DRM_PANEL_MANTIX_MLAF057WE51=m -CONFIG_DRM_PANEL_NOVATEK_NT36672A=m -CONFIG_DRM_PANEL_NOVATEK_NT36672E=m -CONFIG_DRM_PANEL_NOVATEK_NT37801=m -CONFIG_DRM_PANEL_RAYDIUM_RM67191=m -CONFIG_DRM_PANEL_RAYDIUM_RM692E5=m -CONFIG_DRM_PANEL_SAMSUNG_ATNA33XC20=m -CONFIG_DRM_PANEL_SITRONIX_ST7703=m -CONFIG_DRM_PANEL_STARTEK_KD070FHFID015=m -CONFIG_DRM_PANEL_EDP=m -CONFIG_DRM_PANEL_SIMPLE=m -CONFIG_DRM_PANEL_SUMMIT=m -CONFIG_DRM_PANEL_TRULY_NT35597_WQXGA=m -CONFIG_DRM_PANEL_VISIONOX_VTDR6130=m -CONFIG_DRM_PANFROST=m -CONFIG_DRM_PANTHOR=m -CONFIG_DRM_PL111=m -CONFIG_DRM_RCAR_DU=m -CONFIG_DRM_RCAR_DW_HDMI=m -CONFIG_DRM_RZG2L_DU=m -CONFIG_DRM_ROCKCHIP=m -CONFIG_ROCKCHIP_VOP2=y -CONFIG_ROCKCHIP_ANALOGIX_DP=y -CONFIG_ROCKCHIP_CDN_DP=y -CONFIG_ROCKCHIP_DW_DP=y -CONFIG_ROCKCHIP_DW_HDMI=y -CONFIG_ROCKCHIP_DW_HDMI_QP=y -CONFIG_ROCKCHIP_DW_MIPI_DSI=y -CONFIG_ROCKCHIP_INNO_HDMI=y -CONFIG_ROCKCHIP_LVDS=y -CONFIG_DRM_STM=m -CONFIG_DRM_STM_LVDS=m -CONFIG_DRM_SUN4I=m -CONFIG_DRM_SIMPLEDRM=y -CONFIG_DRM_TEGRA=m -CONFIG_DRM_TIDSS=m -CONFIG_DRM_GM12U320=m -CONFIG_DRM_UDL=m -CONFIG_DRM_V3D=m -CONFIG_DRM_VC4=m -CONFIG_DRM_ZYNQMP_DPSUB=m -CONFIG_DRM_ZYNQMP_DPSUB_AUDIO=y -CONFIG_FB=y -CONFIG_FB_EFI=y -CONFIG_BACKLIGHT_CLASS_DEVICE=y -CONFIG_BACKLIGHT_PWM=m -CONFIG_BACKLIGHT_APPLE_DWI=m -CONFIG_BACKLIGHT_QCOM_WLED=m -CONFIG_BACKLIGHT_LP855X=m -CONFIG_BACKLIGHT_GPIO=m -CONFIG_LOGO=y -CONFIG_LOGO_LINUX_CLUT224_FILE="drivers/video/logo/logo_linux_clut224.ppm" -CONFIG_DRM_ACCEL=y -CONFIG_DRM_ACCEL_ANE=m -CONFIG_SOUND=m -CONFIG_SND=m -CONFIG_SND_UMP_LEGACY_RAWMIDI=y -CONFIG_SND_HRTIMER=m -CONFIG_SND_UTIMER=y -CONFIG_SND_SEQUENCER=m -CONFIG_SND_SEQ_DUMMY=m -CONFIG_SND_DUMMY=m -CONFIG_SND_ALOOP=m -CONFIG_SND_VIRMIDI=m -CONFIG_SND_HDA_TEGRA=m -CONFIG_SND_HDA_CODEC_HDMI=m -CONFIG_SND_USB_AUDIO=m -CONFIG_SND_USB_AUDIO_MIDI_V2=y -CONFIG_SND_USB_UA101=m -CONFIG_SND_USB_CAIAQ=m -CONFIG_SND_USB_CAIAQ_INPUT=y -CONFIG_SND_USB_6FIRE=m -CONFIG_SND_USB_HIFACE=m -CONFIG_SND_BCD2000=m -CONFIG_SND_USB_AUDIO_QMI=m -CONFIG_SND_USB_POD=m -CONFIG_SND_USB_PODHD=m -CONFIG_SND_USB_TONEPORT=m -CONFIG_SND_USB_VARIAX=m -CONFIG_SND_SOC=m -CONFIG_SND_SOC_USB=m -CONFIG_SND_SOC_APPLE_AOP_AUDIO=m -CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=m -CONFIG_SND_SOC_APPLE_MACAUDIO=m -CONFIG_SND_BCM2835_SOC_I2S=m -CONFIG_SND_SOC_FSL_ASRC=m -CONFIG_SND_SOC_FSL_MICFIL=m -CONFIG_SND_SOC_FSL_EASRC=m -CONFIG_SND_IMX_SOC=m -CONFIG_SND_SOC_IMX_SGTL5000=m -CONFIG_SND_SOC_FSL_ASOC_CARD=m -CONFIG_SND_SOC_IMX_AUDMIX=m -CONFIG_SND_SOC_MT8183=m -CONFIG_SND_SOC_MT8183_MT6358_TS3A227E_MAX98357A=m -CONFIG_SND_SOC_MT8183_DA7219_MAX98357A=m -CONFIG_SND_SOC_MT8188=m -CONFIG_SND_SOC_MT8188_MT6359=m -CONFIG_SND_SOC_MT8192=m -CONFIG_SND_SOC_MT8192_MT6359_RT1015_RT5682=m -CONFIG_SND_SOC_MT8195=m -CONFIG_SND_SOC_MT8195_MT6359=m -CONFIG_SND_SOC_MT8365=m -CONFIG_SND_SOC_MT8365_MT6357=m -CONFIG_SND_MESON_AXG_SOUND_CARD=m -CONFIG_SND_MESON_GX_SOUND_CARD=m -CONFIG_SND_SOC_QCOM=m -CONFIG_SND_SOC_APQ8016_SBC=m -CONFIG_SND_SOC_QDSP6_USB=m -CONFIG_SND_SOC_MSM8996=m -CONFIG_SND_SOC_SDM845=m -CONFIG_SND_SOC_SM8250=m -CONFIG_SND_SOC_SC8280XP=m -CONFIG_SND_SOC_SC7180=m -CONFIG_SND_SOC_SC7280=m -CONFIG_SND_SOC_X1E80100=m -CONFIG_SND_SOC_RCAR=m -CONFIG_SND_SOC_MSIOF=m -CONFIG_SND_SOC_RZ=m -CONFIG_SND_SOC_ROCKCHIP_I2S_TDM=m -CONFIG_SND_SOC_ROCKCHIP_SAI=m -CONFIG_SND_SOC_ROCKCHIP_SPDIF=m -CONFIG_SND_SOC_ROCKCHIP_RT5645=m -CONFIG_SND_SOC_RK3399_GRU_SOUND=m -CONFIG_SND_SOC_SAMSUNG=m -CONFIG_SND_SUN8I_CODEC=m -CONFIG_SND_SUN8I_CODEC_ANALOG=m -CONFIG_SND_SUN50I_CODEC_ANALOG=m -CONFIG_SND_SUN4I_I2S=m -CONFIG_SND_SUN4I_SPDIF=m -CONFIG_SND_SOC_TEGRA=m -CONFIG_SND_SOC_TEGRA210_AHUB=m -CONFIG_SND_SOC_TEGRA210_DMIC=m -CONFIG_SND_SOC_TEGRA210_I2S=m -CONFIG_SND_SOC_TEGRA210_OPE=m -CONFIG_SND_SOC_TEGRA186_ASRC=m -CONFIG_SND_SOC_TEGRA186_DSPK=m -CONFIG_SND_SOC_TEGRA210_ADMAIF=m -CONFIG_SND_SOC_TEGRA210_MVC=m -CONFIG_SND_SOC_TEGRA210_SFC=m -CONFIG_SND_SOC_TEGRA210_AMX=m -CONFIG_SND_SOC_TEGRA210_ADX=m -CONFIG_SND_SOC_TEGRA210_MIXER=m -CONFIG_SND_SOC_TEGRA_AUDIO_GRAPH_CARD=m -CONFIG_SND_SOC_J721E_EVM=m -CONFIG_SND_SOC_XILINX_I2S=m -CONFIG_SND_SOC_XILINX_AUDIO_FORMATTER=m -CONFIG_SND_SOC_XILINX_SPDIF=m -CONFIG_SND_SOC_SOF_TOPLEVEL=y -CONFIG_SND_SOC_SOF_OF=m -CONFIG_SND_SOC_SOF_MTK_TOPLEVEL=y -CONFIG_SND_SOC_SOF_MT8186=m -CONFIG_SND_SOC_SOF_MT8195=m -CONFIG_SND_SOC_AK4613=m -CONFIG_SND_SOC_AK4619=m -CONFIG_SND_SOC_DA7213=m -CONFIG_SND_SOC_ES7134=m -CONFIG_SND_SOC_ES7241=m -CONFIG_SND_SOC_ES8316=m -CONFIG_SND_SOC_ES8328_I2C=m -CONFIG_SND_SOC_GTM601=m -CONFIG_SND_SOC_MAX98090=m -CONFIG_SND_SOC_MSM8916_WCD_ANALOG=m -CONFIG_SND_SOC_MSM8916_WCD_DIGITAL=m -CONFIG_SND_SOC_RK3308=m -CONFIG_SND_SOC_RK817=m -CONFIG_SND_SOC_RT5640=m -CONFIG_SND_SOC_RT5659=m -CONFIG_SND_SOC_SIMPLE_AMPLIFIER=m -CONFIG_SND_SOC_SIMPLE_MUX=m -CONFIG_SND_SOC_TAS2552=m -CONFIG_SND_SOC_TAS571X=m -CONFIG_SND_SOC_TLV320AIC32X4_I2C=m -CONFIG_SND_SOC_TLV320AIC3X_I2C=m -CONFIG_SND_SOC_WCD9335=m -CONFIG_SND_SOC_WCD934X=m -CONFIG_SND_SOC_WCD937X_SDW=m -CONFIG_SND_SOC_WCD939X_SDW=m -CONFIG_SND_SOC_WM8524=m -CONFIG_SND_SOC_WM8904=m -CONFIG_SND_SOC_WM8960=m -CONFIG_SND_SOC_WM8962=m -CONFIG_SND_SOC_WM8978=m -CONFIG_SND_SOC_WSA881X=m -CONFIG_SND_SOC_WSA883X=m -CONFIG_SND_SOC_WSA884X=m -CONFIG_SND_SOC_NAU8822=m -CONFIG_SND_SOC_LPASS_WSA_MACRO=m -CONFIG_SND_SOC_LPASS_VA_MACRO=m -CONFIG_SND_SIMPLE_CARD=m -CONFIG_SND_AUDIO_GRAPH_CARD=m -CONFIG_SND_AUDIO_GRAPH_CARD2=m -CONFIG_HID_BATTERY_STRENGTH=y -CONFIG_HIDRAW=y -CONFIG_UHID=y -CONFIG_HID_A4TECH=m -CONFIG_HID_ACCUTOUCH=m -CONFIG_HID_ACRUX=m -CONFIG_HID_ACRUX_FF=y -CONFIG_HID_APPLE=y -CONFIG_HID_APPLEIR=m -CONFIG_HID_AUREAL=m -CONFIG_HID_BELKIN=m -CONFIG_HID_BETOP_FF=m -CONFIG_HID_BIGBEN_FF=m -CONFIG_HID_CHERRY=m -CONFIG_HID_CHICONY=m -CONFIG_HID_CORSAIR=m -CONFIG_HID_COUGAR=m -CONFIG_HID_MACALLY=m -CONFIG_HID_PRODIKEYS=m -CONFIG_HID_CMEDIA=m -CONFIG_HID_CP2112=m -CONFIG_HID_CREATIVE_SB0540=m -CONFIG_HID_CYPRESS=m -CONFIG_HID_DRAGONRISE=m -CONFIG_DRAGONRISE_FF=y -CONFIG_HID_EMS_FF=m -CONFIG_HID_ELAN=m -CONFIG_HID_ELECOM=m -CONFIG_HID_ELO=m -CONFIG_HID_EVISION=m -CONFIG_HID_EZKEY=m -CONFIG_HID_FT260=m -CONFIG_HID_GEMBIRD=m -CONFIG_HID_GFRM=m -CONFIG_HID_GLORIOUS=m -CONFIG_HID_HOLTEK=m -CONFIG_HOLTEK_FF=y -CONFIG_HID_GOOGLE_STADIA_FF=m -CONFIG_HID_VIVALDI=m -CONFIG_HID_GT683R=m -CONFIG_HID_KEYTOUCH=m -CONFIG_HID_KYE=m -CONFIG_HID_KYSONA=m -CONFIG_HID_UCLOGIC=m -CONFIG_HID_WALTOP=m -CONFIG_HID_VIEWSONIC=m -CONFIG_HID_XIAOMI=m -CONFIG_HID_GYRATION=m -CONFIG_HID_ICADE=m -CONFIG_HID_ITE=m -CONFIG_HID_JABRA=m -CONFIG_HID_TWINHAN=m -CONFIG_HID_KENSINGTON=m -CONFIG_HID_LCPOWER=m -CONFIG_HID_LENOVO=m -CONFIG_HID_LENOVO_GO=m -CONFIG_HID_LENOVO_GO_S=m -CONFIG_HID_LETSKETCH=m -CONFIG_HID_LOGITECH=m -CONFIG_HID_LOGITECH_DJ=m -CONFIG_LOGITECH_FF=y -CONFIG_LOGIRUMBLEPAD2_FF=y -CONFIG_LOGIG940_FF=y -CONFIG_HID_MAGICMOUSE=y -CONFIG_HID_MALTRON=m -CONFIG_HID_MAYFLASH=m -CONFIG_HID_MEGAWORLD_FF=m -CONFIG_HID_MICROSOFT=m -CONFIG_HID_MONTEREY=m -CONFIG_HID_MULTITOUCH=m -CONFIG_HID_NINTENDO=m -CONFIG_NINTENDO_FF=y -CONFIG_HID_NTI=m -CONFIG_HID_NTRIG=m -CONFIG_HID_NVIDIA_SHIELD=m -CONFIG_NVIDIA_SHIELD_FF=y -CONFIG_HID_ORTEK=m -CONFIG_HID_PANTHERLORD=m -CONFIG_PANTHERLORD_FF=y -CONFIG_HID_PENMOUNT=m -CONFIG_HID_PETALYNX=m -CONFIG_HID_PICOLCD=m -CONFIG_HID_PLANTRONICS=m -CONFIG_HID_PLAYSTATION=m -CONFIG_PLAYSTATION_FF=y -CONFIG_HID_PXRC=m -CONFIG_HID_RAPOO=m -CONFIG_HID_RAZER=m -CONFIG_HID_PRIMAX=m -CONFIG_HID_RETRODE=m -CONFIG_HID_ROCCAT=m -CONFIG_HID_SAITEK=m -CONFIG_HID_SAMSUNG=m -CONFIG_HID_SEMITEK=m -CONFIG_HID_SIGMAMICRO=m -CONFIG_HID_SONY=m -CONFIG_SONY_FF=y -CONFIG_HID_SPEEDLINK=m -CONFIG_HID_STEAM=m -CONFIG_STEAM_FF=y -CONFIG_HID_STEELSERIES=m -CONFIG_HID_SUNPLUS=m -CONFIG_HID_RMI=m -CONFIG_HID_GREENASIA=m -CONFIG_GREENASIA_FF=y -CONFIG_HID_SMARTJOYPLUS=m -CONFIG_SMARTJOYPLUS_FF=y -CONFIG_HID_TIVO=m -CONFIG_HID_TOPSEED=m -CONFIG_HID_TOPRE=m -CONFIG_HID_THINGM=m -CONFIG_HID_THRUSTMASTER=m -CONFIG_THRUSTMASTER_FF=y -CONFIG_HID_UDRAW_PS3=m -CONFIG_HID_U2FZERO=m -CONFIG_HID_UNIVERSAL_PIDFF=m -CONFIG_HID_WACOM=m -CONFIG_HID_WIIMOTE=m -CONFIG_HID_WINWING=m -CONFIG_HID_XINMO=m -CONFIG_HID_ZEROPLUS=m -CONFIG_ZEROPLUS_FF=y -CONFIG_HID_ZYDACRON=m -CONFIG_HID_ALPS=m -CONFIG_HID_MCP2221=m -CONFIG_HID_HUAWEI=m -CONFIG_I2C_HID_ACPI=m -CONFIG_I2C_HID_OF=m -CONFIG_I2C_HID_OF_ELAN=m -CONFIG_HID_PID=y -CONFIG_USB_HIDDEV=y -CONFIG_SPI_HID_APPLE_OF=m -CONFIG_HID_DOCKCHANNEL=y -CONFIG_USB=y -CONFIG_USB_OTG=y -CONFIG_USB_MON=m -CONFIG_USB_XHCI_HCD=y -CONFIG_USB_XHCI_PCI_RENESAS=m -CONFIG_USB_XHCI_RZV2M=y -CONFIG_USB_XHCI_SIDEBAND=y -CONFIG_USB_XHCI_TEGRA=y -CONFIG_USB_BRCMSTB=m -CONFIG_USB_EHCI_HCD=y -CONFIG_USB_EHCI_EXYNOS=y -CONFIG_USB_EHCI_HCD_PLATFORM=y -CONFIG_USB_OHCI_HCD=y -CONFIG_USB_OHCI_EXYNOS=y -CONFIG_USB_OHCI_HCD_PLATFORM=y -CONFIG_USB_RENESAS_USBHS_HCD=m -CONFIG_USB_RENESAS_USBHS=m -CONFIG_USB_PRINTER=m -CONFIG_USB_TMC=m -CONFIG_USB_STORAGE=y -CONFIG_USB_STORAGE_REALTEK=m -CONFIG_USB_STORAGE_DATAFAB=m -CONFIG_USB_STORAGE_FREECOM=m -CONFIG_USB_STORAGE_ISD200=m -CONFIG_USB_STORAGE_USBAT=m -CONFIG_USB_STORAGE_SDDR09=m -CONFIG_USB_STORAGE_SDDR55=m -CONFIG_USB_STORAGE_JUMPSHOT=m -CONFIG_USB_STORAGE_ALAUDA=m -CONFIG_USB_STORAGE_ONETOUCH=m -CONFIG_USB_STORAGE_KARMA=m -CONFIG_USB_STORAGE_CYPRESS_ATACB=m -CONFIG_USB_STORAGE_ENE_UB6250=m -CONFIG_USB_UAS=m -CONFIG_USB_MDC800=m -CONFIG_USB_MICROTEK=m -CONFIG_USBIP_CORE=m -CONFIG_USBIP_VHCI_HCD=m -CONFIG_USBIP_HOST=m -CONFIG_USB_CDNS_SUPPORT=m -CONFIG_USB_CDNS3=m -CONFIG_USB_CDNS3_HOST=y -CONFIG_USB_CDNS3_GADGET=y -# CONFIG_USB_CDNSP_PCI is not set -CONFIG_USB_MTU3=y -CONFIG_USB_MUSB_HDRC=y -CONFIG_USB_MUSB_SUNXI=y -CONFIG_USB_DWC3=y -CONFIG_USB_DWC2=y -CONFIG_USB_CHIPIDEA=y -CONFIG_USB_CHIPIDEA_UDC=y -CONFIG_USB_CHIPIDEA_HOST=y -CONFIG_USB_ISP1760=y -CONFIG_USB_SERIAL=m -CONFIG_USB_SERIAL_GENERIC=y -CONFIG_USB_SERIAL_SIMPLE=m -CONFIG_USB_SERIAL_AIRCABLE=m -CONFIG_USB_SERIAL_ARK3116=m -CONFIG_USB_SERIAL_BELKIN=m -CONFIG_USB_SERIAL_CH341=m -CONFIG_USB_SERIAL_WHITEHEAT=m -CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m -CONFIG_USB_SERIAL_CP210X=m -CONFIG_USB_SERIAL_CYPRESS_M8=m -CONFIG_USB_SERIAL_EMPEG=m -CONFIG_USB_SERIAL_FTDI_SIO=m -CONFIG_USB_SERIAL_VISOR=m -CONFIG_USB_SERIAL_IPAQ=m -CONFIG_USB_SERIAL_IR=m -CONFIG_USB_SERIAL_EDGEPORT=m -CONFIG_USB_SERIAL_EDGEPORT_TI=m -CONFIG_USB_SERIAL_F81232=m -CONFIG_USB_SERIAL_F8153X=m -CONFIG_USB_SERIAL_GARMIN=m -CONFIG_USB_SERIAL_IPW=m -CONFIG_USB_SERIAL_IUU=m -CONFIG_USB_SERIAL_KEYSPAN_PDA=m -CONFIG_USB_SERIAL_KEYSPAN=m -CONFIG_USB_SERIAL_KLSI=m -CONFIG_USB_SERIAL_KOBIL_SCT=m -CONFIG_USB_SERIAL_MCT_U232=m -CONFIG_USB_SERIAL_MOS7720=m -CONFIG_USB_SERIAL_MOS7840=m -CONFIG_USB_SERIAL_NAVMAN=m -CONFIG_USB_SERIAL_PL2303=m -CONFIG_USB_SERIAL_OTI6858=m -CONFIG_USB_SERIAL_QCAUX=m -CONFIG_USB_SERIAL_QUALCOMM=m -CONFIG_USB_SERIAL_SPCP8X5=m -CONFIG_USB_SERIAL_SAFE=m -CONFIG_USB_SERIAL_SAFE_PADDED=y -CONFIG_USB_SERIAL_SIERRAWIRELESS=m -CONFIG_USB_SERIAL_SYMBOL=m -CONFIG_USB_SERIAL_TI=m -CONFIG_USB_SERIAL_CYBERJACK=m -CONFIG_USB_SERIAL_OPTION=m -CONFIG_USB_SERIAL_OMNINET=m -CONFIG_USB_SERIAL_OPTICON=m -CONFIG_USB_SERIAL_XSENS_MT=m -CONFIG_USB_SERIAL_SSU100=m -CONFIG_USB_SERIAL_QT2=m -CONFIG_USB_SERIAL_UPD78F0730=m -CONFIG_USB_SERIAL_XR=m -CONFIG_USB_EMI62=m -CONFIG_USB_EMI26=m -CONFIG_USB_ADUTUX=m -CONFIG_USB_SEVSEG=m -CONFIG_USB_LEGOTOWER=m -CONFIG_USB_LCD=m -CONFIG_USB_IDMOUSE=m -CONFIG_USB_APPLEDISPLAY=m -CONFIG_USB_QCOM_EUD=m -CONFIG_APPLE_MFI_FASTCHARGE=m -CONFIG_USB_SISUSBVGA=m -CONFIG_USB_LD=m -CONFIG_USB_TRANCEVIBRATOR=m -CONFIG_USB_IOWARRIOR=m -CONFIG_USB_ISIGHTFW=m -CONFIG_USB_YUREX=m -CONFIG_USB_HSIC_USB3503=y -CONFIG_USB_CHAOSKEY=m -CONFIG_USB_ONBOARD_DEV=m -CONFIG_USB_ONBOARD_DEV_USB5744=y -CONFIG_NOP_USB_XCEIV=y -CONFIG_USB_MXS_PHY=m -CONFIG_USB_GADGET=y -CONFIG_USB_RENESAS_USBHS_UDC=m -CONFIG_USB_RZV2M_USB3DRD=y -CONFIG_USB_RENESAS_USB3=m -CONFIG_USB_TEGRA_XUDC=m -CONFIG_USB_CONFIGFS=m -CONFIG_USB_CONFIGFS_SERIAL=y -CONFIG_USB_CONFIGFS_ACM=y -CONFIG_USB_CONFIGFS_OBEX=y -CONFIG_USB_CONFIGFS_NCM=y -CONFIG_USB_CONFIGFS_ECM=y -CONFIG_USB_CONFIGFS_ECM_SUBSET=y -CONFIG_USB_CONFIGFS_RNDIS=y -CONFIG_USB_CONFIGFS_EEM=y -CONFIG_USB_CONFIGFS_MASS_STORAGE=y -CONFIG_USB_CONFIGFS_F_FS=y -CONFIG_USB_MASS_STORAGE=m -CONFIG_TYPEC=y -CONFIG_TYPEC_TCPM=m -CONFIG_TYPEC_TCPCI=m -CONFIG_TYPEC_RT1711H=m -CONFIG_TYPEC_MT6360=m -CONFIG_TYPEC_TCPCI_MAXIM=m -CONFIG_TYPEC_FUSB302=m -CONFIG_TYPEC_QCOM_PMIC=m -CONFIG_TYPEC_UCSI=m -CONFIG_UCSI_CCG=m -CONFIG_UCSI_PMIC_GLINK=m -CONFIG_TYPEC_TPS6598X=m -CONFIG_TYPEC_SN201202X=y -CONFIG_TYPEC_HD3SS3220=m -CONFIG_TYPEC_MUX_FSA4480=m -CONFIG_TYPEC_MUX_GPIO_SBU=m -CONFIG_TYPEC_MUX_IT5205=m -CONFIG_TYPEC_MUX_NB7VPQ904M=m -CONFIG_TYPEC_MUX_PS883X=m -CONFIG_TYPEC_MUX_PTN36502=m -CONFIG_TYPEC_MUX_WCD939X_USBSS=m -CONFIG_TYPEC_DP_ALTMODE=m -CONFIG_MMC=y -CONFIG_MMC_BLOCK_MINORS=32 -CONFIG_MMC_ARMMMCI=y -CONFIG_MMC_SDHCI=y -CONFIG_MMC_SDHCI_ACPI=y -CONFIG_MMC_SDHCI_PLTFM=y -CONFIG_MMC_SDHCI_OF_ARASAN=y -CONFIG_MMC_SDHCI_OF_ESDHC=y -CONFIG_MMC_SDHCI_OF_DWCMSHC=y -CONFIG_MMC_SDHCI_OF_SPARX5=y -CONFIG_MMC_SDHCI_CADENCE=y -CONFIG_MMC_SDHCI_ESDHC_IMX=y -CONFIG_MMC_SDHCI_TEGRA=y -CONFIG_MMC_SDHCI_F_SDH30=y -CONFIG_MMC_MESON_GX=y -CONFIG_MMC_SDHCI_MSM=y -CONFIG_MMC_SPI=y -CONFIG_MMC_SDHI=y -CONFIG_MMC_UNIPHIER=y -CONFIG_MMC_DW=y -CONFIG_MMC_DW_EXYNOS=y -CONFIG_MMC_DW_HI3798CV200=y -CONFIG_MMC_DW_K3=y -CONFIG_MMC_DW_ROCKCHIP=y -CONFIG_MMC_VUB300=m -CONFIG_MMC_USHC=m -CONFIG_MMC_REALTEK_USB=m -CONFIG_MMC_SUNXI=y -CONFIG_MMC_BCM2835=y -CONFIG_MMC_MTK=y -CONFIG_MMC_SDHCI_XENON=y -CONFIG_MMC_SDHCI_AM654=y -CONFIG_MMC_OWL=y -CONFIG_SCSI_UFSHCD=y -CONFIG_SCSI_UFS_BSG=y -CONFIG_SCSI_UFS_CRYPTO=y -CONFIG_SCSI_UFSHCD_PLATFORM=y -CONFIG_SCSI_UFS_CDNS_PLATFORM=m -CONFIG_SCSI_UFS_QCOM=m -CONFIG_SCSI_UFS_MEDIATEK=y -CONFIG_SCSI_UFS_HISI=y -CONFIG_SCSI_UFS_RENESAS=m -CONFIG_SCSI_UFS_TI_J721E=m -CONFIG_SCSI_UFS_EXYNOS=y -CONFIG_SCSI_UFS_ROCKCHIP=y -CONFIG_MEMSTICK=m -CONFIG_MSPRO_BLOCK=m -CONFIG_MEMSTICK_REALTEK_USB=m -CONFIG_LEDS_CLASS=y -CONFIG_LEDS_CLASS_FLASH=m -CONFIG_LEDS_CLASS_MULTICOLOR=m -CONFIG_LEDS_LM3692X=m -CONFIG_LEDS_PCA9532=m -CONFIG_LEDS_GPIO=y -CONFIG_LEDS_PWM=y -CONFIG_LEDS_SYSCON=y -CONFIG_LEDS_QCOM_FLASH=m -CONFIG_LEDS_QCOM_LPG=m -CONFIG_LEDS_TRIGGER_TIMER=y -CONFIG_LEDS_TRIGGER_DISK=y -CONFIG_LEDS_TRIGGER_HEARTBEAT=y -CONFIG_LEDS_TRIGGER_CPU=y -CONFIG_LEDS_TRIGGER_DEFAULT_ON=y -CONFIG_LEDS_TRIGGER_PANIC=y -CONFIG_ACCESSIBILITY=y -CONFIG_SPEAKUP=m -CONFIG_SPEAKUP_SYNTH_ACNTSA=m -CONFIG_SPEAKUP_SYNTH_APOLLO=m -CONFIG_SPEAKUP_SYNTH_AUDPTR=m -CONFIG_SPEAKUP_SYNTH_BNS=m -CONFIG_SPEAKUP_SYNTH_DECTLK=m -CONFIG_SPEAKUP_SYNTH_LTLK=m -CONFIG_SPEAKUP_SYNTH_SOFT=m -CONFIG_SPEAKUP_SYNTH_SPKOUT=m -CONFIG_SPEAKUP_SYNTH_TXPRT=m -CONFIG_INFINIBAND=m -CONFIG_INFINIBAND_USER_MAD=m -CONFIG_INFINIBAND_USER_ACCESS=m -CONFIG_RDMA_RXE=m -CONFIG_RDMA_SIW=m -CONFIG_EDAC=y -CONFIG_EDAC_GHES=y -CONFIG_EDAC_LAYERSCAPE=m -CONFIG_EDAC_ZYNQMP=m -CONFIG_EDAC_VERSAL=m -CONFIG_RTC_CLASS=y -CONFIG_RTC_DRV_DS1307=m -CONFIG_RTC_DRV_HYM8563=m -CONFIG_RTC_DRV_MAX77686=y -CONFIG_RTC_DRV_NVIDIA_VRS10=m -CONFIG_RTC_DRV_RK808=m -CONFIG_RTC_DRV_ISL1208=m -CONFIG_RTC_DRV_PCF85363=m -CONFIG_RTC_DRV_PCF8563=m -CONFIG_RTC_DRV_M41T80=m -CONFIG_RTC_DRV_BQ32K=m -CONFIG_RTC_DRV_RX8581=m -CONFIG_RTC_DRV_RV3028=m -CONFIG_RTC_DRV_RV8803=m -CONFIG_RTC_DRV_S5M=y -CONFIG_RTC_DRV_DS3232=y -CONFIG_RTC_DRV_PCF2127=m -CONFIG_RTC_DRV_PCF85063=m -CONFIG_RTC_DRV_DA9063=m -CONFIG_RTC_DRV_EFI=y -CONFIG_RTC_DRV_ZYNQMP=m -CONFIG_RTC_DRV_CROS_EC=y -CONFIG_RTC_DRV_FSL_FTM_ALARM=m -CONFIG_RTC_DRV_S3C=y -CONFIG_RTC_DRV_PL031=y -CONFIG_RTC_DRV_SUN6I=y -CONFIG_RTC_DRV_ARMADA38X=y -CONFIG_RTC_DRV_PM8XXX=m -CONFIG_RTC_DRV_TEGRA=y -CONFIG_RTC_DRV_SNVS=m -CONFIG_RTC_DRV_BBNSM=m -CONFIG_RTC_DRV_IMX_SC=m -CONFIG_RTC_DRV_MT6397=m -CONFIG_RTC_DRV_XGENE=y -CONFIG_RTC_DRV_TI_K3=m -CONFIG_RTC_DRV_RENESAS_RTCA3=m -CONFIG_RTC_DRV_APPLE_PMU=m -CONFIG_RTC_DRV_MACSMC=m -CONFIG_RTC_DRV_S32G=m -CONFIG_DMADEVICES=y -CONFIG_APPLE_SIO=m -CONFIG_DMA_BCM2835=y -CONFIG_DMA_SUN6I=m -CONFIG_FSL_EDMA=y -CONFIG_IMX_SDMA=m -CONFIG_K3_DMA=y -CONFIG_MV_XOR=y -CONFIG_MV_XOR_V2=y -CONFIG_OWL_DMA=y -CONFIG_PL330_DMA=y -CONFIG_TEGRA186_GPC_DMA=y -CONFIG_TEGRA20_APB_DMA=y -CONFIG_TEGRA210_ADMA=m -CONFIG_XILINX_DMA=m -CONFIG_XILINX_ZYNQMP_DMA=m -CONFIG_XILINX_ZYNQMP_DPDMA=m -CONFIG_MTK_UART_APDMA=m -CONFIG_QCOM_BAM_DMA=y -CONFIG_QCOM_GPI_DMA=m -CONFIG_QCOM_HIDMA_MGMT=y -CONFIG_QCOM_HIDMA=y -CONFIG_DW_EDMA=m -CONFIG_RCAR_DMAC=y -CONFIG_RENESAS_USB_DMAC=m -CONFIG_RZ_DMAC=y -CONFIG_TI_K3_UDMA=y -CONFIG_TI_K3_UDMA_GLUE_LAYER=y -CONFIG_STM32_DMA3=m -CONFIG_UDMABUF=y -CONFIG_DMABUF_HEAPS=y -CONFIG_DMABUF_HEAPS_SYSTEM=y -CONFIG_VFIO=y -CONFIG_VFIO_PCI=y -CONFIG_VIRTIO_PCI=y -CONFIG_VIRTIO_BALLOON=y -CONFIG_VIRTIO_MMIO=y -CONFIG_VHOST_NET=m -CONFIG_VHOST_VSOCK=m -CONFIG_XEN_GNTDEV=y -CONFIG_XEN_GRANT_DEV_ALLOC=y -CONFIG_GREYBUS=m -CONFIG_GREYBUS_BEAGLEPLAY=m -CONFIG_GPIB=m -CONFIG_GPIB_AGILENT_82357A=m -CONFIG_GPIB_NI_USB=m -CONFIG_GPIB_LPVO=m -CONFIG_STAGING=y -CONFIG_STAGING_MEDIA=y -CONFIG_VIDEO_MAX96712=m -CONFIG_VIDEO_MESON_VDEC=m -CONFIG_SND_BCM2835=m -CONFIG_CHROME_PLATFORMS=y -CONFIG_CROS_EC=y -CONFIG_CROS_EC_I2C=y -CONFIG_CROS_EC_RPMSG=m -CONFIG_CROS_EC_SPI=y -CONFIG_CROS_KBD_LED_BACKLIGHT=m -CONFIG_CROS_EC_CHARDEV=m -CONFIG_CROS_EC_TYPEC=m -CONFIG_CROS_TYPEC_SWITCH=m -CONFIG_EC_ACER_ASPIRE1=m -CONFIG_EC_HUAWEI_GAOKUN=m -CONFIG_EC_LENOVO_YOGA_C630=m -CONFIG_EC_LENOVO_THINKPAD_T14S=m -CONFIG_COMMON_CLK_RK808=y -CONFIG_COMMON_CLK_SCPI=y -CONFIG_COMMON_CLK_CS2000_CP=y -CONFIG_COMMON_CLK_FSL_SAI=y -CONFIG_COMMON_CLK_S2MPS11=y -CONFIG_COMMON_CLK_PWM=y -CONFIG_COMMON_CLK_RS9_PCIE=y -CONFIG_COMMON_CLK_VC3=y -CONFIG_COMMON_CLK_VC5=y -CONFIG_COMMON_CLK_BD718XX=m -CONFIG_CLK_RASPBERRYPI=y -CONFIG_CLK_IMX8MM=y -CONFIG_CLK_IMX8MN=y -CONFIG_CLK_IMX8MP=y -CONFIG_CLK_IMX8MQ=y -CONFIG_CLK_IMX8QXP=y -CONFIG_CLK_IMX8ULP=y -CONFIG_CLK_IMX93=y -CONFIG_CLK_IMX95_BLK_CTL=y -CONFIG_TI_SCI_CLK=y -CONFIG_COMMON_CLK_QCOM=y -CONFIG_CLK_ELIZA_DISPCC=m -CONFIG_CLK_ELIZA_GCC=y -CONFIG_CLK_ELIZA_TCSRCC=m -CONFIG_CLK_GLYMUR_DISPCC=m -CONFIG_CLK_GLYMUR_GCC=y -CONFIG_CLK_GLYMUR_TCSRCC=m -CONFIG_CLK_KAANAPALI_CAMCC=m -CONFIG_CLK_KAANAPALI_DISPCC=m -CONFIG_CLK_KAANAPALI_GCC=y -CONFIG_CLK_KAANAPALI_GPUCC=m -CONFIG_CLK_KAANAPALI_TCSRCC=m -CONFIG_CLK_KAANAPALI_VIDEOCC=m -CONFIG_CLK_X1E80100_CAMCC=m -CONFIG_CLK_X1E80100_DISPCC=m -CONFIG_CLK_X1E80100_GCC=y -CONFIG_CLK_X1E80100_GPUCC=m -CONFIG_CLK_X1E80100_TCSRCC=y -CONFIG_CLK_X1P42100_GPUCC=m -CONFIG_CLK_QCM2290_GPUCC=m -CONFIG_QCOM_A53PLL=y -CONFIG_QCOM_CLK_APCS_MSM8916=y -CONFIG_QCOM_CLK_APCC_MSM8996=y -CONFIG_QCOM_CLK_SMD_RPM=y -CONFIG_QCOM_CLK_RPMH=y -CONFIG_IPQ_APSS_6018=y -CONFIG_IPQ_CMN_PLL=m -CONFIG_IPQ_GCC_5018=y -CONFIG_IPQ_GCC_5210=y -CONFIG_IPQ_GCC_5332=y -CONFIG_IPQ_GCC_5424=y -CONFIG_IPQ_GCC_6018=y -CONFIG_IPQ_GCC_8074=y -CONFIG_IPQ_GCC_9574=y -CONFIG_IPQ_NSSCC_5424=m -CONFIG_IPQ_NSSCC_9574=m -CONFIG_MSM_GCC_8916=y -CONFIG_MSM_GCC_8953=y -CONFIG_MSM_MMCC_8994=m -CONFIG_MSM_GCC_8994=y -CONFIG_MSM_GCC_8996=y -CONFIG_MSM_MMCC_8996=m -CONFIG_MSM_GCC_8998=y -CONFIG_MSM_MMCC_8998=m -CONFIG_QCM_GCC_2290=y -CONFIG_QCM_DISPCC_2290=m -CONFIG_QCS_DISPCC_615=m -CONFIG_QCS_CAMCC_615=m -CONFIG_QCS_GCC_404=y -CONFIG_SA_CAMCC_8775P=m -CONFIG_QCS_GCC_8300=y -CONFIG_QCS_GCC_615=y -CONFIG_QCS_GPUCC_615=m -CONFIG_QCS_VIDEOCC_615=m -CONFIG_SC_CAMCC_7280=m -CONFIG_SC_CAMCC_8280XP=m -CONFIG_SA_DISPCC_8775P=m -CONFIG_SC_DISPCC_7280=m -CONFIG_SC_DISPCC_8280XP=m -CONFIG_SA_GCC_8775P=y -CONFIG_SA_GPUCC_8775P=m -CONFIG_SC_GCC_7180=y -CONFIG_SC_GCC_7280=y -CONFIG_SC_GCC_8180X=y -CONFIG_SC_GCC_8280XP=y -CONFIG_SC_GPUCC_7280=m -CONFIG_SC_GPUCC_8280XP=m -CONFIG_SC_LPASSCC_8280XP=m -CONFIG_SC_LPASS_CORECC_7280=m -CONFIG_SC_VIDEOCC_7280=m -CONFIG_SDM_CAMCC_845=m -CONFIG_QDU_GCC_1000=y -CONFIG_SDM_GPUCC_845=y -CONFIG_SDM_VIDEOCC_845=y -CONFIG_SDM_DISPCC_845=y -CONFIG_SDM_LPASSCC_845=m -CONFIG_SDX_GCC_75=y -CONFIG_SM_CAMCC_6350=m -CONFIG_SM_CAMCC_MILOS=m -CONFIG_SM_CAMCC_8250=m -CONFIG_SM_CAMCC_8550=m -CONFIG_SM_CAMCC_8650=m -CONFIG_SM_CAMCC_8750=m -CONFIG_SM_DISPCC_6115=m -CONFIG_SM_DISPCC_8250=y -CONFIG_SM_DISPCC_6350=m -CONFIG_SM_DISPCC_MILOS=m -CONFIG_SM_DISPCC_8450=m -CONFIG_SM_DISPCC_8550=m -CONFIG_SM_DISPCC_8750=m -CONFIG_SM_GCC_4450=y -CONFIG_SM_GCC_6115=y -CONFIG_SM_GCC_6350=y -CONFIG_SM_GCC_MILOS=y -CONFIG_SM_GCC_8350=y -CONFIG_SM_GCC_8450=y -CONFIG_SM_GCC_8550=y -CONFIG_SM_GCC_8650=y -CONFIG_SM_GCC_8750=y -CONFIG_SM_GPUCC_6115=m -CONFIG_SM_GPUCC_6350=m -CONFIG_SM_GPUCC_MILOS=m -CONFIG_SM_GPUCC_8150=y -CONFIG_SM_GPUCC_8250=y -CONFIG_SM_GPUCC_8350=m -CONFIG_SM_GPUCC_8450=m -CONFIG_SM_GPUCC_8550=m -CONFIG_SM_GPUCC_8650=m -CONFIG_SM_GPUCC_8750=m -CONFIG_SM_TCSRCC_8550=y -CONFIG_SM_TCSRCC_8650=y -CONFIG_SM_TCSRCC_8750=m -CONFIG_SA_VIDEOCC_8775P=m -CONFIG_SM_VIDEOCC_6350=m -CONFIG_SM_VIDEOCC_MILOS=m -CONFIG_SM_VIDEOCC_8250=y -CONFIG_SM_VIDEOCC_8550=m -CONFIG_SM_VIDEOCC_8750=m -CONFIG_QCOM_HFPLL=y -CONFIG_CLK_GFM_LPASS_SM8250=m -CONFIG_SM_VIDEOCC_8450=m -CONFIG_CLK_RCAR_USB2_CLOCK_SEL=y -CONFIG_CLK_RENESAS_VBATTB=m -CONFIG_EXYNOS_ACPM_CLK=m -CONFIG_CLK_SOPHGO_CV1800=y -CONFIG_XILINX_VCU=m -CONFIG_COMMON_CLK_XLNX_CLKWZRD=m -CONFIG_HWSPINLOCK=y -CONFIG_HWSPINLOCK_OMAP=m -CONFIG_HWSPINLOCK_QCOM=y -CONFIG_TEGRA186_TIMER=y -CONFIG_CLKSRC_STM32_LP=y -CONFIG_RENESAS_OSTM=y -CONFIG_ARM_MHU=y -CONFIG_EXYNOS_MBOX=m -CONFIG_IMX_MBOX=y -CONFIG_PLATFORM_MHU=y -CONFIG_OMAP2PLUS_MBOX=m -CONFIG_BCM2835_MBOX=y -CONFIG_QCOM_APCS_IPC=y -CONFIG_TEGRA_HSP_MBOX=y -CONFIG_MTK_ADSP_MBOX=m -CONFIG_QCOM_CPUCP_MBOX=m -CONFIG_QCOM_IPCC=y -CONFIG_CIX_MBOX=y -CONFIG_ARM_SMMU=y -CONFIG_ARM_SMMU_V3=y -CONFIG_QCOM_IOMMU=y -CONFIG_ROCKCHIP_IOMMU=y -CONFIG_TEGRA_IOMMU_SMMU=y -CONFIG_APPLE_DART=y -CONFIG_MTK_IOMMU=y -CONFIG_REMOTEPROC=y -CONFIG_IMX_REMOTEPROC=y -CONFIG_MTK_SCP=m -CONFIG_QCOM_Q6V5_ADSP=m -CONFIG_QCOM_Q6V5_MSS=m -CONFIG_QCOM_Q6V5_PAS=m -CONFIG_QCOM_SYSMON=m -CONFIG_QCOM_WCNSS_PIL=m -CONFIG_TI_K3_DSP_REMOTEPROC=m -CONFIG_TI_K3_M4_REMOTEPROC=m -CONFIG_TI_K3_R5_REMOTEPROC=m -CONFIG_RPMSG_CHAR=m -CONFIG_RPMSG_CTRL=m -CONFIG_RPMSG_QCOM_GLINK_RPM=y -CONFIG_RPMSG_QCOM_GLINK_SMEM=m -CONFIG_RPMSG_QCOM_SMD=y -CONFIG_RPMSG_VIRTIO=y -CONFIG_SOUNDWIRE=m -CONFIG_SOUNDWIRE_QCOM=m -CONFIG_APPLE_DOCKCHANNEL=y -CONFIG_APPLE_PMGR_MISC=y -CONFIG_APPLE_RTKIT_HELPER=y -CONFIG_APPLE_SART=y -CONFIG_APPLE_CLPC=m -CONFIG_FSL_DPAA=y -CONFIG_FSL_MC_DPIO=y -CONFIG_FSL_RCPM=y -CONFIG_MTK_CMDQ=m -CONFIG_MTK_DEVAPC=m -CONFIG_MTK_PMIC_WRAP=y -CONFIG_MTK_SVS=m -CONFIG_QCOM_AOSS_QMP=y -CONFIG_QCOM_COMMAND_DB=y -CONFIG_QCOM_GENI_SE=y -CONFIG_QCOM_LLCC=m -CONFIG_QCOM_OCMEM=m -CONFIG_QCOM_PMIC_GLINK=m -CONFIG_QCOM_RMTFS_MEM=m -CONFIG_QCOM_RPMH=y -CONFIG_QCOM_SMEM=y -CONFIG_QCOM_SMD_RPM=y -CONFIG_QCOM_SMP2P=y -CONFIG_QCOM_SMSM=y -CONFIG_QCOM_SOCINFO=m -CONFIG_QCOM_SPM=m -CONFIG_QCOM_STATS=m -CONFIG_QCOM_WCNSS_CTRL=m -CONFIG_QCOM_APR=m -CONFIG_QCOM_ICC_BWMON=m -CONFIG_QCOM_PBS=m -CONFIG_ROCKCHIP_IODOMAIN=y -CONFIG_TI_PRUSS=m -CONFIG_OWL_PM_DOMAINS=y -CONFIG_RASPBERRYPI_POWER=y -CONFIG_IMX_SCU_PD=y -CONFIG_QCOM_CPR=y -CONFIG_QCOM_RPMHPD=y -CONFIG_QCOM_RPMPD=y -CONFIG_TI_SCI_PM_DOMAINS=y -CONFIG_ARM_IMX_BUS_DEVFREQ=y -CONFIG_ARM_IMX8M_DDRC_DEVFREQ=m -CONFIG_ARM_MEDIATEK_CCI_DEVFREQ=m -CONFIG_PM_DEVFREQ_EVENT=y -CONFIG_DEVFREQ_EVENT_ROCKCHIP_DFI=m -CONFIG_EXTCON_PTN5150=m -CONFIG_EXTCON_USB_GPIO=y -CONFIG_EXTCON_USBC_CROS_EC=y -CONFIG_FSL_IFC=y -CONFIG_RENESAS_RPCIF=m -CONFIG_STM32_OMM=m -CONFIG_DLN2_ADC=m -CONFIG_EXYNOS_ADC=m -CONFIG_IMX8QXP_ADC=m -CONFIG_IMX93_ADC=m -CONFIG_MAX9611=m -CONFIG_MEDIATEK_MT6359_AUXADC=m -CONFIG_MEDIATEK_MT6577_AUXADC=m -CONFIG_QCOM_SPMI_VADC=m -CONFIG_QCOM_SPMI_ADC5=m -CONFIG_ROCKCHIP_SARADC=m -CONFIG_RZG2L_ADC=m -CONFIG_RZT2H_ADC=m -CONFIG_SOPHGO_CV1800B_ADC=m -CONFIG_TI_ADS1015=m -CONFIG_TI_AM335X_ADC=m -CONFIG_XILINX_XADC=m -CONFIG_XILINX_AMS=m -CONFIG_IIO_AOP_SENSOR_ALS=m -CONFIG_IIO_CROS_EC_SENSORS_CORE=m -CONFIG_IIO_CROS_EC_SENSORS=m -CONFIG_IIO_ST_LSM6DSX=m -CONFIG_IIO_CROS_EC_LIGHT_PROX=m -CONFIG_SENSORS_ISL29018=m -CONFIG_VCNL4000=m -CONFIG_IIO_ST_MAGN_3AXIS=m -CONFIG_IIO_STM32_TIMER_TRIGGER=m -CONFIG_IIO_CROS_EC_BARO=m -CONFIG_MPL3115=m -CONFIG_PWM=y -CONFIG_PWM_ADP5585=m -CONFIG_PWM_APPLE=m -CONFIG_PWM_BCM2835=m -CONFIG_PWM_BRCMSTB=m -CONFIG_PWM_CROS_EC=m -CONFIG_PWM_IMX27=m -CONFIG_PWM_MEDIATEK=m -CONFIG_PWM_MESON=m -CONFIG_PWM_MTK_DISP=m -CONFIG_PWM_RENESAS_RCAR=m -CONFIG_PWM_RENESAS_RZG2L_GPT=m -CONFIG_PWM_RENESAS_RZ_MTU3=m -CONFIG_PWM_RENESAS_TPU=m -CONFIG_PWM_ROCKCHIP=y -CONFIG_PWM_SAMSUNG=y -CONFIG_PWM_SL28CPLD=m -CONFIG_PWM_STM32=m -CONFIG_PWM_SUN4I=m -CONFIG_PWM_TEGRA=m -CONFIG_PWM_TIECAP=m -CONFIG_PWM_TIEHRPWM=m -CONFIG_PWM_VISCONTI=m -CONFIG_PWM_XILINX=m -CONFIG_SL28CPLD_INTC=y -CONFIG_XILINX_INTC=y -CONFIG_QCOM_PDC=y -CONFIG_QCOM_MPM=y -CONFIG_TI_SCI_INTR_IRQCHIP=y -CONFIG_TI_SCI_INTA_IRQCHIP=y -CONFIG_RESET_APPLE_CIO=y -CONFIG_RESET_GPIO=m -CONFIG_RESET_IMX7=y -CONFIG_RESET_QCOM_AOSS=y -CONFIG_RESET_QCOM_PDC=m -CONFIG_RESET_RZG2L_USBPHY_CTRL=y -CONFIG_RESET_RZV2H_USB2PHY=m -CONFIG_RESET_TI_SCI=y -CONFIG_PHY_CAN_TRANSCEIVER=m -CONFIG_PHY_NXP_PTN3222=m -CONFIG_PHY_SNPS_EUSB2=m -CONFIG_PHY_XGENE=y -CONFIG_PHY_SUN4I_USB=y -CONFIG_PHY_APPLE_ATC=y -CONFIG_PHY_APPLE_DPTX=m -CONFIG_PHY_CADENCE_TORRENT=m -CONFIG_PHY_CADENCE_DPHY=m -CONFIG_PHY_CADENCE_DPHY_RX=m -CONFIG_PHY_CADENCE_SIERRA=m -CONFIG_PHY_CADENCE_SALVO=m -CONFIG_PHY_FSL_IMX8MQ_USB=m -CONFIG_PHY_MIXEL_MIPI_DPHY=m -CONFIG_PHY_FSL_IMX8M_PCIE=y -CONFIG_PHY_HI6220_USB=y -CONFIG_PHY_HISTB_COMBPHY=y -CONFIG_PHY_HISI_INNO_USB2=y -CONFIG_PHY_MVEBU_CP110_COMPHY=y -CONFIG_PHY_MTK_PCIE=m -CONFIG_PHY_MTK_TPHY=y -CONFIG_PHY_MTK_HDMI=m -CONFIG_PHY_MTK_MIPI_DSI=m -CONFIG_PHY_MTK_DP=m -CONFIG_PHY_QCOM_EDP=m -CONFIG_PHY_QCOM_PCIE2=m -CONFIG_PHY_QCOM_QMP=m -CONFIG_PHY_QCOM_QUSB2=m -CONFIG_PHY_QCOM_EUSB2_REPEATER=m -CONFIG_PHY_QCOM_M31_USB=m -CONFIG_PHY_QCOM_M31_EUSB=m -CONFIG_PHY_QCOM_USB_HS=m -CONFIG_PHY_QCOM_USB_SNPS_FEMTO_V2=m -CONFIG_PHY_QCOM_USB_HS_28NM=m -CONFIG_PHY_QCOM_USB_SS=m -CONFIG_PHY_QCOM_SGMII_ETH=m -CONFIG_PHY_R8A779F0_ETHERNET_SERDES=y -CONFIG_PHY_RCAR_GEN3_PCIE=y -CONFIG_PHY_RCAR_GEN3_USB2=y -CONFIG_PHY_RCAR_GEN3_USB3=m -CONFIG_PHY_RZ_G3E_USB3=m -CONFIG_PHY_ROCKCHIP_EMMC=y -CONFIG_PHY_ROCKCHIP_INNO_HDMI=m -CONFIG_PHY_ROCKCHIP_INNO_USB2=y -CONFIG_PHY_ROCKCHIP_INNO_DSIDPHY=m -CONFIG_PHY_ROCKCHIP_NANENG_COMBO_PHY=m -CONFIG_PHY_ROCKCHIP_PCIE=m -CONFIG_PHY_ROCKCHIP_SAMSUNG_HDPTX=m -CONFIG_PHY_ROCKCHIP_SNPS_PCIE3=y -CONFIG_PHY_ROCKCHIP_TYPEC=y -CONFIG_PHY_ROCKCHIP_USBDP=m -CONFIG_PHY_SAMSUNG_UFS=y -CONFIG_PHY_EXYNOS5_USBDRD=m -CONFIG_PHY_UNIPHIER_USB2=y -CONFIG_PHY_UNIPHIER_USB3=y -CONFIG_PHY_TEGRA_XUSB=y -CONFIG_OMAP_USB2=m -CONFIG_PHY_AM654_SERDES=m -CONFIG_PHY_J721E_WIZ=m -CONFIG_PHY_XILINX_ZYNQMP=m -CONFIG_ARM_CCI_PMU=m -CONFIG_ARM_CCN=m -CONFIG_ARM_CMN=m -CONFIG_ARM_SMMU_V3_PMU=m -CONFIG_ARM_DSU_PMU=m -CONFIG_FSL_IMX8_DDR_PMU=m -CONFIG_QCOM_L2_PMU=y -CONFIG_QCOM_L3_PMU=y -CONFIG_ARM_SPE_PMU=m -CONFIG_ARM_DMC620_PMU=m -CONFIG_APPLE_M1_CPU_PMU=y -CONFIG_HISI_PMU=y -CONFIG_ARM_CORESIGHT_PMU_ARCH_SYSTEM_PMU=m -CONFIG_NVIDIA_CORESIGHT_PMU_ARCH_SYSTEM_PMU=m -CONFIG_MESON_DDR_PMU=m -CONFIG_NVMEM_LAYOUT_SL28_VPD=m -CONFIG_NVMEM_APPLE_EFUSES=m -CONFIG_NVMEM_APPLE_SPMI=y -CONFIG_NVMEM_IMX_OCOTP=y -CONFIG_NVMEM_IMX_OCOTP_ELE=m -CONFIG_NVMEM_IMX_OCOTP_SCU=y -CONFIG_NVMEM_LAYERSCAPE_SFP=m -CONFIG_NVMEM_MTK_EFUSE=y -CONFIG_NVMEM_QCOM_QFPROM=y -CONFIG_NVMEM_QCOM_SEC_QFPROM=m -CONFIG_NVMEM_RMEM=m -CONFIG_NVMEM_ROCKCHIP_EFUSE=y -CONFIG_NVMEM_ROCKCHIP_OTP=y -CONFIG_NVMEM_SNVS_LPGPR=y -CONFIG_NVMEM_SPMI_SDAM=m -CONFIG_NVMEM_SUNXI_SID=y -CONFIG_NVMEM_UNIPHIER_EFUSE=y -CONFIG_NVMEM_ZYNQMP=m -CONFIG_FPGA=y -CONFIG_FPGA_MGR_ALTERA_CVP=m -CONFIG_FPGA_MGR_STRATIX10_SOC=m -CONFIG_FPGA_BRIDGE=m -CONFIG_ALTERA_FREEZE_BRIDGE=m -CONFIG_XILINX_PR_DECOUPLER=m -CONFIG_FPGA_REGION=m -CONFIG_OF_FPGA_REGION=m -CONFIG_FPGA_MGR_ZYNQMP_FPGA=m -CONFIG_FPGA_MGR_VERSAL_FPGA=m -CONFIG_TEE=y -CONFIG_OPTEE=y -CONFIG_QCOMTEE=m -CONFIG_MUX_APPLE_DPXBAR=y -CONFIG_MUX_GPIO=m -CONFIG_MUX_MMIO=y -CONFIG_SLIM_QCOM_NGD_CTRL=m -CONFIG_INTERCONNECT_IMX=y -CONFIG_INTERCONNECT_IMX8MM=m -CONFIG_INTERCONNECT_IMX8MN=m -CONFIG_INTERCONNECT_IMX8MQ=m -CONFIG_INTERCONNECT_IMX8MP=y -CONFIG_INTERCONNECT_QCOM=y -CONFIG_INTERCONNECT_QCOM_ELIZA=y -CONFIG_INTERCONNECT_QCOM_GLYMUR=y -CONFIG_INTERCONNECT_QCOM_KAANAPALI=y -CONFIG_INTERCONNECT_QCOM_MSM8916=m -CONFIG_INTERCONNECT_QCOM_MSM8953=y -CONFIG_INTERCONNECT_QCOM_MSM8996=y -CONFIG_INTERCONNECT_QCOM_OSM_L3=m -CONFIG_INTERCONNECT_QCOM_QCM2290=y -CONFIG_INTERCONNECT_QCOM_QCS404=m -CONFIG_INTERCONNECT_QCOM_QCS615=y -CONFIG_INTERCONNECT_QCOM_QCS8300=y -CONFIG_INTERCONNECT_QCOM_QDU1000=y -CONFIG_INTERCONNECT_QCOM_SA8775P=y -CONFIG_INTERCONNECT_QCOM_SC7180=y -CONFIG_INTERCONNECT_QCOM_SC7280=y -CONFIG_INTERCONNECT_QCOM_SC8180X=y -CONFIG_INTERCONNECT_QCOM_SC8280XP=y -CONFIG_INTERCONNECT_QCOM_SDM845=y -CONFIG_INTERCONNECT_QCOM_SDX75=y -CONFIG_INTERCONNECT_QCOM_SM6115=y -CONFIG_INTERCONNECT_QCOM_SM6350=y -CONFIG_INTERCONNECT_QCOM_MILOS=y -CONFIG_INTERCONNECT_QCOM_SM8150=y -CONFIG_INTERCONNECT_QCOM_SM8250=y -CONFIG_INTERCONNECT_QCOM_SM8350=y -CONFIG_INTERCONNECT_QCOM_SM8450=y -CONFIG_INTERCONNECT_QCOM_SM8550=y -CONFIG_INTERCONNECT_QCOM_SM8650=y -CONFIG_INTERCONNECT_QCOM_SM8750=y -CONFIG_INTERCONNECT_QCOM_X1E80100=y -CONFIG_COUNTER=m -CONFIG_RZ_MTU3_CNT=m -CONFIG_STM32_TIMER_CNT=m -CONFIG_TI_EQEP=m -CONFIG_HTE=y -CONFIG_HTE_TEGRA194=y -CONFIG_HTE_TEGRA194_TEST=m -CONFIG_EXT4_FS=y -CONFIG_EXT4_FS_POSIX_ACL=y -CONFIG_EXT4_FS_SECURITY=y -CONFIG_XFS_FS=m -CONFIG_XFS_SUPPORT_V4=y -CONFIG_XFS_SUPPORT_ASCII_CI=y -CONFIG_XFS_QUOTA=y -CONFIG_XFS_POSIX_ACL=y -CONFIG_XFS_RT=y -CONFIG_BTRFS_FS=y -CONFIG_BTRFS_FS_POSIX_ACL=y -CONFIG_F2FS_FS=m -CONFIG_F2FS_FS_SECURITY=y -CONFIG_F2FS_FS_COMPRESSION=y -CONFIG_F2FS_UNFAIR_RWSEM=y -CONFIG_FANOTIFY=y -CONFIG_FANOTIFY_ACCESS_PERMISSIONS=y -CONFIG_QUOTA=y -CONFIG_AUTOFS_FS=y -CONFIG_FUSE_FS=m -CONFIG_CUSE=m -CONFIG_OVERLAY_FS=m -CONFIG_ISO9660_FS=m -CONFIG_JOLIET=y -CONFIG_ZISOFS=y -CONFIG_UDF_FS=m -CONFIG_MSDOS_FS=m -CONFIG_VFAT_FS=y -CONFIG_EXFAT_FS=m -CONFIG_NTFS3_FS=m -CONFIG_NTFS3_LZX_XPRESS=y -CONFIG_NTFS3_FS_POSIX_ACL=y -CONFIG_TMPFS_POSIX_ACL=y -CONFIG_HUGETLBFS=y -CONFIG_EFIVAR_FS=y -CONFIG_HFS_FS=m -CONFIG_HFSPLUS_FS=m -CONFIG_UBIFS_FS=m -CONFIG_SQUASHFS=y -CONFIG_SQUASHFS_XATTR=y -CONFIG_SQUASHFS_LZ4=y -CONFIG_SQUASHFS_LZO=y -CONFIG_SQUASHFS_XZ=y -CONFIG_SQUASHFS_ZSTD=y -CONFIG_PSTORE_RAM=m -CONFIG_EROFS_FS=m -CONFIG_EROFS_FS_ZIP_DEFLATE=y -CONFIG_EROFS_FS_ZIP_ZSTD=y -CONFIG_EROFS_FS_ZIP_ACCEL=y -CONFIG_NFS_FS=y -CONFIG_NFS_V4=y -CONFIG_NFS_V4_2=y -CONFIG_ROOT_NFS=y -CONFIG_CIFS=m -CONFIG_CIFS_UPCALL=y -CONFIG_CIFS_XATTR=y -CONFIG_CIFS_POSIX=y -CONFIG_CIFS_DFS_UPCALL=y -CONFIG_CIFS_SWN_UPCALL=y -CONFIG_9P_FS=y -CONFIG_NLS_CODEPAGE_437=y -CONFIG_NLS_CODEPAGE_737=m -CONFIG_NLS_CODEPAGE_775=m -CONFIG_NLS_CODEPAGE_850=m -CONFIG_NLS_CODEPAGE_852=m -CONFIG_NLS_CODEPAGE_855=m -CONFIG_NLS_CODEPAGE_857=m -CONFIG_NLS_CODEPAGE_860=m -CONFIG_NLS_CODEPAGE_861=m -CONFIG_NLS_CODEPAGE_862=m -CONFIG_NLS_CODEPAGE_863=m -CONFIG_NLS_CODEPAGE_864=m -CONFIG_NLS_CODEPAGE_865=m -CONFIG_NLS_CODEPAGE_866=m -CONFIG_NLS_CODEPAGE_869=m -CONFIG_NLS_CODEPAGE_936=m -CONFIG_NLS_CODEPAGE_950=m -CONFIG_NLS_CODEPAGE_932=m -CONFIG_NLS_CODEPAGE_949=m -CONFIG_NLS_CODEPAGE_874=m -CONFIG_NLS_ISO8859_8=m -CONFIG_NLS_CODEPAGE_1250=m -CONFIG_NLS_CODEPAGE_1251=m -CONFIG_NLS_ASCII=m -CONFIG_NLS_ISO8859_1=y -CONFIG_NLS_ISO8859_2=m -CONFIG_NLS_ISO8859_3=m -CONFIG_NLS_ISO8859_4=m -CONFIG_NLS_ISO8859_5=m -CONFIG_NLS_ISO8859_6=m -CONFIG_NLS_ISO8859_7=m -CONFIG_NLS_ISO8859_9=m -CONFIG_NLS_ISO8859_13=m -CONFIG_NLS_ISO8859_14=m -CONFIG_NLS_ISO8859_15=m -CONFIG_NLS_KOI8_R=m -CONFIG_NLS_KOI8_U=m -CONFIG_NLS_MAC_ROMAN=m -CONFIG_NLS_MAC_CELTIC=m -CONFIG_NLS_MAC_CENTEURO=m -CONFIG_NLS_MAC_CROATIAN=m -CONFIG_NLS_MAC_CYRILLIC=m -CONFIG_NLS_MAC_GAELIC=m -CONFIG_NLS_MAC_GREEK=m -CONFIG_NLS_MAC_ICELAND=m -CONFIG_NLS_MAC_INUIT=m -CONFIG_NLS_MAC_ROMANIAN=m -CONFIG_NLS_MAC_TURKISH=m -CONFIG_PERSISTENT_KEYRINGS=y -CONFIG_TRUSTED_KEYS=y -# CONFIG_TRUSTED_KEYS_TPM is not set -# CONFIG_TRUSTED_KEYS_TEE is not set -CONFIG_SECURITY=y -CONFIG_SECURITY_YAMA=y -CONFIG_SECURITY_LANDLOCK=y -CONFIG_CRYPTO_USER=y -CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_BENCHMARK=m -CONFIG_CRYPTO_ECDH=y -CONFIG_CRYPTO_BLOWFISH=m -CONFIG_CRYPTO_CAST5=m -CONFIG_CRYPTO_CAST6=m -CONFIG_CRYPTO_SERPENT=m -CONFIG_CRYPTO_TWOFISH=m -CONFIG_CRYPTO_ADIANTUM=m -CONFIG_CRYPTO_HCTR2=m -CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_AEGIS128=m -CONFIG_CRYPTO_CCM=y -CONFIG_CRYPTO_GCM=y -CONFIG_CRYPTO_ECHAINIV=y -CONFIG_CRYPTO_BLAKE2B=m -CONFIG_CRYPTO_MD4=m -CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_STREEBOG=m -CONFIG_CRYPTO_WP512=m -CONFIG_CRYPTO_XCBC=m -CONFIG_CRYPTO_XXHASH=m -CONFIG_CRYPTO_CRC32C=m -CONFIG_CRYPTO_CRC32=m -CONFIG_CRYPTO_LZ4HC=m -CONFIG_CRYPTO_USER_API_HASH=m -CONFIG_CRYPTO_USER_API_SKCIPHER=m -CONFIG_CRYPTO_USER_API_RNG=m -CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_GHASH_ARM64_CE=y -CONFIG_CRYPTO_AES_ARM64_BS=m -CONFIG_CRYPTO_AES_ARM64_CE_CCM=y -CONFIG_CRYPTO_DEV_SUN8I_CE=m -CONFIG_CRYPTO_DEV_FSL_CAAM=m -CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM=m -CONFIG_CRYPTO_DEV_QCE=m -CONFIG_CRYPTO_DEV_QCOM_RNG=m -CONFIG_CRYPTO_DEV_TEGRA=m -CONFIG_CRYPTO_DEV_ZYNQMP_AES=m -CONFIG_CRYPTO_DEV_ZYNQMP_SHA3=m -CONFIG_CRYPTO_DEV_CCREE=m -CONFIG_CRYPTO_DEV_HISI_SEC2=m -CONFIG_CRYPTO_DEV_HISI_ZIP=m -CONFIG_CRYPTO_DEV_HISI_HPRE=m -CONFIG_CRYPTO_DEV_SA2UL=m -CONFIG_DMA_RESTRICTED_POOL=y -CONFIG_CMA_SIZE_MBYTES=32 -CONFIG_PRINTK_TIME=y -CONFIG_DYNAMIC_DEBUG=y -CONFIG_DEBUG_KERNEL=y -CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y -CONFIG_DEBUG_INFO_REDUCED=y -CONFIG_MAGIC_SYSRQ=y -CONFIG_DEBUG_FS=y -CONFIG_ENABLE_DEFAULT_TRACERS=y -CONFIG_CORESIGHT=m -CONFIG_CORESIGHT_LINK_AND_SINK_TMC=m -CONFIG_CORESIGHT_CATU=m -CONFIG_CORESIGHT_SINK_TPIU=m -CONFIG_CORESIGHT_SINK_ETBV10=m -CONFIG_CORESIGHT_STM=m -CONFIG_CORESIGHT_CPU_DEBUG=m -CONFIG_CORESIGHT_CTI=m -CONFIG_ARM64_USER4K_EXPERIMENTAL=y -CONFIG_MEMTEST=y diff --git a/arch/arm64/configs/j700-device-parity.config b/arch/arm64/configs/j700-device-parity.config deleted file mode 100644 index 55bce7a9ac0af..0000000000000 --- a/arch/arm64/configs/j700-device-parity.config +++ /dev/null @@ -1,1053 +0,0 @@ -# J700 (MacBook Neo, T8140) device-parity fragment. -# -# Brings the J700 configuration to device and feature parity with a general-purpose -# distribution kernel (reference: Fedora Asahi Remix 44, 7.1.13-402.asahi.fc44.aarch64+16k): -# drivers for devices that can attach over USB or Bluetooth, and common protocols, -# filesystems and features. Most entries are modules. Nothing here changes an option -# that defconfig, asahi.config, j700-radio.config or j700.config set. -# -# Merge after j700.config; see that file for the full sequence. - -# Prerequisites of the entries below: nftables core, hidraw, uinput and the ALSA sequencer -CONFIG_NF_TABLES=m -CONFIG_NF_TABLES_INET=y -CONFIG_NF_TABLES_IPV4=y -CONFIG_NF_TABLES_IPV6=y -CONFIG_NF_TABLES_NETDEV=y -CONFIG_NFT_CT=m -CONFIG_NFT_LOG=m -CONFIG_NFT_LIMIT=m -CONFIG_NFT_MASQ=m -CONFIG_NFT_NAT=m -CONFIG_NFT_REJECT=m -CONFIG_NFT_REJECT_INET=m -CONFIG_NFT_FIB_INET=m -CONFIG_NFT_FIB_IPV4=m -CONFIG_NFT_FIB_IPV6=m -CONFIG_HIDRAW=y -CONFIG_INPUT_UINPUT=m -CONFIG_SND_SEQUENCER=m -CONFIG_SND_HRTIMER=m -CONFIG_SND_SEQ_DUMMY=m -CONFIG_SND_SEQ_HRTIMER_DEFAULT=y - -# HID / input / game controllers (USB + Bluetooth HID devices, xpad, joydev, USB tablets/touchscreens) -CONFIG_DRAGONRISE_FF=y -CONFIG_GREENASIA_FF=y -CONFIG_HID_A4TECH=m -CONFIG_HID_ACCUTOUCH=m -CONFIG_HID_ACRUX=m -CONFIG_HID_ACRUX_FF=y -CONFIG_HID_ALPS=m -CONFIG_HID_APPLEIR=m -CONFIG_HID_AUREAL=m -CONFIG_HID_BATTERY_STRENGTH=y -CONFIG_HID_BELKIN=m -CONFIG_HID_BETOP_FF=m -CONFIG_HID_BIGBEN_FF=m -CONFIG_HID_CHERRY=m -CONFIG_HID_CHICONY=m -CONFIG_HID_CMEDIA=m -CONFIG_HID_CORSAIR=m -CONFIG_HID_COUGAR=m -CONFIG_HID_CP2112=m -CONFIG_HID_CREATIVE_SB0540=m -CONFIG_HID_CYPRESS=m -CONFIG_HID_DRAGONRISE=m -CONFIG_HID_ELAN=m -CONFIG_HID_ELECOM=m -CONFIG_HID_ELO=m -CONFIG_HID_EMS_FF=m -CONFIG_HID_EVISION=m -CONFIG_HID_EZKEY=m -CONFIG_HID_FT260=m -CONFIG_HID_GEMBIRD=m -CONFIG_HID_GFRM=m -CONFIG_HID_GLORIOUS=m -CONFIG_HID_GOOGLE_STADIA_FF=m -CONFIG_HID_GREENASIA=m -CONFIG_HID_GT683R=m -CONFIG_HID_GYRATION=m -CONFIG_HID_HOLTEK=m -CONFIG_HID_HUAWEI=m -CONFIG_HID_ICADE=m -CONFIG_HID_ITE=m -CONFIG_HID_JABRA=m -CONFIG_HID_KENSINGTON=m -CONFIG_HID_KEYTOUCH=m -CONFIG_HID_KYE=m -CONFIG_HID_KYSONA=m -CONFIG_HID_LCPOWER=m -CONFIG_HID_LED=m -CONFIG_HID_LENOVO=m -CONFIG_HID_LENOVO_GO=m -CONFIG_HID_LENOVO_GO_S=m -CONFIG_HID_LETSKETCH=m -CONFIG_HID_LOGITECH=m -CONFIG_HID_LOGITECH_DJ=m -CONFIG_HID_LOGITECH_HIDPP=m -CONFIG_HID_MACALLY=m -CONFIG_HID_MALTRON=m -CONFIG_HID_MAYFLASH=m -CONFIG_HID_MCP2221=m -CONFIG_HID_MEGAWORLD_FF=m -CONFIG_HID_MICROSOFT=m -CONFIG_HID_MONTEREY=m -CONFIG_HID_NINTENDO=m -CONFIG_HID_NTI=m -CONFIG_HID_NTRIG=m -CONFIG_HID_NVIDIA_SHIELD=m -CONFIG_HID_ORTEK=m -CONFIG_HID_PANTHERLORD=m -CONFIG_HID_PENMOUNT=m -CONFIG_HID_PETALYNX=m -CONFIG_HID_PICOLCD=m -CONFIG_HID_PICOLCD_BACKLIGHT=y -CONFIG_HID_PICOLCD_FB=y -CONFIG_HID_PICOLCD_LEDS=y -CONFIG_HID_PID=y -CONFIG_HID_PLANTRONICS=m -CONFIG_HID_PLAYSTATION=m -CONFIG_HID_PRIMAX=m -CONFIG_HID_PRODIKEYS=m -CONFIG_HID_PXRC=m -CONFIG_HID_RAPOO=m -CONFIG_HID_RAZER=m -CONFIG_HID_RETRODE=m -CONFIG_HID_RMI=m -CONFIG_HID_ROCCAT=m -CONFIG_HID_SAITEK=m -CONFIG_HID_SAMSUNG=m -CONFIG_HID_SEMITEK=m -CONFIG_HID_SIGMAMICRO=m -CONFIG_HID_SMARTJOYPLUS=m -CONFIG_HID_SONY=m -CONFIG_HID_SPEEDLINK=m -CONFIG_HID_STEAM=m -CONFIG_HID_STEELSERIES=m -CONFIG_HID_SUNPLUS=m -CONFIG_HID_THINGM=m -CONFIG_HID_THRUSTMASTER=m -CONFIG_HID_TIVO=m -CONFIG_HID_TOPRE=m -CONFIG_HID_TOPSEED=m -CONFIG_HID_TWINHAN=m -CONFIG_HID_U2FZERO=m -CONFIG_HID_UCLOGIC=m -CONFIG_HID_UDRAW_PS3=m -CONFIG_HID_UNIVERSAL_PIDFF=m -CONFIG_HID_VIEWSONIC=m -CONFIG_HID_VIVALDI=m -CONFIG_HID_WACOM=m -CONFIG_HID_WALTOP=m -CONFIG_HID_WIIMOTE=m -CONFIG_HID_WINWING=m -CONFIG_HID_XIAOMI=m -CONFIG_HID_XINMO=m -CONFIG_HID_ZEROPLUS=m -CONFIG_HID_ZYDACRON=m -CONFIG_HOLTEK_FF=y -CONFIG_INPUT_JOYDEV=m -CONFIG_INPUT_JOYSTICK=y -CONFIG_INPUT_TABLET=y -CONFIG_JOYSTICK_IFORCE=m -CONFIG_JOYSTICK_IFORCE_USB=m -CONFIG_JOYSTICK_PXRC=m -CONFIG_JOYSTICK_XPAD=m -CONFIG_JOYSTICK_XPAD_FF=y -CONFIG_JOYSTICK_XPAD_LEDS=y -CONFIG_LOGIG940_FF=y -CONFIG_LOGIRUMBLEPAD2_FF=y -CONFIG_LOGITECH_FF=y -CONFIG_LOGIWHEELS_FF=y -CONFIG_MOUSE_SYNAPTICS_USB=m -CONFIG_NINTENDO_FF=y -CONFIG_NVIDIA_SHIELD_FF=y -CONFIG_PANTHERLORD_FF=y -CONFIG_PLAYSTATION_FF=y -CONFIG_RMI4_CORE=m -CONFIG_RMI4_F03=y -CONFIG_RMI4_F11=y -CONFIG_RMI4_F12=y -CONFIG_RMI4_F1A=y -CONFIG_RMI4_F21=y -CONFIG_RMI4_F30=y -CONFIG_RMI4_F34=y -CONFIG_RMI4_F3A=y -CONFIG_RMI4_F55=y -CONFIG_SMARTJOYPLUS_FF=y -CONFIG_SONY_FF=y -CONFIG_STEAM_FF=y -CONFIG_TABLET_USB_ACECAD=m -CONFIG_TABLET_USB_AIPTEK=m -CONFIG_TABLET_USB_HANWANG=m -CONFIG_TABLET_USB_KBTAB=m -CONFIG_TABLET_USB_PEGASUS=m -CONFIG_THRUSTMASTER_FF=y -CONFIG_TOUCHSCREEN_USB_3M=y -CONFIG_TOUCHSCREEN_USB_COMPOSITE=m -CONFIG_TOUCHSCREEN_USB_DMC_TSC10=y -CONFIG_TOUCHSCREEN_USB_E2I=y -CONFIG_TOUCHSCREEN_USB_EASYTOUCH=y -CONFIG_TOUCHSCREEN_USB_EGALAX=y -CONFIG_TOUCHSCREEN_USB_ELO=y -CONFIG_TOUCHSCREEN_USB_ETT_TC45USB=y -CONFIG_TOUCHSCREEN_USB_ETURBO=y -CONFIG_TOUCHSCREEN_USB_GENERAL_TOUCH=y -CONFIG_TOUCHSCREEN_USB_GOTOP=y -CONFIG_TOUCHSCREEN_USB_GUNZE=y -CONFIG_TOUCHSCREEN_USB_IDEALTEK=y -CONFIG_TOUCHSCREEN_USB_IRTOUCH=y -CONFIG_TOUCHSCREEN_USB_ITM=y -CONFIG_TOUCHSCREEN_USB_JASTEC=y -CONFIG_TOUCHSCREEN_USB_NEXIO=y -CONFIG_TOUCHSCREEN_USB_PANJIT=y -CONFIG_TOUCHSCREEN_USB_ZYTRONIC=y -CONFIG_USB_HIDDEV=y -CONFIG_ZEROPLUS_FF=y - -# USB device classes: serial adapters, storage sub-drivers, printers/TMC/WDM, misc USB gadgets, usbmon, USB/IP, card readers, maker adapters -CONFIG_APPLE_MFI_FASTCHARGE=m -CONFIG_DLN2_ADC=m -CONFIG_GNSS_USB=m -CONFIG_GPIB=m -CONFIG_GPIB_AGILENT_82357A=m -CONFIG_GPIB_COMMON=m -CONFIG_GPIB_LPVO=m -CONFIG_GPIB_NI_USB=m -CONFIG_GPIO_DLN2=m -CONFIG_GPIO_MPSSE=m -CONFIG_I2C_CP2615=m -CONFIG_I2C_DIOLAN_U2C=m -CONFIG_I2C_DLN2=m -CONFIG_I2C_TINY_USB=m -CONFIG_MEMSTICK=m -CONFIG_MEMSTICK_REALTEK_USB=m -CONFIG_MFD_DLN2=m -CONFIG_MISC_RTSX_USB=m -CONFIG_MMC_REALTEK_USB=m -CONFIG_MMC_USHC=m -CONFIG_MMC_VUB300=m -CONFIG_MSPRO_BLOCK=m -CONFIG_REALTEK_AUTOPM=y -CONFIG_SENSORS_AQUACOMPUTER_D5NEXT=m -CONFIG_SENSORS_ASUS_ROG_RYUJIN=m -CONFIG_SENSORS_CORSAIR_CPRO=m -CONFIG_SENSORS_CORSAIR_PSU=m -CONFIG_SENSORS_GIGABYTE_WATERFORCE=m -CONFIG_SENSORS_NZXT_KRAKEN2=m -CONFIG_SENSORS_NZXT_KRAKEN3=m -CONFIG_SENSORS_NZXT_SMART2=m -CONFIG_SENSORS_POWERZ=m -CONFIG_SPI_CH341=m -CONFIG_SPI_DLN2=m -CONFIG_USBIP_CORE=m -CONFIG_USBIP_HOST=m -CONFIG_USBIP_VHCI_HCD=m -CONFIG_USBPCWATCHDOG=m -CONFIG_USB_ADUTUX=m -CONFIG_USB_APPLEDISPLAY=m -CONFIG_USB_CHAOSKEY=m -CONFIG_USB_EMI26=m -CONFIG_USB_EMI62=m -CONFIG_USB_EZUSB_FX2=m -CONFIG_USB_IDMOUSE=m -CONFIG_USB_IOWARRIOR=m -CONFIG_USB_ISIGHTFW=m -CONFIG_USB_LCD=m -CONFIG_USB_LD=m -CONFIG_USB_LEGOTOWER=m -CONFIG_USB_MDC800=m -CONFIG_USB_MICROTEK=m -CONFIG_USB_MON=m -CONFIG_USB_PRINTER=m -CONFIG_USB_SERIAL_AIRCABLE=m -CONFIG_USB_SERIAL_ARK3116=m -CONFIG_USB_SERIAL_BELKIN=m -CONFIG_USB_SERIAL_CH341=m -CONFIG_USB_SERIAL_CYBERJACK=m -CONFIG_USB_SERIAL_CYPRESS_M8=m -CONFIG_USB_SERIAL_DIGI_ACCELEPORT=m -CONFIG_USB_SERIAL_EDGEPORT=m -CONFIG_USB_SERIAL_EDGEPORT_TI=m -CONFIG_USB_SERIAL_EMPEG=m -CONFIG_USB_SERIAL_F81232=m -CONFIG_USB_SERIAL_F8153X=m -CONFIG_USB_SERIAL_GARMIN=m -CONFIG_USB_SERIAL_GENERIC=y -CONFIG_USB_SERIAL_IPAQ=m -CONFIG_USB_SERIAL_IPW=m -CONFIG_USB_SERIAL_IR=m -CONFIG_USB_SERIAL_IUU=m -CONFIG_USB_SERIAL_KEYSPAN=m -CONFIG_USB_SERIAL_KEYSPAN_PDA=m -CONFIG_USB_SERIAL_KLSI=m -CONFIG_USB_SERIAL_KOBIL_SCT=m -CONFIG_USB_SERIAL_MCT_U232=m -CONFIG_USB_SERIAL_MOS7720=m -CONFIG_USB_SERIAL_MOS7840=m -CONFIG_USB_SERIAL_NAVMAN=m -CONFIG_USB_SERIAL_OMNINET=m -CONFIG_USB_SERIAL_OPTICON=m -CONFIG_USB_SERIAL_OTI6858=m -CONFIG_USB_SERIAL_PL2303=m -CONFIG_USB_SERIAL_QCAUX=m -CONFIG_USB_SERIAL_QT2=m -CONFIG_USB_SERIAL_QUALCOMM=m -CONFIG_USB_SERIAL_SAFE=m -CONFIG_USB_SERIAL_SAFE_PADDED=y -CONFIG_USB_SERIAL_SIERRAWIRELESS=m -CONFIG_USB_SERIAL_SIMPLE=m -CONFIG_USB_SERIAL_SPCP8X5=m -CONFIG_USB_SERIAL_SSU100=m -CONFIG_USB_SERIAL_SYMBOL=m -CONFIG_USB_SERIAL_TI=m -CONFIG_USB_SERIAL_UPD78F0730=m -CONFIG_USB_SERIAL_VISOR=m -CONFIG_USB_SERIAL_WHITEHEAT=m -CONFIG_USB_SERIAL_XR=m -CONFIG_USB_SERIAL_XSENS_MT=m -CONFIG_USB_SEVSEG=m -CONFIG_USB_SISUSBVGA=m -CONFIG_USB_STORAGE_ALAUDA=m -CONFIG_USB_STORAGE_CYPRESS_ATACB=m -CONFIG_USB_STORAGE_DATAFAB=m -CONFIG_USB_STORAGE_ENE_UB6250=m -CONFIG_USB_STORAGE_FREECOM=m -CONFIG_USB_STORAGE_ISD200=m -CONFIG_USB_STORAGE_JUMPSHOT=m -CONFIG_USB_STORAGE_KARMA=m -CONFIG_USB_STORAGE_ONETOUCH=m -CONFIG_USB_STORAGE_REALTEK=m -CONFIG_USB_STORAGE_SDDR09=m -CONFIG_USB_STORAGE_SDDR55=m -CONFIG_USB_STORAGE_USBAT=m -CONFIG_USB_TMC=m -CONFIG_USB_TRANCEVIBRATOR=m -CONFIG_USB_WDM=m -CONFIG_USB_YUREX=m -CONFIG_W1=m -CONFIG_W1_MASTER_DS2490=m -CONFIG_W1_SLAVE_DS2405=m -CONFIG_W1_SLAVE_DS2406=m -CONFIG_W1_SLAVE_DS2408=m -CONFIG_W1_SLAVE_DS2413=m -CONFIG_W1_SLAVE_DS2423=m -CONFIG_W1_SLAVE_DS2430=m -CONFIG_W1_SLAVE_DS2431=m -CONFIG_W1_SLAVE_DS2433=m -CONFIG_W1_SLAVE_DS2433_CRC=y -CONFIG_W1_SLAVE_DS2438=m -CONFIG_W1_SLAVE_DS2780=m -CONFIG_W1_SLAVE_DS2781=m -CONFIG_W1_SLAVE_DS2805=m -CONFIG_W1_SLAVE_DS28E04=m -CONFIG_W1_SLAVE_SMEM=m -CONFIG_W1_SLAVE_THERM=m -CONFIG_XILLYUSB=m - -# USB network adapters: USB Ethernet/tethering/mobile broadband, USB Wi-Fi dongles, USB CAN adapters -CONFIG_AR5523=m -CONFIG_ATH10K_USB=m -CONFIG_ATH6KL=m -CONFIG_ATH6KL_USB=m -CONFIG_ATH9K_HTC=m -CONFIG_BRCMFMAC_USB=y -CONFIG_CAN_8DEV_USB=m -CONFIG_CAN_EMS_USB=m -CONFIG_CAN_ESD_USB=m -CONFIG_CAN_F81604=m -CONFIG_CAN_GS_USB=m -CONFIG_CAN_ISOTP=m -CONFIG_CAN_J1939=m -CONFIG_CAN_KVASER_USB=m -CONFIG_CAN_MCBA_USB=m -CONFIG_CAN_PEAK_USB=m -CONFIG_CARL9170=m -CONFIG_CARL9170_LEDS=y -CONFIG_LIBERTAS_THINFIRM=m -CONFIG_LIBERTAS_THINFIRM_USB=m -CONFIG_MT7601U=m -CONFIG_MT7663U=m -CONFIG_MT76x0U=m -CONFIG_MT76x2U=m -CONFIG_MT7921U=m -CONFIG_MT7925U=m -CONFIG_RT2500USB=m -CONFIG_RT2800USB=m -CONFIG_RT2800USB_RT33XX=y -CONFIG_RT2800USB_RT3573=y -CONFIG_RT2800USB_RT35XX=y -CONFIG_RT2800USB_RT53XX=y -CONFIG_RT2800USB_RT55XX=y -CONFIG_RT2800USB_UNKNOWN=y -CONFIG_RT2X00=m -CONFIG_RT73USB=m -CONFIG_RTL8187=m -CONFIG_RTL8192DU=m -CONFIG_RTL8XXXU=m -CONFIG_RTW88=m -CONFIG_RTW88_8723DU=m -CONFIG_RTW88_8812AU=m -CONFIG_RTW88_8814AU=m -CONFIG_RTW88_8821AU=m -CONFIG_RTW88_8821CU=m -CONFIG_RTW88_8822BU=m -CONFIG_RTW88_8822CU=m -CONFIG_RTW89=m -CONFIG_RTW89_8851BU=m -CONFIG_RTW89_8852AU=m -CONFIG_RTW89_8852BU=m -CONFIG_RTW89_8852CU=m -CONFIG_USB_ALI_M5632=y -CONFIG_USB_AN2720=y -CONFIG_USB_CATC=m -CONFIG_USB_EPSON2888=y -CONFIG_USB_HSO=m -CONFIG_USB_IPHETH=m -CONFIG_USB_KAWETH=m -CONFIG_USB_KC2190=y -CONFIG_USB_NET_AQC111=m -CONFIG_USB_NET_CDC_EEM=m -CONFIG_USB_NET_CDC_MBIM=m -CONFIG_USB_NET_CH9200=m -CONFIG_USB_NET_CX82310_ETH=m -CONFIG_USB_NET_GL620A=m -CONFIG_USB_NET_HUAWEI_CDC_NCM=m -CONFIG_USB_NET_INT51X1=m -CONFIG_USB_NET_KALMIA=m -CONFIG_USB_NET_QMI_WWAN=m -CONFIG_USB_NET_RNDIS_HOST=m -CONFIG_USB_NET_SR9700=m -CONFIG_USB_SIERRA_NET=m -CONFIG_USB_VL600=m -CONFIG_ZD1211RW=m - -# Storage: optical/SCSI generic, device-mapper (LUKS, LVM, verity, integrity), MD RAID, partition tables -CONFIG_AIX_PARTITION=y -CONFIG_BLK_DEV_SR=m -CONFIG_BSD_DISKLABEL=y -CONFIG_CHR_DEV_SG=m -CONFIG_DM_CACHE=m -CONFIG_DM_CACHE_SMQ=m -CONFIG_DM_CRYPT=m -CONFIG_DM_INTEGRITY=m -CONFIG_DM_RAID=m -CONFIG_DM_SNAPSHOT=m -CONFIG_DM_THIN_PROVISIONING=m -CONFIG_DM_UEVENT=y -CONFIG_DM_VERITY=m -CONFIG_DM_VERITY_FEC=y -CONFIG_DM_WRITECACHE=m -CONFIG_LDM_PARTITION=y -CONFIG_MAC_PARTITION=y -CONFIG_MD_LINEAR=m -CONFIG_MD_RAID0=m -CONFIG_MD_RAID1=m -CONFIG_MD_RAID10=m -CONFIG_MD_RAID456=m -CONFIG_MINIX_SUBPARTITION=y -CONFIG_OSF_PARTITION=y -CONFIG_PARTITION_ADVANCED=y -CONFIG_SGI_PARTITION=y -CONFIG_SOLARIS_X86_PARTITION=y -CONFIG_SUN_PARTITION=y -CONFIG_UNIXWARE_DISKLABEL=y - -# Filesystems for removable media and images, binfmt_misc, SMB client -CONFIG_BINFMT_MISC=m -CONFIG_CIFS=m -CONFIG_CIFS_ALLOW_INSECURE_LEGACY=y -CONFIG_CIFS_DFS_UPCALL=y -CONFIG_CIFS_POSIX=y -CONFIG_CIFS_SWN_UPCALL=y -CONFIG_CIFS_UPCALL=y -CONFIG_CIFS_XATTR=y -CONFIG_EROFS_FS=m -CONFIG_EROFS_FS_BACKED_BY_FILE=y -CONFIG_EROFS_FS_POSIX_ACL=y -CONFIG_EROFS_FS_SECURITY=y -CONFIG_EROFS_FS_XATTR=y -CONFIG_EROFS_FS_ZIP=y -CONFIG_EROFS_FS_ZIP_ACCEL=y -CONFIG_EROFS_FS_ZIP_DEFLATE=y -CONFIG_EROFS_FS_ZIP_LZMA=y -CONFIG_EROFS_FS_ZIP_ZSTD=y -CONFIG_EXFAT_FS=m -CONFIG_F2FS_FS=m -CONFIG_F2FS_FS_COMPRESSION=y -CONFIG_F2FS_FS_LZ4=y -CONFIG_F2FS_FS_LZ4HC=y -CONFIG_F2FS_FS_LZO=y -CONFIG_F2FS_FS_LZORLE=y -CONFIG_F2FS_FS_POSIX_ACL=y -CONFIG_F2FS_FS_SECURITY=y -CONFIG_F2FS_FS_XATTR=y -CONFIG_F2FS_FS_ZSTD=y -CONFIG_F2FS_IOSTAT=y -CONFIG_F2FS_STAT_FS=y -CONFIG_F2FS_UNFAIR_RWSEM=y -CONFIG_HFSPLUS_FS=m -CONFIG_HFS_FS=m -CONFIG_ISO9660_FS=m -CONFIG_JOLIET=y -CONFIG_MSDOS_FS=m -CONFIG_NLS_ASCII=m -CONFIG_NLS_CODEPAGE_1250=m -CONFIG_NLS_CODEPAGE_1251=m -CONFIG_NLS_CODEPAGE_737=m -CONFIG_NLS_CODEPAGE_775=m -CONFIG_NLS_CODEPAGE_850=m -CONFIG_NLS_CODEPAGE_852=m -CONFIG_NLS_CODEPAGE_855=m -CONFIG_NLS_CODEPAGE_857=m -CONFIG_NLS_CODEPAGE_860=m -CONFIG_NLS_CODEPAGE_861=m -CONFIG_NLS_CODEPAGE_862=m -CONFIG_NLS_CODEPAGE_863=m -CONFIG_NLS_CODEPAGE_864=m -CONFIG_NLS_CODEPAGE_865=m -CONFIG_NLS_CODEPAGE_866=m -CONFIG_NLS_CODEPAGE_869=m -CONFIG_NLS_CODEPAGE_874=m -CONFIG_NLS_CODEPAGE_932=m -CONFIG_NLS_CODEPAGE_936=m -CONFIG_NLS_CODEPAGE_949=m -CONFIG_NLS_CODEPAGE_950=m -CONFIG_NLS_ISO8859_13=m -CONFIG_NLS_ISO8859_14=m -CONFIG_NLS_ISO8859_15=m -CONFIG_NLS_ISO8859_2=m -CONFIG_NLS_ISO8859_3=m -CONFIG_NLS_ISO8859_4=m -CONFIG_NLS_ISO8859_5=m -CONFIG_NLS_ISO8859_6=m -CONFIG_NLS_ISO8859_7=m -CONFIG_NLS_ISO8859_8=m -CONFIG_NLS_ISO8859_9=m -CONFIG_NLS_KOI8_R=m -CONFIG_NLS_KOI8_U=m -CONFIG_NLS_MAC_CELTIC=m -CONFIG_NLS_MAC_CENTEURO=m -CONFIG_NLS_MAC_CROATIAN=m -CONFIG_NLS_MAC_CYRILLIC=m -CONFIG_NLS_MAC_GAELIC=m -CONFIG_NLS_MAC_GREEK=m -CONFIG_NLS_MAC_ICELAND=m -CONFIG_NLS_MAC_INUIT=m -CONFIG_NLS_MAC_ROMAN=m -CONFIG_NLS_MAC_ROMANIAN=m -CONFIG_NLS_MAC_TURKISH=m -CONFIG_NLS_UTF8=m -CONFIG_NTFS3_FS=m -CONFIG_NTFS3_FS_POSIX_ACL=y -CONFIG_NTFS3_LZX_XPRESS=y -CONFIG_SQUASHFS_LZ4=y -CONFIG_SQUASHFS_LZO=y -CONFIG_SQUASHFS_XATTR=y -CONFIG_SQUASHFS_XZ=y -CONFIG_SQUASHFS_ZSTD=y -CONFIG_UDF_FS=m -CONFIG_XFS_FS=m -CONFIG_XFS_ONLINE_REPAIR=y -CONFIG_XFS_ONLINE_SCRUB=y -CONFIG_XFS_ONLINE_SCRUB_STATS=y -CONFIG_XFS_POSIX_ACL=y -CONFIG_XFS_QUOTA=y -CONFIG_XFS_RT=y -CONFIG_XFS_SUPPORT_ASCII_CI=y -CONFIG_XFS_SUPPORT_V4=y -CONFIG_ZISOFS=y - -# Network protocols: WireGuard/OpenVPN-DCO/L2TP/PPP/IPsec/kTLS, tunnels, policy routing, nftables+xtables+ipset+ebtables, tc, TCP CC, ss diag, virtual links -CONFIG_BONDING=m -CONFIG_BRIDGE_EBT_802_3=m -CONFIG_BRIDGE_EBT_AMONG=m -CONFIG_BRIDGE_EBT_ARP=m -CONFIG_BRIDGE_EBT_ARPREPLY=m -CONFIG_BRIDGE_EBT_BROUTE=m -CONFIG_BRIDGE_EBT_DNAT=m -CONFIG_BRIDGE_EBT_IP=m -CONFIG_BRIDGE_EBT_IP6=m -CONFIG_BRIDGE_EBT_LIMIT=m -CONFIG_BRIDGE_EBT_LOG=m -CONFIG_BRIDGE_EBT_MARK=m -CONFIG_BRIDGE_EBT_MARK_T=m -CONFIG_BRIDGE_EBT_NFLOG=m -CONFIG_BRIDGE_EBT_PKTTYPE=m -CONFIG_BRIDGE_EBT_REDIRECT=m -CONFIG_BRIDGE_EBT_SNAT=m -CONFIG_BRIDGE_EBT_STP=m -CONFIG_BRIDGE_EBT_T_FILTER=m -CONFIG_BRIDGE_EBT_T_NAT=m -CONFIG_BRIDGE_EBT_VLAN=m -CONFIG_BRIDGE_NF_EBTABLES=m -CONFIG_BRIDGE_NF_EBTABLES_LEGACY=m -CONFIG_CLS_U32_MARK=y -CONFIG_CLS_U32_PERF=y -CONFIG_DEFAULT_CUBIC=y -CONFIG_DUMMY=m -CONFIG_GACT_PROB=y -CONFIG_GENEVE=m -CONFIG_IFB=m -CONFIG_INET6_AH=m -CONFIG_INET6_ESP=m -CONFIG_INET6_ESPINTCP=y -CONFIG_INET6_ESP_OFFLOAD=m -CONFIG_INET6_IPCOMP=m -CONFIG_INET_AH=m -CONFIG_INET_DIAG_DESTROY=y -CONFIG_INET_ESP=m -CONFIG_INET_ESPINTCP=y -CONFIG_INET_ESP_OFFLOAD=m -CONFIG_INET_IPCOMP=m -CONFIG_INET_RAW_DIAG=m -CONFIG_INET_UDP_DIAG=m -CONFIG_IP6_NF_FILTER=m -CONFIG_IP6_NF_IPTABLES_LEGACY=m -CONFIG_IP6_NF_MANGLE=m -CONFIG_IP6_NF_MATCH_AH=m -CONFIG_IP6_NF_MATCH_EUI64=m -CONFIG_IP6_NF_MATCH_FRAG=m -CONFIG_IP6_NF_MATCH_HL=m -CONFIG_IP6_NF_MATCH_IPV6HEADER=m -CONFIG_IP6_NF_MATCH_MH=m -CONFIG_IP6_NF_MATCH_OPTS=m -CONFIG_IP6_NF_MATCH_RPFILTER=m -CONFIG_IP6_NF_MATCH_RT=m -CONFIG_IP6_NF_MATCH_SRH=m -CONFIG_IP6_NF_NAT=m -CONFIG_IP6_NF_RAW=m -CONFIG_IP6_NF_TARGET_HL=m -CONFIG_IP6_NF_TARGET_MASQUERADE=m -CONFIG_IP6_NF_TARGET_NPT=m -CONFIG_IP6_NF_TARGET_REJECT=m -CONFIG_IPV6_GRE=m -CONFIG_IPV6_MULTIPLE_TABLES=y -CONFIG_IPV6_TUNNEL=m -CONFIG_IPV6_VTI=m -CONFIG_IP_ADVANCED_ROUTER=y -CONFIG_IP_MULTIPLE_TABLES=y -CONFIG_IP_NF_ARPFILTER=m -CONFIG_IP_NF_ARPTABLES=m -CONFIG_IP_NF_ARP_MANGLE=m -CONFIG_IP_NF_FILTER=m -CONFIG_IP_NF_IPTABLES_LEGACY=m -CONFIG_IP_NF_MANGLE=m -CONFIG_IP_NF_MATCH_AH=m -CONFIG_IP_NF_MATCH_ECN=m -CONFIG_IP_NF_MATCH_RPFILTER=m -CONFIG_IP_NF_MATCH_TTL=m -CONFIG_IP_NF_NAT=m -CONFIG_IP_NF_RAW=m -CONFIG_IP_NF_TARGET_ECN=m -CONFIG_IP_NF_TARGET_MASQUERADE=m -CONFIG_IP_NF_TARGET_NETMAP=m -CONFIG_IP_NF_TARGET_REDIRECT=m -CONFIG_IP_NF_TARGET_REJECT=m -CONFIG_IP_NF_TARGET_TTL=m -CONFIG_IP_ROUTE_MULTIPATH=y -CONFIG_IP_SET=m -CONFIG_IP_SET_BITMAP_IP=m -CONFIG_IP_SET_BITMAP_IPMAC=m -CONFIG_IP_SET_BITMAP_PORT=m -CONFIG_IP_SET_HASH_IP=m -CONFIG_IP_SET_HASH_IPMAC=m -CONFIG_IP_SET_HASH_IPMARK=m -CONFIG_IP_SET_HASH_IPPORT=m -CONFIG_IP_SET_HASH_IPPORTIP=m -CONFIG_IP_SET_HASH_IPPORTNET=m -CONFIG_IP_SET_HASH_MAC=m -CONFIG_IP_SET_HASH_NET=m -CONFIG_IP_SET_HASH_NETIFACE=m -CONFIG_IP_SET_HASH_NETNET=m -CONFIG_IP_SET_HASH_NETPORT=m -CONFIG_IP_SET_HASH_NETPORTNET=m -CONFIG_IP_SET_LIST_SET=m -CONFIG_L2TP=m -CONFIG_L2TP_ETH=m -CONFIG_L2TP_IP=m -CONFIG_L2TP_V3=y -CONFIG_NETFILTER_NETLINK_ACCT=m -CONFIG_NETFILTER_NETLINK_GLUE_CT=y -CONFIG_NETFILTER_NETLINK_HOOK=m -CONFIG_NETFILTER_NETLINK_LOG=m -CONFIG_NETFILTER_NETLINK_OSF=m -CONFIG_NETFILTER_NETLINK_QUEUE=m -CONFIG_NETFILTER_XTABLES_LEGACY=y -CONFIG_NETFILTER_XT_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_BPF=m -CONFIG_NETFILTER_XT_MATCH_CGROUP=m -CONFIG_NETFILTER_XT_MATCH_CLUSTER=m -CONFIG_NETFILTER_XT_MATCH_COMMENT=m -CONFIG_NETFILTER_XT_MATCH_CONNBYTES=m -CONFIG_NETFILTER_XT_MATCH_CONNLABEL=m -CONFIG_NETFILTER_XT_MATCH_CONNLIMIT=m -CONFIG_NETFILTER_XT_MATCH_CONNMARK=m -CONFIG_NETFILTER_XT_MATCH_CPU=m -CONFIG_NETFILTER_XT_MATCH_DCCP=m -CONFIG_NETFILTER_XT_MATCH_DEVGROUP=m -CONFIG_NETFILTER_XT_MATCH_DSCP=m -CONFIG_NETFILTER_XT_MATCH_ECN=m -CONFIG_NETFILTER_XT_MATCH_ESP=m -CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=m -CONFIG_NETFILTER_XT_MATCH_HELPER=m -CONFIG_NETFILTER_XT_MATCH_HL=m -CONFIG_NETFILTER_XT_MATCH_IPCOMP=m -CONFIG_NETFILTER_XT_MATCH_IPRANGE=m -CONFIG_NETFILTER_XT_MATCH_L2TP=m -CONFIG_NETFILTER_XT_MATCH_LENGTH=m -CONFIG_NETFILTER_XT_MATCH_LIMIT=m -CONFIG_NETFILTER_XT_MATCH_MAC=m -CONFIG_NETFILTER_XT_MATCH_MARK=m -CONFIG_NETFILTER_XT_MATCH_MULTIPORT=m -CONFIG_NETFILTER_XT_MATCH_NFACCT=m -CONFIG_NETFILTER_XT_MATCH_OSF=m -CONFIG_NETFILTER_XT_MATCH_OWNER=m -CONFIG_NETFILTER_XT_MATCH_PHYSDEV=m -CONFIG_NETFILTER_XT_MATCH_PKTTYPE=m -CONFIG_NETFILTER_XT_MATCH_POLICY=m -CONFIG_NETFILTER_XT_MATCH_QUOTA=m -CONFIG_NETFILTER_XT_MATCH_RATEEST=m -CONFIG_NETFILTER_XT_MATCH_REALM=m -CONFIG_NETFILTER_XT_MATCH_RECENT=m -CONFIG_NETFILTER_XT_MATCH_SCTP=m -CONFIG_NETFILTER_XT_MATCH_SOCKET=m -CONFIG_NETFILTER_XT_MATCH_STATE=m -CONFIG_NETFILTER_XT_MATCH_STATISTIC=m -CONFIG_NETFILTER_XT_MATCH_STRING=m -CONFIG_NETFILTER_XT_MATCH_TCPMSS=m -CONFIG_NETFILTER_XT_MATCH_TIME=m -CONFIG_NETFILTER_XT_MATCH_U32=m -CONFIG_NETFILTER_XT_NAT=m -CONFIG_NETFILTER_XT_SET=m -CONFIG_NETFILTER_XT_TARGET_AUDIT=m -CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m -CONFIG_NETFILTER_XT_TARGET_CLASSIFY=m -CONFIG_NETFILTER_XT_TARGET_CONNMARK=m -CONFIG_NETFILTER_XT_TARGET_CT=m -CONFIG_NETFILTER_XT_TARGET_DSCP=m -CONFIG_NETFILTER_XT_TARGET_HL=m -CONFIG_NETFILTER_XT_TARGET_HMARK=m -CONFIG_NETFILTER_XT_TARGET_IDLETIMER=m -CONFIG_NETFILTER_XT_TARGET_LED=m -CONFIG_NETFILTER_XT_TARGET_MARK=m -CONFIG_NETFILTER_XT_TARGET_MASQUERADE=m -CONFIG_NETFILTER_XT_TARGET_NETMAP=m -CONFIG_NETFILTER_XT_TARGET_NFLOG=m -CONFIG_NETFILTER_XT_TARGET_NFQUEUE=m -CONFIG_NETFILTER_XT_TARGET_NOTRACK=m -CONFIG_NETFILTER_XT_TARGET_RATEEST=m -CONFIG_NETFILTER_XT_TARGET_REDIRECT=m -CONFIG_NETFILTER_XT_TARGET_TCPMSS=m -CONFIG_NETFILTER_XT_TARGET_TCPOPTSTRIP=m -CONFIG_NETFILTER_XT_TARGET_TEE=m -CONFIG_NETFILTER_XT_TARGET_TPROXY=m -CONFIG_NETFILTER_XT_TARGET_TRACE=m -CONFIG_NETLINK_DIAG=m -CONFIG_NET_ACT_BPF=m -CONFIG_NET_ACT_CONNMARK=m -CONFIG_NET_ACT_CSUM=m -CONFIG_NET_ACT_CT=m -CONFIG_NET_ACT_CTINFO=m -CONFIG_NET_ACT_MPLS=m -CONFIG_NET_ACT_NAT=m -CONFIG_NET_ACT_PEDIT=m -CONFIG_NET_ACT_POLICE=m -CONFIG_NET_ACT_SAMPLE=m -CONFIG_NET_ACT_SIMP=m -CONFIG_NET_ACT_SKBEDIT=m -CONFIG_NET_ACT_SKBMOD=m -CONFIG_NET_ACT_TUNNEL_KEY=m -CONFIG_NET_ACT_VLAN=m -CONFIG_NET_CLS_BPF=m -CONFIG_NET_CLS_FLOW=m -CONFIG_NET_CLS_FW=m -CONFIG_NET_CLS_MATCHALL=m -CONFIG_NET_CLS_ROUTE4=m -CONFIG_NET_CLS_U32=m -CONFIG_NET_EMATCH=y -CONFIG_NET_EMATCH_CMP=m -CONFIG_NET_EMATCH_IPSET=m -CONFIG_NET_EMATCH_IPT=m -CONFIG_NET_EMATCH_META=m -CONFIG_NET_EMATCH_NBYTE=m -CONFIG_NET_EMATCH_TEXT=m -CONFIG_NET_EMATCH_U32=m -CONFIG_NET_FOU=m -CONFIG_NET_FOU_IP_TUNNELS=y -CONFIG_NET_IPGRE=m -CONFIG_NET_IPGRE_BROADCAST=y -CONFIG_NET_IPGRE_DEMUX=m -CONFIG_NET_IPIP=m -CONFIG_NET_IPVTI=m -CONFIG_NET_KEY=m -CONFIG_NET_KEY_MIGRATE=y -CONFIG_NET_SCH_CAKE=m -CONFIG_NET_SCH_CHOKE=m -CONFIG_NET_SCH_CODEL=m -CONFIG_NET_SCH_DRR=m -CONFIG_NET_SCH_DUALPI2=m -CONFIG_NET_SCH_ETS=m -CONFIG_NET_SCH_FQ=m -CONFIG_NET_SCH_FQ_CODEL=m -CONFIG_NET_SCH_FQ_PIE=m -CONFIG_NET_SCH_GRED=m -CONFIG_NET_SCH_HFSC=m -CONFIG_NET_SCH_HHF=m -CONFIG_NET_SCH_HTB=m -CONFIG_NET_SCH_MULTIQ=m -CONFIG_NET_SCH_NETEM=m -CONFIG_NET_SCH_PIE=m -CONFIG_NET_SCH_PLUG=m -CONFIG_NET_SCH_PRIO=m -CONFIG_NET_SCH_QFQ=m -CONFIG_NET_SCH_RED=m -CONFIG_NET_SCH_SFB=m -CONFIG_NET_SCH_SFQ=m -CONFIG_NET_SCH_TBF=m -CONFIG_NET_SCH_TEQL=m -CONFIG_NET_TEAM=m -CONFIG_NET_TEAM_MODE_ACTIVEBACKUP=m -CONFIG_NET_TEAM_MODE_BROADCAST=m -CONFIG_NET_TEAM_MODE_LOADBALANCE=m -CONFIG_NET_TEAM_MODE_RANDOM=m -CONFIG_NET_TEAM_MODE_ROUNDROBIN=m -CONFIG_NFT_BRIDGE_META=m -CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NFT_COMPAT=m -CONFIG_NFT_CONNLIMIT=m -CONFIG_NFT_DUP_IPV4=m -CONFIG_NFT_DUP_IPV6=m -CONFIG_NFT_DUP_NETDEV=m -CONFIG_NFT_FIB_NETDEV=m -CONFIG_NFT_FLOW_OFFLOAD=m -CONFIG_NFT_FWD_NETDEV=m -CONFIG_NFT_HASH=m -CONFIG_NFT_NUMGEN=m -CONFIG_NFT_OSF=m -CONFIG_NFT_QUEUE=m -CONFIG_NFT_QUOTA=m -CONFIG_NFT_REDIR=m -CONFIG_NFT_REJECT_NETDEV=m -CONFIG_NFT_SOCKET=m -CONFIG_NFT_TPROXY=m -CONFIG_NFT_TUNNEL=m -CONFIG_NFT_XFRM=m -CONFIG_NF_CONNTRACK_AMANDA=m -CONFIG_NF_CONNTRACK_BRIDGE=m -CONFIG_NF_CONNTRACK_FTP=m -CONFIG_NF_CONNTRACK_H323=m -CONFIG_NF_CONNTRACK_IRC=m -CONFIG_NF_CONNTRACK_LABELS=y -CONFIG_NF_CONNTRACK_MARK=y -CONFIG_NF_CONNTRACK_NETBIOS_NS=m -CONFIG_NF_CONNTRACK_PPTP=m -CONFIG_NF_CONNTRACK_PROCFS=y -CONFIG_NF_CONNTRACK_SANE=m -CONFIG_NF_CONNTRACK_SIP=m -CONFIG_NF_CONNTRACK_SNMP=m -CONFIG_NF_CONNTRACK_TFTP=m -CONFIG_NF_CONNTRACK_TIMEOUT=y -CONFIG_NF_CONNTRACK_TIMESTAMP=y -CONFIG_NF_CONNTRACK_ZONES=y -CONFIG_NF_CT_NETLINK=m -CONFIG_NF_CT_NETLINK_HELPER=m -CONFIG_NF_CT_NETLINK_TIMEOUT=m -CONFIG_NF_DUP_IPV4=m -CONFIG_NF_DUP_IPV6=m -CONFIG_NF_DUP_NETDEV=m -CONFIG_NF_FLOW_TABLE=m -CONFIG_NF_FLOW_TABLE_INET=m -CONFIG_NF_FLOW_TABLE_PROCFS=y -CONFIG_NF_LOG_ARP=m -CONFIG_NF_LOG_IPV4=m -CONFIG_NF_NAT_SNMP_BASIC=m -CONFIG_NF_SOCKET_IPV4=m -CONFIG_NF_SOCKET_IPV6=m -CONFIG_NF_TABLES_ARP=y -CONFIG_NF_TABLES_BRIDGE=m -CONFIG_NF_TPROXY_IPV4=m -CONFIG_NF_TPROXY_IPV6=m -CONFIG_OVPN=m -CONFIG_PACKET_DIAG=m -CONFIG_PPP=m -CONFIG_PPPOE=m -CONFIG_PPPOE_HASH_BITS_4=y -CONFIG_PPPOL2TP=m -CONFIG_PPP_ASYNC=m -CONFIG_PPP_BSDCOMP=m -CONFIG_PPP_DEFLATE=m -CONFIG_PPP_FILTER=y -CONFIG_PPP_MPPE=m -CONFIG_PPP_MULTILINK=y -CONFIG_PPP_SYNC_TTY=m -CONFIG_PPTP=m -CONFIG_PSAMPLE=m -CONFIG_TCP_CONG_ADVANCED=y -CONFIG_TCP_CONG_BBR=m -CONFIG_TCP_CONG_BIC=m -CONFIG_TCP_CONG_CDG=m -CONFIG_TCP_CONG_DCTCP=m -CONFIG_TCP_CONG_HSTCP=m -CONFIG_TCP_CONG_HTCP=m -CONFIG_TCP_CONG_HYBLA=m -CONFIG_TCP_CONG_ILLINOIS=m -CONFIG_TCP_CONG_LP=m -CONFIG_TCP_CONG_NV=m -CONFIG_TCP_CONG_SCALABLE=m -CONFIG_TCP_CONG_VEGAS=m -CONFIG_TCP_CONG_VENO=m -CONFIG_TCP_CONG_WESTWOOD=m -CONFIG_TCP_CONG_YEAH=m -CONFIG_TLS=m -CONFIG_UNIX_DIAG=m -CONFIG_VXLAN=m -CONFIG_WIREGUARD=m -CONFIG_XFRM_INTERFACE=m -CONFIG_XFRM_MIGRATE=y -CONFIG_XFRM_STATISTICS=y -CONFIG_XFRM_USER=m - -# Bluetooth: USB dongle drivers, vhci, MSFT/AOSP vendor extensions (USB controllers only) -CONFIG_BT_AOSPEXT=y -CONFIG_BT_ATH3K=m -CONFIG_BT_HCIBCM203X=m -CONFIG_BT_HCIBFUSB=m -CONFIG_BT_HCIBPA10X=m -CONFIG_BT_HCIBTUSB_AUTOSUSPEND=y -CONFIG_BT_HCIVHCI=m -CONFIG_BT_MSFTEXT=y - -# Containers / virtualisation: CRIU/checkpoint, userfaultfd, misc cgroup, KVM host vhost-net/vhost-vsock/vsock -CONFIG_CGROUP_MISC=y -CONFIG_CHECKPOINT_RESTORE=y -CONFIG_USERFAULTFD=y -CONFIG_VHOST_NET=m -CONFIG_VHOST_VSOCK=m -CONFIG_VSOCKETS=m -CONFIG_VSOCKETS_DIAG=m -CONFIG_VSOCKETS_LOOPBACK=m - -# Crypto: AF_ALG user API (cryptsetup/iwd), dm-crypt/fscrypt modes, VeraCrypt/TrueCrypt ciphers+hashes, IPsec AEADs -CONFIG_CRYPTO_ADIANTUM=m -CONFIG_CRYPTO_AEGIS128=m -CONFIG_CRYPTO_AEGIS128_SIMD=y -CONFIG_CRYPTO_BLAKE2B=m -CONFIG_CRYPTO_BLOWFISH=m -CONFIG_CRYPTO_CAMELLIA=m -CONFIG_CRYPTO_CAST5=m -CONFIG_CRYPTO_CAST6=m -CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_CRC32=m -CONFIG_CRYPTO_CRC32C=m -CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_CTS=m -CONFIG_CRYPTO_ESSIV=m -CONFIG_CRYPTO_HCTR2=m -CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_LZ4HC=m -CONFIG_CRYPTO_MD4=m -CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_SEQIV=m -CONFIG_CRYPTO_SERPENT=m -CONFIG_CRYPTO_STREEBOG=m -CONFIG_CRYPTO_TWOFISH=m -CONFIG_CRYPTO_USER_API_AEAD=m -CONFIG_CRYPTO_USER_API_HASH=m -CONFIG_CRYPTO_USER_API_SKCIPHER=m -CONFIG_CRYPTO_WP512=m -CONFIG_CRYPTO_XCBC=m -CONFIG_CRYPTO_XXHASH=m - -# Sound: USB audio/MIDI device drivers, MIDI 2.0, virtual MIDI, dummy card, userspace timers -CONFIG_SND_BCD2000=m -CONFIG_SND_DUMMY=m -CONFIG_SND_SEQ_UMP=y -CONFIG_SND_UMP_LEGACY_RAWMIDI=y -CONFIG_SND_USB_6FIRE=m -CONFIG_SND_USB_AUDIO_MIDI_V2=y -CONFIG_SND_USB_CAIAQ=m -CONFIG_SND_USB_CAIAQ_INPUT=y -CONFIG_SND_USB_HIFACE=m -CONFIG_SND_USB_POD=m -CONFIG_SND_USB_PODHD=m -CONFIG_SND_USB_TONEPORT=m -CONFIG_SND_USB_UA101=m -CONFIG_SND_USB_VARIAX=m -CONFIG_SND_UTIMER=y -CONFIG_SND_VIRMIDI=m - -# Video: non-UVC USB webcams (gspca family, Philips pwc), USB displays (DisplayLink USB2 udl, GUD, GM12U320) -CONFIG_DRM_GM12U320=m -CONFIG_DRM_GUD=m -CONFIG_DRM_UDL=m -CONFIG_USB_GL860=m -CONFIG_USB_GSPCA=m -CONFIG_USB_GSPCA_BENQ=m -CONFIG_USB_GSPCA_CONEX=m -CONFIG_USB_GSPCA_CPIA1=m -CONFIG_USB_GSPCA_DTCS033=m -CONFIG_USB_GSPCA_ETOMS=m -CONFIG_USB_GSPCA_FINEPIX=m -CONFIG_USB_GSPCA_JEILINJ=m -CONFIG_USB_GSPCA_JL2005BCD=m -CONFIG_USB_GSPCA_KINECT=m -CONFIG_USB_GSPCA_KONICA=m -CONFIG_USB_GSPCA_MARS=m -CONFIG_USB_GSPCA_MR97310A=m -CONFIG_USB_GSPCA_NW80X=m -CONFIG_USB_GSPCA_OV519=m -CONFIG_USB_GSPCA_OV534=m -CONFIG_USB_GSPCA_OV534_9=m -CONFIG_USB_GSPCA_PAC207=m -CONFIG_USB_GSPCA_PAC7302=m -CONFIG_USB_GSPCA_PAC7311=m -CONFIG_USB_GSPCA_SE401=m -CONFIG_USB_GSPCA_SN9C2028=m -CONFIG_USB_GSPCA_SN9C20X=m -CONFIG_USB_GSPCA_SONIXB=m -CONFIG_USB_GSPCA_SONIXJ=m -CONFIG_USB_GSPCA_SPCA1528=m -CONFIG_USB_GSPCA_SPCA500=m -CONFIG_USB_GSPCA_SPCA501=m -CONFIG_USB_GSPCA_SPCA505=m -CONFIG_USB_GSPCA_SPCA506=m -CONFIG_USB_GSPCA_SPCA508=m -CONFIG_USB_GSPCA_SPCA561=m -CONFIG_USB_GSPCA_SQ905=m -CONFIG_USB_GSPCA_SQ905C=m -CONFIG_USB_GSPCA_SQ930X=m -CONFIG_USB_GSPCA_STK014=m -CONFIG_USB_GSPCA_STK1135=m -CONFIG_USB_GSPCA_STV0680=m -CONFIG_USB_GSPCA_SUNPLUS=m -CONFIG_USB_GSPCA_T613=m -CONFIG_USB_GSPCA_TOPRO=m -CONFIG_USB_GSPCA_TOUPTEK=m -CONFIG_USB_GSPCA_TV8532=m -CONFIG_USB_GSPCA_VC032X=m -CONFIG_USB_GSPCA_VICAM=m -CONFIG_USB_GSPCA_XIRLINK_CIT=m -CONFIG_USB_GSPCA_ZC3XX=m -CONFIG_USB_M5602=m -CONFIG_USB_PWC=m -CONFIG_USB_PWC_INPUT_EVDEV=y -CONFIG_USB_STV06XX=m - -# Other user-facing: Kerberos persistent keyrings, Speakup screen reader, Wine/Proton ntsync -CONFIG_ACCESSIBILITY=y -CONFIG_NTSYNC=m -CONFIG_PERSISTENT_KEYRINGS=y -CONFIG_SPEAKUP=m -CONFIG_SPEAKUP_SYNTH_ACNTSA=m -CONFIG_SPEAKUP_SYNTH_APOLLO=m -CONFIG_SPEAKUP_SYNTH_AUDPTR=m -CONFIG_SPEAKUP_SYNTH_BNS=m -CONFIG_SPEAKUP_SYNTH_DECTLK=m -CONFIG_SPEAKUP_SYNTH_LTLK=m -CONFIG_SPEAKUP_SYNTH_SOFT=m -CONFIG_SPEAKUP_SYNTH_SPKOUT=m -CONFIG_SPEAKUP_SYNTH_TXPRT=m diff --git a/arch/arm64/configs/j700-radio.config b/arch/arm64/configs/j700-radio.config deleted file mode 100644 index f5c5d3c3ec0bf..0000000000000 --- a/arch/arm64/configs/j700-radio.config +++ /dev/null @@ -1,20 +0,0 @@ -# J700 retained radio bootstrap; merge after defconfig and asahi.config. -# System sleep is retained; active radio firmware cannot cross kexec. -# Firmware/calibration and runtime admission follow the MT7932 documentation. -# CONFIG_KEXEC is not set -# CONFIG_KEXEC_FILE is not set -# CONFIG_KEXEC_HANDOVER is not set -CONFIG_PCI=y -CONFIG_PCIE_APPLE=y -CONFIG_APPLE_DART=y -CONFIG_PCIE_APPLE_PIODMA_DIAG=y -CONFIG_CFG80211=m -CONFIG_WLAN=y -CONFIG_WLAN_VENDOR_MEDIATEK=y -CONFIG_MT7932_FULLMAC=m -CONFIG_BT=m -CONFIG_BT_BREDR=y -CONFIG_BT_LE=y -CONFIG_BT_MTK7932_PCIE=m -CONFIG_CRYPTO_AES=y -CONFIG_CRYPTO_CMAC=y diff --git a/arch/arm64/configs/j700.config b/arch/arm64/configs/j700.config deleted file mode 100644 index 0b81cc8d78d39..0000000000000 --- a/arch/arm64/configs/j700.config +++ /dev/null @@ -1,120 +0,0 @@ -# J700 (MacBook Neo, T8140) base fragment. -# -# Merge after defconfig, asahi.config and j700-radio.config, then add the device-parity fragment: -# make defconfig -# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/asahi.config arch/arm64/configs/j700-radio.config -# make olddefconfig -# scripts/kconfig/merge_config.sh -m .config arch/arm64/configs/j700.config arch/arm64/configs/j700-device-parity.config -# make olddefconfig -# The result is the configuration the J700 kernel is tested with. - -# Boot NOR (nvram) access for boot-volume switching, and the PMU RTC wake alarm -CONFIG_SPI_APPLE_QSPI=m -CONFIG_RTC_DRV_APPLE_PMU=m - -# Classic Bluetooth profiles (headsets, tethering) -CONFIG_BT_RFCOMM=m -CONFIG_BT_RFCOMM_TTY=y -CONFIG_BT_BNEP=m -CONFIG_BT_BNEP_MC_FILTER=y -CONFIG_BT_BNEP_PROTO_FILTER=y - -# Distribution firmware files are compressed -CONFIG_FW_LOADER_COMPRESS=y -CONFIG_FW_LOADER_COMPRESS_ZSTD=y - -# Kernel BTI (ARM64_BTI_KERNEL) depends on clang < 21 since 7.2 -# (https://github.com/llvm/llvm-project/issues/215547); with clang 22 it is -# off by upstream decision and is not forced on here. - -# 48-bit virtual addresses; KASLR is untested on this machine -CONFIG_ARM64_VA_BITS_48=y -# CONFIG_RANDOMIZE_BASE is not set - -# Boot path and platform built in, so storage, keyboard, SMC and USB do not depend on the initramfs -CONFIG_BLK_DEV_NVME=y -CONFIG_NVME_APPLE=y -CONFIG_APPLE_SART=y -CONFIG_APPLE_RTKIT_HELPER=y -CONFIG_BTRFS_FS=y -CONFIG_APPLE_DOCKCHANNEL=y -CONFIG_HID_DOCKCHANNEL=y -CONFIG_HID_APPLE=y -CONFIG_HID_MAGICMOUSE=y -CONFIG_SPMI_APPLE=y -CONFIG_NVMEM_APPLE_SPMI=y -CONFIG_MFD_MACSMC=y -CONFIG_GPIO_MACSMC=y -CONFIG_POWER_RESET_MACSMC=y -CONFIG_SENSORS_MACSMC_HWMON=y -CONFIG_I2C_APPLE=y -CONFIG_PINCTRL_APPLE_GPIO=y -CONFIG_APPLE_WATCHDOG=y -CONFIG_ARM_APPLE_SOC_CPUFREQ=y -CONFIG_PHY_APPLE_ATC=y -CONFIG_RESET_APPLE_CIO=y -CONFIG_USB_DWC3_APPLE=y -CONFIG_TYPEC=y -CONFIG_TYPEC_SN201202X=y -CONFIG_MUX_APPLE_DPXBAR=y -CONFIG_BACKLIGHT_CLASS_DEVICE=y -CONFIG_SYSFB_SIMPLEFB=y -CONFIG_ZRAM=y -CONFIG_UHID=y -CONFIG_RFKILL=y -CONFIG_THERMAL_NETLINK=y -CONFIG_NVME_HWMON=y - -# J700 driver choices -CONFIG_APPLE_CLPC=m -CONFIG_APPLE_PMP=y -# CONFIG_APPLE_T6030_DISPLAY_GATE is not set -CONFIG_DRM_ACCEL=y -CONFIG_DRM_ACCEL_ANE=m -# CONFIG_HID_APPLE_MTP_HAPTIC is not set -CONFIG_IIO=m -# CONFIG_IIO_AOP_SENSOR_LAS is not set -# CONFIG_USB4 is not set - -# Per-process 4K user pages on the 16K kernel (x86 emulation) -CONFIG_ARM64_USER4K_EXPERIMENTAL=y - -# USB Ethernet built in, for a wired link before modules load -CONFIG_USB_USBNET=y -CONFIG_USB_NET_AX8817X=y -CONFIG_USB_NET_AX88179_178A=y -CONFIG_USB_NET_CDC_NCM=y -CONFIG_USB_RTL8152=y -CONFIG_USB_UAS=m - -# Crypto that Bluetooth and Wi-Fi need early -CONFIG_CRYPTO_CCM=y -CONFIG_CRYPTO_ECDH=y -CONFIG_CRYPTO_GCM=y -CONFIG_CRYPTO_HMAC=y -CONFIG_CRYPTO_SHA256=y - -# Utilisation clamping -CONFIG_UCLAMP_TASK=y -CONFIG_UCLAMP_TASK_GROUP=y - -# LSMs named in CONFIG_LSM -CONFIG_SECURITY_LANDLOCK=y -CONFIG_SECURITY_YAMA=y - -# dma-buf heaps and udmabuf (camera and GPU buffer sharing) -CONFIG_DMABUF_HEAPS=y -CONFIG_DMABUF_HEAPS_SYSTEM=y -CONFIG_UDMABUF=y - -# Debugging aids kept in the tested configuration -CONFIG_DYNAMIC_DEBUG=y -CONFIG_FTRACE=y -CONFIG_ENABLE_DEFAULT_TRACERS=y -CONFIG_PM_DEBUG=y -CONFIG_PM_ADVANCED_DEBUG=y - -# Match the tested kernel image: no relative relocation packing, no SCM version suffix -# CONFIG_RELR is not set -# CONFIG_LOCALVERSION_AUTO is not set -CONFIG_LOCALVERSION="-aurora" diff --git a/arch/arm64/configs/j714c.config b/arch/arm64/configs/j714c.config new file mode 100644 index 0000000000000..d38b3be8e8e94 --- /dev/null +++ b/arch/arm64/configs/j714c.config @@ -0,0 +1,405 @@ +# Minimal RAM-only J714c (T6050, M5 Max MacBook Pro 14-inch) bring-up; use with +# KCONFIG_ALLCONFIG and allnoconfig. Derived from j813.config; the kernel boots to a +# shell with keyboard on one core (maxcpus=1). +CONFIG_LOCALVERSION="-j714c-ARCH" +CONFIG_LOCALVERSION_AUTO=n +CONFIG_ARCH_APPLE=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_47=y +CONFIG_SMP=y +CONFIG_NR_CPUS=18 +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=y +CONFIG_ARM64_VHE=y +CONFIG_ARM64_SVE=y +CONFIG_ARM64_PAN=y +CONFIG_ARM64_LSE_ATOMICS=y +CONFIG_EXPERT=y +CONFIG_PRINTK=y +CONFIG_BUG=y +CONFIG_MULTIUSER=y +CONFIG_FUTEX=y +CONFIG_POSIX_TIMERS=y +CONFIG_KALLSYMS=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_RD_GZIP=y +CONFIG_BINFMT_ELF=y +CONFIG_BINFMT_SCRIPT=y +CONFIG_TTY=y +CONFIG_VT=y +CONFIG_VT_CONSOLE=y +CONFIG_UNIX98_PTYS=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_PROC_FS=y +CONFIG_SYSFS=y +CONFIG_TMPFS=y +CONFIG_SHMEM=y +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y +CONFIG_SERIAL_EARLYCON=y +CONFIG_FB=y +CONFIG_FB_DEVICE=y +CONFIG_DRM_SIMPLEDRM=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_DRM=y +CONFIG_DRM_APPLE=y +CONFIG_RUST=y +CONFIG_DRM_ASAHI=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_DRM_FBDEV_EMULATION=y +# CONFIG_FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER is not set +CONFIG_LOG_BUF_SHIFT=20 +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_PM=y +CONFIG_MAILBOX=y +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_RTKIT=y +CONFIG_APPLE_RTKIT_HELPER=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_IOMMU_SUPPORT=y +CONFIG_APPLE_DART=y +CONFIG_MFD_MACSMC=y +CONFIG_HWMON=y +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_INPUT=y +CONFIG_INPUT_EVDEV=y +CONFIG_HID_SUPPORT=y +CONFIG_HID=y +CONFIG_HID_GENERIC=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_APPLE=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_GPIOLIB=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_PWM=y +CONFIG_PWM=y +CONFIG_PWM_APPLE=y +CONFIG_FW_LOADER=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_APPLE_PMGR_PWRSTATE=y +CONFIG_SPMI=y +CONFIG_SPMI_APPLE=y +CONFIG_USB_SUPPORT=y +CONFIG_USB=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_PLATFORM=y +CONFIG_USB_DWC3=y +CONFIG_USB_DWC3_HOST=y +CONFIG_USB_DWC3_APPLE=y +CONFIG_USB_HID=y +CONFIG_BLOCK=y +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_USB_STORAGE=y +CONFIG_USB_UAS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_EFI_PARTITION=y +# Boot through U-Boot and Limine as an EFI application (EFI stub, /sys/firmware/efi) +CONFIG_EFI=y +CONFIG_MSDOS_PARTITION=y +CONFIG_EXFAT_FS=y +CONFIG_NLS_UTF8=y +CONFIG_TYPEC=y +CONFIG_TYPEC_SN201202X=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_PHY_APPLE_T6050_USB2=y +CONFIG_DEBUG_FS=y +# T6050 CPU features and errata +CONFIG_ARM64_SME=y +CONFIG_ARM64_LSUI=y +CONFIG_ARM64_ERRATUM_4193714=y +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_APPLE=y +CONFIG_APPLE_SART=y +CONFIG_FONTS=y +CONFIG_FONT_TER16x32=y +CONFIG_WATCHDOG=y +CONFIG_WATCHDOG_CORE=y +CONFIG_APPLE_WATCHDOG=y +CONFIG_WATCHDOG_SYSFS=y +CONFIG_EXT4_FS=y +CONFIG_VFAT_FS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +CONFIG_BTRFS_FS=y +# systemd +CONFIG_NET=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_IPV6=y +CONFIG_PACKET=y +CONFIG_CGROUPS=y +CONFIG_CGROUP_SCHED=y +CONFIG_FAIR_GROUP_SCHED=y +# Rust reads task_struct: bindgen drops the alignment of an empty sched_statistics, so without +# SCHEDSTATS every Rust field offset after it is 40 bytes early (statistics stay off at run time) +CONFIG_SCHEDSTATS=y +CONFIG_MEMCG=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_BLK_CGROUP=y +CONFIG_NAMESPACES=y +CONFIG_UTS_NS=y +CONFIG_IPC_NS=y +CONFIG_USER_NS=y +CONFIG_PID_NS=y +CONFIG_NET_NS=y +CONFIG_INOTIFY_USER=y +CONFIG_FANOTIFY=y +CONFIG_SIGNALFD=y +CONFIG_TIMERFD=y +CONFIG_EPOLL=y +CONFIG_EVENTFD=y +CONFIG_FHANDLE=y +CONFIG_POSIX_MQUEUE=y +CONFIG_SYSVIPC=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_TMPFS_XATTR=y +CONFIG_SECCOMP=y +CONFIG_SECCOMP_FILTER=y +CONFIG_AUTOFS_FS=y +CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_SHA256=y +CONFIG_BPF_SYSCALL=y +CONFIG_FS_POSIX_ACL=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_BINFMT_MISC=y +CONFIG_OVERLAY_FS=y +CONFIG_ADVISE_SYSCALLS=y +CONFIG_AIO=y +CONFIG_IO_URING=y +CONFIG_MEMBARRIER=y +CONFIG_RSEQ=y +CONFIG_CHECKPOINT_RESTORE=y +# sysctl and pseudo filesystems for systemd/udev +CONFIG_SYSCTL=y +CONFIG_PROC_SYSCTL=y +CONFIG_CONFIGFS_FS=y +CONFIG_SECURITYFS=y +CONFIG_KALLSYMS=y +# file locking, keys, pressure info +CONFIG_FILE_LOCKING=y +CONFIG_KEYS=y +CONFIG_PROC_CHILDREN=y +CONFIG_PSI=y +CONFIG_DNOTIFY=y +# cpufreq: let the scheduler pick the clock; performance for tests +CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y +CONFIG_CPU_FREQ_GOV_PERFORMANCE=y +CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL=y +# CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE is not set +CONFIG_CPU_FREQ_STAT=y +# USB-C host: PD controller, network adapters +CONFIG_NETDEVICES=y +CONFIG_NET_CORE=y +CONFIG_ETHERNET=y +CONFIG_TYPEC_TPS6598X=y +CONFIG_USB_NET_DRIVERS=y +CONFIG_USB_USBNET=y +CONFIG_USB_NET_CDCETHER=y +CONFIG_USB_NET_CDC_NCM=y +CONFIG_USB_RTL8152=y +CONFIG_USB_NET_AX88179_178A=y +CONFIG_USB_NET_AX8817X=y +# bring-up: dyndbg= on the command line +CONFIG_DYNAMIC_DEBUG=y +# Omarchy root: zram swap (zram-generator) and Landlock (pacman's download sandbox) +CONFIG_SWAP=y +CONFIG_BLK_DEV=y +CONFIG_ZRAM=y +CONFIG_ZRAM_BACKEND_ZSTD=y +CONFIG_ZRAM_DEF_COMP_ZSTD=y +CONFIG_SECURITY=y +CONFIG_SECURITY_LANDLOCK=y +CONFIG_LSM="landlock" +# SMC children: lid and power button, RTC (offset in the PMIC NVMEM), GPIO +CONFIG_NVMEM=y +CONFIG_NVMEM_APPLE_SPMI=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_MACSMC=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_MACSMC_INPUT=y +CONFIG_GPIO_MACSMC=y +# Sleep: s2idle when the lid closes; PM_DEBUG gives /sys/power/pm_test for driver sleep tests +CONFIG_SUSPEND=y +CONFIG_PM_DEBUG=y +CONFIG_PM_SLEEP_DEBUG=y +# Battery and charger (SMC) +CONFIG_MACSMC_POWER=y +# Power-off and restart through the SMC (MBSE off1/phra) +CONFIG_POWER_RESET_MACSMC=y +# Firewall: Omarchy's ufw (iptables-nft) needs netfilter, nf_tables and the xtables matches it uses. +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_TABLES=y +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NF_TABLES_IPV6=y +CONFIG_NFT_COMPAT=y +CONFIG_NFT_CT=y +CONFIG_NFT_LOG=y +CONFIG_NFT_LIMIT=y +CONFIG_NFT_REJECT=y +CONFIG_NETFILTER_XTABLES=y +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_HL=y +CONFIG_NETFILTER_XT_MATCH_LIMIT=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y +CONFIG_NETFILTER_XT_MATCH_RECENT=y +CONFIG_NETFILTER_XT_TARGET_LOG=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_IP6_NF_MATCH_RT=y +CONFIG_NF_LOG_SYSLOG=y +CONFIG_PCI=y +# Audio, stage A: the AP GPIO block (codec reset, shutdown and interrupt +# lines), I2C buses 1-3, the six sn012776 speaker amps and the cs42l84 jack +# codec. No sound card yet: T6050 has no AP-side MCA, the I2S ports belong to +# the AOP firmware, so no machine driver. GPIO_CDEV for gpioinfo on the Mac. +CONFIG_PINCTRL=y +CONFIG_PINCTRL_APPLE_GPIO=y +CONFIG_GPIO_CDEV=y +CONFIG_I2C_APPLE=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_SOC=y +CONFIG_SND_SOC_TAS2764=y +CONFIG_SND_SOC_CS42L84=y +# The six amps share shutdown GPIO 107: gpiolib hands shared lines out +# through the proxy chip, without which every consumer defers forever. +CONFIG_GPIO_SHARED_PROXY=y +# Machine driver, for the apple,j714-macaudio config (speakers off, -30 dB +# ceiling). Inert until a sound node with a CPU DAI exists. +CONFIG_SND_SOC_APPLE_MACAUDIO=y +# AOP (always-on processor), step 1 of the audio bring-up: the co-processor, +# its setup port and the base-ns audio DMA. Rust, RTKit and mailbox are on. +CONFIG_APPLE_AOP=y +CONFIG_DMADEVICES=y +CONFIG_APPLE_ADMAC=y +# Step 2: the AOP audio streams (CPU DAIs for the amps and the jack). +CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=y +CONFIG_SND_PROC_FS=y +# Kernel defaults the working builds had on, which allnoconfig turns off: the RTC setting +# the clock at boot, netfilter's advanced matches, ethtool netlink, PCIe ASPM, firmware cache. +CONFIG_AF_UNIX_OOB=y +CONFIG_APPLE_PMP=y +CONFIG_APPLE_PMP_REPORT=y +CONFIG_BPF_UNPRIV_DEFAULT_OFF=y +CONFIG_BQL=y +CONFIG_ETHTOOL_NETLINK=y +CONFIG_EXT4_USE_FOR_EXT2=y +CONFIG_FONT_8x16=y +CONFIG_FONT_8x8=y +CONFIG_FW_CACHE=y +CONFIG_FW_LOADER_DEBUG=y +CONFIG_INET_DIAG=y +CONFIG_INTEGRITY=y +CONFIG_IPV6_SIT=y +CONFIG_MAC80211_STA_HASH_MAX_SIZE=0 +CONFIG_NETFILTER_ADVANCED=y +CONFIG_NETFILTER_EGRESS=y +CONFIG_NETFILTER_INGRESS=y +CONFIG_NET_FLOW_LIMIT=y +CONFIG_NETWORK_FILESYSTEMS=y +CONFIG_NVMEM_SYSFS=y +CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW=y +CONFIG_PCIEASPM_DEFAULT=y +CONFIG_PCIEASPM=y +CONFIG_PCI_QUIRKS=y +CONFIG_PCPU_DEV_REFCNT=y +CONFIG_RAID6_PQ_BENCHMARK=y +CONFIG_RFS_ACCEL=y +CONFIG_RPS=y +CONFIG_RTC_HCTOSYS_DEVICE="rtc0" +CONFIG_RTC_HCTOSYS=y +CONFIG_RTC_INTF_DEV=y +CONFIG_RTC_INTF_PROC=y +CONFIG_RTC_INTF_SYSFS=y +CONFIG_RTC_NVMEM=y +CONFIG_RTC_SYSTOHC_DEVICE="rtc0" +CONFIG_RTC_SYSTOHC=y +CONFIG_RUST_OVERFLOW_CHECKS=y +CONFIG_SND_PCI=y +CONFIG_SND_VERBOSE_PROCFS=y +CONFIG_TIME_NS=y +CONFIG_USB_ARMLINUX=y +CONFIG_USB_BELKIN=y +CONFIG_USB_DWC3_HAPS=y +CONFIG_USB_NET_CDC_SUBSET=y +CONFIG_USB_NET_NET1080=y +CONFIG_USB_NET_ZAURUS=y +CONFIG_USB_PCI=y +CONFIG_USB_XHCI_PCI_ASMEDIA=y +CONFIG_VGA_ARB_MAX_GPUS=16 +CONFIG_VGA_ARB=y +CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED=y +CONFIG_WIRELESS=y +CONFIG_WLAN=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_I2C_CHARDEV=y +# Wi-Fi and Bluetooth: the N1 on apcie0 port 0, through aurora-silicon's T6050 PCIe adopt +# path (apple,t6050-pcie; the port is trained before Linux) and its centauri driver. +CONFIG_PCIE_APPLE=y +CONFIG_CFG80211=y +CONFIG_WLAN_VENDOR_APPLE=y +CONFIG_APPLE_CENTAURI=y +CONFIG_APPLE_CENTAURI_WIFI=y +CONFIG_BT=y +CONFIG_BT_BREDR=y +CONFIG_BT_LE=y +CONFIG_BT_HCICENTAURI=y +CONFIG_RFKILL=y +# iwd, the Wi-Fi service NetworkManager uses on Omarchy, exits at start unless every one of these +# is in the kernel (iwd src/main.c check_crypto: AF_ALG hashes and ciphers, the keyring, the parsers). +CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_USER_API_SKCIPHER=y +CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA512=y +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_CMAC=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_ECB=y +CONFIG_CRYPTO_CBC=y +CONFIG_KEYS=y +CONFIG_KEY_DH_OPERATIONS=y +CONFIG_ASYMMETRIC_KEY_TYPE=y +CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y +CONFIG_X509_CERTIFICATE_PARSER=y +CONFIG_PKCS7_MESSAGE_PARSER=y +CONFIG_PKCS8_PRIVATE_KEY_PARSER=y +# iwd warns at start without MD4 (EAP-MSCHAPv2 needs it). +CONFIG_CRYPTO_MD4=y +# cfg80211 is built in and asks for regulatory.db before the root is mounted; carry the signed +# database in the kernel (from the build host's /lib/firmware, the wireless-regdb package). +CONFIG_EXTRA_FIRMWARE="regulatory.db regulatory.db.p7s" +# tailscaled's normal mode: the /dev/net/tun interface +CONFIG_TUN=y +# The ISP (camera): V4L2 and the apple-isp driver +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_CAMERA_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_MEDIA_PLATFORM_DRIVERS=y +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_APPLE_ISP=y diff --git a/arch/arm64/configs/k114c.config b/arch/arm64/configs/k114c.config new file mode 100644 index 0000000000000..15d73f00d1dce --- /dev/null +++ b/arch/arm64/configs/k114c.config @@ -0,0 +1,406 @@ +# Minimal RAM-only K114c (T6050, M5 Max MacBook Pro 14-inch, OLED) bring-up; use with +# KCONFIG_ALLCONFIG and allnoconfig. Derived from Edi Moldovan's j714c.config with the +# same options (K114c-neutral); the kernel boots with keyboard, USB, NVMe and the N1 +# Wi-Fi/Bluetooth on apcie0 port 0. +CONFIG_LOCALVERSION="-k114c-ARCH" +CONFIG_LOCALVERSION_AUTO=n +CONFIG_ARCH_APPLE=y +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_47=y +CONFIG_SMP=y +CONFIG_NR_CPUS=18 +CONFIG_CPU_FREQ=y +CONFIG_CPU_FREQ_STAT=y +CONFIG_CPU_FREQ_GOV_USERSPACE=y +CONFIG_ARM_APPLE_SOC_CPUFREQ=y +CONFIG_ARM64_VHE=y +CONFIG_ARM64_SVE=y +CONFIG_ARM64_PAN=y +CONFIG_ARM64_LSE_ATOMICS=y +CONFIG_EXPERT=y +CONFIG_PRINTK=y +CONFIG_BUG=y +CONFIG_MULTIUSER=y +CONFIG_FUTEX=y +CONFIG_POSIX_TIMERS=y +CONFIG_KALLSYMS=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_RD_GZIP=y +CONFIG_BINFMT_ELF=y +CONFIG_BINFMT_SCRIPT=y +CONFIG_TTY=y +CONFIG_VT=y +CONFIG_VT_CONSOLE=y +CONFIG_UNIX98_PTYS=y +CONFIG_DEVTMPFS=y +CONFIG_DEVTMPFS_MOUNT=y +CONFIG_PROC_FS=y +CONFIG_SYSFS=y +CONFIG_TMPFS=y +CONFIG_SHMEM=y +CONFIG_SERIAL_SAMSUNG=y +CONFIG_SERIAL_SAMSUNG_CONSOLE=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL=y +CONFIG_SERIAL_APPLE_DOCKCHANNEL_EARLYCON=y +CONFIG_SERIAL_EARLYCON=y +CONFIG_FB=y +CONFIG_FB_DEVICE=y +CONFIG_DRM_SIMPLEDRM=y +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_DRM=y +CONFIG_DRM_APPLE=y +CONFIG_RUST=y +CONFIG_DRM_ASAHI=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_DRM_FBDEV_EMULATION=y +# CONFIG_FRAMEBUFFER_CONSOLE_DEFERRED_TAKEOVER is not set +CONFIG_LOG_BUF_SHIFT=20 +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_PM=y +CONFIG_MAILBOX=y +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_RTKIT=y +CONFIG_APPLE_RTKIT_HELPER=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_IOMMU_SUPPORT=y +CONFIG_APPLE_DART=y +CONFIG_MFD_MACSMC=y +CONFIG_HWMON=y +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_INPUT=y +CONFIG_INPUT_EVDEV=y +CONFIG_HID_SUPPORT=y +CONFIG_HID=y +CONFIG_HID_GENERIC=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_APPLE=y +CONFIG_HID_MAGICMOUSE=y +CONFIG_GPIOLIB=y +CONFIG_NEW_LEDS=y +CONFIG_LEDS_CLASS=y +CONFIG_LEDS_PWM=y +CONFIG_PWM=y +CONFIG_PWM_APPLE=y +CONFIG_FW_LOADER=y +CONFIG_PM_GENERIC_DOMAINS=y +CONFIG_APPLE_PMGR_PWRSTATE=y +CONFIG_SPMI=y +CONFIG_SPMI_APPLE=y +CONFIG_USB_SUPPORT=y +CONFIG_USB=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_PLATFORM=y +CONFIG_USB_DWC3=y +CONFIG_USB_DWC3_HOST=y +CONFIG_USB_DWC3_APPLE=y +CONFIG_USB_HID=y +CONFIG_BLOCK=y +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_USB_STORAGE=y +CONFIG_USB_UAS=y +CONFIG_PARTITION_ADVANCED=y +CONFIG_EFI_PARTITION=y +# Boot through U-Boot and Limine as an EFI application (EFI stub, /sys/firmware/efi) +CONFIG_EFI=y +CONFIG_MSDOS_PARTITION=y +CONFIG_EXFAT_FS=y +CONFIG_NLS_UTF8=y +CONFIG_TYPEC=y +CONFIG_TYPEC_SN201202X=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_PHY_APPLE_T6050_USB2=y +CONFIG_DEBUG_FS=y +# T6050 CPU features and errata +CONFIG_ARM64_SME=y +CONFIG_ARM64_LSUI=y +CONFIG_ARM64_ERRATUM_4193714=y +CONFIG_BLK_DEV_NVME=y +CONFIG_NVME_APPLE=y +CONFIG_APPLE_SART=y +CONFIG_FONTS=y +CONFIG_FONT_TER16x32=y +CONFIG_WATCHDOG=y +CONFIG_WATCHDOG_CORE=y +CONFIG_APPLE_WATCHDOG=y +CONFIG_WATCHDOG_SYSFS=y +CONFIG_EXT4_FS=y +CONFIG_VFAT_FS=y +CONFIG_NLS_CODEPAGE_437=y +CONFIG_NLS_ISO8859_1=y +CONFIG_BTRFS_FS=y +# systemd +CONFIG_NET=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_IPV6=y +CONFIG_PACKET=y +CONFIG_CGROUPS=y +CONFIG_CGROUP_SCHED=y +CONFIG_FAIR_GROUP_SCHED=y +# Rust reads task_struct: bindgen drops the alignment of an empty sched_statistics, so without +# SCHEDSTATS every Rust field offset after it is 40 bytes early (statistics stay off at run time) +CONFIG_SCHEDSTATS=y +CONFIG_MEMCG=y +CONFIG_CPUSETS=y +CONFIG_CGROUP_PIDS=y +CONFIG_CGROUP_FREEZER=y +CONFIG_CGROUP_DEVICE=y +CONFIG_CGROUP_CPUACCT=y +CONFIG_BLK_CGROUP=y +CONFIG_NAMESPACES=y +CONFIG_UTS_NS=y +CONFIG_IPC_NS=y +CONFIG_USER_NS=y +CONFIG_PID_NS=y +CONFIG_NET_NS=y +CONFIG_INOTIFY_USER=y +CONFIG_FANOTIFY=y +CONFIG_SIGNALFD=y +CONFIG_TIMERFD=y +CONFIG_EPOLL=y +CONFIG_EVENTFD=y +CONFIG_FHANDLE=y +CONFIG_POSIX_MQUEUE=y +CONFIG_SYSVIPC=y +CONFIG_TMPFS_POSIX_ACL=y +CONFIG_TMPFS_XATTR=y +CONFIG_SECCOMP=y +CONFIG_SECCOMP_FILTER=y +CONFIG_AUTOFS_FS=y +CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_SHA256=y +CONFIG_BPF_SYSCALL=y +CONFIG_FS_POSIX_ACL=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_BTRFS_FS_POSIX_ACL=y +CONFIG_BINFMT_MISC=y +CONFIG_OVERLAY_FS=y +CONFIG_ADVISE_SYSCALLS=y +CONFIG_AIO=y +CONFIG_IO_URING=y +CONFIG_MEMBARRIER=y +CONFIG_RSEQ=y +CONFIG_CHECKPOINT_RESTORE=y +# sysctl and pseudo filesystems for systemd/udev +CONFIG_SYSCTL=y +CONFIG_PROC_SYSCTL=y +CONFIG_CONFIGFS_FS=y +CONFIG_SECURITYFS=y +CONFIG_KALLSYMS=y +# file locking, keys, pressure info +CONFIG_FILE_LOCKING=y +CONFIG_KEYS=y +CONFIG_PROC_CHILDREN=y +CONFIG_PSI=y +CONFIG_DNOTIFY=y +# cpufreq: let the scheduler pick the clock; performance for tests +CONFIG_CPU_FREQ_GOV_SCHEDUTIL=y +CONFIG_CPU_FREQ_GOV_PERFORMANCE=y +CONFIG_CPU_FREQ_DEFAULT_GOV_SCHEDUTIL=y +# CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE is not set +CONFIG_CPU_FREQ_STAT=y +# USB-C host: PD controller, network adapters +CONFIG_NETDEVICES=y +CONFIG_NET_CORE=y +CONFIG_ETHERNET=y +CONFIG_TYPEC_TPS6598X=y +CONFIG_USB_NET_DRIVERS=y +CONFIG_USB_USBNET=y +CONFIG_USB_NET_CDCETHER=y +CONFIG_USB_NET_CDC_NCM=y +CONFIG_USB_RTL8152=y +CONFIG_USB_NET_AX88179_178A=y +CONFIG_USB_NET_AX8817X=y +# bring-up: dyndbg= on the command line +CONFIG_DYNAMIC_DEBUG=y +# Omarchy root: zram swap (zram-generator) and Landlock (pacman's download sandbox) +CONFIG_SWAP=y +CONFIG_BLK_DEV=y +CONFIG_ZRAM=y +CONFIG_ZRAM_BACKEND_ZSTD=y +CONFIG_ZRAM_DEF_COMP_ZSTD=y +CONFIG_SECURITY=y +CONFIG_SECURITY_LANDLOCK=y +CONFIG_LSM="landlock" +# SMC children: lid and power button, RTC (offset in the PMIC NVMEM), GPIO +CONFIG_NVMEM=y +CONFIG_NVMEM_APPLE_SPMI=y +CONFIG_RTC_CLASS=y +CONFIG_RTC_DRV_MACSMC=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_MACSMC_INPUT=y +CONFIG_GPIO_MACSMC=y +# Sleep: s2idle when the lid closes; PM_DEBUG gives /sys/power/pm_test for driver sleep tests +CONFIG_SUSPEND=y +CONFIG_PM_DEBUG=y +CONFIG_PM_SLEEP_DEBUG=y +# Battery and charger (SMC) +CONFIG_MACSMC_POWER=y +# Power-off and restart through the SMC (MBSE off1/phra) +CONFIG_POWER_RESET_MACSMC=y +# Firewall: Omarchy's ufw (iptables-nft) needs netfilter, nf_tables and the xtables matches it uses. +CONFIG_NETFILTER=y +CONFIG_NF_CONNTRACK=y +CONFIG_NF_TABLES=y +CONFIG_NF_TABLES_INET=y +CONFIG_NF_TABLES_IPV4=y +CONFIG_NF_TABLES_IPV6=y +CONFIG_NFT_COMPAT=y +CONFIG_NFT_CT=y +CONFIG_NFT_LOG=y +CONFIG_NFT_LIMIT=y +CONFIG_NFT_REJECT=y +CONFIG_NETFILTER_XTABLES=y +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y +CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +CONFIG_NETFILTER_XT_MATCH_HL=y +CONFIG_NETFILTER_XT_MATCH_LIMIT=y +CONFIG_NETFILTER_XT_MATCH_STATE=y +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y +CONFIG_NETFILTER_XT_MATCH_RECENT=y +CONFIG_NETFILTER_XT_TARGET_LOG=y +CONFIG_IP_NF_IPTABLES=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_TARGET_REJECT=y +CONFIG_IP6_NF_MATCH_RT=y +CONFIG_NF_LOG_SYSLOG=y +CONFIG_PCI=y +# Audio, stage A: the AP GPIO block (codec reset, shutdown and interrupt +# lines), I2C buses 1-3, the six sn012776 speaker amps and the cs42l84 jack +# codec. No sound card yet: T6050 has no AP-side MCA, the I2S ports belong to +# the AOP firmware, so no machine driver. GPIO_CDEV for gpioinfo on the Mac. +CONFIG_PINCTRL=y +CONFIG_PINCTRL_APPLE_GPIO=y +CONFIG_GPIO_CDEV=y +CONFIG_I2C_APPLE=y +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_SOC=y +CONFIG_SND_SOC_TAS2764=y +CONFIG_SND_SOC_CS42L84=y +# The six amps share shutdown GPIO 107: gpiolib hands shared lines out +# through the proxy chip, without which every consumer defers forever. +CONFIG_GPIO_SHARED_PROXY=y +# Machine driver, for the apple,j714-macaudio config (speakers off, -30 dB +# ceiling). Inert until a sound node with a CPU DAI exists. +CONFIG_SND_SOC_APPLE_MACAUDIO=y +# AOP (always-on processor), step 1 of the audio bring-up: the co-processor, +# its setup port and the base-ns audio DMA. Rust, RTKit and mailbox are on. +CONFIG_APPLE_AOP=y +CONFIG_DMADEVICES=y +CONFIG_APPLE_ADMAC=y +# Step 2: the AOP audio streams (CPU DAIs for the amps and the jack). +CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=y +CONFIG_SND_PROC_FS=y +# Kernel defaults the working builds had on, which allnoconfig turns off: the RTC setting +# the clock at boot, netfilter's advanced matches, ethtool netlink, PCIe ASPM, firmware cache. +CONFIG_AF_UNIX_OOB=y +CONFIG_APPLE_PMP=y +CONFIG_APPLE_PMP_REPORT=y +CONFIG_BPF_UNPRIV_DEFAULT_OFF=y +CONFIG_BQL=y +CONFIG_ETHTOOL_NETLINK=y +CONFIG_EXT4_USE_FOR_EXT2=y +CONFIG_FONT_8x16=y +CONFIG_FONT_8x8=y +CONFIG_FW_CACHE=y +CONFIG_FW_LOADER_DEBUG=y +CONFIG_INET_DIAG=y +CONFIG_INTEGRITY=y +CONFIG_IPV6_SIT=y +CONFIG_MAC80211_STA_HASH_MAX_SIZE=0 +CONFIG_NETFILTER_ADVANCED=y +CONFIG_NETFILTER_EGRESS=y +CONFIG_NETFILTER_INGRESS=y +CONFIG_NET_FLOW_LIMIT=y +CONFIG_NETWORK_FILESYSTEMS=y +CONFIG_NVMEM_SYSFS=y +CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW=y +CONFIG_PCIEASPM_DEFAULT=y +CONFIG_PCIEASPM=y +CONFIG_PCI_QUIRKS=y +CONFIG_PCPU_DEV_REFCNT=y +CONFIG_RAID6_PQ_BENCHMARK=y +CONFIG_RFS_ACCEL=y +CONFIG_RPS=y +CONFIG_RTC_HCTOSYS_DEVICE="rtc0" +CONFIG_RTC_HCTOSYS=y +CONFIG_RTC_INTF_DEV=y +CONFIG_RTC_INTF_PROC=y +CONFIG_RTC_INTF_SYSFS=y +CONFIG_RTC_NVMEM=y +CONFIG_RTC_SYSTOHC_DEVICE="rtc0" +CONFIG_RTC_SYSTOHC=y +CONFIG_RUST_OVERFLOW_CHECKS=y +CONFIG_SND_PCI=y +CONFIG_SND_VERBOSE_PROCFS=y +CONFIG_TIME_NS=y +CONFIG_USB_ARMLINUX=y +CONFIG_USB_BELKIN=y +CONFIG_USB_DWC3_HAPS=y +CONFIG_USB_NET_CDC_SUBSET=y +CONFIG_USB_NET_NET1080=y +CONFIG_USB_NET_ZAURUS=y +CONFIG_USB_PCI=y +CONFIG_USB_XHCI_PCI_ASMEDIA=y +CONFIG_VGA_ARB_MAX_GPUS=16 +CONFIG_VGA_ARB=y +CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED=y +CONFIG_WIRELESS=y +CONFIG_WLAN=y +CONFIG_NF_CT_PROTO_SCTP=y +CONFIG_I2C_CHARDEV=y +# Wi-Fi and Bluetooth: the N1 on apcie0 port 0, through aurora-silicon's T6050 PCIe adopt +# path (apple,t6050-pcie; the port is trained before Linux) and its centauri driver. +CONFIG_PCIE_APPLE=y +CONFIG_CFG80211=y +CONFIG_WLAN_VENDOR_APPLE=y +CONFIG_APPLE_CENTAURI=y +CONFIG_APPLE_CENTAURI_WIFI=y +CONFIG_BT=y +CONFIG_BT_BREDR=y +CONFIG_BT_LE=y +CONFIG_BT_HCICENTAURI=y +CONFIG_RFKILL=y +# iwd, the Wi-Fi service NetworkManager uses on Omarchy, exits at start unless every one of these +# is in the kernel (iwd src/main.c check_crypto: AF_ALG hashes and ciphers, the keyring, the parsers). +CONFIG_CRYPTO_USER_API_HASH=y +CONFIG_CRYPTO_USER_API_SKCIPHER=y +CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_SHA1=y +CONFIG_CRYPTO_SHA256=y +CONFIG_CRYPTO_SHA512=y +CONFIG_CRYPTO_HMAC=y +CONFIG_CRYPTO_CMAC=y +CONFIG_CRYPTO_AES=y +CONFIG_CRYPTO_DES=y +CONFIG_CRYPTO_ECB=y +CONFIG_CRYPTO_CBC=y +CONFIG_KEYS=y +CONFIG_KEY_DH_OPERATIONS=y +CONFIG_ASYMMETRIC_KEY_TYPE=y +CONFIG_ASYMMETRIC_PUBLIC_KEY_SUBTYPE=y +CONFIG_X509_CERTIFICATE_PARSER=y +CONFIG_PKCS7_MESSAGE_PARSER=y +CONFIG_PKCS8_PRIVATE_KEY_PARSER=y +# iwd warns at start without MD4 (EAP-MSCHAPv2 needs it). +CONFIG_CRYPTO_MD4=y +# cfg80211 is built in and asks for regulatory.db before the root is mounted; carry the signed +# database in the kernel (from the build host's /lib/firmware, the wireless-regdb package). +CONFIG_EXTRA_FIRMWARE="regulatory.db regulatory.db.p7s" +# tailscaled's normal mode: the /dev/net/tun interface +CONFIG_TUN=y +# The ISP (camera): V4L2 and the apple-isp driver +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_CAMERA_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_MEDIA_PLATFORM_DRIVERS=y +CONFIG_VIDEO_DEV=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_APPLE_ISP=y diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h index 87560d8b254e6..1da277baacf78 100644 --- a/arch/arm64/include/asm/el2_setup.h +++ b/arch/arm64/include/asm/el2_setup.h @@ -421,8 +421,9 @@ mov x2, xzr mrs x1, id_aa64dfr0_el1 ubfx x1, x1, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 - cmp x1, #ID_AA64DFR0_EL1_PMUVer_V3P9 - b.lt .Lskip_pmuv3p9_\@ + cmp x1, #ID_AA64DFR0_EL1_PMUVer_IMP_DEF + ccmp x1, #ID_AA64DFR0_EL1_PMUVer_V3P9, #8, ne + b.lt .Lskip_pmuv3p9_\@ // Skip if < PMUv3p9 or IMP_DEF orr x0, x0, #HDFGRTR2_EL2_nPMICNTR_EL0 orr x0, x0, #HDFGRTR2_EL2_nPMICFILTR_EL0 diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index cfc2122819d1d..4b9ff74af3091 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -25,6 +25,9 @@ KASAN_SANITIZE_stacktrace.o := n KCOV_INSTRUMENT_entry-common.o := n KCOV_INSTRUMENT_idle.o := n +# Softirq stack switching can run outside interrupt context. +KCOV_INSTRUMENT_irq.o := n + # Object file lists. obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ entry-common.o process.o ptrace.o \ diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index fbf08b543c3fd..8f106e38ac9e4 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -15,6 +15,9 @@ #include #include +/* Explicit publication of retained coprocessor page tables before ASC start (asahi G15). */ +EXPORT_SYMBOL_GPL(dcache_clean_poc); + void sync_icache_aliases(unsigned long start, unsigned long end) { if (icache_is_aliasing()) { diff --git a/arch/arm64/mm/user4k-test.c b/arch/arm64/mm/user4k-test.c index 4f8d5ec709b77..891222d058a40 100644 --- a/arch/arm64/mm/user4k-test.c +++ b/arch/arm64/mm/user4k-test.c @@ -1885,7 +1885,7 @@ static void user4k_vma_copy_shrink_test(struct kunit *test) struct vm_page_offset pos = vm_page_offset_add(initial, i == 2 ? 0 : (i + 1) * SZ_4K); mmap_write_lock(mm); - dest = copy_vma(&source, addr, SZ_4K, pos, &need_locks); + dest = copy_vma(&source, addr, SZ_4K, pos, &need_locks, false); mmap_write_unlock(mm); KUNIT_ASSERT_NOT_NULL(test, dest); KUNIT_EXPECT_EQ(test, mm->map_count, 2); diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index f7028caaeae06..e5166df510bb6 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -450,9 +450,10 @@ config SMP config NR_CPUS int "Maximum number of CPUs (2-512)" depends on SMP - range 2 512 if !RISCV_SBI_V01 - range 2 32 if RISCV_SBI_V01 && 32BIT + range 2 512 if !RISCV_SBI_V01 && 64BIT + range 2 128 if !RISCV_SBI_V01 && 32BIT range 2 64 if RISCV_SBI_V01 && 64BIT + range 2 32 if RISCV_SBI_V01 && 32BIT default "32" if 32BIT default "64" if RISCV_SBI_V01 && 64BIT default "256" if !RISCV_SBI_V01 && 64BIT diff --git a/arch/riscv/Kconfig.errata b/arch/riscv/Kconfig.errata index 3c945d086c7d0..6121ba83cfa6b 100644 --- a/arch/riscv/Kconfig.errata +++ b/arch/riscv/Kconfig.errata @@ -146,6 +146,7 @@ config ERRATA_THEAD_PMU config ERRATA_THEAD_GHOSTWRITE bool "Apply T-Head Ghostwrite errata" depends on ERRATA_THEAD && RISCV_ISA_XTHEADVECTOR + select RISCV_ALTERNATIVE_EARLY default y help The T-Head C9xx cores have a vulnerability in the xtheadvector diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h index 90c86b115e008..6f3bc9bdb85e3 100644 --- a/arch/riscv/include/asm/futex.h +++ b/arch/riscv/include/asm/futex.h @@ -40,6 +40,7 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr) if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; + uaddr = untagged_addr(uaddr); switch (op) { case FUTEX_OP_SET: @@ -82,6 +83,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, if (!access_ok(uaddr, sizeof(u32))) return -EFAULT; + uaddr = untagged_addr(uaddr); __enable_user_access(); __asm__ __volatile__ ( diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h index 023a412fe38d3..49e40eef4ed7b 100644 --- a/arch/riscv/include/asm/scs.h +++ b/arch/riscv/include/asm/scs.h @@ -4,6 +4,7 @@ #ifdef __ASSEMBLER__ #include +#include #ifdef CONFIG_SHADOW_CALL_STACK @@ -22,9 +23,17 @@ REG_L gp, TASK_TI_SCS_SP(tp) .endm -/* Load task_scs_sp(current) to gp, but only if tp has changed. */ -.macro scs_load_current_if_task_changed prev - beq \prev, tp, _skip_scs +/* + * Load task_scs_sp(current) to gp, but only when the trap came from U-mode. + * The source privilege level is taken from the saved sstatus.SPP bit (written + * by hardware on trap entry), not by comparing tp. tp is a user-writable + * register, so gating the shadow call stack reload on it let a user task that + * set tp == ¤t skip the reload and run the kernel with an attacker + * controlled gp (the shadow call stack pointer). + */ +.macro scs_load_current_if_from_user status, tmp + andi \tmp, \status, SR_SPP + bnez \tmp, _skip_scs scs_load_current _skip_scs: .endm @@ -42,7 +51,7 @@ .endm .macro scs_load_current .endm -.macro scs_load_current_if_task_changed prev +.macro scs_load_current_if_from_user status, tmp .endm .macro scs_save_current .endm diff --git a/arch/riscv/include/asm/vector.h b/arch/riscv/include/asm/vector.h index 00cb9c0982b1a..8d736e826d84d 100644 --- a/arch/riscv/include/asm/vector.h +++ b/arch/riscv/include/asm/vector.h @@ -132,8 +132,8 @@ static __always_inline void __vstate_csr_save(struct __riscv_v_ext_state *dest) "csrr %0, " __stringify(CSR_VSTART) "\n\t" "csrr %1, " __stringify(CSR_VTYPE) "\n\t" "csrr %2, " __stringify(CSR_VL) "\n\t" - : "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl), - "=r" (dest->vcsr) : :); + : "=r" (dest->vstart), "=r" (dest->vtype), "=r" (dest->vl) + : :); if (has_xtheadvector()) { unsigned long status; @@ -225,7 +225,7 @@ static inline void __riscv_v_vstate_save(struct __riscv_v_ext_state *save_to, "add %1, %1, %0\n\t" "vse8.v v24, (%1)\n\t" ".option pop\n\t" - : "=&r" (vl) : "r" (datap) : "memory"); + : "=&r" (vl), "+r" (datap) : : "memory"); } riscv_v_disable(); } @@ -261,7 +261,7 @@ static inline void __riscv_v_vstate_restore(struct __riscv_v_ext_state *restore_ "add %1, %1, %0\n\t" "vle8.v v24, (%1)\n\t" ".option pop\n\t" - : "=&r" (vl) : "r" (datap) : "memory"); + : "=&r" (vl), "+r" (datap) : : "memory"); } __vstate_csr_restore(restore_from); riscv_v_disable(); diff --git a/arch/riscv/include/uapi/asm/vendor/mips.h b/arch/riscv/include/uapi/asm/vendor/mips.h index e65ab268b2655..a31c23afe74fb 100644 --- a/arch/riscv/include/uapi/asm/vendor/mips.h +++ b/arch/riscv/include/uapi/asm/vendor/mips.h @@ -1,3 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -#define RISCV_HWPROBE_VENDOR_EXT_XMIPSEXECTL BIT(0) +#include + +#define RISCV_HWPROBE_VENDOR_EXT_XMIPSEXECTL _BITULL(0) diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index d799c4e56f804..61b7a77166971 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -208,7 +208,7 @@ SYM_CODE_START(handle_exception) load_global_pointer /* Load the kernel shadow call stack pointer if coming from userspace */ - scs_load_current_if_task_changed s5 + scs_load_current_if_from_user s1, s5 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE move a0, sp diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 16b243376f362..1e8f1ac5a755f 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -46,7 +46,7 @@ static __always_inline void *patch_map(void *addr, const unsigned int fixmap) if (core_kernel_text(uintaddr) || is_kernel_exittext(uintaddr)) { phys = __pa_symbol(addr); - } else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) { + } else if (IS_ENABLED(CONFIG_EXECMEM)) { struct page *page = vmalloc_to_page(addr); BUG_ON(!page); diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index f336a183667eb..a0055743d3ba5 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -230,6 +230,7 @@ static int riscv_vr_set(struct task_struct *target, return -ENODATA; /* Copy rest of the vstate except datap */ + memset(&ptrace_vstate, 0, sizeof(ptrace_vstate)); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ptrace_vstate, 0, sizeof(struct __riscv_v_regset_state)); if (unlikely(ret)) @@ -299,6 +300,9 @@ static int riscv_cfi_get(struct task_struct *target, struct user_cfi_state user_cfi; struct pt_regs *regs; + if (!is_user_lpad_enabled() && !is_user_shstk_enabled()) + return -EINVAL; + memset(&user_cfi, 0, sizeof(user_cfi)); regs = task_pt_regs(target); @@ -337,8 +341,12 @@ static int riscv_cfi_set(struct task_struct *target, struct user_cfi_state user_cfi; struct pt_regs *regs; + if (!is_user_lpad_enabled() && !is_user_shstk_enabled()) + return -EINVAL; + regs = task_pt_regs(target); + memset(&user_cfi, 0, sizeof(user_cfi)); ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &user_cfi, 0, -1); if (ret) return ret; diff --git a/arch/riscv/lib/strnlen.S b/arch/riscv/lib/strnlen.S index a8911605c2480..5b3bbf0a30987 100644 --- a/arch/riscv/lib/strnlen.S +++ b/arch/riscv/lib/strnlen.S @@ -87,9 +87,25 @@ strnlen_zbb: * Aligned boundary. Use the address of the last valid byte * (s + count - 1) to avoid loading a word past the count * boundary in the loop below. count == 0 is handled above. + * + * Saturate the boundary when s + count would wrap around (very + * large counts, e.g. SIZE_MAX passed by FORTIFY strcat/strlcat + * with a destination whose size is unknown at compile time). + * Without this, the wrapped boundary lands before s and the + * pre-loop guard below always exits, returning a truncated + * length. + * + * Clamping the increment to ~s (== SIZE_MAX - s) keeps the + * computation branchless and wrap-free: + * + * s + min(count - 1, ~s) == saturating_add(s, count - 1) + * + * Saturating makes the scan equivalent to strlen(). */ - add t4, a0, a1 - addi t4, t4, -1 + addi t4, a1, -1 /* count - 1 */ + not t1, a0 /* SIZE_MAX - s */ + minu t4, t4, t1 /* clamp, so s + t4 can never wrap */ + add t4, a0, t4 /* saturated s + count - 1 */ andi t4, t4, -SZREG /* Get the first word. */ diff --git a/arch/riscv/mm/extable.c b/arch/riscv/mm/extable.c index dd1530af3ef15..e252eba55edec 100644 --- a/arch/riscv/mm/extable.c +++ b/arch/riscv/mm/extable.c @@ -68,8 +68,8 @@ ex_handler_load_unaligned_zeropad(const struct exception_table_entry *ex, addr = regs_get_gpr(regs, reg_addr * sizeof(unsigned long)); - offset = addr & 0x7UL; - addr &= ~0x7UL; + offset = addr & (sizeof(data) - 1); + addr &= ~(sizeof(data) - 1); data = *(unsigned long *)addr >> (offset * 8); diff --git a/arch/um/drivers/vector_transports.c b/arch/um/drivers/vector_transports.c index ddd127ee96785..e1fb2a76fdff5 100644 --- a/arch/um/drivers/vector_transports.c +++ b/arch/um/drivers/vector_transports.c @@ -209,6 +209,7 @@ static int raw_verify_header( if ((vheader->flags & VIRTIO_NET_HDR_F_DATA_VALID) > 0) return 1; + skb_set_network_header(skb, ETH_HLEN); virtio_net_hdr_to_skb(skb, vheader, virtio_legacy_is_little_endian()); return 0; } diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 1b4a48bff18f7..caca69a4e6d5a 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -465,6 +465,8 @@ #define X86_FEATURE_NO_NESTED_DATA_BP (20*32+ 0) /* No Nested Data Breakpoints */ #define X86_FEATURE_WRMSR_XX_BASE_NS (20*32+ 1) /* WRMSR to {FS,GS,KERNEL_GS}_BASE is non-serializing */ #define X86_FEATURE_LFENCE_RDTSC (20*32+ 2) /* LFENCE always serializing / synchronizes RDTSC */ + +#define X86_FEATURE_TLBSYNC_SFW_NO (20*32+ 4) /* TLBI Erratum #1718 fixed in microcode */ #define X86_FEATURE_VERW_CLEAR (20*32+ 5) /* The memory form of VERW mitigates TSA */ #define X86_FEATURE_NULL_SEL_CLR_BASE (20*32+ 6) /* Null Selector Clears Base */ @@ -575,4 +577,5 @@ #define X86_BUG_TSA X86_BUG( 1*32+ 9) /* "tsa" CPU is affected by Transient Scheduler Attacks */ #define X86_BUG_VMSCAPE X86_BUG( 1*32+10) /* "vmscape" CPU is affected by VMSCAPE attacks from guests */ #define X86_BUG_SEAMRET_INVD_VMCS X86_BUG( 1*32+11) /* "seamret_invd_vmcs" SEAMRET from P-SEAMLDR clears the current VMCS */ +#define X86_BUG_TLBI X86_BUG( 1*32+12) /* "tlbi" AMD TLBI Erratum #1718 */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h index 866ea78ba156a..eba925c85f522 100644 --- a/arch/x86/include/asm/tlb.h +++ b/arch/x86/include/asm/tlb.h @@ -110,6 +110,12 @@ static inline void __tlbsync(void) /* TLBSYNC: supported in binutils >= 0.36. */ asm volatile(".byte 0x0f, 0x01, 0xff" ::: "memory"); + + if (!static_cpu_has_bug(X86_BUG_TLBI)) + return; + + /* TLBSYNC: supported in binutils >= 0.36. */ + asm volatile(".byte 0x0f, 0x01, 0xff" ::: "memory"); } #else /* Some compilers (I'm looking at you clang!) simply can't do DCE */ diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index d61df70d6875e..89e3280475a9d 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -1047,6 +1047,10 @@ static void init_amd_zen5(struct cpuinfo_x86 *c) msr_clear_bit(MSR_AMD64_CPUID_FN_7, 18); pr_emerg_once("RDSEED32 is broken. Disabling the corresponding CPUID bit.\n"); } + + if (cpu_has(c, X86_FEATURE_INVLPGB) && + !cpu_has(c, X86_FEATURE_TLBSYNC_SFW_NO)) + set_cpu_bug(c, X86_BUG_TLBI); } static void init_amd(struct cpuinfo_x86 *c) diff --git a/drivers/android/binder/node.rs b/drivers/android/binder/node.rs index c10148e9069f3..8f7aa867ce68b 100644 --- a/drivers/android/binder/node.rs +++ b/drivers/android/binder/node.rs @@ -49,9 +49,9 @@ pub(crate) struct CouldNotDeliverCriticalIncrement; /// about to drop the weak reference, then the strong increment could be processed after the /// other thread has already exited, which would be too late. /// -/// Note that trying to create a `ListArc` to the node can succeed even if `has_normal_push` is +/// Note that trying to create a `ListArc` to the node can succeed even if `has_pushed_node` is /// set. This is because another thread might just have popped the node from a todo list, but not -/// yet called `do_work`. However, if `has_normal_push` is false, then creating a `ListArc` should +/// yet called `do_work`. However, if `has_pushed_node` is false, then creating a `ListArc` should /// always succeed. /// /// Like the other fields in `NodeInner`, the delivery state is protected by the process lock. @@ -733,7 +733,21 @@ impl DeliverToRead for Node { self.do_work_locked(writer, owner_inner) } - fn cancel(self: DArc) {} + fn cancel(self: DArc) { + let _drop_outside_lock; + let mut owner_inner = self.owner.inner.lock(); + + // We only do something on BINDER_THREAD_EXIT, not process exit. + if owner_inner.is_dead { + return; + } + + // If BINDER_THREAD_EXIT is invoked on a thread with a pending node refcount update, we + // should move ourselves to ensure the refcount update is still delivered. + if let Some(node) = ListArc::try_from_arc_borrow(self.as_arc_borrow()) { + _drop_outside_lock = owner_inner.push_work(node); + } + } fn should_sync_wakeup(&self) -> bool { false diff --git a/drivers/android/binder/node/wrapper.rs b/drivers/android/binder/node/wrapper.rs index 43294c0505029..aeda07ff7218b 100644 --- a/drivers/android/binder/node/wrapper.rs +++ b/drivers/android/binder/node/wrapper.rs @@ -57,7 +57,39 @@ impl DeliverToRead for NodeWrapper { node.do_work_locked(writer, owner_inner) } - fn cancel(self: DArc) {} + fn cancel(self: DArc) { + let _drop_outside_lock; + let node = &self.node; + let mut owner_inner = node.owner.inner.lock(); + + // We only do something on BINDER_THREAD_EXIT, not process exit. + if owner_inner.is_dead { + return; + } + + // We transfer the responsibility of the node refcount update to the scheduled Node because + // NodeWrapper has no way to re-create the ListArc. + let inner = node.inner.access_mut(&mut owner_inner); + + let ds = &mut inner.delivery_state; + assert!(ds.has_pushed_wrapper); + assert!(ds.has_strong_zero2one); + ds.has_pushed_wrapper = false; + + // We are changing the state to one where the Node is the strong zero2one update instead of + // the wrapper. + ds.has_weak_zero2one = false; + + if !ds.has_pushed_node { + if let Some(node2) = ListArc::try_from_arc_borrow(node.as_arc_borrow()) { + ds.has_pushed_node = true; + _drop_outside_lock = owner_inner.push_work(node2); + } else { + // This can't actually happen. + ds.has_strong_zero2one = false; + } + } + } fn should_sync_wakeup(&self) -> bool { false diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 3559485b50349..d06062abca9ce 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -3026,10 +3026,15 @@ static void atapi_qc_complete(struct ata_queued_cmd *qc) if (qc->cdb[0] == ALLOW_MEDIUM_REMOVAL && qc->dev->sdev) qc->dev->sdev->locked = 0; - if (cmd->result) - ata_scsi_qc_done(qc, false, 0); - else - ata_scsi_qc_done(qc, true, SAM_STAT_CHECK_CONDITION); + /* + * Report CHECK CONDITION unless the command was terminated + * with a host byte set. Only set the status byte, so that the + * SCSI midlayer internal byte, which scsi_check_sense() may + * have set from the sense data, is preserved. + */ + if (get_host_byte(cmd) == DID_OK) + set_status_byte(cmd, SAM_STAT_CHECK_CONDITION); + ata_scsi_qc_done(qc, false, 0); goto schedule_deferred; } diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 1333fae05ac5d..a011f3e518a55 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -799,6 +799,27 @@ static void zloop_cmd_workfn(struct work_struct *work) current->flags = orig_flags; } +static void zloop_fill_zero_rq(struct request *rq, unsigned int zero_from) +{ + struct req_iterator iter; + struct bio_vec bv; + unsigned int pos = 0; + unsigned int end, off; + + rq_for_each_segment(bv, rq, iter) { + end = pos + bv.bv_len; + if (end > zero_from) { + if (pos < zero_from) + off = zero_from - pos; + else + off = 0; + memzero_page(bv.bv_page, bv.bv_offset + off, + bv.bv_len - off); + } + pos = end; + } +} + static void zloop_complete_rq(struct request *rq) { struct zloop_cmd *cmd = blk_mq_rq_to_pdu(rq); @@ -813,13 +834,9 @@ static void zloop_complete_rq(struct request *rq) pr_err("Zone %u: failed read sector %llu, %llu sectors\n", zone_no, cmd->sector, cmd->nr_sectors); - if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq)) { - /* short read */ - struct bio *bio; - - __rq_for_each_bio(bio, rq) - zero_fill_bio(bio); - } + /* Short read. Zero out the unread tail beyond cmd->ret. */ + if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq)) + zloop_fill_zero_rq(rq, cmd->ret); break; case REQ_OP_WRITE: case REQ_OP_ZONE_APPEND: diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index b35637d575a34..c83daad4264f5 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -507,3 +507,11 @@ config BT_INTEL_PCIE Say Y here to compiler support for Intel Bluetooth PCIe device into the kernel or say M to compile it as module (btintel_pcie) endmenu + +config BT_HCICENTAURI + tristate "Apple N1 Centauri Bluetooth" + depends on BT && PCI && APPLE_CENTAURI + help + Protocol-27 Beta HCI and ACL transport using the shared N1 firmware. + Requires the Control supplier and a DT-provided device address. + SCO and ISO delivery are not implemented by this driver. diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index d6062472f1621..862b139014896 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -52,3 +52,4 @@ hci_uart-$(CONFIG_BT_HCIUART_AML) += hci_aml.o hci_uart-objs := $(hci_uart-y) obj-$(CONFIG_BT_MTK7932_PCIE) += mt7932_bt_pcie.o +obj-$(CONFIG_BT_HCICENTAURI) += btcentauri.o diff --git a/drivers/bluetooth/btcentauri.c b/drivers/bluetooth/btcentauri.c new file mode 100644 index 0000000000000..9e0e2d786356e --- /dev/null +++ b/drivers/bluetooth/btcentauri.c @@ -0,0 +1,957 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CEN_ARENA_SIZE SZ_1M +#define CEN_MTR 0x2000 +#define CEN_MCR 0x2200 +#define CEN_MESSAGES 0x4000 +#define CEN_RX 0x8000 +#define CEN_TX 0xa000 +#define CEN_FOOTER 260 +#define CEN_STRIDE (16 + CEN_FOOTER) +#define CEN_RX_SLOTS 17 +#define CEN_TX_SLOTS 5 +#define CEN_LE_RX 0xc000 +#define CEN_LE_TX 0xe000 +#define CEN_LE_FOOTER 256 +#define CEN_LE_STRIDE (16 + CEN_LE_FOOTER) +#define CEN_LE_SLOTS 17 +#define CEN_MESSAGE_SLOTS 16 +#define CEN_AUX_BASE 0x20000 +#define CEN_CAPTURE_SIZE SZ_1M + +#define CEN_OP_READ_PROP_FEATURES 0xffa3 +#define CEN_OP_CREATE_CONN 0xff7b +#define CEN_OP_FLUSH_DUPLICATES 0xff3f +#define CEN_PROP_CREATE_CONN BIT(9) + +struct cen_aux_spec { + u16 id, slots, footer; + bool receive; +}; + +static const struct cen_aux_spec cen_aux_specs[] = { + { 2, 9, 2592, false }, { 3, 9, 724, false }, { 5, 9, 260, false }, + { 6, 5, 1024, false }, { 8, 9, 2592, true }, { 9, 9, 724, true }, + { 11, 9, 268, true }, { 12, 128, 1024, true }, { 13, 17, 260, true }, + { 14, 9, 1304, false }, { 15, 9, 1304, true }, +}; + +struct cen_aux_state { + u32 offset; + u16 head, post; + bool opened; +}; + +struct cen_bt { + struct pci_dev *pdev; + struct hci_dev *hdev; + void __iomem *bar; + u8 *arena; + dma_addr_t dma; + struct mutex lock; + struct work_struct io_work; + unsigned int irq_count; + bool irq_vectors; + struct sk_buff_head txq; + struct sk_buff_head acl_txq, le_txq; + u16 acl_tx_head, le_tx_head; + u16 message_head, message_tag, tx_head, rx_head, rx_post; + u16 le_rx_head, le_rx_post; + struct cen_aux_state aux[ARRAY_SIZE(cen_aux_specs)]; + bool ipc_requested, running; + u8 native_startup_steps; + bool connection_command_supported, connection_status_pending; +}; + +static u16 cen_tail(struct cen_bt *bt, u16 ring) +{ + return le16_to_cpu(READ_ONCE(*(__le16 *)(bt->arena + 0x7a + ring * 2))); +} + +static void cen_post(struct cen_bt *bt, u16 ring, u16 value) +{ + put_unaligned_le16(value, bt->arena + 0xa8 + ring * 2); + dma_wmb(); + writel(value, bt->bar + 0x1000 + ring * 4); +} + +static int cen_message(struct cen_bt *bt, const void *data, size_t length) +{ + u16 head = bt->message_head, tail, + next = (head + 1) % CEN_MESSAGE_SLOTS; + u16 tag = bt->message_tag + 1; + u8 *descriptor = bt->arena + CEN_MTR + head * 16; + u8 *completion = bt->arena + CEN_MCR + head * 32; + int ret; + + if (head >= CEN_MESSAGE_SLOTS || !length || length > 512) + return -EINVAL; + memset(descriptor, 0, 16); + memset(completion, 0, 32); + memcpy(bt->arena + CEN_MESSAGES + head * 512, data, length); + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(bt->dma + CEN_MESSAGES + head * 512, descriptor + 4); + put_unaligned_le16(tag, descriptor + 12); + cen_post(bt, 0, next); + ret = read_poll_timeout(cen_tail, tail, tail == next, 100, 1500000, + false, bt, 0); + if (ret) + return ret; + + ret = read_poll_timeout(le16_to_cpu, tail, tail == next, 100, 1500000, + false, + READ_ONCE(*(__le16 *)(bt->arena + 0x78))); + if (ret) + return ret; + dma_rmb(); + if (get_unaligned_le16(completion + 4) != tag || + get_unaligned_le16(completion + 6) != length || + get_unaligned_le32(completion + 8)) + return -EPROTO; + bt->message_head = next; + bt->message_tag = tag; + put_unaligned_le16(bt->message_head, bt->arena + 0xa6); + dma_wmb(); + return 0; +} + +static int cen_open_ring(struct cen_bt *bt, u16 id, u16 slots, u32 offset, + u16 footer) +{ + u8 message[0x34] = { 1 }; + u32 words = footer / 4, shift = 0; + + if (!words || footer % 4) + return -EINVAL; + if (words > 255) + shift = min_t(u32, __ffs(words), 4); + if ((words >> shift) > 255) + return -EINVAL; + message[2] = words >> shift; + message[3] = shift << 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le64(bt->dma + offset, message + 8); + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(slots, message + 0x18); + put_unaligned_le16(0xffff, message + 0x1a); + put_unaligned_le16(id, message + 0x1c); + put_unaligned_le16(0x40, message + 0x1e); + return cen_message(bt, message, sizeof(message)); +} + +static void cen_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_RX + slot * CEN_STRIDE; + + memset(descriptor, 0, CEN_STRIDE); + put_unaligned_le32(1 | (CEN_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static void cen_le_rx_buffer(struct cen_bt *bt, u16 slot) +{ + u8 *descriptor = bt->arena + CEN_LE_RX + slot * CEN_LE_STRIDE; + + memset(descriptor, 0, CEN_LE_STRIDE); + put_unaligned_le32(1 | (CEN_LE_FOOTER << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static int cen_receive_acl(struct cen_bt *bt, const u8 *data, u32 length) +{ + struct sk_buff *skb; + u32 packet_length, pos; + + if (length < HCI_ACL_HDR_SIZE) + return -EPROTO; + packet_length = HCI_ACL_HDR_SIZE + get_unaligned_le16(data + 2); + if (packet_length > length) + return -EPROTO; + + for (pos = packet_length; pos < length;) { + if (!data[pos]) + break; + if (data[pos] == 1) + pos++; + else if (data[pos] == 2 && length - pos >= 9) + pos += 9; + else + return -EPROTO; + } + skb = bt_skb_alloc(packet_length, GFP_KERNEL); + if (!skb) { + bt->hdev->stat.err_rx++; + return 0; + } + skb_put_data(skb, data, packet_length); + hci_skb_pkt_type(skb) = HCI_ACLDATA_PKT; + bt->hdev->stat.byte_rx += packet_length; + hci_recv_frame(bt->hdev, skb); + return 0; +} + +static void cen_aux_rx_buffer(struct cen_bt *bt, unsigned int index, u16 slot) +{ + const struct cen_aux_spec *spec = &cen_aux_specs[index]; + u8 *descriptor = + bt->arena + bt->aux[index].offset + slot * (16 + spec->footer); + + memset(descriptor, 0, 16 + spec->footer); + put_unaligned_le32(1 | (spec->footer << 8), descriptor); + put_unaligned_le16(slot + 1, descriptor + 12); +} + +static bool cen_aux_required(u16 id) +{ + return id == 2 || id == 8; +} + +static int cen_aux_start(struct cen_bt *bt) +{ + u32 offset = CEN_AUX_BASE; + unsigned int i, slot; + int ret; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u32 length = spec->slots * (16U + spec->footer); + + if (offset > CEN_ARENA_SIZE || length > CEN_ARENA_SIZE - offset) + return -ENOSPC; + state->offset = offset; + offset = ALIGN(offset + length, SZ_4K); + if (!cen_aux_required(spec->id)) + continue; + ret = cen_open_ring(bt, spec->id, spec->slots, state->offset, + spec->footer); + if (ret) { + dev_err(&bt->pdev->dev, + "Auxiliary ring %u open failed: %d\n", spec->id, + ret); + return ret; + } + state->opened = true; + if (!spec->receive) + continue; + for (slot = 0; slot < spec->slots - 1; slot++) + cen_aux_rx_buffer(bt, i, slot); + state->post = spec->slots - 1; + cen_post(bt, spec->id, state->post); + } + return 0; +} + +static int cen_aux_drain(struct cen_bt *bt, bool repost) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cen_aux_specs); i++) { + const struct cen_aux_spec *spec = &cen_aux_specs[i]; + struct cen_aux_state *state = &bt->aux[i]; + u16 tail; + int budget = spec->slots; + + if (!state->opened || !spec->receive) + continue; + tail = cen_tail(bt, spec->id); + if (tail >= spec->slots) + return -EPROTO; + while (state->head != tail && budget--) { + u8 *descriptor = bt->arena + state->offset + + state->head * (16 + spec->footer); + u32 length; + + dma_rmb(); + length = get_unaligned_le32(descriptor) >> 8; + if (length > spec->footer) + return -EPROTO; + if (repost && spec->id == 8 && + cen_receive_acl(bt, descriptor + 16, length)) + return -EPROTO; + state->head = (state->head + 1) % spec->slots; + if (repost) { + cen_aux_rx_buffer(bt, i, state->post); + state->post = (state->post + 1) % spec->slots; + cen_post(bt, spec->id, state->post); + } + } + } + return 0; +} + +static int cen_start(struct cen_bt *bt) +{ + static const u16 indices[] = { 0x78, 0x7a, 0xa6, 0xa8 }; + u8 config[4] = { 9, 0, 0xff, 0xff }; + u32 status; + int i, ret; + + put_unaligned_le16(0x300, bt->arena); + put_unaligned_le16(0x68, bt->arena + 2); + put_unaligned_le64(bt->dma + 0x68, bt->arena + 8); + for (i = 0; i < ARRAY_SIZE(indices); i++) + put_unaligned_le64(bt->dma + indices[i], + bt->arena + 0x10 + i * 8); + put_unaligned_le16(1, bt->arena + 0x30); + put_unaligned_le16(22, bt->arena + 0x32); + put_unaligned_le64(bt->dma + CEN_MCR, bt->arena + 0x34); + put_unaligned_le64(bt->dma + CEN_MTR, bt->arena + 0x3c); + put_unaligned_le16(16, bt->arena + 0x44); + put_unaligned_le16(16, bt->arena + 0x46); + + bt->arena[0x53] = 4; + put_unaligned_le64(bt->dma + 0x10000, bt->arena + 0x58); + pci_set_master(bt->pdev); + writel(lower_32_bits(bt->dma), bt->bar + 0xc); + writel(upper_32_bits(bt->dma), bt->bar + 0x10); + writel(CEN_ARENA_SIZE, bt->bar + 0x14); + bt->ipc_requested = true; + writel(1, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 1, 100, 2000000); + if (ret) + return ret; + dma_wmb(); + writel(lower_32_bits(bt->dma), bt->bar + 4); + writel(upper_32_bits(bt->dma), bt->bar + 8); + writel(2, bt->bar + 0x107c); + ret = readl_poll_timeout(bt->bar, status, status == 2, 100, 2000000); + if (ret) + return ret; + ret = cen_message(bt, config, sizeof(config)); + if (!ret) + ret = cen_open_ring(bt, 7, CEN_RX_SLOTS, CEN_RX, CEN_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 1, CEN_TX_SLOTS, CEN_TX, CEN_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 10, CEN_LE_SLOTS, CEN_LE_RX, + CEN_LE_FOOTER); + if (!ret) + ret = cen_open_ring(bt, 4, CEN_LE_SLOTS, CEN_LE_TX, + CEN_LE_FOOTER); + if (ret) + return ret; + for (i = 0; i < CEN_RX_SLOTS - 1; i++) + cen_rx_buffer(bt, i); + bt->rx_post = CEN_RX_SLOTS - 1; + cen_post(bt, 7, bt->rx_post); + { + for (i = 0; i < CEN_LE_SLOTS - 1; i++) + cen_le_rx_buffer(bt, i); + bt->le_rx_post = CEN_LE_SLOTS - 1; + cen_post(bt, 10, bt->le_rx_post); + } + return cen_aux_start(bt); +} + +static int cen_transmit(struct cen_bt *bt, struct sk_buff_head *queue, u16 ring, + u16 slots, u32 offset, u16 footer, u16 *head) +{ + struct sk_buff *skb; + u16 tail, next; + u8 *descriptor; + u32 wire_length; + + while (!skb_queue_empty(queue)) { + tail = cen_tail(bt, ring); + if (tail >= slots) + return -EPROTO; + next = (*head + 1) % slots; + if (next == tail) + break; + dma_rmb(); + skb = skb_dequeue(queue); + if (!skb) + break; + descriptor = bt->arena + offset + *head * (16 + footer); + memset(descriptor, 0, 16 + footer); + put_unaligned_le16(*head + 1, descriptor + 12); + memcpy(descriptor + 16, skb->data, skb->len); + wire_length = skb->len; + if (ring == 1 && READ_ONCE(bt->connection_command_supported) && + get_unaligned_le16(skb->data) == HCI_OP_CREATE_CONN) { + u8 *command = descriptor + 16; + + put_unaligned_le16(CEN_OP_CREATE_CONN, command); + command[2] = 15; + command[HCI_COMMAND_HDR_SIZE + 9] = 0; + command[HCI_COMMAND_HDR_SIZE + 12] = 0; + wire_length = HCI_COMMAND_HDR_SIZE + 15; + bt->connection_status_pending = true; + } + put_unaligned_le32(1 | (wire_length << 8), descriptor); + *head = next; + cen_post(bt, ring, next); + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT) + bt->hdev->stat.cmd_tx++; + else + bt->hdev->stat.acl_tx++; + bt->hdev->stat.byte_tx += wire_length; + kfree_skb(skb); + } + return 0; +} + +static void cen_fixup_scan_response(u8 *event, size_t length) +{ + struct hci_ev_le_ext_adv_info *info; + size_t offset = HCI_EVENT_HDR_SIZE + 2; + u8 count; + + if (length < offset || event[0] != HCI_EV_LE_META || + event[HCI_EVENT_HDR_SIZE] != HCI_EV_LE_EXT_ADV_REPORT) + return; + count = event[HCI_EVENT_HDR_SIZE + 1]; + while (count-- && length - offset >= sizeof(*info)) { + u16 type; + + info = (void *)(event + offset); + if (info->length > length - offset - sizeof(*info)) + return; + type = get_unaligned_le16(&info->type); + if ((type & LE_EXT_ADV_EVT_TYPE_MASK) == + (LE_EXT_ADV_LEGACY_PDU | LE_EXT_ADV_SCAN_RSP)) + put_unaligned_le16(type | LE_EXT_ADV_SCAN_IND, + &info->type); + offset += sizeof(*info) + info->length; + } +} + +static void cen_io_work(struct work_struct *work) +{ + struct cen_bt *bt = container_of(work, struct cen_bt, io_work); + struct hci_dev *hdev = bt->hdev; + struct sk_buff *skb; + u16 tail; + u8 *descriptor; + u32 length; + int budget = CEN_RX_SLOTS; + + mutex_lock(&bt->lock); + if (!bt->running) + goto unlock; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 2) { + if (readl(bt->bar) == 2) + cen_aux_drain(bt, false); + goto protocol_error; + } + if (cen_aux_drain(bt, true)) + goto protocol_error; + { + tail = cen_tail(bt, 10); + if (tail >= CEN_LE_SLOTS) + goto protocol_error; + while (bt->le_rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_LE_RX + + bt->le_rx_head * CEN_LE_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length > CEN_LE_FOOTER) + goto protocol_error; + if (cen_receive_acl(bt, descriptor + 16, length)) + goto protocol_error; + bt->le_rx_head = (bt->le_rx_head + 1) % CEN_LE_SLOTS; + cen_le_rx_buffer(bt, bt->le_rx_post); + bt->le_rx_post = (bt->le_rx_post + 1) % CEN_LE_SLOTS; + cen_post(bt, 10, bt->le_rx_post); + } + budget = CEN_RX_SLOTS; + } + tail = cen_tail(bt, 7); + if (tail >= CEN_RX_SLOTS) + goto protocol_error; + while (bt->rx_head != tail && budget--) { + dma_rmb(); + descriptor = bt->arena + CEN_RX + bt->rx_head * CEN_STRIDE; + length = get_unaligned_le32(descriptor) >> 8; + if (length < HCI_EVENT_HDR_SIZE || length > CEN_FOOTER || + length != descriptor[17] + HCI_EVENT_HDR_SIZE) + goto protocol_error; + skb = bt_skb_alloc(length, GFP_KERNEL); + if (skb) { + skb_put_data(skb, descriptor + 16, length); + cen_fixup_scan_response(skb->data, skb->len); + + if (bt->connection_status_pending && length == 6 && + skb->data[0] == HCI_EV_CMD_STATUS && + get_unaligned_le16(skb->data + 4) == + CEN_OP_CREATE_CONN) { + put_unaligned_le16(HCI_OP_CREATE_CONN, + skb->data + 4); + bt->connection_status_pending = false; + } + + if (READ_ONCE(bt->connection_command_supported) && + length == + HCI_EVENT_HDR_SIZE + + sizeof(struct hci_ev_conn_complete) && + skb->data[0] == HCI_EV_CONN_COMPLETE && + (skb->data[11] & 0xfe) == 0xf0) { + skb->data[11] = ACL_LINK; + } + hci_skb_pkt_type(skb) = HCI_EVENT_PKT; + hdev->stat.byte_rx += length; + hci_recv_frame(hdev, skb); + } else { + hdev->stat.err_rx++; + } + bt->rx_head = (bt->rx_head + 1) % CEN_RX_SLOTS; + cen_rx_buffer(bt, bt->rx_post); + bt->rx_post = (bt->rx_post + 1) % CEN_RX_SLOTS; + cen_post(bt, 7, bt->rx_post); + } + if (cen_transmit(bt, &bt->txq, 1, CEN_TX_SLOTS, CEN_TX, CEN_FOOTER, + &bt->tx_head)) + goto protocol_error; + + if ((cen_transmit(bt, &bt->acl_txq, 2, cen_aux_specs[0].slots, + bt->aux[0].offset, cen_aux_specs[0].footer, + &bt->acl_tx_head) || + cen_transmit(bt, &bt->le_txq, 4, CEN_LE_SLOTS, CEN_LE_TX, + CEN_LE_FOOTER, &bt->le_tx_head))) + goto protocol_error; + goto unlock; +protocol_error: + bt->running = false; + hdev->stat.err_rx++; + dev_err(&bt->pdev->dev, + "Beta transport stopped (exec=%u ipc=%u); interrupt processing stopped\n", + readl(bt->bar + 0x18), readl(bt->bar)); +unlock: + mutex_unlock(&bt->lock); +} + +static int cen_hci_open(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = true; + schedule_work(&bt->io_work); + mutex_unlock(&bt->lock); + return 0; +} + +static int cen_hci_close(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + mutex_lock(&bt->lock); + bt->running = false; + bt->connection_status_pending = false; + mutex_unlock(&bt->lock); + cancel_work_sync(&bt->io_work); + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_vendor_command(struct hci_dev *hdev, u16 opcode, + const void *data, u8 length) +{ + struct sk_buff *skb; + int ret; + + skb = __hci_cmd_sync(hdev, opcode, length, data, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len < 1 ? -EPROTO : -bt_to_errno(skb->data[0]); + kfree_skb(skb); + return ret; +} + +static int cen_hci_reset_dup_filter(struct hci_dev *hdev) +{ + static const u8 flags; + int ret; + + ret = __hci_cmd_sync_status(hdev, CEN_OP_FLUSH_DUPLICATES, + sizeof(flags), &flags, HCI_CMD_TIMEOUT); + if (ret) { + bt_dev_warn(hdev, "Duplicate filter reset failed: %d", ret); + return ret; + } + return 0; +} + +static int cen_hci_post_init(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff *skb; + u32 features; + int ret; + + WRITE_ONCE(bt->connection_command_supported, false); + skb = __hci_cmd_sync(hdev, CEN_OP_READ_PROP_FEATURES, 0, NULL, + HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + ret = skb->len == 5 ? -bt_to_errno(skb->data[0]) : -EPROTO; + features = skb->len == 5 ? get_unaligned_le32(skb->data + 1) : 0; + kfree_skb(skb); + if (!ret && !(features & CEN_PROP_CREATE_CONN)) + ret = -EOPNOTSUPP; + if (ret) { + bt_dev_warn(hdev, "Firmware connection command unavailable: %d", + ret); + return 0; + } + WRITE_ONCE(bt->connection_command_supported, true); + return 0; +} + +/* + * The factory address in network byte order, for the firmware's provisioning command: from + * the standard local-bd-address (least significant byte first, as boot loaders fill it from + * the bluetooth0 alias), else from local-mac-address (already in network order). + */ +static int cen_bt_address(struct device *dev, u8 address[6]) +{ + u8 bd[6]; + int i, ret; + + if (device_property_count_u8(dev, "local-bd-address") == 6) { + ret = device_property_read_u8_array(dev, "local-bd-address", bd, sizeof(bd)); + if (ret) + return ret; + for (i = 0; i < 6; i++) + address[i] = bd[5 - i]; + return 0; + } + if (device_property_count_u8(dev, "local-mac-address") != 6) + return -EINVAL; + return device_property_read_u8_array(dev, "local-mac-address", address, 6); +} + +static int cen_hci_setup(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + u8 address[6], init_done[16], seed[24]; + int ret; + + ret = cen_bt_address(&bt->pdev->dev, address); + if (ret) + return ret; + if (is_zero_ether_addr(address) || is_broadcast_ether_addr(address)) + return -EINVAL; + ret = get_random_bytes_wait(seed, sizeof(seed)); + if (ret) + return ret; + memset(init_done, 0xff, sizeof(init_done)); + ret = cen_vendor_command(hdev, 0xfc06, address, sizeof(address)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 1); + ret = cen_vendor_command(hdev, 0xff0b, init_done, sizeof(init_done)); + if (ret) + goto out; + WRITE_ONCE(bt->native_startup_steps, 2); + ret = cen_vendor_command(hdev, 0xfe44, seed, sizeof(seed)); + if (!ret) + WRITE_ONCE(bt->native_startup_steps, 3); +out: + memzero_explicit(seed, sizeof(seed)); + if (ret) + bt_dev_err( + hdev, + "Firmware initialization stopped after %u commands: %d", + bt->native_startup_steps, ret); + return ret; +} + +static int cen_hci_flush(struct hci_dev *hdev) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + + skb_queue_purge(&bt->txq); + skb_queue_purge(&bt->acl_txq); + skb_queue_purge(&bt->le_txq); + return 0; +} + +static int cen_hci_send(struct hci_dev *hdev, struct sk_buff *skb) +{ + struct cen_bt *bt = hci_get_drvdata(hdev); + struct sk_buff_head *queue = &bt->txq; + struct hci_conn *conn; + u16 handle; + u8 link_type; + int ret = 0; + + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT && + (skb->len < HCI_COMMAND_HDR_SIZE || + skb->len != HCI_COMMAND_HDR_SIZE + skb->data[2])) + return -EMSGSIZE; + if (hci_skb_pkt_type(skb) == HCI_ACLDATA_PKT) { + if (skb->len < HCI_ACL_HDR_SIZE || + skb->len != HCI_ACL_HDR_SIZE + + get_unaligned_le16(skb->data + 2)) { + ret = -EMSGSIZE; + goto out; + } + handle = get_unaligned_le16(skb->data); + rcu_read_lock(); + conn = hci_conn_hash_lookup_handle(hdev, hci_handle(handle)); + link_type = conn ? conn->type : 0xff; + rcu_read_unlock(); + if (link_type == LE_LINK && (handle & 0x3000) != 0x3000) { + queue = &bt->le_txq; + if (skb->len > CEN_LE_FOOTER) + ret = -EMSGSIZE; + } else if (link_type == ACL_LINK) { + queue = &bt->acl_txq; + if (skb->len > cen_aux_specs[0].footer) + ret = -EMSGSIZE; + } else { + ret = -EOPNOTSUPP; + } + if (ret) + goto out; + } else if (hci_skb_pkt_type(skb) != HCI_COMMAND_PKT) { + ret = -EOPNOTSUPP; + goto out; + } else if (skb->len < HCI_COMMAND_HDR_SIZE || skb->len > CEN_FOOTER || + skb->len != HCI_COMMAND_HDR_SIZE + skb->data[2]) { + ret = -EMSGSIZE; + goto out; + } + if (hci_skb_pkt_type(skb) == HCI_COMMAND_PKT) { + u16 opcode = get_unaligned_le16(skb->data); + + if (opcode == CEN_OP_CREATE_CONN) { + ret = -EOPNOTSUPP; + goto out; + } + if (opcode == HCI_OP_CREATE_CONN && + skb->len != HCI_COMMAND_HDR_SIZE + + sizeof(struct hci_cp_create_conn)) { + ret = -EMSGSIZE; + goto out; + } + } + mutex_lock(&bt->lock); + if (!bt->running) + ret = -ENETDOWN; + else if (skb_queue_len(queue) >= 64) + ret = -ENOBUFS; + else { + skb_queue_tail(queue, skb); + schedule_work(&bt->io_work); + } + mutex_unlock(&bt->lock); + if (!ret) + return 0; +out: + + return ret; +} + +static irqreturn_t cen_irq(int irq, void *data) +{ + struct cen_bt *bt = data; + + if (READ_ONCE(bt->running)) + schedule_work(&bt->io_work); + return IRQ_HANDLED; +} + +static void cen_free_irqs(struct cen_bt *bt) +{ + while (bt->irq_count) + free_irq(pci_irq_vector(bt->pdev, --bt->irq_count), bt); + if (bt->irq_vectors) { + pci_free_irq_vectors(bt->pdev); + bt->irq_vectors = false; + } +} + +static int cen_request_irqs(struct cen_bt *bt) +{ + int ret = pci_alloc_irq_vectors(bt->pdev, 2, 2, PCI_IRQ_MSI); + + if (ret < 0) + return ret; + bt->irq_vectors = true; + for (bt->irq_count = 0; bt->irq_count < 2; bt->irq_count++) { + ret = request_irq(pci_irq_vector(bt->pdev, bt->irq_count), + cen_irq, 0, "btcentauri", bt); + if (ret) { + cen_free_irqs(bt); + return ret; + } + } + return 0; +} + +static void cen_stop(struct cen_bt *bt) +{ + u16 command; + int ret; + + pci_clear_master(bt->pdev); + cen_free_irqs(bt); + cancel_work_sync(&bt->io_work); + + ret = pci_read_config_word(bt->pdev, PCI_COMMAND, &command); + if (bt->ipc_requested || ret || (command & PCI_COMMAND_MASTER) || + !pci_wait_for_pending_transaction(bt->pdev)) + dev_warn( + &bt->pdev->dev, + "Beta detach requires fresh firmware boot; retaining DMA arena\n"); + else + dma_free_coherent(&bt->pdev->dev, CEN_ARENA_SIZE, bt->arena, + bt->dma); +} + +static int cen_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct cen_bt *bt; + struct hci_dev *hdev; + int ret; + struct pci_dev *control; + struct device_link *link; + + if (PCI_FUNC(pdev->devfn) != 2) + return -ENODEV; + control = pci_get_domain_bus_and_slot( + pci_domain_nr(pdev->bus), pdev->bus->number, + PCI_DEVFN(PCI_SLOT(pdev->devfn), 0)); + if (!control) + return -EPROBE_DEFER; + link = device_link_add(&pdev->dev, &control->dev, + DL_FLAG_AUTOREMOVE_CONSUMER); + if (!link) { + pci_dev_put(control); + return -ENOMEM; + } + ret = apple_centauri_control_ready(control) ? 0 : -EPROBE_DEFER; + pci_dev_put(control); + if (ret) + return ret; + bt = devm_kzalloc(&pdev->dev, sizeof(*bt), GFP_KERNEL); + if (!bt) + return -ENOMEM; + bt->pdev = pdev; + mutex_init(&bt->lock); + skb_queue_head_init(&bt->txq); + skb_queue_head_init(&bt->acl_txq); + skb_queue_head_init(&bt->le_txq); + INIT_WORK(&bt->io_work, cen_io_work); + ret = pcim_enable_device(pdev); + if (ret) + return ret; + if (pci_resource_len(pdev, 0) != SZ_64K) + return -ENODEV; + ret = pcim_iomap_regions(pdev, BIT(0), "btcentauri"); + if (ret) + return ret; + bt->bar = pcim_iomap_table(pdev)[0]; + if (readl(bt->bar + 0x18) != 2 || readl(bt->bar) != 0) + return dev_err_probe( + &pdev->dev, -EBUSY, + "Beta requires running firmware and unowned IPC (exec=%u ipc=%u)\n", + readl(bt->bar + 0x18), readl(bt->bar)); + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + bt->arena = dma_alloc_coherent(&pdev->dev, CEN_ARENA_SIZE, &bt->dma, + GFP_KERNEL); + if (!bt->arena) + return -ENOMEM; + memset(bt->arena, 0, CEN_ARENA_SIZE); + ret = cen_request_irqs(bt); + if (ret) + goto stop; + ret = cen_start(bt); + if (ret) { + dev_err(&pdev->dev, "Beta IPC initialization failed: %d\n", + ret); + goto stop; + } + hdev = hci_alloc_dev(); + if (!hdev) { + ret = -ENOMEM; + goto stop; + } + bt->hdev = hdev; + hdev->bus = HCI_PCI; + hdev->open = cen_hci_open; + hdev->close = cen_hci_close; + hdev->flush = cen_hci_flush; + hdev->send = cen_hci_send; + hdev->setup = cen_hci_setup; + hdev->post_init = cen_hci_post_init; + hdev->reset_dup_filter = cen_hci_reset_dup_filter; + + hci_set_quirk(hdev, HCI_QUIRK_FIXUP_LE_EXT_ADV_REPORT_PHY); + SET_HCIDEV_DEV(hdev, &pdev->dev); + hci_set_drvdata(hdev, bt); + pci_set_drvdata(pdev, bt); + ret = hci_register_dev(hdev); + if (ret) { + hci_free_dev(hdev); + goto stop; + } + return 0; +stop: + cen_stop(bt); + return ret; +} + +static void cen_remove(struct pci_dev *pdev) +{ + struct cen_bt *bt = pci_get_drvdata(pdev); + hci_unregister_dev(bt->hdev); + cen_hci_close(bt->hdev); + cen_free_irqs(bt); + cancel_work_sync(&bt->io_work); + hci_free_dev(bt->hdev); + cen_stop(bt); +} + +static int cen_suspend(struct device *dev) +{ + return pci_save_state(to_pci_dev(dev)); +} + +static int cen_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(cen_pm_ops, cen_suspend, cen_resume); + +static const struct pci_device_id cen_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1903) }, + {} +}; +MODULE_DEVICE_TABLE(pci, cen_ids); + +static struct pci_driver cen_driver = { + .name = "btcentauri", + .id_table = cen_ids, + .probe = cen_probe, + .remove = cen_remove, + .driver.pm = pm_sleep_ptr(&cen_pm_ops), +}; +module_pci_driver(cen_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 protocol-27 Bluetooth transport"); diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c index fdc1fd2586b9e..ed4ba2e469ac7 100644 --- a/drivers/bluetooth/btintel_pcie.c +++ b/drivers/bluetooth/btintel_pcie.c @@ -401,7 +401,7 @@ static int btintel_pcie_send_sync(struct btintel_pcie_data *data, tfd_index = data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM]; - if (tfd_index > txq->count) + if (tfd_index >= txq->count) return -ERANGE; if (skb->len > BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN) { @@ -1715,7 +1715,7 @@ static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id) static int btintel_pcie_setup_irq(struct btintel_pcie_data *data) { int err; - int num_irqs, i; + int num_irqs, i, j; for (i = 0; i < BTINTEL_PCIE_MSIX_VEC_MAX; i++) data->msix_entries[i].entry = i; @@ -1744,6 +1744,10 @@ static int btintel_pcie_setup_irq(struct btintel_pcie_data *data) KBUILD_MODNAME, msix_entry); if (err) { + for (j = 0; j < i; j++) + devm_free_irq(&data->pdev->dev, + data->msix_entries[j].vector, + &data->msix_entries[j]); pci_free_irq_vectors(data->pdev); data->alloc_vecs = 0; return err; @@ -1815,6 +1819,13 @@ static int btintel_pcie_config_pcie(struct pci_dev *pdev, if (IS_ERR(data->base_addr)) return PTR_ERR(data->base_addr); + /* Do shared hardware reset to ensure a clean start before + * configuring interrupts. + */ + err = btintel_pcie_reset_bt(data); + if (err) + return err; + err = btintel_pcie_setup_irq(data); if (err) return err; @@ -3003,7 +3014,7 @@ static void btintel_pcie_remove(struct pci_dev *pdev) struct msix_entry *msix_entry; msix_entry = &data->msix_entries[i]; - free_irq(msix_entry->vector, msix_entry); + devm_free_irq(&pdev->dev, msix_entry->vector, msix_entry); } pci_free_irq_vectors(pdev); diff --git a/drivers/clk/qcom/gcc-eliza.c b/drivers/clk/qcom/gcc-eliza.c index 3e26c7a1e5b54..fe18942a59ac4 100644 --- a/drivers/clk/qcom/gcc-eliza.c +++ b/drivers/clk/qcom/gcc-eliza.c @@ -3010,8 +3010,6 @@ static const u32 gcc_eliza_critical_cbcrs[] = { 0x26034, /* GCC_CAMERA_XO_CLK */ 0x27004, /* GCC_DISP_AHB_CLK */ 0x71004, /* GCC_GPU_CFG_AHB_CLK */ - 0x52010, /* GCC_PCIE_RSCC_CFG_AHB_CLK */ - 0x52010, /* GCC_PCIE_RSCC_XO_CLK */ 0x32004, /* GCC_VIDEO_AHB_CLK */ 0x32038, /* GCC_VIDEO_XO_CLK */ }; @@ -3045,6 +3043,10 @@ static const struct regmap_config gcc_eliza_regmap_config = { static void clk_eliza_regs_configure(struct device *dev, struct regmap *regmap) { + /* Keep clocks always enabled */ + regmap_update_bits(regmap, 0x52010, BIT(20), BIT(20)); /* GCC_PCIE_RSCC_CFG_AHB_CLK */ + regmap_update_bits(regmap, 0x52010, BIT(21), BIT(21)); /* GCC_PCIE_RSCC_XO_CLK */ + /* FORCE_MEM_CORE_ON for ufs phy ice core and gcc ufs phy axi clocks */ qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true); qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_axi_clk, true); diff --git a/drivers/clk/qcom/gcc-kaanapali.c b/drivers/clk/qcom/gcc-kaanapali.c index 842c1a70c6916..5bcbcd1786577 100644 --- a/drivers/clk/qcom/gcc-kaanapali.c +++ b/drivers/clk/qcom/gcc-kaanapali.c @@ -3464,8 +3464,6 @@ static const u32 gcc_kaanapali_critical_cbcrs[] = { 0x9f004, /* GCC_EVA_AHB_CLK */ 0x9f024, /* GCC_EVA_XO_CLK */ 0x71004, /* GCC_GPU_CFG_AHB_CLK */ - 0x52010, /* GCC_PCIE_RSCC_CFG_AHB_CLK */ - 0x52010, /* GCC_PCIE_RSCC_XO_CLK */ 0x32004, /* GCC_VIDEO_AHB_CLK */ 0x32040, /* GCC_VIDEO_XO_CLK */ }; @@ -3480,6 +3478,10 @@ static const struct regmap_config gcc_kaanapali_regmap_config = { static void clk_kaanapali_regs_configure(struct device *dev, struct regmap *regmap) { + /* Keep clocks always enabled */ + regmap_update_bits(regmap, 0x52010, BIT(20), BIT(20)); /* GCC_PCIE_RSCC_CFG_AHB_CLK */ + regmap_update_bits(regmap, 0x52010, BIT(21), BIT(21)); /* GCC_PCIE_RSCC_XO_CLK */ + /* FORCE_MEM_CORE_ON for ufs phy ice core clocks */ qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true); } diff --git a/drivers/clk/qcom/gpucc-glymur.c b/drivers/clk/qcom/gpucc-glymur.c index 001b2454786a0..cac51de20b295 100644 --- a/drivers/clk/qcom/gpucc-glymur.c +++ b/drivers/clk/qcom/gpucc-glymur.c @@ -510,7 +510,7 @@ static struct gdsc gpu_cc_cx_gdsc = { .name = "gpu_cc_cx_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, + .flags = RETAIN_FF_ENABLE | VOTABLE, }; static struct clk_regmap *gpu_cc_glymur_clocks[] = { diff --git a/drivers/clk/qcom/gpucc-kaanapali.c b/drivers/clk/qcom/gpucc-kaanapali.c index ae5563e516f67..f75f2caa2205c 100644 --- a/drivers/clk/qcom/gpucc-kaanapali.c +++ b/drivers/clk/qcom/gpucc-kaanapali.c @@ -380,7 +380,7 @@ static struct gdsc gpu_cc_cx_gdsc = { .name = "gpu_cc_cx_gdsc", }, .pwrsts = PWRSTS_OFF_ON, - .flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE, + .flags = RETAIN_FF_ENABLE | VOTABLE, }; static struct clk_regmap *gpu_cc_kaanapali_clocks[] = { diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index d5172ddd346f3..6cf49c2f40a15 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -306,6 +306,11 @@ static const struct of_device_id apple_soc_cpufreq_of_match[] __maybe_unused = { .compatible = "apple,t8142-cluster-cpufreq", .data = &soc_t8142_info, }, + { + /* T6050: same command/readback model as T8142, CLPC/PMP keep the rest. */ + .compatible = "apple,t6050-cluster-cpufreq", + .data = &soc_t8142_info, + }, { .compatible = "apple,t8152-cluster-cpufreq", .data = &soc_t8152_info, diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index b65fcd35fa689..94f9c2e2860d9 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -99,10 +99,45 @@ config APPLE_SIO depends on APPLE_RTKIT depends on OF_ADDRESS select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS help Enable support for the SIO coprocessor found on Apple Silicon SoCs where it provides DMA services. +config APPLE_SIO_KUNIT_TEST + bool "KUnit tests for the Apple SIO probe" if !KUNIT_ALL_TESTS + depends on APPLE_SIO=y && KUNIT=y + depends on OF_OVERLAY + default KUNIT_ALL_TESTS + help + Check which SIO device-tree nodes the probe goes on with: the + generic SIO of the M1 and M2 in every configuration, and a T6030 + SIO only with APPLE_SIO_T6030 and protocol-10 boot tables. Run it + both with and without APPLE_SIO_T6030. + +config APPLE_SIO_T6030 + bool "Apple SIO on the M3 Pro (T6030)" + depends on APPLE_SIO + help + Configure the SIO channels of the T6030 (apple,t6030-sio) the way + its firmware expects them: the DisplayPort audio channels refill + their FIFO at the threshold the system's device tree records for + them (0x600 bytes, not 0x800), and every period descriptor is + published whole. The firmware's boot tables come from the boot + loader, as on the other SoCs; the SIO is only started when they + are protocol-10 tables, which a boot loader without T6030 support + does not build. Without this option a T6030 SIO is left stopped. + +config APPLE_SIO_T6030_KUNIT_TEST + bool "KUnit tests for the T6030 SIO boot-table check" if !KUNIT_ALL_TESTS + depends on APPLE_SIO=y && APPLE_SIO_T6030 && KUNIT=y + depends on OF_OVERLAY + default KUNIT_ALL_TESTS + help + Check the protocol-10 boot-table admission of the T6030 SIO + against tables a T6030-aware boot loader builds, an earlier + loader's tables, and malformed, truncated and out-of-window ones. + config ARM_DMA350 tristate "Arm DMA-350 support" depends on ARM || ARM64 || COMPILE_TEST diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index edea48f908e96..b3b59d224e190 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -18,6 +18,8 @@ obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ obj-$(CONFIG_APPLE_ADMAC) += apple-admac.o obj-$(CONFIG_APPLE_SIO) += apple-sio.o +obj-$(CONFIG_APPLE_SIO_KUNIT_TEST) += apple-sio-probe-test.dtbo.o +obj-$(CONFIG_APPLE_SIO_T6030_KUNIT_TEST) += apple-sio-t6030-test.dtbo.o obj-$(CONFIG_ARM_DMA350) += arm-dma350.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_AT_XDMAC) += at_xdmac.o diff --git a/drivers/dma/apple-sio-probe-test.c b/drivers/dma/apple-sio-probe-test.c new file mode 100644 index 0000000000000..7049c8acb532a --- /dev/null +++ b/drivers/dma/apple-sio-probe-test.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by apple-sio.c: which SIO nodes the probe goes on with, with and + * without APPLE_SIO_T6030. The test devices have no register range, so a probe + * that goes on toward the hardware stops there with -EINVAL; a refused node + * returns -ENODEV first. + */ +#include +#include +#include + +OF_OVERLAY_DECLARE(apple_sio_probe_test); + +static void sio_probe_nodes_test(struct kunit *test) +{ + const bool t6030 = IS_ENABLED(CONFIG_APPLE_SIO_T6030); + const struct { + const char *path; + int err; + } cases[] = { + /* M1/M2: the generic SIO, either way */ + { "/sio-probe-test/sio-probe-test-t8103", -EINVAL }, + /* T6030: only with its support built, and with protocol-10 tables */ + { "/sio-probe-test/sio-probe-test-t6030-v10", t6030 ? -EINVAL : -ENODEV }, + { "/sio-probe-test/sio-probe-test-t6030-earlier", -ENODEV }, + }; + unsigned int i; + + KUNIT_ASSERT_EQ(test, 0, of_overlay_apply_kunit(test, apple_sio_probe_test)); + for (i = 0; i < ARRAY_SIZE(cases); i++) { + struct platform_device *pdev; + u64 mask; + + pdev = kunit_platform_device_alloc(test, "apple-sio-probe-test", i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + /* The device's release drops this reference */ + pdev->dev.of_node = of_find_node_by_path(cases[i].path); + KUNIT_ASSERT_NOT_NULL_MSG(test, pdev->dev.of_node, "%s", cases[i].path); + mask = pdev->dev.coherent_dma_mask; + KUNIT_EXPECT_EQ_MSG(test, sio_probe(pdev), cases[i].err, "%s", cases[i].path); + /* Without its support, a T6030 is refused before its DMA is set up */ + if (!t6030 && cases[i].err == -ENODEV) + KUNIT_EXPECT_EQ_MSG(test, pdev->dev.coherent_dma_mask, mask, "%s", + cases[i].path); + } +} + +static struct kunit_case sio_probe_test_cases[] = { + KUNIT_CASE(sio_probe_nodes_test), + {} +}; + +static struct kunit_suite sio_probe_test_suite = { + .name = "apple-sio-probe", + .test_cases = sio_probe_test_cases, +}; + +kunit_test_suite(sio_probe_test_suite); diff --git a/drivers/dma/apple-sio-probe-test.dtso b/drivers/dma/apple-sio-probe-test.dtso new file mode 100644 index 0000000000000..79f3346beaad8 --- /dev/null +++ b/drivers/dma/apple-sio-probe-test.dtso @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/dts-v1/; +/plugin/; + +/* + * SIO nodes for the probe test: an M1 one, and T6030 ones with the tables of a + * T6030-aware boot loader and of an earlier one. The container is not a bus, so + * nothing binds to them; the test calls the probe itself. + */ +&{/} { + sio-probe-test { + #address-cells = <2>; + #size-cells = <2>; + + sio-probe-test-t8103 { + compatible = "apple,t8103-sio", "apple,sio"; + dma-channels = <8>; + }; + + v10_w: v10-w { + iommu-addresses = <&v10 0x0 0x30000 0x0 0x4000>, <&v10 0x0 0x34000 0x0 0x20000>, + <&v10 0x0 0x54000 0x0 0x4000>, <&v10 0x0 0x58000 0x0 0x4000>; + }; + + v10: sio-probe-test-t6030-v10 { + compatible = "apple,t6030-sio", "apple,sio"; + dma-channels = <8>; + memory-region = <&v10_w>; + apple,sio-firmware-params = <0x1e 0x30 0x1f 0x300 0xb 0x31 0xc 0x1b80 + 0xf 0x34 0x10 0x1e000 0x30d 0x54 0x30e 0x4000 + 0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xd8 + 0x1c 0x5a 0x1d 0x48>; + }; + + old_w: old-w { iommu-addresses = <&earlier 0x0 0x30000 0x0 0x2c000>; }; + + earlier: sio-probe-test-t6030-earlier { + compatible = "apple,t6030-sio", "apple,sio"; + dma-channels = <8>; + memory-region = <&old_w>; + apple,sio-firmware-params = <0x1e 0x30 0x1f 0x300 0xb 0x31 0xc 0x1b80 + 0xf 0x34 0x10 0x1e000 0x30d 0x54 0x30e 0x4000 + 0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xa0 + 0x1c 0x5a 0x1d 0x40>; + }; + }; +}; diff --git a/drivers/dma/apple-sio-t6030-test.dtso b/drivers/dma/apple-sio-t6030-test.dtso new file mode 100644 index 0000000000000..df42f31a2a8d1 --- /dev/null +++ b/drivers/dma/apple-sio-t6030-test.dtso @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/dts-v1/; +/plugin/; + +/* + * T6030 SIO nodes as boot loaders leave them: parameters in apple,sio-firmware-params and the + * firmware data mapped through memory regions with iommu-addresses. The v10 node carries the + * tables a T6030-aware loader builds; the earlier node the tables of a loader without + * protocol-10 support, in the same windows. The container is not a bus, so nothing binds to + * them; the probe test calls the probe itself, on a device without registers. + */ +&{/} { + sio-t6030-test { + #address-cells = <2>; + #size-cells = <2>; + + v10_w0: v10-w0 { iommu-addresses = <&v10 0x0 0x30000 0x0 0x4000>; }; + v10_w1: v10-w1 { iommu-addresses = <&v10 0x0 0x34000 0x0 0x20000>; }; + v10_w2: v10-w2 { iommu-addresses = <&v10 0x0 0x54000 0x0 0x4000>; }; + v10_w3: v10-w3 { iommu-addresses = <&v10 0x0 0x58000 0x0 0x4000>; }; + + v10: sio-t6030-test-v10 { + compatible = "apple,t6030-sio", "apple,sio"; + dma-channels = <8>; + memory-region = <&v10_w0 &v10_w1 &v10_w2 &v10_w3>; + apple,sio-firmware-params = <0x1e 0x30 0x1f 0x300 0xb 0x31 0xc 0x1b80 + 0xf 0x34 0x10 0x1e000 0x30d 0x54 0x30e 0x4000 + 0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xd8 + 0x1c 0x5a 0x1d 0x48>; + }; + + old_w: old-w { iommu-addresses = <&earlier 0x0 0x30000 0x0 0x2c000>; }; + + earlier: sio-t6030-test-earlier { + compatible = "apple,t6030-sio", "apple,sio"; + dma-channels = <8>; + memory-region = <&old_w>; + apple,sio-firmware-params = <0x1e 0x30 0x1f 0x300 0xb 0x31 0xc 0x1b80 + 0xf 0x34 0x10 0x1e000 0x30d 0x54 0x30e 0x4000 + 0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xa0 + 0x1c 0x5a 0x1d 0x40>; + }; + + sio-t6030-test-unmapped { + apple,sio-firmware-params = <0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xd8 + 0x1c 0x5a 0x1d 0x48>; + }; + + other_w: other-w { iommu-addresses = <&v10 0x0 0x30000 0x0 0x2c000>; }; + + sio-t6030-test-other-owner { + memory-region = <&other_w>; + apple,sio-firmware-params = <0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xd8 + 0x1c 0x5a 0x1d 0x48>; + }; + + nop_w: nop-w { iommu-addresses = <&no_params 0x0 0x30000 0x0 0x2c000>; }; + + no_params: sio-t6030-test-no-params { + memory-region = <&nop_w>; + }; + + odd_w: odd-w { iommu-addresses = <&odd 0x0 0x30000 0x0 0x2c000>; }; + + odd: sio-t6030-test-odd { + memory-region = <&odd_w>; + apple,sio-firmware-params = <0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xd8 + 0x1c 0x5a 0x1d>; + }; + + /* A window entry cut short, after the window that holds the tables */ + bad_w: bad-w { iommu-addresses = <&bad 0x0 0x58000 0x0 0x4000>, <&bad 0x0 0x30000 0x0>; }; + + bad: sio-t6030-test-bad-window { + memory-region = <&bad_w>; + apple,sio-firmware-params = <0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xd8 + 0x1c 0x5a 0x1d 0x48>; + }; + + /* + * An entry whose owner phandle does not resolve (none, or no such + * node), after the window that holds the tables + */ + noown0_w: noown0-w { iommu-addresses = <&noown0 0x0 0x58000 0x0 0x4000>, + <0x0 0x0 0x30000 0x0 0x1000>; }; + + noown0: sio-t6030-test-owner-none { + memory-region = <&noown0_w>; + apple,sio-firmware-params = <0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xd8 + 0x1c 0x5a 0x1d 0x48>; + }; + + noown1_w: noown1-w { iommu-addresses = <&noown1 0x0 0x58000 0x0 0x4000>, + <0xfffffff0 0x0 0x30000 0x0 0x1000>; }; + + noown1: sio-t6030-test-owner-unknown { + memory-region = <&noown1_w>; + apple,sio-firmware-params = <0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xd8 + 0x1c 0x5a 0x1d 0x48>; + }; + + /* More windows than any loader maps; the first one holds the tables */ + many_w: many-w { + iommu-addresses = <&many 0x0 0x58000 0x0 0x4000>, <&many 0x0 0x10000 0x0 0x1000>, + <&many 0x0 0x11000 0x0 0x1000>, <&many 0x0 0x12000 0x0 0x1000>, + <&many 0x0 0x13000 0x0 0x1000>, <&many 0x0 0x14000 0x0 0x1000>, + <&many 0x0 0x15000 0x0 0x1000>, <&many 0x0 0x16000 0x0 0x1000>, + <&many 0x0 0x17000 0x0 0x1000>; + }; + + many: sio-t6030-test-many-windows { + memory-region = <&many_w>; + apple,sio-firmware-params = <0x1a 0x58 0x1b 0x40 0x22 0x59 0x23 0xd8 + 0x1c 0x5a 0x1d 0x48>; + }; + }; +}; diff --git a/drivers/dma/apple-sio-test.c b/drivers/dma/apple-sio-test.c new file mode 100644 index 0000000000000..1088b96768638 --- /dev/null +++ b/drivers/dma/apple-sio-test.c @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by apple-sio.c: the T6030 SIO boot-table check. + * + * sio_t6030_build() lays the tables out the way the boot loader does (m1n1's add_fwdata(): a + * table shares the previous mapping while it fits in that mapping's 16K, the mappings start at + * IOVA 0x30000 and end up 16K-aligned). It builds both a T6030-aware loader's protocol-10 tables + * and an earlier loader's. + */ +#include +#include +#include + +#define SIO_TEST_PARAMS 32 + +struct sio_t6030_layout { + u32 params[SIO_TEST_PARAMS]; + unsigned int nparams; + struct sio_t6030_window windows[SIO_T6030_WINDOWS_MAX]; + unsigned int nwindows; +}; + +static void sio_t6030_add(struct sio_t6030_layout *l, u32 key, u32 size) +{ + struct sio_t6030_window *last = l->nwindows ? &l->windows[l->nwindows - 1] : NULL; + u64 iova; + + if (last && ALIGN(last->size, SZ_16K) >= last->size + size) { + iova = last->start + last->size; + last->size = ALIGN(last->size + size, SZ_4K); + } else { + iova = last ? ALIGN(last->start + last->size, SZ_16K) : 0x30000; + l->windows[l->nwindows].start = iova; + l->windows[l->nwindows].size = ALIGN(size, SZ_4K); + l->nwindows++; + } + l->params[l->nparams++] = key; + l->params[l->nparams++] = iova >> 12; + l->params[l->nparams++] = key + 1; + l->params[l->nparams++] = size; +} + +/* shims and types: the DMA shim and device-type table sizes this loader builds */ +static void sio_t6030_build(struct sio_t6030_layout *l, u32 shims, u32 types) +{ + unsigned int i; + + memset(l, 0, sizeof(*l)); + sio_t6030_add(l, 0x1e, 0x300); /* tunables, from the ADT */ + sio_t6030_add(l, 0x0b, 0x1b80); + sio_t6030_add(l, 0x0f, 0x1e000); /* panic buffer */ + sio_t6030_add(l, 0x30d, 0x4000); + sio_t6030_add(l, 0x1a, 4 * 16); /* map ranges */ + sio_t6030_add(l, 0x22, shims); + sio_t6030_add(l, 0x1c, types); + for (i = 0; i < l->nwindows; i++) + l->windows[i].size = ALIGN(l->windows[i].size, SZ_16K); +} + +static int sio_t6030_check(const struct sio_t6030_layout *l) +{ + return sio_t6030_check_params(l->params, l->nparams, l->windows, l->nwindows); +} + +static u32 *sio_t6030_value(struct sio_t6030_layout *l, u32 key) +{ + unsigned int i; + + for (i = 0; i < l->nparams; i += 2) + if (l->params[i] == key) + return &l->params[i + 1]; + return NULL; +} + +/* A T6030-aware loader's tables (six 36-byte shims, nine types) are admitted. */ +static void sio_t6030_protocol10_test(struct kunit *test) +{ + struct sio_t6030_layout l; + + sio_t6030_build(&l, 6 * 36, 9 * 8); + KUNIT_EXPECT_EQ(test, l.nwindows, 4U); + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), 0); +} + +/* An earlier loader's tables (five 32-byte shims, eight types) are refused. */ +static void sio_t6030_earlier_loader_test(struct kunit *test) +{ + struct sio_t6030_layout l; + + sio_t6030_build(&l, 5 * 32, 8 * 8); + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + /* Either table alone of the earlier size refuses too */ + sio_t6030_build(&l, 5 * 32, 9 * 8); + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + sio_t6030_build(&l, 6 * 36, 8 * 8); + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); +} + +/* Other record counts, such as a later protocol's, are refused. */ +static void sio_t6030_wrong_counts_test(struct kunit *test) +{ + static const u32 sizes[][2] = { + { 5 * 36, 9 * 8 }, { 7 * 36, 9 * 8 }, { 6 * 32, 9 * 8 }, { 6 * 40, 9 * 8 }, + { 6 * 36, 10 * 8 }, { 6 * 36, 9 * 16 }, { 0, 9 * 8 }, { 6 * 36, 0 }, + }; + struct sio_t6030_layout l; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(sizes); i++) { + sio_t6030_build(&l, sizes[i][0], sizes[i][1]); + KUNIT_EXPECT_EQ_MSG(test, sio_t6030_check(&l), -EINVAL, "case %u", i); + } + sio_t6030_build(&l, 6 * 36, 9 * 8); + *sio_t6030_value(&l, 0x1b) = 5 * 16; /* map ranges */ + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); +} + +/* Malformed and truncated parameter lists are refused. */ +static void sio_t6030_malformed_test(struct kunit *test) +{ + struct sio_t6030_layout l; + unsigned int i; + + sio_t6030_build(&l, 6 * 36, 9 * 8); + KUNIT_EXPECT_EQ(test, sio_t6030_check_params(l.params, 0, l.windows, l.nwindows), -EINVAL); + KUNIT_EXPECT_EQ(test, sio_t6030_check_params(l.params, l.nparams - 1, l.windows, + l.nwindows), -EINVAL); + /* Truncated after any pair: the last table's size is always lost */ + for (i = 2; i < l.nparams; i += 2) + KUNIT_EXPECT_EQ_MSG(test, sio_t6030_check_params(l.params, i, l.windows, + l.nwindows), + -EINVAL, "cut at %u", i); + /* A key twice, even with the same value, for a table or any other parameter */ + sio_t6030_build(&l, 6 * 36, 9 * 8); + l.params[l.nparams] = 0x22; + l.params[l.nparams + 1] = *sio_t6030_value(&l, 0x22); + l.nparams += 2; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + sio_t6030_build(&l, 6 * 36, 9 * 8); + l.params[l.nparams] = 0x1e; + l.params[l.nparams + 1] = *sio_t6030_value(&l, 0x1e); + l.nparams += 2; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + /* A table without its size, or its size without the table */ + sio_t6030_build(&l, 6 * 36, 9 * 8); + l.params[l.nparams - 2] = 0x40; /* 0x1d replaced */ + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + sio_t6030_build(&l, 6 * 36, 9 * 8); + l.params[l.nparams - 4] = 0x41; /* 0x1c replaced */ + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + /* More pairs than the controller takes */ + { + static u32 many[2 * (SIO_T6030_PARAMS_MAX + 1)]; + + sio_t6030_build(&l, 6 * 36, 9 * 8); + memset(many, 0, sizeof(many)); + memcpy(many, l.params, l.nparams * sizeof(u32)); + for (i = l.nparams; i < ARRAY_SIZE(many); i += 2) + many[i] = 0x1000 + i; + KUNIT_EXPECT_EQ(test, sio_t6030_check_params(many, ARRAY_SIZE(many), l.windows, + l.nwindows), -EINVAL); + KUNIT_EXPECT_EQ(test, sio_t6030_check_params(many, ARRAY_SIZE(many) - 2, + l.windows, l.nwindows), 0); + } +} + +/* + * A key wider than 16 bits is refused: the firmware gets its low 16 bits as + * the endpoint and parameter bytes, so it would overwrite a checked parameter. + */ +static void sio_t6030_key_alias_test(struct kunit *test) +{ + static const u32 aliases[] = { 0x1001a, 0xff00001a, 0x1000022, 0x10023, 0x1001c, 0x1001d }; + struct sio_t6030_layout l; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(aliases); i++) { + u32 key = aliases[i], checked; + + /* A run-time value, as the driver has: FIELD_PREP() masks it */ + OPTIMIZER_HIDE_VAR(key); + checked = key & 0xffff; + /* The setup message sio_send_dt_params() sends is that of the checked key */ + KUNIT_EXPECT_EQ(test, FIELD_PREP(SIOMSG_PARAM, key & 0xff) | + FIELD_PREP(SIOMSG_EP, key >> 8), + FIELD_PREP(SIOMSG_PARAM, checked & 0xff) | + FIELD_PREP(SIOMSG_EP, checked >> 8)); + sio_t6030_build(&l, 6 * 36, 9 * 8); + KUNIT_ASSERT_NOT_NULL(test, sio_t6030_value(&l, checked)); + l.params[l.nparams] = key; + l.params[l.nparams + 1] = 0x80; + l.nparams += 2; + KUNIT_EXPECT_EQ_MSG(test, sio_t6030_check(&l), -EINVAL, "key %#x", key); + } + /* Wider than 16 bits but aliasing no listed key: refused all the same */ + sio_t6030_build(&l, 6 * 36, 9 * 8); + l.params[l.nparams] = 0x10099; + l.params[l.nparams + 1] = 0; + l.nparams += 2; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + /* Any 16-bit key is its own message, and is admitted */ + sio_t6030_build(&l, 6 * 36, 9 * 8); + l.params[l.nparams] = 0xffff; + l.params[l.nparams + 1] = 0; + l.nparams += 2; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), 0); +} + +/* Each table must lie inside one mapped window; spans must not overflow. */ +static void sio_t6030_spans_test(struct kunit *test) +{ + struct sio_t6030_layout l; + + sio_t6030_build(&l, 6 * 36, 9 * 8); + KUNIT_EXPECT_EQ(test, sio_t6030_check_params(l.params, l.nparams, l.windows, 0), -EINVAL); + /* The shims' page outside every window */ + *sio_t6030_value(&l, 0x22) = 0x80; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + /* + * Ending exactly at the end of their window, and one byte past it. The window + * keeps the map ranges and the shims; the types get a window of their own. + */ + sio_t6030_build(&l, 6 * 36, 9 * 8); + l.windows[3].size = 0x59000 - 0x58000 + 6 * 36; + l.windows[4].start = 0x5a000; + l.windows[4].size = SZ_16K; + l.nwindows = 5; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), 0); + l.windows[3].size--; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + /* Starting at the start of their window, and one byte before it */ + l.windows[3].size++; + l.windows[3].start += SZ_4K; + l.windows[3].size -= SZ_4K; + *sio_t6030_value(&l, 0x1a) = 0x59; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), 0); + l.windows[3].start += 1; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + /* Beyond the controller's 42-bit DMA range, or in a window that wraps around */ + sio_t6030_build(&l, 6 * 36, 9 * 8); + *sio_t6030_value(&l, 0x22) = U32_MAX; + l.windows[3].start = 0; + l.windows[3].size = U64_MAX; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); + sio_t6030_build(&l, 6 * 36, 9 * 8); + l.windows[3].start = U64_MAX - 0xfff; + l.windows[3].size = 0x2000; + KUNIT_EXPECT_EQ(test, sio_t6030_check(&l), -EINVAL); +} + +OF_OVERLAY_DECLARE(apple_sio_t6030_test); + +/* The device-tree side: the parameters and windows a loader writes, read back. */ +static void sio_t6030_device_tree_test(struct kunit *test) +{ + static const char *const nodes[][2] = { + { "/sio-t6030-test/sio-t6030-test-v10", "admitted" }, + { "/sio-t6030-test/sio-t6030-test-earlier", "refused" }, + { "/sio-t6030-test/sio-t6030-test-unmapped", "refused" }, + { "/sio-t6030-test/sio-t6030-test-other-owner", "refused" }, + { "/sio-t6030-test/sio-t6030-test-no-params", "refused" }, + { "/sio-t6030-test/sio-t6030-test-odd", "refused" }, + { "/sio-t6030-test/sio-t6030-test-bad-window", "refused" }, + { "/sio-t6030-test/sio-t6030-test-owner-none", "refused" }, + { "/sio-t6030-test/sio-t6030-test-owner-unknown", "refused" }, + { "/sio-t6030-test/sio-t6030-test-many-windows", "refused" }, + }; + unsigned int i; + + KUNIT_ASSERT_EQ(test, 0, of_overlay_apply_kunit(test, apple_sio_t6030_test)); + for (i = 0; i < ARRAY_SIZE(nodes); i++) { + struct device_node *np __free(device_node) = of_find_node_by_path(nodes[i][0]); + int ret; + + KUNIT_ASSERT_NOT_NULL_MSG(test, np, "%s", nodes[i][0]); + ret = sio_t6030_check_node(np); + if (!strcmp(nodes[i][1], "admitted")) + KUNIT_EXPECT_EQ_MSG(test, ret, 0, "%s", nodes[i][0]); + else + KUNIT_EXPECT_NE_MSG(test, ret, 0, "%s", nodes[i][0]); + } +} + +/* + * The probe refuses an earlier loader's tables before it maps a register. The test devices + * have no register range, so a probe that gets past the check stops there with -EINVAL. + */ +static void sio_t6030_probe_test(struct kunit *test) +{ + static const struct { + const char *path; + int err; + } cases[] = { + { "/sio-t6030-test/sio-t6030-test-earlier", -ENODEV }, + { "/sio-t6030-test/sio-t6030-test-v10", -EINVAL }, + }; + unsigned int i; + + KUNIT_ASSERT_EQ(test, 0, of_overlay_apply_kunit(test, apple_sio_t6030_test)); + for (i = 0; i < ARRAY_SIZE(cases); i++) { + struct platform_device *pdev; + + pdev = kunit_platform_device_alloc(test, "apple-sio-t6030-test", i); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pdev); + /* The device's release drops this reference */ + pdev->dev.of_node = of_find_node_by_path(cases[i].path); + KUNIT_ASSERT_NOT_NULL_MSG(test, pdev->dev.of_node, "%s", cases[i].path); + KUNIT_EXPECT_EQ_MSG(test, sio_probe(pdev), cases[i].err, "%s", cases[i].path); + } +} + +static struct kunit_case sio_t6030_test_cases[] = { + KUNIT_CASE(sio_t6030_protocol10_test), + KUNIT_CASE(sio_t6030_earlier_loader_test), + KUNIT_CASE(sio_t6030_wrong_counts_test), + KUNIT_CASE(sio_t6030_malformed_test), + KUNIT_CASE(sio_t6030_key_alias_test), + KUNIT_CASE(sio_t6030_spans_test), + KUNIT_CASE(sio_t6030_device_tree_test), + KUNIT_CASE(sio_t6030_probe_test), + {} +}; + +static struct kunit_suite sio_t6030_test_suite = { + .name = "apple-sio-t6030", + .test_cases = sio_t6030_test_cases, +}; + +kunit_test_suite(sio_t6030_test_suite); diff --git a/drivers/dma/apple-sio.c b/drivers/dma/apple-sio.c index 511f91999ed3d..0c06e0264b7b6 100644 --- a/drivers/dma/apple-sio.c +++ b/drivers/dma/apple-sio.c @@ -13,8 +13,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -128,6 +130,9 @@ struct sio_data { } tags; int nchannels; +#if IS_ENABLED(CONFIG_APPLE_SIO_T6030) + bool t6030; +#endif struct sio_chan channels[]; }; @@ -150,6 +155,173 @@ static int sio_send_siomsg_atomic(struct sio_data *sio, u64 msg, void *cookie); static int sio_call(struct sio_data *sio, u64 msg); +static inline bool sio_is_t6030(struct sio_data *sio) +{ +#if IS_ENABLED(CONFIG_APPLE_SIO_T6030) + return sio->t6030; +#else + return false; +#endif +} + +#if IS_ENABLED(CONFIG_APPLE_SIO_T6030) +/* + * Boot tables of the T6030 SIO firmware, protocol 10. The boot loader builds them from the ADT + * and passes each one as two apple,sio-firmware-params: the table's IOVA page under its key and + * its size in bytes under key + 1. Protocol 10 takes six 36-byte DMA shim records and nine device + * types. A loader without protocol-10 support builds five 32-byte records and eight types, which + * this firmware cannot use, so the SIO is not started with them. + */ +#define SIO_T6030_PARAMS_MAX 64 /* key and value pairs */ +#define SIO_T6030_WINDOWS_MAX 8 +#define SIO_T6030_IOVA_LIMIT BIT_ULL(42) +/* + * A parameter goes to the firmware as an endpoint byte (key >> 8) and a + * parameter byte (key & 0xff), so a wider key would alias another one. + */ +#define SIO_T6030_KEY_MAX 0xffff + +struct sio_t6030_window { + u64 start; + u64 size; +}; + +static const struct { + u32 key; + u32 size; +} sio_t6030_tables[] = { + { 0x1a, 4 * 16 }, /* map ranges */ + { 0x1c, 9 * 8 }, /* device types */ + { 0x22, 6 * 36 }, /* DMA shims */ +}; + +/* + * 0 when the parameters hold each protocol-10 table once, at its exact size, entirely inside one + * of the windows the boot loader mapped for the SIO; -EINVAL otherwise. + */ +static int sio_t6030_check_params(const u32 *params, unsigned int nparams, + const struct sio_t6030_window *windows, unsigned int nwindows) +{ + unsigned int i, j, t; + + if (!nparams || nparams % 2 || nparams > 2 * SIO_T6030_PARAMS_MAX) + return -EINVAL; + for (i = 0; i < nparams; i += 2) + if (params[i] > SIO_T6030_KEY_MAX) + return -EINVAL; + for (i = 0; i < nparams; i += 2) + for (j = i + 2; j < nparams; j += 2) + if (params[i] == params[j]) + return -EINVAL; + + for (t = 0; t < ARRAY_SIZE(sio_t6030_tables); t++) { + bool have_page = false, have_size = false, inside = false; + u64 start = 0, size = 0, end; + + for (i = 0; i < nparams; i += 2) { + if (params[i] == sio_t6030_tables[t].key) { + start = (u64)params[i + 1] << 12; + have_page = true; + } else if (params[i] == sio_t6030_tables[t].key + 1) { + size = params[i + 1]; + have_size = true; + } + } + if (!have_page || !have_size || size != sio_t6030_tables[t].size) + return -EINVAL; + if (check_add_overflow(start, size, &end) || end > SIO_T6030_IOVA_LIMIT) + return -EINVAL; + for (j = 0; j < nwindows && !inside; j++) { + u64 window_end; + + if (check_add_overflow(windows[j].start, windows[j].size, &window_end)) + continue; + inside = start >= windows[j].start && end <= window_end; + } + if (!inside) + return -EINVAL; + } + return 0; +} + +/* The IOVA windows of this device's memory regions, from their iommu-addresses */ +static int sio_t6030_windows(struct device_node *np, struct sio_t6030_window *windows, + unsigned int max) +{ + struct of_phandle_iterator it; + unsigned int n = 0; + int err; + + of_for_each_phandle(&it, err, np, "memory-region", NULL, 0) { + const __be32 *maps, *end; + int size; + + maps = of_get_property(it.node, "iommu-addresses", &size); + if (!maps) + continue; + if (size % sizeof(*maps)) { + of_node_put(it.node); + return -EINVAL; + } + end = maps + size / sizeof(*maps); + while (maps < end) { + struct device_node *owner __free(device_node) = + of_find_node_by_phandle(be32_to_cpup(maps++)); + phys_addr_t iova; + size_t length; + + /* An entry whose owner does not resolve is malformed, as a short one is */ + if (!owner) { + of_node_put(it.node); + return -EINVAL; + } + maps = of_translate_dma_region_checked(owner, maps, end - maps, &iova, + &length); + if (!maps) { + of_node_put(it.node); + return -EINVAL; + } + if (owner != np) + continue; + if (n == max) { + of_node_put(it.node); + return -EINVAL; + } + windows[n].start = iova; + windows[n].size = length; + n++; + } + } + return err == -ENOENT ? n : err; +} + +/* 0 when the node's boot tables are protocol-10 tables inside its mapped windows */ +static int sio_t6030_check_node(struct device_node *np) +{ + struct sio_t6030_window windows[SIO_T6030_WINDOWS_MAX]; + u32 params[2 * SIO_T6030_PARAMS_MAX]; + int nparams, nwindows; + + nparams = of_property_count_u32_elems(np, "apple,sio-firmware-params"); + if (nparams <= 0 || nparams > ARRAY_SIZE(params) || + of_property_read_u32_array(np, "apple,sio-firmware-params", params, nparams)) + return -EINVAL; + nwindows = sio_t6030_windows(np, windows, ARRAY_SIZE(windows)); + if (nwindows < 0) + return nwindows; + return sio_t6030_check_params(params, nparams, windows, nwindows); +} + +static int sio_t6030_check_boot_tables(struct device *dev) +{ + if (sio_t6030_check_node(dev->of_node)) { + dev_err(dev, "boot tables are not for SIO protocol 10; leaving the SIO stopped\n"); + return -ENODEV; + } + return 0; +} +#endif + static struct sio_chan *to_sio_chan(struct dma_chan *chan) { return container_of(chan, struct sio_chan, vc.chan); @@ -283,6 +455,12 @@ static struct dma_async_tx_descriptor *sio_prep_dma_cyclic( return NULL; } + /* + * T6030: publish the whole descriptor, its other fields zero; + * a slot may come back from an earlier transfer. + */ + if (sio_is_t6030(siochan->host)) + memset(d, 0, sizeof(*d)); d->flag = 1; /* not sure what's up with this */ d->iova = buf_addr + period_len * i; d->size = period_len; @@ -656,7 +834,12 @@ static int sio_device_config(struct dma_chan *chan, cfg.timeout = 0; cfg.fifo = 0x800; cfg.limit = 0x800; - cfg.threshold = 0x800; + /* + * The ADT records each client's channel configuration with it (the + * dp-audioN dma-channels: channel, data shape, timeout, FIFO size, + * threshold). T6030's DisplayPort audio channels refill at 0x600. + */ + cfg.threshold = sio_is_t6030(sio) ? 0x600 : 0x800; /* * Dmaengine prescribes we ought to apply the new configuration only @@ -789,6 +972,16 @@ static int sio_probe(struct platform_device *pdev) int nchannels; int err, i; + /* + * Only APPLE_SIO_T6030 checks a T6030's boot tables and configures its + * channels: without it, leave the SIO stopped rather than start it as + * the generic one. + */ + if (!IS_ENABLED(CONFIG_APPLE_SIO_T6030) && + of_device_is_compatible(np, "apple,t6030-sio")) + return dev_err_probe(&pdev->dev, -ENODEV, + "T6030 SIO support (APPLE_SIO_T6030) not built; leaving the SIO stopped\n"); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(42)); if (err) return dev_err_probe(&pdev->dev, err, "Failed to set DMA mask\n"); @@ -805,6 +998,15 @@ static int sio_probe(struct platform_device *pdev) platform_set_drvdata(pdev, sio); sio->dev = &pdev->dev; sio->nchannels = nchannels; +#if IS_ENABLED(CONFIG_APPLE_SIO_T6030) + sio->t6030 = of_device_is_compatible(np, "apple,t6030-sio"); + /* Before any register, power or firmware access */ + if (sio->t6030) { + err = sio_t6030_check_boot_tables(&pdev->dev); + if (err) + return err; + } +#endif sio->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sio->base)) @@ -940,3 +1142,11 @@ module_platform_driver(apple_sio_driver); MODULE_AUTHOR("Martin PoviÅ¡er "); MODULE_DESCRIPTION("Driver for SIO coprocessor on Apple SoCs"); MODULE_LICENSE("Dual MIT/GPL"); + +#if IS_ENABLED(CONFIG_APPLE_SIO_T6030_KUNIT_TEST) +#include "apple-sio-test.c" +#endif + +#if IS_ENABLED(CONFIG_APPLE_SIO_KUNIT_TEST) +#include "apple-sio-probe-test.c" +#endif diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 475235c402e88..0c0d11c729259 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -2796,13 +2796,13 @@ static inline void decode_bus_error(int node_id, struct mce *m) * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel * number. * - * For DRAM ECC errors, the Chip Select number is given in bits [2:0] of + * For DRAM ECC errors, the Chip Select number is given in bits [1:0] of * the MCA_SYND[ErrorInformation] field. */ static void umc_get_err_info(struct mce *m, struct err_info *err) { err->channel = (m->ipid & GENMASK(31, 0)) >> 20; - err->csrow = m->synd & 0x7; + err->csrow = m->synd & 0x3; } static void decode_umc_error(int node_id, struct mce *m) diff --git a/drivers/gpio/gpio-exar.c b/drivers/gpio/gpio-exar.c index 9053662f18177..f629cc526f3ee 100644 --- a/drivers/gpio/gpio-exar.c +++ b/drivers/gpio/gpio-exar.c @@ -199,6 +199,7 @@ static int gpio_exar_probe(struct platform_device *pdev) index = ida_alloc(&ida_index, GFP_KERNEL); if (index < 0) return index; + exar_gpio->index = index; ret = devm_add_action_or_reset(dev, exar_devm_ida_free, exar_gpio); if (ret) @@ -214,7 +215,6 @@ static int gpio_exar_probe(struct platform_device *pdev) exar_gpio->gpio_chip.set = exar_set_value; exar_gpio->gpio_chip.base = -1; exar_gpio->gpio_chip.ngpio = ngpios; - exar_gpio->index = index; exar_gpio->first_pin = first_pin; ret = devm_gpiochip_add_data(dev, &exar_gpio->gpio_chip, exar_gpio); diff --git a/drivers/gpio/gpio-macsmc.c b/drivers/gpio/gpio-macsmc.c index 424f19b1c4a5d..ceab13aa5f503 100644 --- a/drivers/gpio/gpio-macsmc.c +++ b/drivers/gpio/gpio-macsmc.c @@ -18,20 +18,31 @@ #define MAX_GPIO 64 /* - * J700 resets its trackpad analog front end through the SMC "pcIO" key - * rather than a gPxx GPIO key. The Apple device tree describes the reset as - * the platform function pKW8("pcIO", 0x10000, 0x130000): an eight-byte write - * with 0x10000 in the low word, 0x130000 in the high word and the requested - * level in bits 15:0. Bits 23:16 of the high word hold the pin number. + * J700 and J714c reset their trackpad analog front end through the SMC + * "pcIO" key rather than a gPxx GPIO key. The Apple device tree describes the + * reset as a platform function such as pKW8("pcIO", 0x10000, 0x130000): an + * eight-byte write with 0x10000 in the low word, 0x130000 in the high word + * and the requested level in bits 15:0. Bits 23:16 of the high word hold the + * pin number. */ -#define MACSMC_J700_AFE_RESET_GPIO 19 -#define MACSMC_J700_AFE_RESET_LO 0x10000 -#define MACSMC_J700_AFE_RESET_HI 0x130000 +struct macsmc_pcio_line { + unsigned int gpio; + u32 lo, hi; +}; -/* J700 amfm-sunrise function-reg_on: pKW8("pcIO", 0x800000, 0x80000). */ -#define MACSMC_J700_RADIO_REG_ON_GPIO 8 -#define MACSMC_J700_RADIO_REG_ON_LO 0x800000 -#define MACSMC_J700_RADIO_REG_ON_HI 0x80000 +static const struct macsmc_pcio_line macsmc_j700_pcio[] = { + /* multi-touch function-afe-reset */ + { 19, 0x10000, 0x130000 }, + /* amfm-sunrise function-reg_on */ + { 8, 0x800000, 0x80000 }, + { } +}; + +static const struct macsmc_pcio_line macsmc_j714c_pcio[] = { + /* multi-touch function-afe-reset: pKW8("pcIO", 0x10000, 0x11c0000) */ + { 28, 0x10000, 0x11c0000 }, + { } +}; /* * Commands 0-6 are, presumably, the intended API. @@ -94,16 +105,19 @@ struct macsmc_gpio { int first_index; smc_key base_key; - bool j700_pcio; - DECLARE_BITMAP(j700_pcio_level, MAX_GPIO); + const struct macsmc_pcio_line *pcio; + DECLARE_BITMAP(pcio_level, MAX_GPIO); }; -static bool macsmc_gpio_is_j700_pcio(struct macsmc_gpio *smcgp, - unsigned int offset) +static const struct macsmc_pcio_line *macsmc_gpio_pcio(struct macsmc_gpio *smcgp, + unsigned int offset) { - return smcgp->j700_pcio && - (offset == MACSMC_J700_AFE_RESET_GPIO || - offset == MACSMC_J700_RADIO_REG_ON_GPIO); + const struct macsmc_pcio_line *line; + + for (line = smcgp->pcio; line && line->hi; line++) + if (line->gpio == offset) + return line; + return NULL; } static int macsmc_gpio_nr(smc_key key) @@ -176,7 +190,7 @@ static int macsmc_gpio_get_direction(struct gpio_chip *gc, unsigned int offset) u32 val; int ret; - if (macsmc_gpio_is_j700_pcio(smcgp, offset)) + if (macsmc_gpio_pcio(smcgp, offset)) return GPIO_LINE_DIRECTION_OUT; /* First try reading the explicit pin mode register */ @@ -200,8 +214,8 @@ static int macsmc_gpio_get(struct gpio_chip *gc, unsigned int offset) int ret; /* pcIO cannot be read back; report the last level written. */ - if (macsmc_gpio_is_j700_pcio(smcgp, offset)) - return test_bit(offset, smcgp->j700_pcio_level); + if (macsmc_gpio_pcio(smcgp, offset)) + return test_bit(offset, smcgp->pcio_level); ret = macsmc_gpio_get_direction(gc, offset); if (ret < 0) @@ -223,20 +237,11 @@ static int macsmc_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) { struct macsmc_gpio *smcgp = gpiochip_get_data(gc); smc_key key = macsmc_gpio_key(smcgp->base_key, offset); + const struct macsmc_pcio_line *line = macsmc_gpio_pcio(smcgp, offset); int ret; - if (macsmc_gpio_is_j700_pcio(smcgp, offset)) { - u32 lo, hi; - u64 payload; - - if (offset == MACSMC_J700_RADIO_REG_ON_GPIO) { - lo = MACSMC_J700_RADIO_REG_ON_LO; - hi = MACSMC_J700_RADIO_REG_ON_HI; - } else { - lo = MACSMC_J700_AFE_RESET_LO; - hi = MACSMC_J700_AFE_RESET_HI; - } - payload = ((u64)hi << 32) | lo | (value ? 1 : 0); + if (line) { + u64 payload = ((u64)line->hi << 32) | line->lo | (value ? 1 : 0); ret = apple_smc_write_u64(smcgp->smc, SMC_KEY(pcIO), payload); if (ret < 0) { @@ -245,7 +250,7 @@ static int macsmc_gpio_set(struct gpio_chip *gc, unsigned int offset, int value) offset, !!value, ret); return ret; } - assign_bit(offset, smcgp->j700_pcio_level, value); + assign_bit(offset, smcgp->pcio_level, value); return 0; } @@ -262,16 +267,15 @@ static int macsmc_gpio_init_valid_mask(struct gpio_chip *gc, unsigned long *valid_mask, unsigned int ngpios) { struct macsmc_gpio *smcgp = gpiochip_get_data(gc); + const struct macsmc_pcio_line *line; int count; int i; count = min(smcgp->smc->key_count, MAX_GPIO); bitmap_zero(valid_mask, ngpios); - if (smcgp->j700_pcio) { - set_bit(MACSMC_J700_AFE_RESET_GPIO, valid_mask); - set_bit(MACSMC_J700_RADIO_REG_ON_GPIO, valid_mask); - } + for (line = smcgp->pcio; line && line->hi; line++) + set_bit(line->gpio, valid_mask); for (i = 0; i < count; i++) { int ret, gpio_nr; @@ -315,8 +319,13 @@ static int macsmc_gpio_probe(struct platform_device *pdev) smcgp->dev = &pdev->dev; smcgp->smc = smc; smcgp->base_key = data ? data->base_key : _SMC_KEY("gP\0\0"); - smcgp->j700_pcio = of_machine_is_compatible("apple,j700") && - device_is_compatible(&pdev->dev, "apple,smc-gpio"); + if (device_is_compatible(&pdev->dev, "apple,smc-gpio")) { + if (of_machine_is_compatible("apple,j700")) + smcgp->pcio = macsmc_j700_pcio; + else if (of_machine_is_compatible("apple,j714c") || + of_machine_is_compatible("apple,k114c")) + smcgp->pcio = macsmc_j714c_pcio; + } smcgp->first_index = macsmc_gpio_find_first_gpio_index(smcgp); if (smcgp->first_index < 0) diff --git a/drivers/gpio/gpio-mpsse.c b/drivers/gpio/gpio-mpsse.c index a859deab2bca8..5ce822c0ddcf4 100644 --- a/drivers/gpio/gpio-mpsse.c +++ b/drivers/gpio/gpio-mpsse.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -24,6 +25,7 @@ struct mpsse_priv { raw_spinlock_t irq_spin; /* protects worker list */ atomic_t irq_type[16]; /* pin -> edge detection type */ atomic_t irq_enabled; + atomic_t dying; /* no new workers after disconnect */ int id; u8 gpio_outputs[2]; /* Output states for GPIOs [L, H] */ @@ -46,6 +48,7 @@ struct mpsse_worker { atomic_t cancelled; struct list_head list; /* linked list */ struct list_head destroy; /* teardown linked list */ + struct rcu_head rcu; /* deferred free for dying path */ }; struct bulk_desc { @@ -525,10 +528,24 @@ static void gpio_mpsse_irq_enable(struct irq_data *irqd) worker->priv = priv; INIT_LIST_HEAD(&worker->list); INIT_WORK(&worker->work, gpio_mpsse_poll); - schedule_work(&worker->work); - scoped_guard(raw_spinlock_irqsave, &priv->irq_spin) + scoped_guard(raw_spinlock_irqsave, &priv->irq_spin) { + if (atomic_read(&priv->dying)) { + /* + * Cannot kfree() here: this callback + * runs with the IRQ descriptor's raw + * spinlock held and kfree() may sleep + * on PREEMPT_RT. The worker is not + * yet published, so the deferred free + * is unobservable. + */ + kfree_rcu(worker, rcu); + return; + } + list_add(&worker->list, &priv->workers); + schedule_work(&worker->work); + } } } @@ -704,6 +721,9 @@ static void gpio_mpsse_disconnect(struct usb_interface *intf) { struct mpsse_priv *priv = usb_get_intfdata(intf); + scoped_guard(raw_spinlock_irqsave, &priv->irq_spin) + atomic_set(&priv->dying, 1); + /* * Lock prevents double-free of worker from here and the teardown * step at the beginning of gpio_mpsse_poll diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c index 532205175827a..9406ce85f9330 100644 --- a/drivers/gpio/gpio-xilinx.c +++ b/drivers/gpio/gpio-xilinx.c @@ -271,14 +271,7 @@ static void xgpio_save_regs(struct xgpio_instance *chip) static int xgpio_request(struct gpio_chip *chip, unsigned int offset) { - int ret; - - ret = pm_runtime_get_sync(chip->parent); - /* - * If the device is already active pm_runtime_get() will return 1 on - * success, but gpio_request still needs to return 0. - */ - return ret < 0 ? ret : 0; + return pm_runtime_resume_and_get(chip->parent); } static void xgpio_free(struct gpio_chip *chip, unsigned int offset) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 4b29e7eee6105..299baecb83ebd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -597,14 +597,18 @@ int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, /* first get metadata_size by buffer = NULL */ r = amdgpu_bo_get_metadata(bo, NULL, 0, metadata_size, NULL); + if (r) + goto out_put; /* user buf_size is bigger than bo metadata_size * allocate a buf at kernel space and copy */ if (*metadata_size <= buffer_size) { *metadata_buffer = kzalloc(*metadata_size, GFP_KERNEL); - if (!*metadata_buffer) - return -ENOMEM; + if (!*metadata_buffer) { + r = -ENOMEM; + goto out_put; + } r = amdgpu_bo_get_metadata(bo, *metadata_buffer, *metadata_size, NULL, &metadata_flags); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index 4536d46b7f21b..40b34770414b7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -120,6 +120,20 @@ static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, *pos &= (1UL << 22) - 1; + if (size > PAGE_SIZE) + return -EINVAL; + + /* + * Access the user buffer only while none of the locks below are held: + * a fault takes mmap_lock, and grbm_idx_mutex and srbm_mutex nest + * inside it. + */ + u32 *data __free(kfree) = kmalloc(size, GFP_KERNEL); + if (!data) + return -ENOMEM; + if (!read && copy_from_user(data, buf, size)) + return -EFAULT; + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); @@ -151,28 +165,16 @@ static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, mutex_lock(&adev->pm.mutex); while (size) { - uint32_t value; - - if (read) { - value = RREG32(*pos >> 2); - r = put_user(value, (uint32_t *)buf); - } else { - r = get_user(value, (uint32_t *)buf); - if (!r) - amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0); - } - if (r) { - result = r; - goto end; - } + if (read) + data[result >> 2] = RREG32(*pos >> 2); + else + amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, data[result >> 2], 0); result += 4; - buf += 4; *pos += 4; size -= 4; } -end: if (use_bank) { amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); mutex_unlock(&adev->grbm_idx_mutex); @@ -187,6 +189,10 @@ static int amdgpu_debugfs_process_reg_op(bool read, struct file *f, pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); + + if (read && copy_to_user(buf, data, result)) + return -EFAULT; + return result; } @@ -237,11 +243,24 @@ static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 off struct amdgpu_device *adev = rd->adev; ssize_t result = 0; int r; - uint32_t value; if (size & 0x3 || offset & 0x3) return -EINVAL; + if (size > PAGE_SIZE) + return -EINVAL; + + /* + * Access the user buffer only while none of the locks below are held: + * a fault takes mmap_lock, and grbm_idx_mutex and srbm_mutex nest + * inside it. + */ + u32 *data __free(kfree) = kmalloc(size, GFP_KERNEL); + if (!data) + return -ENOMEM; + if (write_en && copy_from_user(data, buf, size)) + return -EFAULT; + r = pm_runtime_get_sync(adev_to_drm(adev)->dev); if (r < 0) { pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); @@ -280,24 +299,16 @@ static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 off mutex_lock(&adev->pm.mutex); while (size) { - if (!write_en) { - value = RREG32(offset >> 2); - r = put_user(value, (uint32_t *)buf); - } else { - r = get_user(value, (uint32_t *)buf); - if (!r) - amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value, rd->id.xcc_id); - } - if (r) { - result = r; - goto end; - } + if (!write_en) + data[result >> 2] = RREG32(offset >> 2); + else + amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, data[result >> 2], + rd->id.xcc_id); offset += 4; size -= 4; result += 4; - buf += 4; } -end: + if (rd->id.use_grbm) { amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id); mutex_unlock(&adev->grbm_idx_mutex); @@ -316,6 +327,10 @@ static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 off pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); amdgpu_virt_disable_access_debugfs(adev); + + if (!write_en && copy_to_user(buf, data, result)) + return -EFAULT; + return result; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 5d6149ba7ab78..8fb3ebe4c2990 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -1089,20 +1089,17 @@ void amdgpu_gmc_init_vga_resv_regions(struct amdgpu_device *adev) case CHIP_RENOIR: adev->mman.keep_stolen_vga_memory = true; break; - case CHIP_POLARIS10: - case CHIP_POLARIS11: - case CHIP_POLARIS12: - /* MacBookPros with switchable graphics put VRAM at 0 when - * the iGPU is enabled which results in cursor issues if - * the cursor ends up at 0. Reserve vram at 0 in that case. - */ - if (adev->gmc.vram_start == 0) - adev->mman.keep_stolen_vga_memory = true; - break; default: adev->mman.keep_stolen_vga_memory = false; break; } + /* MacBookPros with switchable graphics put VRAM at 0 when + * the iGPU is enabled which results in cursor issues if + * the cursor ends up at 0. Reserve vram at 0 in that case. + */ + if ((adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) && + (adev->gmc.vram_start == 0)) + adev->mman.keep_stolen_vga_memory = true; if (amdgpu_sriov_vf(adev) || !amdgpu_device_has_display_hardware(adev)) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 8bb9c198ddec1..872cce1115a91 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "amdgpu_uvd.h" #include "amdgpu_vce.h" #include "atom.h" @@ -1631,6 +1632,9 @@ u32 amdgpu_get_vblank_counter_kms(struct drm_crtc *crtc) DRM_ERROR("Invalid crtc %u\n", pipe); return -EINVAL; } + if (!down_read_trylock(&adev->reset_domain->sem)) + return drm_crtc_vblank_count(crtc); + /* The hw increments its frame counter at start of vsync, not at start * of vblank, as is required by DRM core vblank counter handling. @@ -1676,6 +1680,7 @@ u32 amdgpu_get_vblank_counter_kms(struct drm_crtc *crtc) DRM_DEBUG_VBL("NULL mode info! Returned count may be wrong.\n"); } + up_read(&adev->reset_domain->sem); return count; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 655429c403518..0c3fa2d60555a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -439,6 +439,7 @@ amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, u64 doorbell_index; struct drm_gem_object *gobj; struct amdgpu_userq_obj *db_obj = db_info->db_obj; + struct amdgpu_bo *abo; int r, db_size; gobj = drm_gem_object_lookup(filp, db_info->doorbell_handle); @@ -447,7 +448,17 @@ amdgpu_userq_get_doorbell_index(struct amdgpu_userq_mgr *uq_mgr, return -EINVAL; } - db_obj->obj = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); + /* + * Pinning a regular BO into the doorbell domain would discard its + * contents, possibly those of a buffer shared by another client. + */ + abo = gem_to_amdgpu_bo(gobj); + if (!(abo->preferred_domains & AMDGPU_GEM_DOMAIN_DOORBELL)) { + drm_gem_object_put(gobj); + return -EINVAL; + } + + db_obj->obj = amdgpu_bo_ref(abo); drm_gem_object_put(gobj); r = amdgpu_bo_reserve(db_obj->obj, true); diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c index 4e44a581a78a2..e4bd6e5509b52 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c @@ -304,8 +304,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, compute_mqd = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); if (IS_ERR(compute_mqd)) { - DRM_ERROR("Failed to read user MQD\n"); - r = -ENOMEM; + r = PTR_ERR(compute_mqd); goto free_mqd; } @@ -349,8 +348,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, mqd_gfx_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); if (IS_ERR(mqd_gfx_v11)) { - DRM_ERROR("Failed to read user MQD\n"); - r = -ENOMEM; + r = PTR_ERR(mqd_gfx_v11); goto free_mqd; } @@ -394,8 +392,7 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, mqd_sdma_v11 = memdup_user(u64_to_user_ptr(mqd_user->mqd), mqd_user->mqd_size); if (IS_ERR(mqd_sdma_v11)) { - DRM_ERROR("Failed to read sdma user MQD\n"); - r = -ENOMEM; + r = PTR_ERR(mqd_sdma_v11); goto free_mqd; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 487cf746c4504..ed93442590790 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -599,12 +599,32 @@ static int allocate_doorbell(struct qcm_process_device *qpd, * we need the physical sdma engine id in order to get the * correct doorbell offset. */ - uint32_t valid_id = idx_offset[qpd->dqm->dev->node_id * - get_num_all_sdma_engines(qpd->dqm) + - q->properties.sdma_engine_id] - + (q->properties.sdma_queue_id & 1) - * KFD_QUEUE_DOORBELL_MIRROR_OFFSET - + (q->properties.sdma_queue_id >> 1); + u32 engine_base = idx_offset[dev->node_id * + get_num_all_sdma_engines(qpd->dqm) + + q->properties.sdma_engine_id]; + u32 valid_id; + + /* + * With the aqua_vanjaram doorbell layout, a shader doorbell + * write whose 32-byte block starts in the previous engine's + * range rings that engine instead. Engine ranges are 10 + * indices apart (sdma_doorbell_range = 20 dwords, halved for + * 8-byte doorbells, in aqua_vanjaram_doorbell_index_init()), + * which puts odd engines 2 indices into a shared block. + * Aligning costs at most 2 indices, and 8 queues per engine use + * only 4 of the 10, so the aligned base stays in range. The + * 512-index mirror offset keeps the alignment. + */ + if (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3) || + KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4) || + KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)) + engine_base = ALIGN(engine_base, + KFD_SDMA_SHADER_DOORBELL_GRANULARITY); + + valid_id = engine_base + + (q->properties.sdma_queue_id & 1) * + KFD_QUEUE_DOORBELL_MIRROR_OFFSET + + (q->properties.sdma_queue_id >> 1); if (restore_id && *restore_id != valid_id) return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 4d65f94da4d81..993d98374253b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -132,6 +132,13 @@ */ #define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512 +/* + * On GC 9.4.3, 9.4.4 and 9.5.0, doorbell writes issued from shaders are + * routed to an SDMA engine at 32-byte granularity, i.e. 4 indices with the + * 8-byte SDMA doorbells used on SOC15. + */ +#define KFD_SDMA_SHADER_DOORBELL_GRANULARITY 4 + /** * enum kfd_ioctl_flags - KFD ioctl flags * Various flags that can be set in &amdkfd_ioctl_desc.flags to control how diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 4ee7db8a6a226..b92d129962b2a 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -7260,10 +7260,15 @@ decide_crtc_timing_for_drm_display_mode(struct drm_display_mode *drm_mode, const struct drm_display_mode *native_mode, bool scale_enabled) { + /* Inserted modes copy the native sync; a sink mode with equal totals may not */ if (scale_enabled || ( native_mode->clock == drm_mode->clock && native_mode->htotal == drm_mode->htotal && - native_mode->vtotal == drm_mode->vtotal)) { + native_mode->vtotal == drm_mode->vtotal && + native_mode->hsync_start == drm_mode->hsync_start && + native_mode->hsync_end == drm_mode->hsync_end && + native_mode->vsync_start == drm_mode->vsync_start && + native_mode->vsync_end == drm_mode->vsync_end)) { if (native_mode->crtc_clock) copy_crtc_timing_for_drm_display_mode(native_mode, drm_mode); } else { @@ -10397,6 +10402,13 @@ static void amdgpu_dm_enable_self_refresh(struct amdgpu_display_manager *dm, struct amdgpu_dm_connector *aconn = (struct amdgpu_dm_connector *)acrtc_state->stream->dm_stream_context; + if (!acrtc_state->base.async_flip) { + amdgpu_dm_psr_set_event(dm, acrtc_state->stream, false, + psr_event_immediate_flip, false); + amdgpu_dm_replay_set_event(dm, acrtc_state->stream, false, + replay_event_immediate_flip, false); + } + /* Decrement skip count when SR is enabled and we're doing fast updates. */ if (acrtc_state->update_type == UPDATE_TYPE_FAST && (psr->psr_feature_enabled || pr->replay_feature_enabled)) { @@ -10810,6 +10822,18 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state, spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); } mutex_lock(&dm->dc_lock); + /* + * Tearing (immediate) flips cannot work with panel self-refresh + * features. The event is cleared by + * amdgpu_dm_enable_self_refresh() once async flips stop. + */ + if (immediate_flip) { + dc_exit_ips_for_hw_access(dm->dc); + amdgpu_dm_psr_set_event(dm, acrtc_state->stream, true, + psr_event_immediate_flip, true); + amdgpu_dm_replay_set_event(dm, acrtc_state->stream, true, + replay_event_immediate_flip, true); + } update_planes_and_stream_adapter(dm->dc, acrtc_state->update_type, planes_count, diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index cfd76c54f652b..1d69ecae10a47 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -110,25 +110,7 @@ void amdgpu_dm_plane_fill_blending_from_plane_state(const struct drm_plane_state if (plane_state->pixel_blend_mode == DRM_MODE_BLEND_PREMULTI || plane_state->pixel_blend_mode == DRM_MODE_BLEND_COVERAGE) { - static const uint32_t alpha_formats[] = { - DRM_FORMAT_ARGB8888, - DRM_FORMAT_RGBA8888, - DRM_FORMAT_ABGR8888, - DRM_FORMAT_ARGB2101010, - DRM_FORMAT_ABGR2101010, - DRM_FORMAT_ARGB16161616, - DRM_FORMAT_ABGR16161616, - DRM_FORMAT_ARGB16161616F, - }; - uint32_t format = plane_state->fb->format->format; - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(alpha_formats); ++i) { - if (format == alpha_formats[i]) { - *per_pixel_alpha = true; - break; - } - } + *per_pixel_alpha = plane_state->fb->format->has_alpha; if (*per_pixel_alpha && plane_state->pixel_blend_mode == DRM_MODE_BLEND_COVERAGE) *pre_multiplied_alpha = false; diff --git a/drivers/gpu/drm/apple/Kconfig b/drivers/gpu/drm/apple/Kconfig index 57a1ce69e7447..9940e4b48452a 100644 --- a/drivers/gpu/drm/apple/Kconfig +++ b/drivers/gpu/drm/apple/Kconfig @@ -6,6 +6,8 @@ config DRM_APPLE depends on APPLE_RTKIT depends on TYPEC || !TYPEC depends on OF_ADDRESS + # The firmware-27 remote buffer path calls into the DART driver. + depends on APPLE_DART || !APPLE_DART select APPLE_PMP_REPORT select OF_DYNAMIC select DRM_CLIENT_SELECTION @@ -29,6 +31,18 @@ config DRM_APPLE_AUDIO select SND_PCM select SND_DMAENGINE_PCM +config DRM_APPLE_DCP_T6030_AUDIO + bool "HDMI audio through the T6030 external display processor" + default y + depends on DRM_APPLE_AUDIO + help + DP/HDMI audio of the external display processors of the M3 Pro + (T6030, MacBook Pro J514S/J516S built-in HDMI port), whose DCP + firmware 14.7 external role takes 32-byte audio link data and the + native driver's link lifecycle. The audio service of every other + DCP is driven unchanged. Say N to build the audio and element + parser code exactly as before. + config DRM_APPLE_DEBUG bool "Enable additional driver debugging" depends on DRM_APPLE diff --git a/drivers/gpu/drm/apple/Makefile b/drivers/gpu/drm/apple/Makefile index e21650d949859..95809e410d7c3 100644 --- a/drivers/gpu/drm/apple/Makefile +++ b/drivers/gpu/drm/apple/Makefile @@ -26,6 +26,7 @@ appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_announce_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_external_relink_test.o appledrm-$(CONFIG_DRM_APPLE_KUNIT_TEST) += dcp_dropped_swap_test.o appledrm-y += iomfb_v26_6.o iomfb_v26_6_link.o j613_dcp_activate.o +appledrm-y += iomfb_v27.o appledrm-y += epic/dpavservep.o appledrm-y += plane.o @@ -44,3 +45,4 @@ obj-$(CONFIG_DRM_APPLE_PARSER_KUNIT_TEST) += afkv2-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += parser-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += hotplug-test.o appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += v14-board-test.o +appledrm-$(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) += audio-no-sio-test.dtbo.o diff --git a/drivers/gpu/drm/apple/afk.c b/drivers/gpu/drm/apple/afk.c index b97ec275e25ea..7cc4946e2b91c 100644 --- a/drivers/gpu/drm/apple/afk.c +++ b/drivers/gpu/drm/apple/afk.c @@ -2,6 +2,8 @@ /* Copyright 2022 Sven Peter */ #include +#include +#include #include #include #include @@ -14,6 +16,8 @@ #include #include #include +#include +#include #include #include @@ -102,6 +106,7 @@ struct apple_dcp_afkep *afk_init(struct apple_dcp *dcp, u32 endpoint, afkep->ops = ops; afkep->dcp = dcp; + afkep->wire_v2 = dcp->fw_compat == DCP_FIRMWARE_V_27_0; afkep->endpoint = endpoint; afkep->wq = alloc_ordered_workqueue("apple-dcp-afkep%02x", WQ_MEM_RECLAIM, endpoint); @@ -462,6 +467,8 @@ static void afk_service_worker(struct work_struct *work) afk_retry_service_init(ep); } +static void afk_recv_handle_teardown(struct apple_dcp_afkep *ep, u32 channel); + static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, u8 *payload, size_t payload_size) { @@ -479,6 +486,15 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, size_t props_size = payload_size - sizeof(name); previous = afk_epic_find_service(ep, channel); + /* + * The 14.7 external role announces its AV service again on the same + * channel after a replug: that is a new instance, and the old one is + * gone with the sink it served. + */ + if (previous && dcp_is_v14_7_external(ep->dcp) && ep == ep->dcp->dcpavservep) { + afk_recv_handle_teardown(ep, channel); + previous = afk_epic_find_service(ep, channel); + } if (previous) { spin_lock_irqsave(&previous->lock, flags); retired = previous->torndown; @@ -521,7 +537,7 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, return; } ret = parse_epic_service_init(&ctx, &epic_name, &epic_class, &epic_unit, - ep->dcp->fw_compat == DCP_FIRMWARE_H17P); + (ep->dcp->fw_compat == DCP_FIRMWARE_H17P || ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0)); if (ret) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: failed to extract init props: %d\n", @@ -614,6 +630,8 @@ static void afk_recv_handle_init(struct apple_dcp_afkep *ep, u32 channel, } if (ops->prepare) afk_service_publish(service); + if (ops->init_props) + ops->init_props(service, epic_unit, props, props_size); if (ops->init) ops->init(service, epic_name, epic_class, epic_unit); dev_info(ep->dcp->dev, "AFK[ep:%02x]: new service %s on channel %d\n", @@ -899,6 +917,53 @@ static void afk_v2_status(struct apple_dcp_afkep *ep, dev_warn(ep->dcp->dev, "AFKV2 error reply failed: %d\n", ret); } +/* + * A buffer the DCP firmware names that is none of the host's pending command + * buffers. Only the firmware's own retained mappings are followed, with the + * permission they grant: a reply is written only where the firmware's leaf + * allows writes, and a DVA that Linux mapped itself, or that names Linux's RAM + * or a device's registers, is refused. + */ +static int afk_v27_access_remote(struct apple_dcp_afkep *ep, u64 dva, void *buffer, + size_t size, bool to_remote, bool copy) +{ + size_t off = 0; + + if (!dva || size > SZ_1M || size > U64_MAX - dva) + return -EINVAL; + while (off < size) { + phys_addr_t pa = 0; + size_t chunk = min_t(size_t, size - off, + PAGE_SIZE - ((dva + off) & ~PAGE_MASK)); + int ret = apple_dart_fw_remote(ep->dcp->dev, dva + off, to_remote, &pa); + void *cpu; + + if (!ret && region_intersects(pa, chunk, IORESOURCE_MEM, + IORES_DESC_NONE) != REGION_DISJOINT) + ret = -EACCES; + if (ret) { + dev_err_ratelimited(ep->dcp->dev, + "AFK[ep:%02x]: refused remote buffer %#llx + %#zx (%s): %d\n", + ep->endpoint, dva, off, to_remote ? "reply" : "request", ret); + return ret; + } + if (!copy) { + off += chunk; + continue; + } + cpu = memremap(pa, chunk, MEMREMAP_WB); + if (!cpu) + return -ENOMEM; + if (to_remote) + memcpy(cpu, buffer + off, chunk); + else + memcpy(buffer + off, cpu, chunk); + memunmap(cpu); + off += chunk; + } + return 0; +} + static int afk_v2_access_owned(struct apple_epic_service *service, u64 dva, void *buffer, size_t size, bool to_peer, bool copy) { @@ -942,6 +1007,9 @@ static int afk_v2_access_owned(struct apple_epic_service *service, u64 dva, break; } spin_unlock_irqrestore(&service->lock, flags); + /* T6050 callbacks may refer to firmware-owned buffers in the retained DART. */ + if (ret && service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + ret = afk_v27_access_remote(service->ep, dva, buffer, size, to_peer, copy); if (ret) dev_warn_ratelimited(service->ep->dcp->dev, "AFKV2 unowned buffer: ep=%u interface=%u size=%zu write=%u\n", @@ -1157,7 +1225,8 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, hdr_len = sizeof(*ehdr) + sizeof(*c); if (data_size < hdr_len) goto too_small; - if (ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + if (ep->dcp->fw_compat == DCP_FIRMWARE_H17P && + ep->dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !afk_validate_h17p_header(data, data_size)) { dev_err(ep->dcp->dev, "AFK[ep:%02x]: invalid compact message length\n", @@ -1175,7 +1244,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, * queue channel at face value binds "system" to 0 and makes * every later announce and report look like a duplicate. */ - channel = le16_to_cpup((__le16 *)(data + 2)); + channel = get_unaligned_le16(data + 2); /* * Normalise into the wide sub-header the rest of this file @@ -1197,6 +1266,7 @@ static void afk_recv_handle(struct apple_dcp_afkep *ep, u32 channel, u32 type, payload = data + hdr_len; payload_size = data_size - hdr_len; + trace_afk_recv_handle(ep, channel, type, data_size, ehdr, eshdr); @@ -1522,7 +1592,7 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, } spin_lock_irqsave(&ep->lock, flags); - if (ep->stopping) { + if (ep->stopping || !ep->txbfr.ready) { ret = -ESHUTDOWN; goto out; } @@ -1536,6 +1606,12 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, dma_rmb(); rptr = le32_to_cpu(*ep->txbfr.rptr); wptr = le32_to_cpu(*ep->txbfr.wptr); + if (rptr >= ep->txbfr.bufsz || wptr >= ep->txbfr.bufsz || + !IS_ALIGNED(rptr, ep->txbfr.stride) || + !IS_ALIGNED(wptr, ep->txbfr.stride)) { + ret = -EPROTO; + goto out; + } trace_afk_send_rwptr_pre(ep, rptr, wptr); total_epic_size = sizeof(*ehdr) + (ep->wire_v2 ? sizeof(struct epic_sub_hdr_h17p) : sizeof(*eshdr)) + @@ -1675,6 +1751,9 @@ static int afk_send_epic_token(struct apple_dcp_afkep *ep, u32 channel, u16 tag, eshdr->type = cpu_to_le16(stype); eshdr->timestamp = cpu_to_le64(0); eshdr->tag = cpu_to_le16(tag); + /* The 14.7 external role's callback replies carry 8 in byte 35. */ + if (ecat == EPIC_CAT_REPLY && dcp_is_v14_7_external(ep->dcp)) + eshdr->unk = cpu_to_le16(8 << 8); if (ecat == EPIC_CAT_REPLY) eshdr->inline_len = cpu_to_le32(payload_len - 4); else diff --git a/drivers/gpu/drm/apple/afk.h b/drivers/gpu/drm/apple/afk.h index 2b6609abe361a..d7e64a78e5899 100644 --- a/drivers/gpu/drm/apple/afk.h +++ b/drivers/gpu/drm/apple/afk.h @@ -98,6 +98,9 @@ struct apple_epic_service_ops { const char *class, s64 unit); void (*init)(struct apple_epic_service *service, const char *name, const char *class, s64 unit); + /* Consumed synchronously before init publishes service readiness. */ + void (*init_props)(struct apple_epic_service *service, s64 unit, + const void *props, size_t size); int (*call)(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size, void *reply, size_t reply_size); diff --git a/drivers/gpu/drm/apple/apple_drv.c b/drivers/gpu/drm/apple/apple_drv.c index d1cda95aa9833..8bf6d56f708e0 100644 --- a/drivers/gpu/drm/apple/apple_drv.c +++ b/drivers/gpu/drm/apple/apple_drv.c @@ -13,11 +13,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -54,6 +56,55 @@ struct apple_drm_private { struct drm_device drm; }; +/* Component readiness is later than platform probe/bind. */ +static DEFINE_MUTEX(apple_t6030_display_lock); +static struct device *apple_t6030_display; + +static int apple_t6030_display_publish(struct device *dev) +{ + if (!of_device_is_compatible(dev->of_node, "apple,t6030-display-subsystem")) + return 0; + guard(mutex)(&apple_t6030_display_lock); + if (apple_t6030_display) + return apple_t6030_display == dev ? 0 : -EBUSY; + apple_t6030_display = get_device(dev); + return 0; +} + +static void apple_t6030_display_withdraw(struct device *dev) +{ + struct device *old = NULL; + + scoped_guard(mutex, &apple_t6030_display_lock) { + if (apple_t6030_display == dev) { + old = apple_t6030_display; + apple_t6030_display = NULL; + } + } + put_device(old); +} + +/* Initial G15 firmware startup must follow successful native display setup. + * The component's device lock does not serialize component bind/unbind, so + * publish separately. This lock protects only publication and a device link; + * no display calls or firmware RPC run while it is held. + */ +int apple_drm_t6030_link_device(struct device *consumer) +{ + struct device_link *link; + + if (!of_machine_is_compatible("apple,t6030") || !consumer->of_node || + !of_device_is_compatible(consumer->of_node, "apple,agx-t6030")) + return -ENODEV; + guard(mutex)(&apple_t6030_display_lock); + if (!apple_t6030_display) + return -EPROBE_DEFER; + link = device_link_add(consumer, apple_t6030_display, + DL_FLAG_AUTOREMOVE_CONSUMER); + return link ? 0 : -EINVAL; +} +EXPORT_SYMBOL_GPL(apple_drm_t6030_link_device); + DEFINE_DRM_GEM_DMA_FOPS(apple_fops); #define DART_PAGE_SIZE 16384 @@ -551,6 +602,7 @@ static int apple_probe_per_dcp(struct device *dev, struct apple_encoder *enc; struct drm_plane *planes[DCP_MAX_PLANES]; unsigned long *iomfb_surfaces = dcp_get_iomfb_surfaces(dcp); + unsigned long top_surf = find_last_bit(iomfb_surfaces, DCP_MAX_PLANES); int ret; int connector_type; u32 surf; @@ -559,9 +611,16 @@ static int apple_probe_per_dcp(struct device *dev, struct apple_dcp *dcp_data = platform_get_drvdata(dcp); bool supports_xrgb2101010; enum drm_plane_type plane_type; + bool interchange; + bool supports_interchange; supports_xrgb2101010 = dcp_data->fw_compat != DCP_FIRMWARE_H17P || dcp_data->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G; + /* + * Interchange scanout has not been qualified with the H17P firmware + * (T8140 and T8122): advertise linear framebuffers only there. + */ + supports_interchange = dcp_data->fw_compat != DCP_FIRMWARE_H17P; if (bitmap_empty(iomfb_surfaces, DCP_MAX_PLANES)) return dev_err_probe(dev, -EINVAL, "No usable display surfaces\n"); @@ -576,14 +635,27 @@ static int apple_probe_per_dcp(struct device *dev, zpos = 1; } + /* + * BUG: + * Some DCPs support more than two surfaces. For some reason, these + * will crash if the topmost layer is Interchange *and* the surface + * is cleared *while content is being scanned out on the middle one*. + * They do not do this with linear framebuffers, nor do they do this + * when the middle surface is not in use. We therefore hide Interchange + * from the surface. + */ for_each_set_bit(surf, iomfb_surfaces, DCP_MAX_PLANES) { if (dcp_fw_compat_is_14_7(dcp)) break; plane_type = (zpos == 0) ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY; + interchange = supports_interchange && + (zpos <= 1 || surf != top_surf); + planes[zpos] = apple_plane_init(drm, 1U << num, surf, supports_l10r, supports_xrgb2101010, - plane_type); + plane_type, interchange); + if (IS_ERR(planes[zpos])) return PTR_ERR(planes[zpos]); @@ -708,15 +780,13 @@ static int apple_probe_typec_ports(struct drm_device *drm, * lowest-indexed Type-C-only pipeline (dcpext1). Fix the * connector to it: compositors read possible_crtcs once * and would otherwise pair it with a pipeline the stream - * can never be routed to. + * can never be routed to. T6030 instead narrows both + * connectors to the native processor holding their route. */ - if (secondary) { - if (dcp_is_typec_only(dcp[i]) && - (!mask || crtc_mask[i] < mask)) - mask = crtc_mask[i]; - continue; - } - mask |= crtc_mask[i]; + mask = dcp_fabric_port_crtc_mask(secondary, + dcp_typec_ordered_dual_stream(), + dcp_is_typec_only(dcp[i]), + mask, crtc_mask[i]); } if (!mask) { @@ -1018,6 +1088,13 @@ static int apple_drm_init(struct device *dev) drm_client_setup_with_fourcc(&apple->drm, DRM_FORMAT_XRGB8888); + /* A subsequent successful platform probe retries deferred GPU probes. */ + ret = apple_t6030_display_publish(dev); + if (ret) { + drm_dev_unregister(&apple->drm); + goto err_unbind; + } + return 0; err_unbind: @@ -1034,6 +1111,8 @@ static void apple_drm_uninit(struct device *dev) { struct apple_drm_private *apple = dev_get_drvdata(dev); + apple_t6030_display_withdraw(dev); + apple_drm_quiesce_connectors(&apple->drm); drm_dev_unplug(&apple->drm); drm_atomic_helper_shutdown(&apple->drm); diff --git a/drivers/gpu/drm/apple/audio-link-test.c b/drivers/gpu/drm/apple/audio-link-test.c new file mode 100644 index 0000000000000..a90d85f221ead --- /dev/null +++ b/drivers/gpu/drm/apple/audio-link-test.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by audio.c: a Terminate's link release ends only the link of the + * instance it is for. The production dcpaud_release_link() runs; the audio + * service's answer to which instance the link is on, and its link calls, + * are stubbed. + */ +#include +#include +#include + +struct dcpaud_link_test { + bool link_on; + u64 asked; + unsigned int completes; + unsigned int stops; +}; + +static struct dcpaud_link_test *dcpaud_link_current; + +static bool dcpaud_link_test_on(struct device *dev, u64 gen) +{ + dcpaud_link_current->asked = gen; + return dcpaud_link_current->link_on; +} + +static int dcpaud_link_test_call(struct device *dev, enum dcp_audio_link op, + const struct dcp_sound_cookie *cookie) +{ + if (op == DCP_AUDIO_LINK_COMPLETE) + dcpaud_link_current->completes++; + else if (op == DCP_AUDIO_LINK_STOP) + dcpaud_link_current->stops++; + return 0; +} + +static struct dcp_audio *dcpaud_link_setup(struct kunit *test, struct platform_device **pdev) +{ + struct dcpaud_link_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct dcp_audio *dcpaud = kunit_kzalloc(test, sizeof(*dcpaud), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, t); + KUNIT_ASSERT_NOT_NULL(test, dcpaud); + *pdev = kunit_platform_device_alloc(test, "dcpaud-link", PLATFORM_DEVID_AUTO); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, *pdev); + platform_set_drvdata(*pdev, dcpaud); + dcpaud->dev = &(*pdev)->dev; + dcpaud->dcp_dev = &(*pdev)->dev; + dcpaud->native_link = true; + mutex_init(&dcpaud->link_lock); + dcpaud_link_current = t; + kunit_activate_static_stub(test, dcp_audiosrv_link_on, dcpaud_link_test_on); + kunit_activate_static_stub(test, dcp_audiosrv_link, dcpaud_link_test_call); + return dcpaud; +} + +static void dcpaud_link_release_instance(struct kunit *test) +{ + struct platform_device *pdev; + struct dcp_audio *dcpaud = dcpaud_link_setup(test, &pdev); + u64 gen = 7; + + /* The link is on another instance: left as it is */ + dcpaud->link_prepared = true; + dcpaud->link_started = true; + dcpaud_release_link(pdev, &gen); + KUNIT_EXPECT_EQ(test, dcpaud_link_current->asked, 7ULL); + KUNIT_EXPECT_TRUE(test, dcpaud->link_prepared); + KUNIT_EXPECT_TRUE(test, dcpaud->link_started); + KUNIT_EXPECT_EQ(test, dcpaud_link_current->stops + dcpaud_link_current->completes, 0U); + + /* On the instance asked for: stopped and completed */ + dcpaud_link_current->link_on = true; + dcpaud_release_link(pdev, &gen); + KUNIT_EXPECT_FALSE(test, dcpaud->link_prepared); + KUNIT_EXPECT_FALSE(test, dcpaud->link_started); + KUNIT_EXPECT_EQ(test, dcpaud_link_current->stops, 1U); + KUNIT_EXPECT_EQ(test, dcpaud_link_current->completes, 1U); + + /* Without an instance (the display's own power-off): whichever link */ + dcpaud_link_current->link_on = false; + dcpaud->link_prepared = true; + dcpaud_release_link(pdev, NULL); + KUNIT_EXPECT_FALSE(test, dcpaud->link_prepared); + KUNIT_EXPECT_EQ(test, dcpaud_link_current->completes, 2U); +} + +static struct kunit_case dcpaud_link_cases[] = { + KUNIT_CASE(dcpaud_link_release_instance), + {} +}; + +static struct kunit_suite dcpaud_link_suite = { + .name = "apple-dcpaud-release-link", + .test_cases = dcpaud_link_cases, +}; + +kunit_test_suite(dcpaud_link_suite); diff --git a/drivers/gpu/drm/apple/audio-no-sio-test.c b/drivers/gpu/drm/apple/audio-no-sio-test.c new file mode 100644 index 0000000000000..8ea2387ff4741 --- /dev/null +++ b/drivers/gpu/drm/apple/audio-no-sio-test.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by audio.c: a DP audio interface whose SIO never registers its + * DMA controller (refused for an earlier boot loader's tables) gets no TX + * channel; a PCM open fails with -ENXIO, each time, and nothing is kept. + */ +#include +#include +#include + +OF_OVERLAY_DECLARE(audio_no_sio_test); + +static void dcpaud_no_sio_put(void *data) +{ + struct device *dev = data; + + of_node_put(dev->of_node); + dev->of_node = NULL; +} + +static void dcpaud_no_sio_test(struct kunit *test) +{ + struct dcp_audio *dcpaud = kunit_kzalloc(test, sizeof(*dcpaud), GFP_KERNEL); + struct device *dev; + + KUNIT_ASSERT_NOT_NULL(test, dcpaud); + KUNIT_ASSERT_EQ(test, 0, of_overlay_apply_kunit(test, audio_no_sio_test)); + dev = kunit_device_register(test, "dcpaud-no-sio"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + dev->of_node = of_find_node_by_path("/dcpaud-no-sio-test/dpaudio-no-sio"); + KUNIT_ASSERT_NOT_NULL(test, dev->of_node); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dcpaud_no_sio_put, dev), 0); + dcpaud->dev = dev; + mutex_init(&dcpaud->data_lock); + + KUNIT_EXPECT_EQ(test, dcpaud_init_dma(dcpaud), -ENXIO); + KUNIT_EXPECT_NULL(test, dcpaud->chan); + /* Every later open fails the same way, with nothing left behind */ + KUNIT_EXPECT_EQ(test, dcpaud_init_dma(dcpaud), -ENXIO); + KUNIT_EXPECT_NULL(test, dcpaud->chan); +} + +static struct kunit_case dcpaud_no_sio_cases[] = { + KUNIT_CASE(dcpaud_no_sio_test), + {} +}; + +static struct kunit_suite dcpaud_no_sio_suite = { + .name = "apple-dcpaud-no-sio", + .test_cases = dcpaud_no_sio_cases, +}; + +kunit_test_suite(dcpaud_no_sio_suite); diff --git a/drivers/gpu/drm/apple/audio-no-sio-test.dtso b/drivers/gpu/drm/apple/audio-no-sio-test.dtso new file mode 100644 index 0000000000000..9719e7078b392 --- /dev/null +++ b/drivers/gpu/drm/apple/audio-no-sio-test.dtso @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/dts-v1/; +/plugin/; + +/* + * A DP audio interface whose DMA controller never registers, as the T6030 + * SIO with an earlier boot loader's tables: its probe refuses it. Nothing + * binds to these nodes; the test requests the channel itself. + */ +&{/} { + dcpaud-no-sio-test { + #address-cells = <2>; + #size-cells = <2>; + + sio_stub: sio-stub { + #dma-cells = <1>; + }; + + dpaudio-no-sio { + dmas = <&sio_stub 0x66>; + dma-names = "tx"; + }; + }; +}; diff --git a/drivers/gpu/drm/apple/audio.c b/drivers/gpu/drm/apple/audio.c index cc54ae2a9b587..2b507dc1fa1b0 100644 --- a/drivers/gpu/drm/apple/audio.c +++ b/drivers/gpu/drm/apple/audio.c @@ -11,6 +11,7 @@ #define DEBUG +#include #include #include #include @@ -32,6 +33,8 @@ #define DCPAUD_ELEMENTS_MAXSIZE 16384 #define DCPAUD_PRODUCTATTRS_MAXSIZE 1024 +/* The native element catalog capacity (14.7 external role). */ +#define DCPAUD_ELEMENTS_V14_7 SZ_256K /* * SIO takes one of its 64 descriptor slots per period, shared by all its @@ -68,8 +71,34 @@ struct dcp_audio { void *productattrs; struct snd_pcm_chmap *chmap_info; +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + /* + * DCP firmware 14.7, external role (T6030 external processor): the + * native link lifecycle, one link per stream start + * (dcp_native_playback_ops). + */ + bool native_link; + struct mutex link_lock; + bool link_prepared; + bool link_started; +#endif }; +/* The 14.7 external role's audio service (DRM_APPLE_DCP_T6030_AUDIO only). */ +static inline bool dcpaud_native_link(struct dcp_audio *dcpaud) +{ +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + return dcpaud->native_link; +#else + return false; +#endif +} + +static inline size_t dcpaud_elements_size(struct dcp_audio *dcpaud) +{ + return dcpaud_native_link(dcpaud) ? DCPAUD_ELEMENTS_V14_7 : DCPAUD_ELEMENTS_MAXSIZE; +} + static const struct snd_pcm_hardware dcp_pcm_hw = { .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_INTERLEAVED, @@ -118,7 +147,7 @@ static int dcpaud_read_remote_info(struct dcp_audio *dcpaud) int ret; ret = dcp_audiosrv_get_elements(dcpaud->dcp_dev, dcpaud->elements, - DCPAUD_ELEMENTS_MAXSIZE); + dcpaud_elements_size(dcpaud)); if (ret < 0) return ret; @@ -175,8 +204,11 @@ static void dcpaud_consult_elements(struct dcp_audio *dcpaud, struct dcp_parse_ctx elements = { .dcp = dev_get_drvdata(dcpaud->dcp_dev), .blob = dcpaud->elements + 4, - .len = DCPAUD_ELEMENTS_MAXSIZE - 4, + .len = dcpaud_elements_size(dcpaud) - 4, .pos = 0, +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + .v14_7_audio = dcpaud->native_link, +#endif }; dcpaud_fill_fmt_sieve(params, &sieve); @@ -192,8 +224,11 @@ static int dcpaud_select_cookie(struct dcp_audio *dcpaud, struct dcp_parse_ctx elements = { .dcp = dev_get_drvdata(dcpaud->dcp_dev), .blob = dcpaud->elements + 4, - .len = DCPAUD_ELEMENTS_MAXSIZE - 4, + .len = dcpaud_elements_size(dcpaud) - 4, .pos = 0, +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + .v14_7_audio = dcpaud->native_link, +#endif }; dcpaud_fill_fmt_sieve(params, &sieve); @@ -557,6 +592,239 @@ struct snd_pcm_ops dcp_playback_ops = { .pointer = dcp_pcm_pointer, }; +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +/* + * DCP firmware 14.7, external role (the T6030 external processor, J514S + * built-in HDMI): the link lifecycle of the native audio driver: + * + * prepare: the DMA channel configured, then the link prepared, once; + * START: the cyclic DMA issued, then the link started; + * STOP: the DMA terminated; + * sync_stop: the DMA drained, then the link stopped and completed; + * hw_free: a link prepared but never started, completed. + * + * Every link method carries the selected element's link data. A link of an + * earlier connection went with its service: its state is dropped unsent. + * The audio service itself is open only while the PCM is. + */ +static int dcp_pcm_native_open(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + bool live; + int ret; + + scoped_guard(mutex, &dcpaud->data_lock) + live = dcpaud->connected; + if (live) { + ret = dcp_audiosrv_open(dcpaud->dcp_dev); + if (ret) + return ret; + } + + ret = dcp_pcm_open(substream); + if (ret && live) + dcp_audiosrv_close(dcpaud->dcp_dev); + + return ret; +} + +static int dcp_pcm_native_close(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + int ret = dcp_pcm_close(substream); + + dcp_audiosrv_close(dcpaud->dcp_dev); + + return ret; +} + +/* + * Stop and complete the link held, on the open audio service (@live), or + * drop both unsent with a service that went away. + */ +static void dcpaud_link_teardown(struct dcp_audio *dcpaud, bool live) +{ + int ret; + + lockdep_assert_held(&dcpaud->link_lock); + if (dcpaud->link_started && live) { + ret = dcp_audiosrv_link(dcpaud->dcp_dev, DCP_AUDIO_LINK_STOP, + &dcpaud->selected_cookie); + if (ret) + dev_err(dcpaud->dev, "audio link stop failed: %d\n", ret); + } + dcpaud->link_started = false; + if (dcpaud->link_prepared && live) { + ret = dcp_audiosrv_link(dcpaud->dcp_dev, DCP_AUDIO_LINK_COMPLETE, + &dcpaud->selected_cookie); + if (ret) + dev_err(dcpaud->dev, "audio link complete failed: %d\n", ret); + } + dcpaud->link_prepared = false; +} + +/* New parameters select a new element: the link of the old one ends first. */ +static int dcp_pcm_native_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + + scoped_guard(mutex, &dcpaud->link_lock) + dcpaud_link_teardown(dcpaud, dcpaud_connection_up(dcpaud)); + + return dcp_pcm_hw_params(substream, params); +} + +static int dcp_pcm_native_hw_free(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + + guard(mutex)(&dcpaud->link_lock); + dcpaud_link_teardown(dcpaud, dcpaud_connection_up(dcpaud)); + + return 0; +} + +static int dcp_pcm_native_prepare(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + struct dma_slave_config slave_config = { + .direction = DMA_MEM_TO_DEV, + /* One 32-bit DPA entry per sample word, as in hw_params. */ + .dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES, + }; + int ret; + + if (dcpaud->open_unplugged) + return 0; + + if (!dcpaud_connection_up(dcpaud)) + return -ENXIO; + + guard(mutex)(&dcpaud->link_lock); + if (dcpaud->link_prepared) + return 0; + + /* The SIO channel is configured again before every link prepare. */ + ret = dmaengine_slave_config(snd_dmaengine_pcm_get_chan(substream), + &slave_config); + if (ret) + return ret; + + ret = dcp_audiosrv_link(dcpaud->dcp_dev, DCP_AUDIO_LINK_PREPARE, + &dcpaud->selected_cookie); + if (ret) + return ret; + + dcpaud->link_prepared = true; + return 0; +} + +static int dcp_pcm_native_start(struct snd_pcm_substream *substream, int cmd) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + int ret; + + if (!dcpaud_connection_up(dcpaud)) + return -ENXIO; + + /* The first period descriptor is issued before the link starts. */ + ret = snd_dmaengine_pcm_trigger(substream, cmd); + if (ret) + return ret; + + guard(mutex)(&dcpaud->link_lock); + if (!dcpaud->link_prepared) + return -EBADFD; + if (dcpaud->link_started) + return 0; + ret = dcp_audiosrv_link(dcpaud->dcp_dev, DCP_AUDIO_LINK_START, + &dcpaud->selected_cookie); + dcpaud->link_started = !ret; + return ret; +} + +static int dcp_pcm_native_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + int ret; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_RESUME: + /* + * Balanced by STOP, which the PCM core also sends after a + * failed start; sync_stop then drains the DMA and completes + * the link. + */ + WARN_ON(pm_runtime_get_sync(dcpaud->dev) < 0); + return dcp_pcm_native_start(substream, cmd); + + case SNDRV_PCM_TRIGGER_STOP: + case SNDRV_PCM_TRIGGER_SUSPEND: + ret = snd_dmaengine_pcm_trigger(substream, cmd); + pm_runtime_mark_last_busy(dcpaud->dev); + __pm_runtime_put_autosuspend(dcpaud->dev); + return ret; + + default: + return -EINVAL; + } +} + +static int dcp_pcm_native_sync_stop(struct snd_pcm_substream *substream) +{ + struct dcp_audio *dcpaud = substream->pcm->private_data; + int ret; + + /* SIO TERMINATE acknowledged and the issued periods reported back. */ + ret = snd_dmaengine_pcm_sync_stop(substream); + + guard(mutex)(&dcpaud->link_lock); + dcpaud_link_teardown(dcpaud, dcpaud_connection_up(dcpaud)); + + return ret; +} + +/* + * The card was disconnected (its stream stopped) while the audio service + * stays: the display was powered off or the sink released, and the service + * is about to be closed. Its link is completed first, the DMA drained + * before: the firmware keeps a link prepared by a client that closes without + * completing it, and refuses every later link prepare of that service + * (0xe00002ea), so every later open failed. + */ +void dcpaud_release_link(struct platform_device *pdev, const u64 *gen) +{ + struct dcp_audio *dcpaud; + + KUNIT_STATIC_STUB_REDIRECT(dcpaud_release_link, pdev, gen); + dcpaud = platform_get_drvdata(pdev); + if (!dcpaud->native_link) + return; + guard(mutex)(&dcpaud->link_lock); + if (!dcpaud->link_prepared && !dcpaud->link_started) + return; + /* Only that instance's link: a later instance's PCM keeps its own. */ + if (gen && !dcp_audiosrv_link_on(dcpaud->dcp_dev, *gen)) + return; + if (dcpaud->chan) + dmaengine_synchronize(dcpaud->chan); + dcpaud_link_teardown(dcpaud, true); +} + +static const struct snd_pcm_ops dcp_native_playback_ops = { + .open = dcp_pcm_native_open, + .close = dcp_pcm_native_close, + .hw_params = dcp_pcm_native_hw_params, + .hw_free = dcp_pcm_native_hw_free, + .prepare = dcp_pcm_native_prepare, + .trigger = dcp_pcm_native_trigger, + .sync_stop = dcp_pcm_native_sync_stop, + .pointer = snd_dmaengine_pcm_pointer, +}; +#endif + // Transitional workaround: for the chmap control TLV, advertise options // copied from hdmi-codec.c #include "hdmi-codec-chmap.h" @@ -608,6 +876,11 @@ static int dcpaud_create_pcm(struct dcp_audio *dcpaud) if (ret) return ret; +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + if (dcpaud_native_link(dcpaud)) + snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &dcp_native_playback_ops); + else +#endif snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &dcp_playback_ops); dcpaud->substream = pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; pcm->nonatomic = true; @@ -726,8 +999,10 @@ void dcpaud_connect(struct platform_device *pdev, bool connected) void dcpaud_disconnect(struct platform_device *pdev) { - struct dcp_audio *dcpaud = platform_get_drvdata(pdev); + struct dcp_audio *dcpaud; + KUNIT_STATIC_STUB_REDIRECT(dcpaud_disconnect, pdev); + dcpaud = platform_get_drvdata(pdev); mutex_lock(&dcpaud->data_lock); dcpaud_report_hotplug(dcpaud, false); @@ -781,6 +1056,19 @@ static int dcpaud_comp_bind(struct device *dev, struct device *main, void *data) } dcpaud->dcp_dev = &dcp_pdev->dev; +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + /* The 14.7 external role: its native links and element catalog. */ + if (dcp_audiosrv_native_link(dcpaud->dcp_dev)) { + void *elements = devm_kzalloc(dev, DCPAUD_ELEMENTS_V14_7, GFP_KERNEL); + + if (!elements) + goto rpm_put; + devm_kfree(dev, dcpaud->elements); + dcpaud->elements = elements; + dcpaud->native_link = true; + } +#endif + dma_pdev = of_find_device_by_node(dma_spec.np); of_node_put(dma_spec.np); if (!dma_pdev) { @@ -801,7 +1089,7 @@ static int dcpaud_comp_bind(struct device *dev, struct device *main, void *data) dcpaud_expose_debugfs_blob(dcpaud, "selected_cookie", &dcpaud->selected_cookie, sizeof(dcpaud->selected_cookie)); dcpaud_expose_debugfs_blob(dcpaud, "elements", dcpaud->elements, - DCPAUD_ELEMENTS_MAXSIZE); + dcpaud_elements_size(dcpaud)); dcpaud_expose_debugfs_blob(dcpaud, "product_attrs", dcpaud->productattrs, DCPAUD_PRODUCTATTRS_MAXSIZE); } @@ -849,6 +1137,9 @@ static int dcpaud_probe(struct platform_device *pdev) dcpaud->dev = &pdev->dev; mutex_init(&dcpaud->data_lock); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + mutex_init(&dcpaud->link_lock); +#endif INIT_DELAYED_WORK(&dcpaud->void_work, dcpaud_void_work); platform_set_drvdata(pdev, dcpaud); @@ -908,3 +1199,9 @@ void dcp_audio_unregister(void) platform_driver_unregister(&dcpaud_driver); } +#if IS_ENABLED(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) +#include "audio-no-sio-test.c" +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +#include "audio-link-test.c" +#endif +#endif diff --git a/drivers/gpu/drm/apple/audio.h b/drivers/gpu/drm/apple/audio.h index 83b990dc6c343..58633598253ea 100644 --- a/drivers/gpu/drm/apple/audio.h +++ b/drivers/gpu/drm/apple/audio.h @@ -17,4 +17,24 @@ int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsi void dcpaud_connect(struct platform_device *pdev, bool connected); void dcpaud_disconnect(struct platform_device *pdev); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +/* The native link methods of the DCP firmware 14.7 external role. */ +enum dcp_audio_link { + DCP_AUDIO_LINK_PREPARE, + DCP_AUDIO_LINK_START, + DCP_AUDIO_LINK_STOP, + DCP_AUDIO_LINK_COMPLETE, +}; + +bool dcp_audiosrv_native_link(struct device *dev); +int dcp_audiosrv_open(struct device *dev); +void dcp_audiosrv_close(struct device *dev); +int dcp_audiosrv_link(struct device *dev, enum dcp_audio_link op, + const struct dcp_sound_cookie *cookie); +/* @gen: only a link prepared on that instance, still the open one (NULL: any) */ +void dcpaud_release_link(struct platform_device *pdev, const u64 *gen); +/* Whether the link prepared last is on instance @gen, still the open one */ +bool dcp_audiosrv_link_on(struct device *dev, u64 gen); +#endif + #endif /* __AUDIO_H__ */ diff --git a/drivers/gpu/drm/apple/av-lifetime-test.c b/drivers/gpu/drm/apple/av-lifetime-test.c new file mode 100644 index 0000000000000..0232dfb670bbd --- /dev/null +++ b/drivers/gpu/drm/apple/av-lifetime-test.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by av.c. Real workqueue lifetime for audiosrv_data: quiesce drains + * owner work before the owner is marked dead, an ERR_PTR endpoint is not + * dereferenced, and skipping the drain lets the callback observe the dead + * owner. The deliberate no-drain control validates the test boundary; + * production-body mutation runs separately verify the real drain. + */ +#include +#include +#include +#include +#include + +struct av_life_test { + struct device *dev; + void *group; + struct apple_dcp *dcp; + struct audiosrv_data *asrv; + struct apple_dcp_afkep *ep; + struct workqueue_struct *wq; + struct work_struct ep_work; + struct work_struct stop_work; + struct workqueue_struct *stop_wq; + atomic_t owner_runs; + atomic_t owner_saw; + int ep_runs; + int ep_saw; + int dead; + struct completion entered; + struct completion release; + struct completion owner_done; + struct completion stop_entered; + struct completion stop_done; + bool stop_saw_owner_done; + bool stop_unbind; + bool stop_release_group; + struct audiosrv_data *release_owner; + struct apple_dcp_afkep *release_ep; + bool endpoint_released; + bool release_saw_stopped; + bool release_saw_quiesced; + bool group_released; +}; + +static struct av_life_test *av_life_current; +static void av_life_release_group(struct av_life_test *t); + +static void av_life_owner_fn(struct work_struct *work) +{ + struct av_life_test *t = av_life_current; + + atomic_set(&t->owner_saw, READ_ONCE(t->dead)); + atomic_inc(&t->owner_runs); + complete(&t->owner_done); +} + +static void av_life_stop_fn(struct work_struct *work) +{ + struct av_life_test *t = container_of(work, struct av_life_test, stop_work); + + complete(&t->stop_entered); + if (t->stop_release_group) + av_life_release_group(t); + else if (t->stop_unbind) + dcp_comp_unbind(t->dcp->dev, NULL, NULL); + else + av_audiosrv_quiesce(t->dcp); + t->stop_saw_owner_done = completion_done(&t->owner_done); + complete(&t->stop_done); +} + +static void av_life_cleanup(void *data) +{ + struct av_life_test *t = data; + + /* Assertion failure must not strand a blocked system worker. */ + complete_all(&t->release); + if (t->stop_unbind) + cancel_work_sync(&t->dcp->external_audio_work); + if (t->stop_wq) { + flush_work(&t->stop_work); + destroy_workqueue(t->stop_wq); + } + if (t->group) + av_life_release_group(t); + if (t->group_released) { + t->dcp->audiosrv = NULL; + t->dcp->avep = NULL; + } + av_audiosrv_quiesce(t->dcp); + /* Keep deliberately broken production mutants safe after assertions. */ + if (t->asrv && t->asrv->works_live) { + disable_work_sync(&t->asrv->start_av_service_wq); + disable_work_sync(&t->asrv->terminate_wq); + } + if (t->wq) + destroy_workqueue(t->wq); + if (t->ep && !IS_ERR(t->ep)) + t->ep->wq = NULL; + av_life_current = NULL; +} + +static void av_life_ep_fn(struct work_struct *work) +{ + struct av_life_test *t = av_life_current; + + t->ep_saw = READ_ONCE(t->dead); + t->ep_runs++; +} + +static void av_life_blocked_fn(struct work_struct *work) +{ + struct av_life_test *t = av_life_current; + + complete(&t->entered); + wait_for_completion(&t->release); + atomic_set(&t->owner_saw, READ_ONCE(t->dead)); + atomic_inc(&t->owner_runs); + complete(&t->owner_done); +} + +static void av_life_queue_blocker_fn(struct work_struct *work) +{ + struct av_life_test *t = av_life_current; + + complete(&t->entered); + wait_for_completion(&t->release); +} + +static void av_life_producer_fn(struct work_struct *work) +{ + struct av_life_test *t = av_life_current; + + complete(&t->entered); + wait_for_completion(&t->release); + /* Model only the initializer publication boundary, with real works. */ + t->dcp->audiosrv = t->asrv; + schedule_work(&t->asrv->start_av_service_wq); +} + +static struct av_life_test *av_life_setup(struct kunit *test, work_func_t owner_fn) +{ + struct av_life_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, t); + t->dev = kunit_device_register(test, "av-lifetime"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->dev); + t->dcp = kunit_kzalloc(test, sizeof(*t->dcp), GFP_KERNEL); + t->asrv = kunit_kzalloc(test, sizeof(*t->asrv), GFP_KERNEL); + t->ep = kunit_kzalloc(test, sizeof(*t->ep), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->dcp); + KUNIT_ASSERT_NOT_NULL(test, t->asrv); + KUNIT_ASSERT_NOT_NULL(test, t->ep); + t->dcp->dev = t->dev; + dev_set_drvdata(t->dev, t->dcp); + spin_lock_init(&t->ep->lock); + init_rwsem(&t->asrv->srv_rwsem); + INIT_WORK(&t->asrv->start_av_service_wq, owner_fn); + INIT_WORK(&t->asrv->terminate_wq, owner_fn); + INIT_WORK(&t->dcp->external_audio_work, av_life_owner_fn); + t->asrv->works_live = true; + t->dcp->audiosrv = t->asrv; + atomic_set(&t->owner_saw, -1); + init_completion(&t->entered); + init_completion(&t->release); + init_completion(&t->owner_done); + init_completion(&t->stop_entered); + init_completion(&t->stop_done); + INIT_WORK(&t->stop_work, av_life_stop_fn); + av_life_current = t; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, av_life_cleanup, t), 0); + return t; +} + +static bool av_life_wait_reports_stopped(struct av_life_test *t) +{ + unsigned long deadline = jiffies + HZ; + bool stopped; + + do { + down_read(&t->asrv->srv_rwsem); + stopped = t->asrv->reports_stopped; + up_read(&t->asrv->srv_rwsem); + if (stopped) + return true; + usleep_range(50, 100); + } while (time_before(jiffies, deadline)); + return false; +} + +static void av_life_parallel_stop(struct kunit *test, struct av_life_test *t) +{ + unsigned long left; + + t->stop_wq = alloc_ordered_workqueue("av-lifetime-stop", 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->stop_wq); + KUNIT_ASSERT_TRUE(test, queue_work(t->stop_wq, &t->stop_work)); + left = wait_for_completion_timeout(&t->stop_entered, HZ); + KUNIT_ASSERT_NE(test, left, 0UL); + /* Observe entry into the real drain, rather than just its caller. */ + KUNIT_ASSERT_TRUE(test, av_life_wait_reports_stopped(t)); + /* The owner is deliberately still running: quiesce must wait. */ + left = wait_for_completion_timeout(&t->stop_done, msecs_to_jiffies(20)); + KUNIT_EXPECT_EQ(test, left, 0UL); + complete(&t->release); + left = wait_for_completion_timeout(&t->stop_done, HZ); + KUNIT_ASSERT_NE(test, left, 0UL); + KUNIT_EXPECT_TRUE(test, t->stop_saw_owner_done); +} + +/* Quiesce waits for owner work and flushes a live endpoint before dead. */ +static void av_lifetime_drain_before_dead(struct kunit *test) +{ + struct av_life_test *t = av_life_setup(test, av_life_blocked_fn); + unsigned long left; + + t->wq = alloc_ordered_workqueue("av-lifetime-test", 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->wq); + t->ep->wq = t->wq; + t->dcp->avep = t->ep; + INIT_WORK(&t->ep_work, av_life_ep_fn); + + KUNIT_EXPECT_TRUE(test, queue_work(t->wq, &t->ep_work)); + KUNIT_EXPECT_TRUE(test, schedule_work(&t->asrv->terminate_wq)); + left = wait_for_completion_timeout(&t->entered, HZ); + KUNIT_ASSERT_NE(test, left, 0UL); + av_life_parallel_stop(test, t); + KUNIT_EXPECT_EQ(test, t->ep_runs, 1); + KUNIT_EXPECT_EQ(test, t->ep_saw, 0); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 1); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_saw), 0); + WRITE_ONCE(t->dead, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_saw), 0); + KUNIT_EXPECT_EQ(test, t->ep_saw, 0); + KUNIT_EXPECT_TRUE(test, t->asrv->work_disabled); + KUNIT_EXPECT_TRUE(test, t->asrv->reports_stopped); +} + +/* + * afk_init publishes ERR_PTR. Quiesce still drains owner work and must not + * dereference that pointer or clear it. + */ +static void av_lifetime_err_ptr_endpoint(struct kunit *test) +{ + struct av_life_test *t = av_life_setup(test, av_life_blocked_fn); + unsigned long left; + + t->dcp->avep = ERR_PTR(-EINVAL); + KUNIT_EXPECT_TRUE(test, schedule_work(&t->asrv->start_av_service_wq)); + left = wait_for_completion_timeout(&t->entered, HZ); + KUNIT_ASSERT_NE(test, left, 0UL); + av_life_parallel_stop(test, t); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 1); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_saw), 0); + WRITE_ONCE(t->dead, 1); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_saw), 0); + KUNIT_EXPECT_TRUE(test, IS_ERR(t->dcp->avep)); + KUNIT_EXPECT_EQ(test, PTR_ERR(t->dcp->avep), -EINVAL); + KUNIT_EXPECT_TRUE(test, t->asrv->work_disabled); +} + +/* Pending items are cancelled; only already-running callbacks must finish. */ +static void av_lifetime_pending_owner_cancelled(struct kunit *test) +{ + struct av_life_test *t = av_life_setup(test, av_life_owner_fn); + unsigned long left; + + t->wq = alloc_ordered_workqueue("av-lifetime-pending", 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->wq); + INIT_WORK(&t->ep_work, av_life_queue_blocker_fn); + KUNIT_ASSERT_TRUE(test, queue_work(t->wq, &t->ep_work)); + left = wait_for_completion_timeout(&t->entered, HZ); + KUNIT_ASSERT_NE(test, left, 0UL); + KUNIT_ASSERT_TRUE(test, queue_work(t->wq, &t->asrv->start_av_service_wq)); + KUNIT_ASSERT_TRUE(test, queue_work(t->wq, &t->asrv->terminate_wq)); + av_audiosrv_quiesce(t->dcp); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 0); + KUNIT_EXPECT_FALSE(test, work_pending(&t->asrv->start_av_service_wq)); + KUNIT_EXPECT_FALSE(test, work_pending(&t->asrv->terminate_wq)); + WRITE_ONCE(t->dead, 1); + complete(&t->release); + flush_workqueue(t->wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 0); +} + +/* The real common-unbind entry must stop initialization before owner drain. */ +static void av_lifetime_unbind_waits_for_initializer(struct kunit *test) +{ + struct av_life_test *t = av_life_setup(test, av_life_owner_fn); + unsigned long left; + unsigned long deadline; + bool disabled = false; + + t->dcp->external_native = true; + t->dcp->ext_dead = true; + t->dcp->audiosrv = NULL; + t->stop_unbind = true; + INIT_WORK(&t->dcp->external_audio_work, av_life_producer_fn); + KUNIT_ASSERT_TRUE(test, schedule_work(&t->dcp->external_audio_work)); + left = wait_for_completion_timeout(&t->entered, HZ); + KUNIT_ASSERT_NE(test, left, 0UL); + t->stop_wq = alloc_ordered_workqueue("av-lifetime-unbind", 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->stop_wq); + KUNIT_ASSERT_TRUE(test, queue_work(t->stop_wq, &t->stop_work)); + left = wait_for_completion_timeout(&t->stop_entered, HZ); + KUNIT_ASSERT_NE(test, left, 0UL); + /* + * The running producer cannot finish yet. Queue rejection with no + * pending requeue proves unbind reached disable_work_sync, rather + * than merely pausing before its first statement. Any admitted + * requeue is cancelled by the real disable before this succeeds. + */ + deadline = jiffies + HZ; + do { + if (!schedule_work(&t->dcp->external_audio_work) && + !work_pending(&t->dcp->external_audio_work)) { + disabled = true; + break; + } + usleep_range(50, 100); + } while (time_before(jiffies, deadline)); + KUNIT_EXPECT_TRUE(test, disabled); + left = wait_for_completion_timeout(&t->stop_done, msecs_to_jiffies(20)); + KUNIT_EXPECT_EQ(test, left, 0UL); + complete(&t->release); + left = wait_for_completion_timeout(&t->stop_done, HZ); + KUNIT_ASSERT_NE(test, left, 0UL); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->audiosrv, t->asrv); + KUNIT_EXPECT_TRUE(test, t->asrv->reports_stopped); + KUNIT_EXPECT_TRUE(test, t->asrv->work_disabled); + KUNIT_EXPECT_FALSE(test, work_busy(&t->asrv->start_av_service_wq)); + KUNIT_EXPECT_FALSE(test, schedule_work(&t->asrv->start_av_service_wq)); + KUNIT_EXPECT_FALSE(test, schedule_work(&t->asrv->terminate_wq)); +} + +/* Two quiesces are one disable; one restart arms a single queue. */ +static void av_lifetime_quiesce_once_restart_once(struct kunit *test) +{ + struct av_life_test *t = av_life_setup(test, av_life_owner_fn); + + av_audiosrv_quiesce(t->dcp); + av_audiosrv_quiesce(t->dcp); + KUNIT_EXPECT_FALSE(test, schedule_work(&t->asrv->terminate_wq)); + av_audiosrv_reports_restart(t->dcp); + /* A second restart must not underflow either work's disable count. */ + av_audiosrv_reports_restart(t->dcp); + KUNIT_EXPECT_FALSE(test, t->asrv->reports_stopped); + KUNIT_EXPECT_FALSE(test, t->asrv->work_disabled); + KUNIT_EXPECT_TRUE(test, schedule_work(&t->asrv->terminate_wq)); + flush_work(&t->asrv->terminate_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 1); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_saw), 0); + av_audiosrv_quiesce(t->dcp); +} + +static void av_lifetime_report_after_stop(struct kunit *test) +{ + struct av_life_test *t = av_life_setup(test, av_life_owner_fn); + struct apple_epic_service svc = { }; + struct epic_service_call call = { + .group = cpu_to_le16(2), + .command = cpu_to_le32(1), + .magic = cpu_to_le32(EPIC_SERVICE_CALL_MAGIC), + }; + t->dcp->fw_compat = DCP_FIRMWARE_V_14_7; + t->dcp->external_native = true; + t->ep->dcp = t->dcp; + svc.ep = t->ep; + t->asrv->srv = &svc; + + av_audiosrv_quiesce(t->dcp); + av_audiosrv_quiesce(t->dcp); + av_audiosrv_report(&svc, 0, &call, sizeof(call)); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 0); + KUNIT_EXPECT_FALSE(test, work_busy(&t->asrv->terminate_wq)); + + av_audiosrv_reports_restart(t->dcp); + av_audiosrv_report(&svc, 0, &call, sizeof(call)); + flush_work(&t->asrv->terminate_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 1); + av_audiosrv_quiesce(t->dcp); +} + +/* + * Negative control. The owner is marked dead with the callback blocked and + * without quiesce. The callback must observe that dead owner. + */ +static void av_lifetime_control_without_drain(struct kunit *test) +{ + struct av_life_test *t = av_life_setup(test, av_life_blocked_fn); + unsigned long left; + + KUNIT_EXPECT_TRUE(test, schedule_work(&t->asrv->terminate_wq)); + left = wait_for_completion_timeout(&t->entered, HZ); + if (!left) { + complete(&t->release); + flush_work(&t->asrv->terminate_wq); + KUNIT_FAIL(test, "terminate work did not start"); + return; + } + WRITE_ONCE(t->dead, 1); + complete(&t->release); + flush_work(&t->asrv->terminate_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 1); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_saw), 1); +} + +/* Never-initialized owner work is not passed to the workqueue. */ +static void av_lifetime_never_inited(struct kunit *test) +{ + struct av_life_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, t); + t->dcp = kunit_kzalloc(test, sizeof(*t->dcp), GFP_KERNEL); + t->asrv = kunit_kzalloc(test, sizeof(*t->asrv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->dcp); + KUNIT_ASSERT_NOT_NULL(test, t->asrv); + t->dcp->audiosrv = t->asrv; + t->dcp->avep = ERR_PTR(-ENODEV); + av_audiosrv_quiesce(t->dcp); + av_audiosrv_reports_restart(t->dcp); + KUNIT_EXPECT_FALSE(test, t->asrv->works_live); + KUNIT_EXPECT_FALSE(test, t->asrv->work_disabled); + KUNIT_EXPECT_FALSE(test, t->asrv->reports_stopped); + KUNIT_EXPECT_EQ(test, PTR_ERR(t->dcp->avep), -ENODEV); +} + +/* Failed start stays disarmed across a bind restart. */ +static void av_lifetime_failed_start_stays_stopped(struct kunit *test) +{ + struct av_life_test *t = av_life_setup(test, av_life_owner_fn); + + t->dcp->avep = ERR_PTR(-EIO); + av_audiosrv_block_owner_restart(t->asrv); + av_audiosrv_quiesce(t->dcp); + av_audiosrv_reports_restart(t->dcp); + KUNIT_EXPECT_TRUE(test, t->asrv->work_disabled); + KUNIT_EXPECT_TRUE(test, t->asrv->reports_stopped); + KUNIT_EXPECT_FALSE(test, schedule_work(&t->asrv->terminate_wq)); + KUNIT_EXPECT_EQ(test, PTR_ERR(t->dcp->avep), -EIO); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 0); +} + + +/* The fake context action runs where afk_release_context would run. */ +static void av_life_endpoint_release(void *data) +{ + struct av_life_test *t = data; + + t->release_saw_stopped = t->release_owner->reports_stopped && + t->release_owner->work_disabled; + t->release_saw_quiesced = !t->release_ep->wq; + t->endpoint_released = true; + /* Keep a missing-quiesce mutant from leaking the actual queue. */ + afk_quiesce(t->release_ep); +} + +static void av_life_release_group(struct av_life_test *t) +{ + devres_release_group(t->dev, t->group); + t->group = NULL; + t->group_released = true; + t->asrv = NULL; + t->ep = NULL; + t->wq = NULL; +} + +static struct av_life_test *av_life_group_setup(struct kunit *test, + work_func_t fn, int endpoint) +{ + struct av_life_test *t = av_life_setup(test, fn); + + t->group = devres_open_group(t->dev, NULL, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->group); + t->asrv = devm_kzalloc(t->dev, sizeof(*t->asrv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->asrv); + init_rwsem(&t->asrv->srv_rwsem); + INIT_WORK(&t->asrv->start_av_service_wq, fn); + INIT_WORK(&t->asrv->terminate_wq, fn); + t->asrv->works_live = true; + t->dcp->audiosrv = t->asrv; + if (endpoint > 0) { + t->ep = devm_kzalloc(t->dev, sizeof(*t->ep), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->ep); + spin_lock_init(&t->ep->lock); + t->ep->dcp = t->dcp; + t->wq = alloc_ordered_workqueue("av-lifetime-devres", 0); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, t->wq); + t->ep->wq = t->wq; + t->release_ep = t->ep; + t->release_owner = t->asrv; + KUNIT_ASSERT_EQ(test, devm_add_action_or_reset(t->dev, + av_life_endpoint_release, t), 0); + } else { + t->ep = endpoint < 0 ? ERR_PTR(endpoint) : NULL; + } + t->dcp->avep = t->ep; + return t; +} + +static void av_lifetime_devres_order(struct kunit *test) +{ + struct av_life_test *t = av_life_group_setup(test, av_life_blocked_fn, 1); + unsigned long left; + + KUNIT_ASSERT_EQ(test, av_audiosrv_register_cleanup(t->dcp, t->asrv, t->ep), 0); + devres_close_group(t->dev, t->group); + KUNIT_ASSERT_TRUE(test, schedule_work(&t->asrv->terminate_wq)); + left = wait_for_completion_timeout(&t->entered, HZ); + KUNIT_ASSERT_NE(test, left, 0UL); + t->stop_release_group = true; + av_life_parallel_stop(test, t); + KUNIT_EXPECT_TRUE(test, t->endpoint_released); + KUNIT_EXPECT_TRUE(test, t->release_saw_stopped); + KUNIT_EXPECT_TRUE(test, t->release_saw_quiesced); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->audiosrv, NULL); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->avep, NULL); + if (!t->dcp->audiosrv) + av_audiosrv_quiesce(t->dcp); +} + +static void av_lifetime_devres_null_endpoint(struct kunit *test) +{ + struct av_life_test *t = av_life_group_setup(test, av_life_owner_fn, 0); + + KUNIT_ASSERT_EQ(test, av_audiosrv_register_cleanup(t->dcp, t->asrv, NULL), 0); + devres_close_group(t->dev, t->group); + av_life_release_group(t); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->audiosrv, NULL); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->avep, NULL); +} + +static void av_lifetime_devres_error_endpoint(struct kunit *test) +{ + struct av_life_test *t = av_life_group_setup(test, av_life_owner_fn, -ENOMEM); + + KUNIT_ASSERT_EQ(test, av_audiosrv_register_cleanup(t->dcp, t->asrv, t->ep), 0); + devres_close_group(t->dev, t->group); + av_life_release_group(t); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->audiosrv, NULL); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->avep, NULL); +} + +/* Model only devm's allocation boundary; execute its real reset callback. */ +static int av_life_fail_add_action(struct device *dev, void (*action)(void *), void *data) +{ + action(data); + return -ENOMEM; +} + +static void av_lifetime_add_action_reset(struct kunit *test) +{ + struct av_life_test *t = av_life_group_setup(test, av_life_owner_fn, 1); + + kunit_activate_static_stub(test, av_audiosrv_add_action, av_life_fail_add_action); + KUNIT_EXPECT_EQ(test, av_audiosrv_register_cleanup(t->dcp, t->asrv, t->ep), -ENOMEM); + KUNIT_EXPECT_TRUE(test, t->asrv->restart_blocked); + KUNIT_EXPECT_TRUE(test, t->asrv->work_disabled); + KUNIT_EXPECT_TRUE(test, t->asrv->reports_stopped); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->audiosrv, NULL); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->avep, NULL); + KUNIT_EXPECT_PTR_EQ(test, t->ep->wq, NULL); + /* The endpoint action still belongs to this group and is idempotent. */ + devres_close_group(t->dev, t->group); + av_life_release_group(t); + KUNIT_EXPECT_TRUE(test, t->release_saw_stopped); + KUNIT_EXPECT_TRUE(test, t->release_saw_quiesced); +} + +static void av_lifetime_stale_owner_shared_error(struct kunit *test) +{ + struct av_life_test *t = av_life_group_setup(test, av_life_owner_fn, -ENOMEM); + struct audiosrv_data *next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, next); + KUNIT_ASSERT_EQ(test, av_audiosrv_register_cleanup(t->dcp, t->asrv, t->ep), 0); + devres_close_group(t->dev, t->group); + t->dcp->audiosrv = next; + /* A shared ERR_PTR value identifies no particular endpoint owner. */ + t->dcp->avep = ERR_PTR(-ENOMEM); + av_life_release_group(t); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->audiosrv, next); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->avep, ERR_PTR(-ENOMEM)); +} + +static void av_lifetime_devres_preserves_successor(struct kunit *test) +{ + struct av_life_test *t = av_life_group_setup(test, av_life_owner_fn, 1); + struct audiosrv_data *next = kunit_kzalloc(test, sizeof(*next), GFP_KERNEL); + struct apple_dcp_afkep *ep = kunit_kzalloc(test, sizeof(*ep), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, next); + KUNIT_ASSERT_NOT_NULL(test, ep); + KUNIT_ASSERT_EQ(test, av_audiosrv_register_cleanup(t->dcp, t->asrv, t->ep), 0); + devres_close_group(t->dev, t->group); + t->dcp->audiosrv = next; + t->dcp->avep = ep; + av_life_release_group(t); + KUNIT_EXPECT_TRUE(test, t->release_saw_stopped); + KUNIT_EXPECT_TRUE(test, t->release_saw_quiesced); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->audiosrv, next); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->avep, ep); + KUNIT_EXPECT_FALSE(test, next->reports_stopped); + KUNIT_EXPECT_FALSE(test, next->work_disabled); +} + + +/* Releasing one group cannot leave restart state on the next owner. */ +static void av_lifetime_devres_fresh_rebind(struct kunit *test) +{ + struct av_life_test *t = av_life_group_setup(test, av_life_owner_fn, 0); + + KUNIT_ASSERT_EQ(test, av_audiosrv_register_cleanup(t->dcp, t->asrv, NULL), 0); + devres_close_group(t->dev, t->group); + av_life_release_group(t); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->audiosrv, NULL); + t->group = devres_open_group(t->dev, NULL, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->group); + t->group_released = false; + t->asrv = devm_kzalloc(t->dev, sizeof(*t->asrv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->asrv); + init_rwsem(&t->asrv->srv_rwsem); + INIT_WORK(&t->asrv->start_av_service_wq, av_life_owner_fn); + INIT_WORK(&t->asrv->terminate_wq, av_life_owner_fn); + t->asrv->works_live = true; + t->dcp->audiosrv = t->asrv; + KUNIT_ASSERT_EQ(test, av_audiosrv_register_cleanup(t->dcp, t->asrv, NULL), 0); + devres_close_group(t->dev, t->group); + av_audiosrv_reports_restart(t->dcp); + KUNIT_EXPECT_FALSE(test, t->asrv->work_disabled); + KUNIT_EXPECT_FALSE(test, t->asrv->restart_blocked); + KUNIT_ASSERT_TRUE(test, schedule_work(&t->asrv->start_av_service_wq)); + flush_work(&t->asrv->start_av_service_wq); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 1); + av_life_release_group(t); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->audiosrv, NULL); +} + +/* Common unbind owns one producer disable, balanced by one bind enable. */ +static void av_lifetime_producer_rebind_balance(struct kunit *test) +{ + struct av_life_test *t = av_life_setup(test, av_life_owner_fn); + + t->dcp->external_native = true; + t->dcp->ext_dead = true; + t->stop_unbind = true; + dcp_comp_unbind(t->dev, NULL, NULL); + KUNIT_EXPECT_FALSE(test, schedule_work(&t->dcp->external_audio_work)); + KUNIT_EXPECT_TRUE(test, enable_work(&t->dcp->external_audio_work)); + av_audiosrv_reports_restart(t->dcp); + KUNIT_EXPECT_TRUE(test, schedule_work(&t->dcp->external_audio_work)); + flush_work(&t->dcp->external_audio_work); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 1); + dcp_comp_unbind(t->dev, NULL, NULL); + KUNIT_EXPECT_FALSE(test, schedule_work(&t->dcp->external_audio_work)); + KUNIT_EXPECT_TRUE(test, enable_work(&t->dcp->external_audio_work)); + av_audiosrv_reports_restart(t->dcp); + KUNIT_EXPECT_TRUE(test, schedule_work(&t->dcp->external_audio_work)); + flush_work(&t->dcp->external_audio_work); + KUNIT_EXPECT_EQ(test, atomic_read(&t->owner_runs), 2); +} + +static struct kunit_case av_lifetime_cases[] = { + KUNIT_CASE(av_lifetime_devres_fresh_rebind), + KUNIT_CASE(av_lifetime_producer_rebind_balance), + KUNIT_CASE(av_lifetime_devres_order), + KUNIT_CASE(av_lifetime_devres_null_endpoint), + KUNIT_CASE(av_lifetime_devres_error_endpoint), + KUNIT_CASE(av_lifetime_add_action_reset), + KUNIT_CASE(av_lifetime_stale_owner_shared_error), + KUNIT_CASE(av_lifetime_devres_preserves_successor), + KUNIT_CASE(av_lifetime_drain_before_dead), + KUNIT_CASE(av_lifetime_err_ptr_endpoint), + KUNIT_CASE(av_lifetime_pending_owner_cancelled), + KUNIT_CASE(av_lifetime_unbind_waits_for_initializer), + KUNIT_CASE(av_lifetime_quiesce_once_restart_once), + KUNIT_CASE(av_lifetime_report_after_stop), + KUNIT_CASE(av_lifetime_control_without_drain), + KUNIT_CASE(av_lifetime_never_inited), + KUNIT_CASE(av_lifetime_failed_start_stays_stopped), + {} +}; + +static struct kunit_suite av_lifetime_suite = { + .name = "apple-av-audio-lifetime", + .test_cases = av_lifetime_cases, +}; + +kunit_test_suite(av_lifetime_suite); diff --git a/drivers/gpu/drm/apple/av-v14-test.c b/drivers/gpu/drm/apple/av-v14-test.c new file mode 100644 index 0000000000000..8d78113fb8f17 --- /dev/null +++ b/drivers/gpu/drm/apple/av-v14-test.c @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by av.c: which instance the 14.7 external role's audio client + * opens and closes. One reusable service slot is published, torn down and + * published again by the production init and teardown; the production open, + * close, Terminate report and Terminate work run on it, with only the + * service calls stubbed. The works are disabled for the workqueue and run + * here, in test context, where the stubs apply. + */ +#include +#include +#include + +struct av_v14_test { + struct apple_dcp *dcp; + struct audiosrv_data *asrv; + struct apple_dcp_afkep ep; + struct apple_epic_service svc; + struct apple_epic_service other; + /* Service calls: the command, and the instance generation it went to */ + u32 calls[8]; + u64 call_gens[8]; + unsigned int ncalls; + /* The card's disconnects, and link releases with the instance asked for */ + unsigned int disconnects; + unsigned int releases; + u64 release_gen; + /* Within a link release: the old PCM closes, a successor is published and opened */ + bool successor_during_release; +}; + +static struct av_v14_test *av_v14_current; + +static int av_v14_call(struct apple_epic_service *srv, u32 command, const void *body, + size_t body_len, size_t body_pad, void *reply, size_t reply_len) +{ + struct av_v14_test *t = av_v14_current; + + if (t->ncalls < ARRAY_SIZE(t->calls)) { + t->calls[t->ncalls] = command; + t->call_gens[t->ncalls] = t->asrv->gen; + } + t->ncalls++; + memset(reply, 0, reply_len); + /* Open: status 0 at +0, opened at +16 */ + if (command == t->asrv->cmds.open && reply_len >= 24) + put_unaligned_le64(1, reply + 16); + return 0; +} + +static void av_v14_card_disconnect(struct platform_device *pdev) +{ + av_v14_current->disconnects++; +} + +static void av_v14_replace(struct av_v14_test *t); + +static void av_v14_release_link(struct platform_device *pdev, const u64 *gen) +{ + struct av_v14_test *t = av_v14_current; + + t->releases++; + t->release_gen = gen ? *gen : 0; + if (!t->successor_during_release) + return; + t->successor_during_release = false; + dcp_audiosrv_close(t->dcp->dev); + av_v14_replace(t); + KUNIT_EXPECT_EQ(kunit_get_current_test(), dcp_audiosrv_open(t->dcp->dev), 0); +} + +static void av_v14_publish(struct av_v14_test *t) +{ + t->svc.enabled = true; + t->svc.torndown = false; + av_audiosrv_init(&t->svc, "DCPAVAudioInterface", "test", 0); +} + +/* The instance removed, and its successor published in the same slot */ +static void av_v14_replace(struct av_v14_test *t) +{ + av_audiosrv_teardown(&t->svc); + av_v14_publish(t); +} + +static void av_v14_terminate(struct av_v14_test *t, struct apple_epic_service *service) +{ + struct epic_service_call call = { + .group = cpu_to_le16(2), + .command = cpu_to_le32(1), + .magic = cpu_to_le32(EPIC_SERVICE_CALL_MAGIC), + }; + + av_audiosrv_report(service, 0, &call, sizeof(call)); +} + +static void av_v14_run_terminate_work(struct av_v14_test *t) +{ + av_work_service_terminate(&t->asrv->terminate_wq); +} + +static struct av_v14_test *av_v14_setup(struct kunit *test) +{ + struct av_v14_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct audiosrv_data *asrv; + struct device *dev; + + KUNIT_ASSERT_NOT_NULL(test, t); + t->dcp = kunit_kzalloc(test, sizeof(*t->dcp), GFP_KERNEL); + asrv = kunit_kzalloc(test, sizeof(*asrv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->dcp); + KUNIT_ASSERT_NOT_NULL(test, asrv); + dev = kunit_device_register(test, "av-v14"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + dev_set_drvdata(dev, t->dcp); + t->dcp->dev = dev; + t->dcp->fw_compat = DCP_FIRMWARE_V_14_7; + t->dcp->external = true; + t->dcp->external_native = true; + + init_rwsem(&asrv->srv_rwsem); + mutex_init(&asrv->plug_lock); + mutex_init(&asrv->open_lock); + asrv->cmds = dcp_av_audio_cmds_v13_5; + INIT_WORK(&asrv->start_av_service_wq, av_work_service_start); + disable_work(&asrv->start_av_service_wq); + INIT_WORK(&asrv->terminate_wq, av_work_service_terminate); + disable_work(&asrv->terminate_wq); + t->asrv = asrv; + t->dcp->audiosrv = asrv; + + t->ep.dcp = t->dcp; + spin_lock_init(&t->svc.lock); + t->svc.ep = &t->ep; + spin_lock_init(&t->other.lock); + t->other.ep = &t->ep; + t->other.enabled = true; + av_v14_current = t; + kunit_activate_static_stub(test, av_audiosrv_call, av_v14_call); + kunit_activate_static_stub(test, dcpaud_disconnect, av_v14_card_disconnect); + kunit_activate_static_stub(test, dcpaud_release_link, av_v14_release_link); + /* A card, never dereferenced: its disconnect and link release are stubbed */ + asrv->audio_dev = (struct platform_device *)t; + av_v14_publish(t); + return t; +} + +/* Serial: one Open per instance while a PCM is open, one Close after it. */ +static void av_v14_open_close(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + KUNIT_EXPECT_EQ(test, t->svc.users, 2U); /* published, and open */ + dcp_audiosrv_close(t->dcp->dev); + dcp_audiosrv_close(t->dcp->dev); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->calls[0], t->asrv->cmds.open); + KUNIT_EXPECT_EQ(test, t->calls[1], t->asrv->cmds.close); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* A successor in the slot of an instance left open is opened, not skipped. */ +static void av_v14_successor_opened(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + av_v14_replace(t); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->calls[1], t->asrv->cmds.open); + KUNIT_EXPECT_EQ(test, t->call_gens[1], t->asrv->gen); + KUNIT_EXPECT_NE(test, t->call_gens[1], t->call_gens[0]); + /* Its Close goes to it */ + dcp_audiosrv_close(t->dcp->dev); + KUNIT_ASSERT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_EQ(test, t->calls[2], t->asrv->cmds.close); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* The Close of an instance gone is not sent to its successor in the same slot. */ +static void av_v14_old_close_not_to_successor(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + av_v14_replace(t); + dcp_audiosrv_close(t->dcp->dev); + KUNIT_EXPECT_EQ(test, t->ncalls, 1U); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* + * A Terminate's work, queued for an instance, runs after its successor took + * the slot and was opened: the successor stays open. The successor's own + * Terminate closes it. + */ +static void av_v14_old_terminate_vs_successor(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + av_v14_terminate(t, &t->svc); + av_v14_replace(t); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + av_v14_run_terminate_work(t); + KUNIT_EXPECT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_PTR_EQ(test, t->asrv->open_srv, &t->svc); + KUNIT_EXPECT_EQ(test, t->asrv->open_gen, t->asrv->gen); + + av_v14_terminate(t, &t->svc); + av_v14_run_terminate_work(t); + KUNIT_ASSERT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_EQ(test, t->calls[2], t->asrv->cmds.close); + KUNIT_EXPECT_EQ(test, t->call_gens[2], t->asrv->gen); + KUNIT_EXPECT_NULL(test, t->asrv->open_srv); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* An instance published over the open one, without a teardown, is opened too. */ +static void av_v14_republished(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + av_v14_publish(t); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + KUNIT_EXPECT_EQ(test, t->ncalls, 2U); + dcp_audiosrv_close(t->dcp->dev); + KUNIT_EXPECT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* + * A Terminate's disconnect is past its instance check when, within its link + * release, the old PCM closes and a successor is published in the same slot + * and opened: the Terminate's final Close leaves the successor open. + */ +static void av_v14_successor_during_release(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + u64 old; + + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + old = t->asrv->gen; + av_v14_terminate(t, &t->svc); + t->successor_during_release = true; + av_v14_run_terminate_work(t); + /* The Terminate's, and the old instance's teardown's */ + KUNIT_EXPECT_EQ(test, t->disconnects, 2U); + KUNIT_EXPECT_EQ(test, t->releases, 1U); + KUNIT_EXPECT_EQ(test, t->release_gen, old); + /* Open, the old PCM's Close, the successor's Open; no Close after it */ + KUNIT_ASSERT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_EQ(test, t->calls[1], t->asrv->cmds.close); + KUNIT_EXPECT_EQ(test, t->call_gens[1], old); + KUNIT_EXPECT_EQ(test, t->calls[2], t->asrv->cmds.open); + KUNIT_EXPECT_NE(test, t->call_gens[2], old); + KUNIT_EXPECT_PTR_EQ(test, t->asrv->open_srv, &t->svc); + KUNIT_EXPECT_EQ(test, t->asrv->open_gen, t->asrv->gen); + dcp_audiosrv_close(t->dcp->dev); + KUNIT_EXPECT_EQ(test, t->ncalls, 4U); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* A Terminate that finds a successor published does nothing to it or the card. */ +static void av_v14_terminate_after_successor(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + av_v14_terminate(t, &t->svc); + av_v14_replace(t); + av_v14_run_terminate_work(t); + /* The old instance's teardown's only */ + KUNIT_EXPECT_EQ(test, t->disconnects, 1U); + KUNIT_EXPECT_EQ(test, t->releases, 0U); +} + +/* + * A link ends on the instance it was prepared on: once a successor is open, + * the old link's stop and complete are not sent, and its link is not on any + * open instance; a link prepared on the successor is. + */ +static void av_v14_link_instance(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + struct dcp_sound_cookie cookie = {}; + u64 old; + + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + old = t->asrv->gen; + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, &cookie), 0); + KUNIT_EXPECT_TRUE(test, dcp_audiosrv_link_on(t->dcp->dev, old)); + av_v14_replace(t); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + KUNIT_ASSERT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_STOP, &cookie), -ENODEV); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_COMPLETE, &cookie), + -ENODEV); + KUNIT_EXPECT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_FALSE(test, dcp_audiosrv_link_on(t->dcp->dev, old)); + KUNIT_EXPECT_FALSE(test, dcp_audiosrv_link_on(t->dcp->dev, t->asrv->gen)); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_PREPARE, &cookie), 0); + KUNIT_EXPECT_TRUE(test, dcp_audiosrv_link_on(t->dcp->dev, t->asrv->gen)); + KUNIT_EXPECT_EQ(test, dcp_audiosrv_link(t->dcp->dev, DCP_AUDIO_LINK_COMPLETE, &cookie), 0); + KUNIT_EXPECT_EQ(test, t->ncalls, 5U); +} + +/* A Terminate from an instance that is not the published one does nothing. */ +static void av_v14_terminate_unpublished(struct kunit *test) +{ + struct av_v14_test *t = av_v14_setup(test); + + KUNIT_EXPECT_EQ(test, dcp_audiosrv_open(t->dcp->dev), 0); + av_v14_terminate(t, &t->other); + av_v14_run_terminate_work(t); + KUNIT_EXPECT_EQ(test, t->ncalls, 1U); + KUNIT_EXPECT_PTR_EQ(test, t->asrv->open_srv, &t->svc); + dcp_audiosrv_close(t->dcp->dev); + KUNIT_EXPECT_EQ(test, t->ncalls, 2U); +} + +static struct kunit_case av_v14_cases[] = { + KUNIT_CASE(av_v14_open_close), + KUNIT_CASE(av_v14_successor_opened), + KUNIT_CASE(av_v14_old_close_not_to_successor), + KUNIT_CASE(av_v14_old_terminate_vs_successor), + KUNIT_CASE(av_v14_republished), + KUNIT_CASE(av_v14_successor_during_release), + KUNIT_CASE(av_v14_terminate_after_successor), + KUNIT_CASE(av_v14_link_instance), + KUNIT_CASE(av_v14_terminate_unpublished), + {} +}; + +static struct kunit_suite av_v14_suite = { + .name = "apple-av-v14-audio", + .test_cases = av_v14_cases, +}; + +kunit_test_suite(av_v14_suite); diff --git a/drivers/gpu/drm/apple/av.c b/drivers/gpu/drm/apple/av.c index 8a19f2a295bbc..461d162e2c6e9 100644 --- a/drivers/gpu/drm/apple/av.c +++ b/drivers/gpu/drm/apple/av.c @@ -4,7 +4,10 @@ // #define DEBUG #include +#include #include +#include +#include #include #include #include @@ -53,7 +56,16 @@ static const struct dcp_av_audio_cmds dcp_av_audio_cmds_v13_5 = { .get_product_attrs = 18, }; +struct audiosrv_data; + +struct av_audiosrv_cleanup { + struct apple_dcp *dcp; + struct audiosrv_data *owner; + struct apple_dcp_afkep *ep; +}; + struct audiosrv_data { + struct av_audiosrv_cleanup cleanup; struct platform_device *audio_dev; bool plugged; struct mutex plug_lock; @@ -62,12 +74,41 @@ struct audiosrv_data { struct rw_semaphore srv_rwsem; /* Workqueue for starting the audio service */ struct work_struct start_av_service_wq; +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + /* The 14.7 external role: let a terminated audio interface go. */ + struct work_struct terminate_wq; + /* + * The instance generation, changed when an instance is published or + * removed (srv_rwsem), so a slot reused for the next instance is not + * taken for the old one; the Terminate work's instance. + */ + u64 gen; + u64 terminate_gen; + /* + * The 14.7 external role: the instance the PCM opened and its + * generation (open_lock); open_srv holds a pin, so its slot is not + * reused while it is open. + */ + struct mutex open_lock; + struct apple_epic_service *open_srv; + u64 open_gen; + /* The instance the link was last prepared on (open_lock) */ + u64 link_gen; +#endif struct dcp_av_audio_cmds cmds; bool warned_get_elements; bool warned_get_product_attrs; bool is_open; + /* INIT_WORK has run for the embedded owner works. */ + bool works_live; + /* Report and service-start admission. Writers hold srv_rwsem. */ + bool reports_stopped; + /* One disable_work_sync is outstanding. Restart enables once. */ + bool work_disabled; + /* afk_init/afk_start failed. Rebind must not enable these works. */ + bool restart_blocked; }; static void av_interface_init(struct apple_epic_service *service, const char *name, @@ -97,6 +138,7 @@ static void av_audiosrv_init(struct apple_epic_service *service, const char *nam struct apple_dcp *dcp = service->ep->dcp; struct audiosrv_data *asrv = dcp->audiosrv; struct apple_epic_service *old; + bool queue_start; /* The owner pin keeps the slot from being recycled while published. */ service = afk_service_get(service); @@ -108,13 +150,18 @@ static void av_audiosrv_init(struct apple_epic_service *service, const char *nam down_write(&asrv->srv_rwsem); old = asrv->srv; asrv->srv = service; +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + asrv->gen++; +#endif + queue_start = !asrv->reports_stopped; up_write(&asrv->srv_rwsem); if (old) afk_service_put(old); asrv->plugged = true; mutex_unlock(&asrv->plug_lock); - schedule_work(&asrv->start_av_service_wq); + if (queue_start) + schedule_work(&asrv->start_av_service_wq); } static void av_audiosrv_teardown(struct apple_epic_service *service) @@ -127,8 +174,12 @@ static void av_audiosrv_teardown(struct apple_epic_service *service) down_write(&asrv->srv_rwsem); owned = asrv->srv == service; - if (owned) + if (owned) { asrv->srv = NULL; +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + asrv->gen++; +#endif + } up_write(&asrv->srv_rwsem); /* A stale generation must not unplug its successor. */ @@ -202,6 +253,190 @@ int dcp_audiosrv_unprepare(struct device *dev) return ret; } +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +/* The audio service of the 14.7 external role, known before it starts. */ +bool dcp_audiosrv_native_link(struct device *dev) +{ + return dcp_is_v14_7_external(dev_get_drvdata(dev)); +} + +/* The 14.7 external role's audio service calls; KUnit stubs them. */ +static int av_audiosrv_call(struct apple_epic_service *srv, u32 command, const void *body, + size_t body_len, size_t body_pad, void *reply, size_t reply_len) +{ + KUNIT_STATIC_STUB_REDIRECT(av_audiosrv_call, srv, command, body, body_len, body_pad, + reply, reply_len); + return afk_service_call(srv, 0, command, body, body_len, body_pad, reply, reply_len, 0); +} + +/* + * The 14.7 external role: the service is open while a PCM is (see + * av_service_connect()). The instance the PCM opened is kept under open_lock, + * which the endpoint's receive path never takes, and the calls are made + * without srv_rwsem: a teardown arriving meanwhile goes through, and so do + * the replies. The audio slots are reusable, so the open instance is kept + * pinned, with its generation: its slot cannot be reused while it is open, + * a successor is opened even in the same slot, and a removed instance is + * never closed (it took its open with it), so no Close goes to a successor. + */ +int dcp_audiosrv_open(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct audiosrv_data *asrv = dcp->audiosrv; + struct apple_epic_service *srv; + u8 reply[32] = {}; + u64 gen; + int ret; + + if (!asrv) + return -ENODEV; + + guard(mutex)(&asrv->open_lock); + scoped_guard(rwsem_read, &asrv->srv_rwsem) { + srv = afk_service_get(asrv->srv); + gen = asrv->gen; + } + if (!srv) + return -ENODEV; + if (asrv->open_srv) { + if (asrv->open_gen == gen) { + afk_service_put(srv); + return 0; + } + /* An earlier instance's open went with that instance. */ + afk_service_put(asrv->open_srv); + asrv->open_srv = NULL; + } + + /* Open: a zero body; status at +0, the opened Boolean at +16. */ + ret = av_audiosrv_call(srv, asrv->cmds.open, NULL, 0, 32, reply, sizeof(reply)); + if (!ret && (get_unaligned_le32(reply) || !get_unaligned_le64(reply + 16))) { + dev_err(dev, "audiosrv: open answered %#x, opened %llu\n", + get_unaligned_le32(reply), get_unaligned_le64(reply + 16)); + ret = -EIO; + } + if (ret) { + afk_service_put(srv); + return ret; + } + asrv->open_srv = srv; + asrv->open_gen = gen; + return 0; +} + +/* @gen: only the client of that instance (NULL: whichever is open) */ +static void av_audiosrv_close(struct apple_dcp *dcp, struct audiosrv_data *asrv, + const u64 *gen) +{ + struct apple_epic_service *srv; + u8 reply[16] = {}; + bool published; + int ret; + + guard(mutex)(&asrv->open_lock); + /* Checked here, at the Close: a later instance may be open by now. */ + if (gen && asrv->open_gen != *gen) + return; + srv = asrv->open_srv; + asrv->open_srv = NULL; + if (!srv) + return; + scoped_guard(rwsem_read, &asrv->srv_rwsem) + published = asrv->srv == srv && asrv->gen == asrv->open_gen; + + /* Close: a zero body; status at +0. Only the instance still published. */ + ret = published ? av_audiosrv_call(srv, asrv->cmds.close, NULL, 0, 16, reply, + sizeof(reply)) : -ENODEV; + afk_service_put(srv); + if (ret == -ENODEV) + return; + if (ret || get_unaligned_le32(reply)) + dev_err(dcp->dev, "audiosrv: close failed: %d, answered %#x\n", ret, + get_unaligned_le32(reply)); +} + +void dcp_audiosrv_close(struct device *dev) +{ + struct apple_dcp *dcp = dev_get_drvdata(dev); + + if (dcp->audiosrv) + av_audiosrv_close(dcp, dcp->audiosrv, NULL); +} + +bool dcp_audiosrv_link_on(struct device *dev, u64 gen) +{ + struct apple_dcp *dcp; + struct audiosrv_data *asrv; + + KUNIT_STATIC_STUB_REDIRECT(dcp_audiosrv_link_on, dev, gen); + dcp = dev_get_drvdata(dev); + asrv = dcp->audiosrv; + if (!asrv) + return false; + guard(mutex)(&asrv->open_lock); + return asrv->open_srv && asrv->open_gen == gen && asrv->link_gen == gen; +} + +/* + * The link commands of the 14.7 external role's audio service, the native + * audio driver's: prepare, start, stop and complete all take the selected + * element's 32-byte link data at body+0 and link source 0 at +32, and + * answer a status at body+48. + */ +int dcp_audiosrv_link(struct device *dev, enum dcp_audio_link op, + const struct dcp_sound_cookie *cookie) +{ + struct apple_dcp *dcp; + struct audiosrv_data *asrv; + u8 body[64] = {}, reply[64]; + u32 command, status; + int ret; + + KUNIT_STATIC_STUB_REDIRECT(dcp_audiosrv_link, dev, op, cookie); + dcp = dev_get_drvdata(dev); + asrv = dcp->audiosrv; + + switch (op) { + case DCP_AUDIO_LINK_PREPARE: + command = asrv->cmds.prepare; + break; + case DCP_AUDIO_LINK_START: + command = asrv->cmds.start_link; + break; + case DCP_AUDIO_LINK_STOP: + command = asrv->cmds.stop_link; + break; + case DCP_AUDIO_LINK_COMPLETE: + command = asrv->cmds.unprepare; + break; + default: + return -EINVAL; + } + static_assert(sizeof(*cookie) == DCP_SOUND_COOKIE_SIZE_V14_7); + memcpy(body, cookie, sizeof(*cookie)); + + scoped_guard(mutex, &asrv->open_lock) { + if (!asrv->open_srv) + return -ENODEV; + /* A link ends on the instance it was prepared on; one gone took it along. */ + if (op != DCP_AUDIO_LINK_PREPARE && asrv->link_gen != asrv->open_gen) + return -ENODEV; + ret = av_audiosrv_call(asrv->open_srv, command, body, sizeof(body), 0, + reply, sizeof(reply)); + if (!ret && op == DCP_AUDIO_LINK_PREPARE) + asrv->link_gen = asrv->open_gen; + } + if (ret) + return ret; + status = get_unaligned_le32(reply + 48); + if (status) { + dev_err(dev, "audiosrv: link method %u answered %#x\n", command, status); + return -EIO; + } + return 0; +} +#endif + static int dcp_audiosrv_osobject_call(struct apple_epic_service *service, u16 group, u32 command, void *output, size_t output_maxsize, @@ -283,9 +518,46 @@ int dcp_audiosrv_get_product_attrs(struct device *dev, void *attrs, size_t maxsi return ret; } +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +/* A DCPAV report (group 2) with command 1: Terminate. */ +static bool av_report_terminate(const void *data, size_t size) +{ + const struct epic_service_call *call = data; + + return size >= sizeof(*call) && le32_to_cpu(call->magic) == EPIC_SERVICE_CALL_MAGIC && + le16_to_cpu(call->group) == 2 && le32_to_cpu(call->command) == 1; +} +#endif + static int av_audiosrv_report(struct apple_epic_service *service, u32 idx, const void *data, size_t data_size) { +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + struct apple_dcp *dcp = service->ep->dcp; + + /* + * The 14.7 external role terminates an open audio interface when the + * sink changes (after it reports the sink's physical address). Its + * client must close it (Close, command 5): the firmware waits up to + * 10 s for that before it removes the interface, and the sink's link + * with it. A later publication brings the new instance. + */ + if (dcp_is_v14_7_external(dcp) && av_report_terminate(data, data_size)) { + struct audiosrv_data *asrv = dcp->audiosrv; + bool published; + + /* The work acts on this instance only, not on a later one. */ + scoped_guard(rwsem_read, &asrv->srv_rwsem) { + published = asrv->srv == service && !asrv->reports_stopped; + if (published) + WRITE_ONCE(asrv->terminate_gen, asrv->gen); + } + if (published) { + dev_info(dcp->dev, "audio interface terminated\n"); + schedule_work(&asrv->terminate_wq); + } + } +#endif dev_dbg(service->ep->dcp->dev, "got audio report %d size %zx\n", idx, data_size); #ifdef DEBUG print_hex_dump(KERN_DEBUG, "audio report: ", DUMP_PREFIX_NONE, 16, 1, data, data_size, true); @@ -294,6 +566,30 @@ static int av_audiosrv_report(struct apple_epic_service *service, u32 idx, return 0; } +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +/* + * The 14.7 external role behind a DP-to-HDMI converter (the J514S HDMI + * port) also publishes a CEC interface on this endpoint. The host makes no + * use of CEC; the interface's event reports (group 2: the sink's physical + * address, link training, link start and stop) are taken and dropped. + */ +static void av_cec_init(struct apple_epic_service *service, const char *name, + const char *class, s64 unit) +{ +} + +static int av_cec_report(struct apple_epic_service *service, enum epic_subtype type, + const void *data, size_t data_size) +{ + return 0; +} + +static void av_cec_teardown(struct apple_epic_service *service) +{ + afk_service_disable(service); +} +#endif + static const struct apple_epic_service_ops avep_ops[] = { { .name = "DCPAVSimpleVideoInterface", @@ -308,6 +604,14 @@ static const struct apple_epic_service_ops avep_ops[] = { .report = av_audiosrv_report, .teardown = av_audiosrv_teardown, }, +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + { + .name = "DCPDPCECInterface", + .init = av_cec_init, + .report = av_cec_report, + .teardown = av_cec_teardown, + }, +#endif {} }; @@ -328,6 +632,26 @@ void av_service_connect(struct apple_dcp *dcp) if (asrv->is_open) goto out; +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + /* + * The 14.7 external role opens its audio service only while a PCM is + * (dcp_audiosrv_open()): an open audio interface is a client the + * firmware must wait for whenever the sink changes (see + * av_audiosrv_report()); an idle one need not be. The card follows + * the display's power (A472 on): the CRTC enable connects before the + * mode is set, and the modeset connects again once the display is on. + */ + if (dcp_is_v14_7_external(dcp)) { + if (!iomfb_v14_7_external_powered(dcp)) + goto out; + scoped_guard(mutex, &asrv->plug_lock) { + if (asrv->audio_dev) + dcpaud_connect(asrv->audio_dev, asrv->plugged); + } + goto out; + } +#endif + ret = afk_service_call(service, 0, asrv->cmds.open, NULL, 0, 32, NULL, 0, 32); if (ret) { @@ -344,13 +668,18 @@ void av_service_connect(struct apple_dcp *dcp) afk_service_put(service); } -void av_service_disconnect(struct apple_dcp *dcp) +/* @gen: only while that instance is published (NULL: whichever is) */ +static void __av_service_disconnect(struct apple_dcp *dcp, const u64 *gen) { struct apple_epic_service *service; struct audiosrv_data *asrv = dcp->audiosrv; int ret; scoped_guard(rwsem_write, &asrv->srv_rwsem) { +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + if (gen && asrv->gen != *gen) + return; +#endif service = afk_service_pin(asrv->srv); if (!service) return; @@ -358,6 +687,39 @@ void av_service_disconnect(struct apple_dcp *dcp) /* close AV audio service */ dev_info(dcp->dev, "%s: stopping audio service\n", __func__); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + /* The 14.7 external role's card goes with the display, and an open PCM's service. */ + if (dcp_is_v14_7_external(dcp)) { + struct platform_device *audio_dev = NULL; + bool published = true; + + /* + * Each step acts on instance @gen only, checked as it is taken: + * a successor may be published and opened in between. + */ + scoped_guard(mutex, &asrv->plug_lock) { + if (gen) { + scoped_guard(rwsem_read, &asrv->srv_rwsem) + published = asrv->gen == *gen; + } + if (published) { + audio_dev = asrv->audio_dev; + if (audio_dev) + dcpaud_disconnect(audio_dev); + } + } + if (!published) + goto out; + /* + * The open PCM's link ends on the service, before its Close; + * outside plug_lock, which the endpoint's receive path takes. + */ + if (audio_dev) + dcpaud_release_link(audio_dev, gen); + av_audiosrv_close(dcp, asrv, gen); + goto out; + } +#endif if (!asrv->is_open) goto out; @@ -380,6 +742,28 @@ void av_service_disconnect(struct apple_dcp *dcp) afk_service_put(service); } +void av_service_disconnect(struct apple_dcp *dcp) +{ + __av_service_disconnect(dcp, NULL); +} + +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +/* The instance the Terminate came from, if it is still published. */ +static void av_work_service_terminate(struct work_struct *work) +{ + struct audiosrv_data *asrv = container_of(work, struct audiosrv_data, terminate_wq); + u64 gen = READ_ONCE(asrv->terminate_gen); + struct apple_dcp *dcp = NULL; + + scoped_guard(rwsem_read, &asrv->srv_rwsem) { + if (asrv->srv && asrv->srv->ep && asrv->gen == gen) + dcp = asrv->srv->ep->dcp; + } + if (dcp) + __av_service_disconnect(dcp, &gen); +} +#endif + static void av_work_service_start(struct work_struct *work) { struct audiosrv_data *audiosrv_data; @@ -400,12 +784,146 @@ static void av_work_service_start(struct work_struct *work) av_service_connect(dcp); } +static void av_audiosrv_flush_owner(struct audiosrv_data *asrv) +{ + flush_work(&asrv->start_av_service_wq); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + flush_work(&asrv->terminate_wq); +#endif +} + +/* Drain the captured owner, without following a later dcp publication. */ +static bool av_audiosrv_drain_owner(struct audiosrv_data *asrv) +{ + bool first_stop; + + if (!asrv || !asrv->works_live) + return false; + + down_write(&asrv->srv_rwsem); + first_stop = !asrv->work_disabled; + asrv->reports_stopped = true; + asrv->work_disabled = true; + up_write(&asrv->srv_rwsem); + + /* + * Queue admission is already refused. disable_work_sync then raises + * the work disable count and waits. No audio lock is held: the + * callback takes srv_rwsem and may RPC. A later call only flushes, + * so the disable count stays one for the rebind enable. + */ + if (first_stop) { + disable_work_sync(&asrv->start_av_service_wq); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + disable_work_sync(&asrv->terminate_wq); +#endif + } else { + av_audiosrv_flush_owner(asrv); + } + return true; +} + +static void av_audiosrv_block_owner_restart(struct audiosrv_data *asrv) +{ + if (!asrv) + return; + down_write(&asrv->srv_rwsem); + asrv->restart_blocked = true; + up_write(&asrv->srv_rwsem); +} + +void av_audiosrv_quiesce(struct apple_dcp *dcp) +{ + struct audiosrv_data *asrv; + struct apple_dcp_afkep *ep; + struct workqueue_struct *wq; + + if (!dcp) + return; + /* Lifecycle callers stop the native initializer before this snapshot. */ + asrv = dcp->audiosrv; + ep = dcp->avep; + if (!av_audiosrv_drain_owner(asrv)) + return; + if (IS_ERR_OR_NULL(ep)) + return; + wq = ep->wq; + if (!wq) + return; + flush_workqueue(wq); + av_audiosrv_flush_owner(asrv); +} + +static void av_audiosrv_cleanup(void *data) +{ + struct av_audiosrv_cleanup *cleanup = data; + struct audiosrv_data *asrv = cleanup->owner; + struct apple_dcp *dcp = cleanup->dcp; + + /* Keep RX alive until any owner work that may make an RPC has left. */ + if (asrv && asrv->works_live) { + av_audiosrv_block_owner_restart(asrv); + av_audiosrv_drain_owner(asrv); + } + afk_quiesce(cleanup->ep); + /* An old devres group must not unpublish a later session. */ + if (dcp->audiosrv == asrv) { + if (dcp->avep == cleanup->ep) + dcp->avep = NULL; + dcp->audiosrv = NULL; + } +} + +static int av_audiosrv_add_action(struct device *dev, void (*action)(void *), + void *data) +{ + KUNIT_STATIC_STUB_REDIRECT(av_audiosrv_add_action, dev, action, data); + return devm_add_action_or_reset(dev, action, data); +} + +static int av_audiosrv_register_cleanup(struct apple_dcp *dcp, + struct audiosrv_data *owner, + struct apple_dcp_afkep *ep) +{ + owner->cleanup = (struct av_audiosrv_cleanup) { + .dcp = dcp, + .owner = owner, + .ep = ep, + }; + return av_audiosrv_add_action(dcp->dev, av_audiosrv_cleanup, &owner->cleanup); +} + +void av_audiosrv_reports_restart(struct apple_dcp *dcp) +{ + struct audiosrv_data *asrv; + + if (!dcp) + return; + asrv = dcp->audiosrv; + if (!asrv || !asrv->works_live || asrv->restart_blocked) + return; + + down_write(&asrv->srv_rwsem); + if (!asrv->work_disabled || asrv->restart_blocked) { + up_write(&asrv->srv_rwsem); + return; + } + asrv->work_disabled = false; + enable_work(&asrv->start_av_service_wq); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + enable_work(&asrv->terminate_wq); +#endif + asrv->reports_stopped = false; + up_write(&asrv->srv_rwsem); +} + int avep_init(struct apple_dcp *dcp) { struct audiosrv_data *audiosrv_data; struct platform_device *audio_pdev; struct device *dev = dcp->dev; struct device_node *endpoint, *audio_node = NULL; + int ret; audiosrv_data = devm_kzalloc(dcp->dev, sizeof(*audiosrv_data), GFP_KERNEL); if (!audiosrv_data) @@ -422,12 +940,28 @@ int avep_init(struct apple_dcp *dcp) audiosrv_data->cmds = dcp_av_audio_cmds_v13_5; break; default: +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + /* + * The 14.7 external role (T6030 external processor) numbers its + * methods as 13.5 does: open 4, close 5, prepare 6, start 7, + * stop 10, complete 11, elements 1/16, product attributes 1/18. + */ + if (dcp_is_v14_7_external(dcp)) { + audiosrv_data->cmds = dcp_av_audio_cmds_v13_5; + break; + } +#endif dev_err(dcp->dev, "Audio not supported for firmware\n"); return -ENODEV; } - dcp->audiosrv = audiosrv_data; INIT_WORK(&audiosrv_data->start_av_service_wq, av_work_service_start); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + INIT_WORK(&audiosrv_data->terminate_wq, av_work_service_terminate); + mutex_init(&audiosrv_data->open_lock); +#endif + audiosrv_data->works_live = true; + dcp->audiosrv = audiosrv_data; endpoint = of_graph_get_endpoint_by_regs(dev->of_node, 0, 0); if (endpoint) { @@ -437,14 +971,14 @@ int avep_init(struct apple_dcp *dcp) if (!audio_node || !of_device_is_available(audio_node)) { of_node_put(audio_node); dev_info(dev, "No audio support\n"); - return 0; + return av_audiosrv_register_cleanup(dcp, audiosrv_data, NULL); } audio_pdev = of_find_device_by_node(audio_node); of_node_put(audio_node); if (!audio_pdev) { dev_info(dev, "No DP/HDMI audio device not ready\n"); - return 0; + return av_audiosrv_register_cleanup(dcp, audiosrv_data, NULL); } dcp->audiosrv->audio_dev = audio_pdev; @@ -452,8 +986,24 @@ int avep_init(struct apple_dcp *dcp) DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME); dcp->avep = afk_init(dcp, AV_ENDPOINT, avep_ops); - if (IS_ERR(dcp->avep)) - return PTR_ERR(dcp->avep); + if (IS_ERR(dcp->avep)) { + ret = PTR_ERR(dcp->avep); + if (!av_audiosrv_register_cleanup(dcp, audiosrv_data, dcp->avep)) + av_audiosrv_cleanup(&audiosrv_data->cleanup); + return ret; + } + /* Newer than afk_release_context, so owner work drains before RX dies. */ + ret = av_audiosrv_register_cleanup(dcp, audiosrv_data, dcp->avep); + if (ret) + return ret; dcp->avep->debugfs_entry = dcp->ep_debugfs[AV_ENDPOINT - 0x20]; - return afk_start(dcp->avep); + ret = afk_start(dcp->avep); + if (ret) + av_audiosrv_cleanup(&audiosrv_data->cleanup); + return ret; } + +#if IS_ENABLED(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) && IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +#include "av-v14-test.c" +#include "av-lifetime-test.c" +#endif diff --git a/drivers/gpu/drm/apple/av.h b/drivers/gpu/drm/apple/av.h index c00cbef549fd2..1d710b0f706f3 100644 --- a/drivers/gpu/drm/apple/av.h +++ b/drivers/gpu/drm/apple/av.h @@ -6,12 +6,21 @@ //int avep_audiosrv_startlink(struct apple_dcp *dcp, struct dcp_sound_cookie *cookie); //int avep_audiosrv_stoplink(struct apple_dcp *dcp); +/* dcp.c: the opt-in to DP/HDMI audio (appledrm.hdmi_audio). */ +extern bool hdmi_audio; + #if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) void av_service_connect(struct apple_dcp *dcp); void av_service_disconnect(struct apple_dcp *dcp); +/* Stop and drain owner work after the caller stops its native initializer. */ +void av_audiosrv_quiesce(struct apple_dcp *dcp); +/* Pair of av_audiosrv_quiesce after a later successful bind. */ +void av_audiosrv_reports_restart(struct apple_dcp *dcp); #else static inline void av_service_connect(struct apple_dcp *dcp) { } static inline void av_service_disconnect(struct apple_dcp *dcp) { } +static inline void av_audiosrv_quiesce(struct apple_dcp *dcp) { } +static inline void av_audiosrv_reports_restart(struct apple_dcp *dcp) { } #endif #endif /* __AV_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.c b/drivers/gpu/drm/apple/dcp-fabric-core.c index dccb400a9ff0f..8c952dc024560 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.c +++ b/drivers/gpu/drm/apple/dcp-fabric-core.c @@ -28,8 +28,9 @@ EXPORT_SYMBOL_GPL(dcp_fabric_t6020_flow); u8 dcp_fabric_tunnel_clock_blocked(bool t6030, unsigned int source) { - return t6030 ? BIT(1) | - (dcp_fabric_t6030_pclk(source, 0) < 0 ? BIT(0) : 0) : 0; + return t6030 ? + (dcp_fabric_t6030_pclk(source, 0) < 0 ? BIT(0) : 0) | + (dcp_fabric_t6030_pclk(source, 1) < 0 ? BIT(1) : 0) : 0; } EXPORT_SYMBOL_GPL(dcp_fabric_tunnel_clock_blocked); @@ -155,6 +156,13 @@ bool dcp_fabric_keep_order(bool dual_stream, bool ready, bool frozen) } EXPORT_SYMBOL_GPL(dcp_fabric_keep_order); +/* T6030 native pipelines retain their route; the encoder follows that owner. */ +bool dcp_fabric_ordered_dual_stream(bool wired, bool t6030) +{ + return wired && !t6030; +} +EXPORT_SYMBOL_GPL(dcp_fabric_ordered_dual_stream); + bool dcp_fabric_waiting(const struct dcp_fabric_port *port) { return !port->owner[0] && port->wanted && port->hpd; @@ -501,6 +509,15 @@ u32 dcp_fabric_connector_mask(bool dual_stream, bool bound, bool routed, } EXPORT_SYMBOL_GPL(dcp_fabric_connector_mask); +u32 dcp_fabric_port_crtc_mask(bool secondary, bool ordered_dual, bool typec_only, + u32 mask, u32 candidate) +{ + if (secondary && ordered_dual) + return typec_only && (!mask || candidate < mask) ? candidate : mask; + return mask | candidate; +} +EXPORT_SYMBOL_GPL(dcp_fabric_port_crtc_mask); + enum dcp_fabric_attach_action dcp_fabric_attach_action(bool external) { return external ? DCP_FABRIC_ATTACH_WORK : DCP_FABRIC_ATTACH_OOB; diff --git a/drivers/gpu/drm/apple/dcp-fabric-core.h b/drivers/gpu/drm/apple/dcp-fabric-core.h index e4eae1f785efa..1b8c59ac15a54 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-core.h +++ b/drivers/gpu/drm/apple/dcp-fabric-core.h @@ -77,13 +77,13 @@ enum dcp_fabric_wiring dcp_fabric_wiring(bool dpin0, bool dpin1, bool legacy, unsigned int endpoints); bool dcp_fabric_t6020_flow(bool usb4, bool soc_support, bool connector_wired); /* - * T6030's even external sources feed DP IN0 through DPTX PCLK1. A local copy - * of apple_dp_tunnel_t6030_pclk() (linux/soc/apple/dp-tunnel.h, 170-usb), - * which the dp-tunnel.h this tree builds against does not have yet. + * T6030's even external sources feed DP IN0/1 through DPTX PCLK1/2. Keep this + * pure policy helper independent of OF/PHY header dependencies. The fabric + * tests check parity with apple_dp_tunnel_t6030_pclk() in dp-tunnel.h. */ static inline int dcp_fabric_t6030_pclk(unsigned int source, unsigned int dpin) { - return !dpin && (source == 0 || source == 2) ? 0 : -EOPNOTSUPP; + return dpin < 2 && (source == 0 || source == 2) ? (int)dpin : -EOPNOTSUPP; } u8 dcp_fabric_tunnel_clock_blocked(bool t6030, unsigned int source); @@ -170,6 +170,7 @@ bool dcp_fabric_fits(const struct dcp_fabric_pipeline *pipeline, const struct dcp_fabric_policy *policy, u32 mask, bool connector_present); bool dcp_fabric_keep_order(bool dual_stream, bool ready, bool frozen); +bool dcp_fabric_ordered_dual_stream(bool wired, bool t6030); bool dcp_fabric_waiting(const struct dcp_fabric_port *port); bool dcp_fabric_movable(const struct dcp_fabric_route *owner, const struct dcp_fabric_route *target); @@ -191,6 +192,8 @@ unsigned int dcp_fabric_fixed_steps(bool has_fixed, bool owned, bool needs_mux, enum dcp_fabric_fixed_step steps[2]); u32 dcp_fabric_connector_mask(bool dual_stream, bool bound, bool routed, unsigned int index, u32 candidates); +u32 dcp_fabric_port_crtc_mask(bool secondary, bool ordered_dual, bool typec_only, + u32 mask, u32 candidate); enum dcp_fabric_attach_action dcp_fabric_attach_action(bool external); bool dcp_fabric_t6030_link(bool tunnel, bool has_xbar, bool t6030_dpin); diff --git a/drivers/gpu/drm/apple/dcp-fabric-test.c b/drivers/gpu/drm/apple/dcp-fabric-test.c index bdf42c35bd361..2d2fab58136b6 100644 --- a/drivers/gpu/drm/apple/dcp-fabric-test.c +++ b/drivers/gpu/drm/apple/dcp-fabric-test.c @@ -1910,8 +1910,9 @@ static void fabric_t6030_init(struct fabric_fixture *f) { unsigned int p; - fabric_init(f, false); + fabric_init(f, dcp_fabric_ordered_dual_stream(true, true)); for (p = 0; p < 3; p++) { + f->port[p].candidate_crtcs[1] = BIT(1) | BIT(2); f->route[p][0].tunnel_clock_blocked = dcp_fabric_tunnel_clock_blocked(true, 0); f->route[p][1].tunnel_clock_blocked = @@ -1919,6 +1920,73 @@ static void fabric_t6030_init(struct fabric_fixture *f) } } +static void fabric_t6030_dual_policy_test(struct kunit *test) +{ + struct fabric_fake f; + u32 mask; + + fabric_fake_init(&f, test, dcp_fabric_ordered_dual_stream(true, true)); + KUNIT_EXPECT_FALSE(test, f.f.policy.dual_stream); + KUNIT_EXPECT_TRUE(test, dcp_fabric_ordered_dual_stream(true, false)); + KUNIT_EXPECT_FALSE(test, dcp_fabric_ordered_dual_stream(false, true)); + /* A primary dock stream leaves the fixed processor free for HDMI. */ + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f.f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[0], &f.f.route[0][1]); + f.borrowed = f.f.pipeline[0].owned; + f.frozen = true; + f.levels[0] = f.levels[1] = 1; + dcp_fabric_run_hdmi(&fabric_fake_hdmi_ops, &f); + KUNIT_ASSERT_EQ(test, f.count, 4U); + KUNIT_EXPECT_EQ(test, f.trace[3], FX_CONNECT_FIXED); + /* HDMI owns one processor; the other still takes the first tunnel. */ + fabric_release(&f.f, 0, 0); + f.f.pipeline[0].fixed_busy = true; + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f.f, 0, 0), 0); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[0], &f.f.route[0][1]); + f.f.pipeline[0].fixed_busy = false; + f.f.port[0].candidate_crtcs[1] = BIT(1) | BIT(2); + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f.f, 0, 1), 0); + KUNIT_EXPECT_PTR_EQ(test, f.f.port[0].owner[1], &f.f.route[0][0]); + /* Both encoders expose candidates, then narrow to the actual owner. */ + mask = dcp_fabric_port_crtc_mask(true, false, false, 0, BIT(1)); + mask = dcp_fabric_port_crtc_mask(true, false, true, mask, BIT(2)); + KUNIT_EXPECT_EQ(test, mask, (u32)(BIT(1) | BIT(2))); + KUNIT_EXPECT_EQ(test, dcp_fabric_connector_mask(false, true, true, 1, mask), (u32)BIT(1)); + KUNIT_EXPECT_EQ(test, dcp_fabric_connector_mask(false, true, true, 2, mask), (u32)BIT(2)); + /* Ordered M2 secondary connectors retain the Type-C-only processor. */ + mask = dcp_fabric_port_crtc_mask(true, true, false, 0, BIT(1)); + mask = dcp_fabric_port_crtc_mask(true, true, true, mask, BIT(2)); + KUNIT_EXPECT_EQ(test, mask, (u32)BIT(2)); + mask = dcp_fabric_port_crtc_mask(false, true, false, mask, BIT(1)); + KUNIT_EXPECT_EQ(test, mask, (u32)(BIT(1) | BIT(2))); +} + +static void fabric_t6030_clock_header_parity_test(struct kunit *test) +{ + static const struct { + unsigned int source; + unsigned int dpin; + int pclk; + } cases[] = { + { 0, 0, 0 }, { 0, 1, 1 }, { 2, 0, 0 }, { 2, 1, 1 }, + { 1, 0, -EOPNOTSUPP }, { 1, 1, -EOPNOTSUPP }, + { 3, 0, -EOPNOTSUPP }, { 4, 0, -EOPNOTSUPP }, + { UINT_MAX, 0, -EOPNOTSUPP }, + { 0, 2, -EOPNOTSUPP }, { 2, 2, -EOPNOTSUPP }, + { 0, UINT_MAX, -EOPNOTSUPP }, + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + KUNIT_EXPECT_EQ(test, + apple_dp_tunnel_t6030_pclk(cases[i].source, cases[i].dpin), + cases[i].pclk); + KUNIT_EXPECT_EQ(test, + dcp_fabric_t6030_pclk(cases[i].source, cases[i].dpin), + cases[i].pclk); + } +} + static void fabric_t6030_clock_routes_test(struct kunit *test) { struct fabric_fixture f; @@ -1959,9 +2027,16 @@ static void fabric_t6030_clock_routes_test(struct kunit *test) KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); fabric_release(&f, 0, 0); - /* A second stream on the same dock is still unsupported on T6030. */ - KUNIT_EXPECT_EQ(test, fabric_tunnel(&f, 0, 1), -EOPNOTSUPP); - KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[1], NULL); + /* Separate dock monitors hold separate processors and pixel clocks. */ + f.pipeline[0].services_ready = true; + f.pipeline[1].services_ready = true; + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 0), 0); + KUNIT_ASSERT_EQ(test, fabric_tunnel(&f, 0, 1), 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[0], &f.route[0][1]); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[1], &f.route[0][0]); + fabric_release(&f, 0, 0); + KUNIT_EXPECT_PTR_EQ(test, f.port[0].owner[1], &f.route[0][0]); + fabric_release(&f, 0, 1); KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_clock_blocked(true, 1), (u8)3); KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_clock_blocked(true, 4), (u8)3); KUNIT_EXPECT_EQ(test, dcp_fabric_tunnel_clock_blocked(false, 0), (u8)0); @@ -1998,7 +2073,7 @@ static void fabric_t6030_follow_test(struct kunit *test) NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_MOVE); f.route[0][0].dpin = 1; KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][0], &f.route[0][1], - NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_REFUSE); + NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_MOVE); f.route[0][0].tunnel = false; KUNIT_EXPECT_EQ(test, dcp_fabric_follow(&f.route[0][0], &f.route[0][1], NULL, NULL, false, &f.policy), DCP_FABRIC_FOLLOW_MOVE); @@ -2275,7 +2350,9 @@ static struct kunit_case fabric_tests[] = { KUNIT_CASE(relink_budget_test), KUNIT_CASE(relink_disabled_test), KUNIT_CASE(relink_stamp_test), + KUNIT_CASE(fabric_t6030_dual_policy_test), KUNIT_CASE(fabric_t6030_clock_routes_test), + KUNIT_CASE(fabric_t6030_clock_header_parity_test), KUNIT_CASE(fabric_t6030_plan_test), KUNIT_CASE(fabric_t6030_follow_test), KUNIT_CASE(fabric_reclaim_transaction_test), diff --git a/drivers/gpu/drm/apple/dcp-fabric.c b/drivers/gpu/drm/apple/dcp-fabric.c index c0e469592eac4..38f7d8a12634e 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.c +++ b/drivers/gpu/drm/apple/dcp-fabric.c @@ -191,7 +191,7 @@ static bool dcp_typec_route_available(struct apple_dcp_typec_route *route) .fixed_busy = dcp_typec_route_busy(route), }; - struct dcp_fabric_policy policy = { .dual_stream = dcp_typec_dual_stream() }; + struct dcp_fabric_policy policy = { .dual_stream = dcp_typec_ordered_dual_stream() }; pipeline.presence = dcp_hdmi_presence(route->dcp); return dcp_fabric_available(&pipeline, &policy); @@ -207,6 +207,13 @@ bool dcp_typec_dual_stream(void) return atomic_read(&dcp_dual_stream_routes) > 0; } +/* Keep native T6030 routes pinned to their owning processor, on either DPIN. */ +bool dcp_typec_ordered_dual_stream(void) +{ + return dcp_fabric_ordered_dual_stream(dcp_typec_dual_stream(), + of_machine_is_compatible("apple,t6030")); +} + /* * Do the Type-C routes of @dcp follow the CRTC a modeset pairs them with * (see dcp_typec_follow_crtc())? Direct routes and Thunderbolt tunnels, @@ -602,7 +609,7 @@ static bool dcp_typec_keep_order(void) { struct drm_device *drm; - if (!dcp_typec_dual_stream()) + if (!dcp_typec_ordered_dual_stream()) return false; drm = dcp_typec_drm(); if (!drm || !READ_ONCE(drm->registered)) @@ -617,7 +624,7 @@ static bool dcp_hdmi_settle_enabled(struct apple_dcp *dcp) { return dcp->hdmi_hpd && dcp_fabric_hdmi_settle_enabled(!!dcp->fixed_phy, dcp->nr_typec_routes, - dcp_typec_dual_stream()); + dcp_typec_ordered_dual_stream()); } static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *dcp) @@ -982,7 +989,7 @@ static struct apple_dcp_typec_route * dcp_typec_free_route(struct apple_dcp_typec_port *port) { struct dcp_fabric_policy policy = { - .dual_stream = dcp_typec_dual_stream(), + .dual_stream = dcp_typec_ordered_dual_stream(), .follow = dcp_typec_port_follows(port), }; @@ -1259,7 +1266,7 @@ static void dcp_typec_pipeline_freed(void) */ void dcp_typec_reorder(void) { - if (!dcp_typec_dual_stream()) + if (!dcp_typec_ordered_dual_stream()) return; guard(mutex)(&dcp_typec_fabric_lock); @@ -1750,12 +1757,27 @@ int dcp_tunnel_crossbar_down(struct apple_dcp *dcp, u32 unit) * The output is the borrowed Type-C route's, or the fixed route's. Other * SoCs clock it at select. Runs inside a DCP apcall: only tb_lock. */ +static int dcp_direct_crossbar_link_t6050(struct apple_dcp *dcp, bool up) +{ + struct mux_control *xbar; + + guard(mutex)(&dcp->tb_lock); + xbar = dcp->active_typec_route ? + dcp->active_typec_route->active_xbar : dcp->xbar; + if (dcp->dptx_tunnel || !dcp->phy || !xbar) + return 0; + /* No T6050 DisplayPort PHY or crossbar link driver in this tree yet. */ + return dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? -ENODEV : 0; +} + int dcp_direct_crossbar_link(struct apple_dcp *dcp, bool up) { struct apple_dcp_typec_route *route; struct mux_control *xbar = NULL; int ret; + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dcp_direct_crossbar_link_t6050(dcp, up); if (!of_machine_is_compatible("apple,t6030")) return 0; guard(mutex)(&dcp->tb_lock); @@ -1799,7 +1821,9 @@ int dcp_tunnel_set_rate(struct apple_dcp *dcp, struct phy *phy, u32 unit, if (!dcp_fabric_callback_valid(route->tunnel_generation, dcp->tb_generation)) return -ESTALE; if (of_machine_is_compatible("apple,t6030") && - (unit || dcp->dptx_dfp_port != 1)) + (unit || dcp->dptx_dfp_port != 1 + route->tunnel_dpin || + apple_dp_tunnel_t6030_pclk(route->mux_index, + route->tunnel_dpin) < 0)) return -EOPNOTSUPP; if (unit) { /* Each DP IN owns a reference, including on a shared clock. */ @@ -1928,7 +1952,7 @@ static int dcp_tb_candidate(void *data) { struct dcp_fabric_policy policy = { - .dual_stream = dcp_typec_dual_stream(), + .dual_stream = dcp_typec_ordered_dual_stream(), .follow = dcp_typec_port_follows(port), }; struct dcp_fabric_route *chosen; @@ -2189,6 +2213,24 @@ static int dcp_tb_split_join_locked(struct apple_dcp_typec_port *port, return ret; } +/* + * Can no pipeline that drives @port take the second half of a tiled display? + * A tile joins DPTX port 1 of the pipeline that drives dpin0, which an + * external display processor never does (see dcp_tb_split_locked()). That is + * fixed when the processor probes, before its routes are registered, so it + * holds before any of its services are up. On T6030 every Type-C pipeline is + * an external processor. A port with no routes is not judged here. + */ +static bool dcp_typec_port_untileable(struct apple_dcp_typec_port *port) +{ + struct apple_dcp_typec_route *route; + + list_for_each_entry(route, &port->routes, port_link) + if (!route->dcp->external) + return false; + return !list_empty(&port->routes); +} + /* -EOPNOTSUPP: not a tile, route dpin1 as a display of its own. */ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, const struct dcp_tb_attach_context *req) @@ -2221,6 +2263,9 @@ static int dcp_tb_split_locked(struct apple_dcp_typec_port *port, } if (!req->active) return -EOPNOTSUPP; + /* No tile is possible here: there is no dpin0 to wait for. */ + if (dcp_typec_port_untileable(port)) + return -EOPNOTSUPP; /* dpin0 first: the Thunderbolt side retries on -ENODEV */ if (!route || !route->tunnel || route->tunnel_dpin != 0) return -ENODEV; @@ -2640,7 +2685,7 @@ static int dcp_typec_follow_decide(struct apple_dcp *dcp, struct drm_crtc *crtc, struct apple_connector *connector, struct dcp_typec_follow *follow) { - struct dcp_fabric_policy policy = { .dual_stream = dcp_typec_dual_stream() }; + struct dcp_fabric_policy policy = { .dual_stream = dcp_typec_ordered_dual_stream() }; struct apple_dcp_typec_route *from, *to, *holder, *back = NULL; struct apple_dcp_typec_port *port, *other = NULL; bool secondary, off = false; @@ -3413,6 +3458,13 @@ static int dcp_typec_route_wiring(struct apple_dcp_typec_route *route, return -EINVAL; } } + if (!named[0] && !named[1] && !legacy && + apple_dp_tunnel_t6030_legacy_mux(dev->of_node, index)) { + if (route->xbar != &route->xbar->chip->mux[0] || + route->xbar->chip->controllers < 3) + return -EINVAL; + legacy = &route->xbar->chip->mux[1]; + } wiring = dcp_fabric_wiring(named[0], named[1], !!legacy, dcp_typec_route_endpoints(connector_np)); if (wiring == DCP_FABRIC_INVALID_WIRING) @@ -3422,7 +3474,7 @@ static int dcp_typec_route_wiring(struct apple_dcp_typec_route *route, /* Old j414/j416 ESP DTBs name cell 1 but never name cell 2. */ route->dpin[0] = legacy; route->dpin[1] = &route->xbar->chip->mux[2]; - dev_warn_once(dev, "legacy typecN-usb4 DPIN wiring; update the device tree\n"); + dev_warn_once(dev, "implicit DPIN wiring; update the device tree\n"); } return 0; } @@ -3676,7 +3728,7 @@ static bool dcp_hdmi_keep_order(void *ctx) static bool dcp_hdmi_dual_stream(void *ctx) { - return dcp_typec_dual_stream(); + return dcp_typec_ordered_dual_stream(); } static int dcp_hdmi_read_hpd(void *ctx) @@ -3860,7 +3912,7 @@ static bool dcp_typec_hdmi_reclaim(struct apple_dcp *dcp) struct dcp_hdmi_reclaim_context ctx; lockdep_assert_held(&dcp_typec_fabric_lock); - if (!owner || dcp_typec_dual_stream()) + if (!owner || dcp_typec_ordered_dual_stream()) return false; port = owner->port; /* A port in SAFE keeps its owner until the mux set phase completes. */ @@ -4065,3 +4117,10 @@ void dcp_fabric_shutdown_dptx(struct apple_dcp *dcp) } } } + +int dcp_typec_test_link(struct platform_device *pdev, bool enabled) +{ + guard(mutex)(&dcp_typec_fabric_lock); + return enabled ? dcp_dptx_connect_oob(pdev, 0) : + dcp_dptx_disconnect_oob(pdev, 0); +} diff --git a/drivers/gpu/drm/apple/dcp-fabric.h b/drivers/gpu/drm/apple/dcp-fabric.h index 62b1a72d5946a..e621d6341acbf 100644 --- a/drivers/gpu/drm/apple/dcp-fabric.h +++ b/drivers/gpu/drm/apple/dcp-fabric.h @@ -71,6 +71,7 @@ bool dcp_typec_port_unusable(unsigned int idx); void dcp_typec_port_set_connector(unsigned int idx, bool secondary, struct apple_connector *connector); bool dcp_typec_dual_stream(void); +bool dcp_typec_ordered_dual_stream(void); void dcp_typec_reorder(void); bool dcp_typec_follows_crtc(struct apple_dcp *dcp); bool dcp_typec_follow_holder_off(struct drm_atomic_commit *state, struct drm_crtc *crtc, @@ -118,4 +119,6 @@ int dcp_dptx_connect(struct apple_dcp *dcp, u32 port); int dcp_dptx_disconnect(struct apple_dcp *dcp, u32 port); bool dcp_is_external_h17p(struct apple_dcp *dcp); +int dcp_typec_test_link(struct platform_device *pdev, bool enabled); + #endif /* __APPLE_DCP_FABRIC_H__ */ diff --git a/drivers/gpu/drm/apple/dcp-internal.h b/drivers/gpu/drm/apple/dcp-internal.h index 5aedd1acb5945..374563a68e938 100644 --- a/drivers/gpu/drm/apple/dcp-internal.h +++ b/drivers/gpu/drm/apple/dcp-internal.h @@ -27,12 +27,15 @@ #include "dcp-180hz.h" #include "iomfb.h" #include "iomfb_h17p.h" +#include "iomfb_v27.h" #include "iomfb-state.h" #include "iomfb_v12_3.h" #include "iomfb_v13_3.h" #include "iomfb_v14_7.h" #include "iomfb_v14_7_tpl.h" #include "iomfb_v26_6.h" +#include "dcp-peer-v26_6.h" +#include "dcp-peer-frames-v27.h" #include "epic/dpavservep.h" #include "version_utils.h" @@ -183,6 +186,7 @@ enum dcp_iomfb_method_profile { }; struct apple_dcp_hw_data { + u8 firmware_dma_bits; u32 num_dptx_ports; enum dcp_iomfb_method_profile iomfb_method_profile; /* @@ -204,6 +208,19 @@ struct apple_dcp_hw_data { * (the T8122 dcpext), see dcp_fw_is_14_7(). */ bool fw_14_7_endpoints; + /* + * Firmware 27.3 (T6050): the IOMFB call tags of macOS 27.3's + * IOMobileGraphicsFamily-DCP, its swap record and the endpoints macOS + * starts. + */ + bool fw_27_3; + /* + * The firmware reads swap timestamp 6 as the time to present at, in its + * own clock rather than the AP's counter: leave the swap timestamps zero. + */ + bool swap_no_timestamps; + /* Use the full panel height, the notch rows included. */ + bool show_notch; }; enum dcp_test_point { @@ -225,6 +242,20 @@ struct apple_dcp { bool vblank_works_disabled; struct dcp_fabric_pipeline fabric; struct device *dev; + /* Peer DCP for a dual-pipe display: the primary reserves an idle peer + * pipe; the peer records its owner. Protected by dcp_peer_lock. + */ + struct apple_dcp *peer_reserved; + struct apple_dcp *peer_owner; + bool peer_starved; + struct dcp_peer_tiling peer_tiling; + bool peer_unbinding; + /* Firmware display endpoint (its Disp_Index), apple,dcp-endpoint. */ + u32 peer_endpoint; + /* Paired DRM scanout on the primary; under dcp_peer_lock. */ + bool peer_active, peer_failed; + u32 peer_mode_timing, peer_mode_color, peer_min_vrr; + bool peer_complete_logged; struct platform_device *piodma; bool piodma_created; struct iommu_domain *iommu_dom; @@ -233,6 +264,10 @@ struct apple_dcp { struct apple_rtkit *rtk; struct apple_crtc *crtc; struct apple_connector *connector; + /* Protected by dcp_peer_lock; never take the physical route lock in + * an AFK callback (a route operation may be waiting for that reply). + */ + struct dcp_peer_frames_v27 peer_frames; struct apple_dcp_hw_data hw; bool external_prepared; @@ -303,6 +338,12 @@ struct apple_dcp { */ bool external_detached; atomic_t external_requested; + /* + * Native: the DP/HDMI audio endpoint (EP 0x29) starts once per + * session, after the first display's EDID; see dcp_external_audio_start(). + */ + struct work_struct external_audio_work; + bool external_audio_started; /* DCP_FIRMWARE_V_14_7 state; outlives this device once RTKit runs. */ struct apple_dcp_v14 *v14; @@ -313,6 +354,8 @@ struct apple_dcp { /* DCP has crashed */ bool crashed; + /* CoreAnalytics events dumped in full (dcpep_cb_core_analytics_v27). */ + u32 analytics_dumps; DECLARE_BITMAP(iomfb_surfaces, DCP_MAX_PLANES); @@ -324,6 +367,20 @@ struct apple_dcp { /* clock rate request by dcp in */ struct clk *clk; struct clk *clk_194; + struct clk_bulk_data *firmware_clocks; + unsigned int num_firmware_clocks; + u64 disp_firmware_base[MAX_DISP_REGISTERS]; + u32 dfb_format, dfb_compression; + u64 dfb_region_size; + bool dfb_format_valid, dfb_compression_valid; + struct list_head iomfb_boolean_properties; + u32 iomfb_boolean_property_count; + u32 external_pipes_v27, external_displays_v27, external_suppression_v27; + struct dcp_swap_submit_req_v27 present_v27; + /* The present request as firmware 27.3 takes it, see dcp_swap_submit() */ + u8 present_27_3[0x1144]; + /* Both paired wire records belong to the primary endpoint lock. */ + struct dcp_swap_submit_req_v27 peer_present_v27; /* DCP shared memory */ void *shmem; @@ -409,6 +466,8 @@ struct apple_dcp { /* completion for active turning true */ struct completion start_done; + /* IOMFB has published fresh modes and acknowledged a Type-C display. */ + struct completion typec_display_ready; /* Is the DCP booted? */ bool active; @@ -477,6 +536,8 @@ struct apple_dcp { struct dcp_panel panel; struct apple_dcp_afkep *systemep; + /* Endpoints started for the firmware only, by endpoint - 0x20 */ + struct apple_dcp_afkep *plain_afk[0x20]; struct completion systemep_done; struct apple_dcp_afkep *ibootep; @@ -609,6 +670,10 @@ struct apple_dcp { struct iomfb_transaction *active; struct task_struct *owner; struct work_struct work; + spinlock_t callback_lock; + struct list_head callbacks; + struct work_struct callback_work; + unsigned int callback_count; struct delayed_work timeout; struct delayed_work backlight_retry; unsigned long deadline; @@ -636,6 +701,31 @@ struct apple_dcp { struct dcp_hdmi *hdmi; }; +/* T8132 firmware uses its own wire codec, with the common DRM/KMS owner. */ +static inline bool iomfb_is_m4(const struct apple_dcp *dcp) +{ + return IS_ENABLED(CONFIG_DRM_APPLE_DCP_T8132) && + dcp->fw_compat == DCP_FIRMWARE_M4; +} + +/* Presents carry nits; one serialized IOMFB owner. */ +static inline bool dcp_presents_nits(const struct apple_dcp *dcp) +{ + return ((dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0) && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || + iomfb_is_m4(dcp); +} + +/* An adopted firmware session that fails stop and is never torn down. */ +static inline bool dcp_fail_stop(const struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + iomfb_is_m4(dcp); +} + +bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp); void dcp_firmware_metadata_connection(struct apple_dcp *dcp, u64 generation, bool connected); void dcp_firmware_metadata_hotplug(struct apple_dcp *dcp, bool connected); void dcp_firmware_metadata_announced(struct apple_dcp *dcp); @@ -661,6 +751,8 @@ void iomfb_opaque_x_reset_h17p(struct apple_dcp *dcp); int iomfb_configure_backlight_h17p(struct apple_dcp *dcp, u32 maximum, bool inherited_valid, u32 inherited, bool default_valid, u32 default_nits); +int iomfb_queue_callback(struct apple_dcp *dcp, dcp_callback_t callback, + const void *data, size_t size, void *cookie); void iomfb_queue_init(struct apple_dcp *dcp); void iomfb_queue_stop(struct apple_dcp *dcp); bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout); @@ -721,6 +813,27 @@ static inline bool dcp_fw_is_14_7(const struct apple_dcp *dcp) dcp->hw.fw_14_7_endpoints); } +/* + * The T6030 external processor (the J514S/J516S built-in HDMI port and + * USB-C displays) driven natively through Aurora's DCPLink session: DCP + * firmware 14.7's external role, whose DP/HDMI audio service differs from + * the M1/M2 one (see av.c). The manual diagnostic pipe (dcpext_scanout) and + * the T8122 template path are not this. + */ +static inline bool dcp_is_v14_7_external(const struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_V_14_7 && dcp->external_native; +} + +/* The 14.7 internal panel (J514S/J516S), presented through DCPLink. */ +static inline bool dcp_is_v14_7_panel(const struct apple_dcp *dcp) +{ + return dcp->fw_compat == DCP_FIRMWARE_V_14_7 && !dcp->external; +} + +/* Native external pipe: start its audio endpoint, once, after the first EDID. */ +void dcp_external_audio_start(struct apple_dcp *dcp); + /* * The T8122 dcpext connects its DPTX port as AsahiLinux/linux#607 does, * the kernel this firmware was first driven with: no validation of the diff --git a/drivers/gpu/drm/apple/dcp-peer-frames-v27.h b/drivers/gpu/drm/apple/dcp-peer-frames-v27.h new file mode 100644 index 0000000000000..abb7798a1764f --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-frames-v27.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_PEER_FRAMES_V27_H +#define APPLE_DCP_PEER_FRAMES_V27_H + +#include + +/* Protected by dcp_peer_lock. Sequence comparisons cover less than 2^31 + * outstanding swaps, as in the existing single-pipe retirement path. + */ +struct dcp_peer_frames_v27 { + u32 first, submitted, engine[2], retired; + bool active, armed, failed, have[2], have_retired; +}; + +static inline void dcp_peer_frame_submit_v27(struct dcp_peer_frames_v27 *s, u32 id) +{ + if (!s->armed) { + s->first = id; + s->armed = true; + } + s->submitted = id; +} + +static inline bool dcp_peer_frame_complete_v27(struct dcp_peer_frames_v27 *s, + unsigned int engine, u32 id, + u32 *watermark) +{ + u32 done; + + if (!s->active || s->failed || !s->armed || engine > 1 || + (s32)(id - s->first) < 0 || (s32)(s->submitted - id) < 0 || + (s->have[engine] && (s32)(id - s->engine[engine]) <= 0)) + return false; + s->engine[engine] = id; + s->have[engine] = true; + if (!s->have[0] || !s->have[1]) + return false; + done = (s32)(s->engine[0] - s->engine[1]) < 0 ? s->engine[0] : s->engine[1]; + if (s->have_retired && (s32)(done - s->retired) <= 0) + return false; + s->retired = *watermark = done; + s->have_retired = true; + return true; +} +#endif diff --git a/drivers/gpu/drm/apple/dcp-peer-modeset-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-modeset-probe-v27.inc new file mode 100644 index 0000000000000..a5aa449b446dd --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-modeset-probe-v27.inc @@ -0,0 +1,227 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Paired modeset bring-up, opt-in only. Scanout is a separate step. */ +/* The power-state step starts one asynchronous AP request per engine. + * Submit both before waiting: either firmware may rendezvous with its peer. + */ +static int dcp_peer_probe_power(struct apple_dcp *dcp, struct apple_dcp *peer, + bool on) +{ + struct dcp_peer_video_wait *wait; + struct dcp_method_entry method = { .name = "peer_power_test", + .tag = {'A', '4', '8', '4'} }; + u8 power[16] = {}, response[8]; + int ret; + + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + kref_init(&wait->refs); + init_completion(&wait->done); + wait->size = sizeof(response); + put_unaligned_le64(on, power); + kref_get(&wait->refs); + dcp_push(dcp, false, &method, sizeof(power), sizeof(response), power, + dcp_peer_video_done, wait); + ret = dcp_peer_video_call(peer, "A484", power, sizeof(power), + response, sizeof(response)); + if (!ret && get_unaligned_le32(response + 4)) + ret = -EIO; + if (!wait_for_completion_timeout(&wait->done, msecs_to_jiffies(2000))) + ret = ret ?: -ETIMEDOUT; + else if (!ret && get_unaligned_le32(wait->reply + 4)) + ret = -EIO; + kref_put(&wait->refs, dcp_peer_video_free); + dev_info(dcp->dev, "peer power %u result=%d\n", on, ret); + return ret; +} + +static int dcp_peer_probe_quiesce(struct apple_dcp *dcp, struct apple_dcp *peer) +{ + __le32 manage = 0; + int ret; + + /* Setting the digital-out mode: pair API power0, 100ms, + * then manage_elements(false), A362, on each engine before A365. + * Only discard AP state after both engines have acknowledged power0. + */ + { + guard(mutex)(&dcp_peer_lock); + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = false; + } + ret = dcp_peer_probe_power(dcp, peer, false); + if (ret) + return ret; + msleep(100); + ret = dcp_peer_video_call(dcp, "A362", &manage, sizeof(manage), NULL, 0); + if (ret) + return ret; + ret = dcp_peer_video_call(peer, "A362", &manage, sizeof(manage), NULL, 0); + if (ret) + return ret; + { + guard(mutex)(&dcp_peer_lock); + dcp->peer_frames = (struct dcp_peer_frames_v27){}; + dcp->peer_tiling.member = peer->peer_tiling.member = false; + dcp->peer_tiling.vftg = peer->peer_tiling.vftg = false; + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = false; + dcp->peer_tiling.phase = peer->peer_tiling.phase = 0; + dcp->peer_tiling.modeset_state = peer->peer_tiling.modeset_state = 0; + dcp->peer_tiling.ready = peer->peer_tiling.ready = false; + } + return 0; +} + +static int dcp_peer_probe_prepare_vftg(struct apple_dcp *dcp, struct apple_dcp *peer, + const struct dcp_peer_insert_video_req_v27 *video) +{ + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_reserved != peer || peer->peer_owner != dcp) + return -ENODEV; + /* These are set BEFORE A366/A365, but membership is marked only after + * both merge-mode insertions have accepted the derived video data. + */ + dcp->peer_tiling.vftg = peer->peer_tiling.vftg = true; + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = false; + dcp->peer_tiling.primary = true; + peer->peer_tiling.primary = false; + dcp->peer_tiling.width = peer->peer_tiling.width = + get_unaligned_le32(video->video.timing + 8); + dcp->peer_tiling.height = peer->peer_tiling.height = + get_unaligned_le32(video->video.timing + 0x28); + dcp->peer_tiling.x = dcp->peer_tiling.y = peer->peer_tiling.y = 0; + peer->peer_tiling.x = dcp->peer_tiling.width; + return 0; +} + +static int dcp_peer_probe_copy_dsc(struct apple_dcp *primary, struct apple_dcp *peer) +{ + u8 caps_req[36] = {}, caps[32], bpp_req[4] = {}, bpp[8]; + u8 hints_req[12] = {}, hints[12]; + u32 required; + int ret; + + ret = dcp_peer_video_call_stream(primary, true, "A369", NULL, 0, &required, 4); + if (ret || !(required & 1)) + return ret; + ret = dcp_peer_video_call_stream(primary, true, "A367", caps_req, 36, caps, 32); + if (ret) + return ret; + memcpy(caps_req, caps, 32); + ret = dcp_peer_video_call(peer, "A368", caps_req, 36, caps, 32); + if (ret) + return ret; + ret = dcp_peer_video_call_stream(primary, true, "A370", bpp_req, 4, bpp, 8); + if (ret || get_unaligned_le32(bpp + 2)) + return ret ?: -EIO; + memcpy(bpp_req, bpp, 2); + ret = dcp_peer_video_call(peer, "A371", bpp_req, 4, bpp, 8); + if (ret || get_unaligned_le32(bpp + 2)) + return ret ?: -EIO; + ret = dcp_peer_video_call_stream(primary, true, "A372", hints_req, 12, hints, 12); + if (ret || get_unaligned_le32(hints + 8)) + return ret ?: -EIO; + memcpy(hints_req, hints, 8); + ret = dcp_peer_video_call(peer, "A373", hints_req, 12, hints, 12); + if (ret || get_unaligned_le32(hints + 8)) + return ret ?: -EIO; + dev_info(primary->dev, "peer DSC copied: bpp=%u hints=%*ph\n", + get_unaligned_le16(bpp), 8, hints); + return 0; +} + +static int dcp_peer_probe_modeset(struct apple_dcp *dcp, struct apple_dcp *peer, + const struct dcp_peer_insert_video_req_v27 *video, + bool firmware_requires_merge) +{ + struct dcp_peer_video_wait *wait; + struct dcp_method_entry method = { .name = "peer_modeset_test", + .tag = {'A', '4', '1', '3'} }; + __le32 mode[2] = {video->color_id, video->timing_id}; + u32 primary_id, peer_id, phase, status; + unsigned long deadline; + int ret; + + { + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_reserved != peer || peer->peer_owner != dcp || + !dcp->peer_tiling.display_id_valid || !peer->peer_tiling.display_id_valid) + return -ENODEV; + dcp->peer_tiling.member = peer->peer_tiling.member = true; + /* The original A364 result is preserved even when the host + * allocation requires a pair for a lower-bandwidth timing. + */ + dcp->peer_tiling.mode_flag = peer->peer_tiling.mode_flag = + !firmware_requires_merge; + } + ret = dcp_peer_probe_power(dcp, peer, true); + if (ret) + return ret; + /* Wait 100ms after both API-power transitions, before swap IDs. */ + msleep(100); + ret = dcp_peer_video_call(dcp, "A360", NULL, 0, &primary_id, 4); + if (ret) + return ret; + ret = dcp_peer_video_call(peer, "A360", NULL, 0, &peer_id, 4); + if (ret) + return ret; + if (primary_id != peer_id) { + status = max(primary_id, peer_id); + ret = dcp_peer_video_call(primary_id < peer_id ? dcp : peer, + "A361", &status, 4, NULL, 0); + if (ret) + return ret; + } + phase = 1; + dcp_peer_tiling_state_v27(dcp, true, 1, 10, &phase); + dcp_peer_tiling_state_v27(peer, true, 1, 10, &phase); + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + init_completion(&wait->done); + kref_init(&wait->refs); + kref_get(&wait->refs); + wait->size = 4; + dcp_push(dcp, false, &method, sizeof(mode), 4, mode, + dcp_peer_video_done, wait); + /* The primary negotiates the link, signals phase2, and then waits for + * its peer. Waiting for its final ACK first would deadlock the pair. + */ + deadline = jiffies + msecs_to_jiffies(8500); + for (;;) { + phase = 0; + if (!dcp_peer_tiling_state_v27(dcp, false, 1, 10, &phase)) { + ret = -ENODEV; + goto put; + } + if (phase >= 2) + break; + if (completion_done(&wait->done) || time_after_eq(jiffies, deadline)) { + ret = completion_done(&wait->done) ? -EIO : -ETIMEDOUT; + dev_err(dcp->dev, "peer primary ended before phase2: phase=%u result=%#x\n", + phase, get_unaligned_le32(wait->reply)); + goto put; + } + msleep(2); + } + dev_info(dcp->dev, "peer primary reached modeset phase%u\n", phase); + ret = dcp_peer_probe_copy_dsc(dcp, peer); + if (ret) + goto put; + ret = dcp_peer_video_call(peer, "A413", mode, sizeof(mode), &status, 4); + if (ret || status) { + ret = ret ?: -EIO; + goto put; + } + if (!wait_for_completion_timeout(&wait->done, msecs_to_jiffies(8500))) + ret = -ETIMEDOUT; + else if (get_unaligned_le32(wait->reply)) + ret = -EIO; +put: + kref_put(&wait->refs, dcp_peer_video_free); + if (!ret) { + guard(mutex)(&dcp_peer_lock); + dcp->peer_tiling.modeset_complete = peer->peer_tiling.modeset_complete = true; + } + dev_info(dcp->dev, "peer paired modeset result=%d (scanout not submitted)\n", ret); + return ret; +} diff --git a/drivers/gpu/drm/apple/dcp-peer-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-probe-v27.inc new file mode 100644 index 0000000000000..33def11c752ea --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-probe-v27.inc @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Explicit, opt-in firmware bring-up command. This is not the final hotplug + * lifecycle: use only the dedicated, otherwise unrouted peer reservation. + */ +struct dcp_peer_probe_wait { + struct kref refs; + struct completion done; +}; + +static void dcp_peer_probe_free(struct kref *ref) +{ + kfree(container_of(ref, struct dcp_peer_probe_wait, refs)); +} + +static void dcp_peer_probe_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_peer_probe_wait *wait = cookie; + + complete(&wait->done); + kref_put(&wait->refs, dcp_peer_probe_free); +} + +static ssize_t peer_setup_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + static const struct dcp_method_entry method = { + .name = "setup_peer_dcp", .tag = {'A', '3', '6', '6'}, + }; + struct apple_dcp *dcp = dev_get_drvdata(dev); + struct dcp_peer_probe_wait *wait; + __le32 enabled = cpu_to_le32(1); + bool value; + long ret; + + if (kstrtobool(buf, &value) || !value) + return -EINVAL; + if (!peer_allocation_probe || !dcp) + return -EOPNOTSUPP; + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + init_completion(&wait->done); + kref_init(&wait->refs); + { + guard(mutex)(&dcp_peer_lock); + if (!dcp->peer_owner || dcp->peer_owner->peer_reserved != dcp || + dcp->peer_unbinding || dcp->crashed || !dcp->active || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + dcp->fixed_phy || dcp->nr_typec_routes || + (dcp->connector && dcp->connector->connected) || + dcp->ch_cmd.depth || dcp->ch_cb.depth || dcp->ch_async.depth || + dcp->ch_oobcmd.depth || dcp->ch_oobcb.depth || + dcp->ch_oobasync.depth) { + kref_put(&wait->refs, dcp_peer_probe_free); + return -EBUSY; + } + kref_get(&wait->refs); + /* Peer-DCP setup (one bool argument): four input + * bytes (boolean followed by zero padding), no output. + */ + dcp_push(dcp, false, &method, sizeof(enabled), 0, &enabled, + dcp_peer_probe_done, wait); + } + ret = wait_for_completion_timeout(&wait->done, msecs_to_jiffies(2000)); + kref_put(&wait->refs, dcp_peer_probe_free); + dev_info(dev, "peer A366 setup test %s\n", ret ? "acknowledged" : "timed out"); + return ret ? count : -ETIMEDOUT; +} +static DEVICE_ATTR_WO(peer_setup_test); + +#include "dcp-peer-video-probe-v27.inc" +#include "dcp-peer-swap-probe-v27.inc" +#include "dcp-peer-runtime-v27.inc" + +static struct attribute *dcp_peer_probe_attrs[] = { + &dev_attr_peer_setup_test.attr, + &dev_attr_peer_video_test.attr, + &dev_attr_peer_link_test.attr, + &dev_attr_peer_swap_test.attr, + &dev_attr_peer_stop_test.attr, + NULL, +}; + +static umode_t dcp_peer_probe_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + return peer_allocation_probe ? attr->mode : 0; +} + +static const struct attribute_group dcp_peer_probe_group = { + .attrs = dcp_peer_probe_attrs, + .is_visible = dcp_peer_probe_visible, +}; +static const struct attribute_group *dcp_peer_probe_groups[] = { + &dcp_peer_probe_group, + NULL, +}; diff --git a/drivers/gpu/drm/apple/dcp-peer-runtime-v27.inc b/drivers/gpu/drm/apple/dcp-peer-runtime-v27.inc new file mode 100644 index 0000000000000..b1f99d6ddd651 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-runtime-v27.inc @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Kept opt-in while paired DRM lifecycle is being validated on T6050. */ +static bool peer_scanout; +module_param(peer_scanout, bool, 0444); +MODULE_PARM_DESC(peer_scanout, "Enable firmware27 paired DRM scanout bring-up"); + +static bool dcp_peer_transport_idle(struct apple_dcp *dcp) +{ + return dcp->active && !dcp->crashed && !dcp->peer_unbinding && + !dcp->ch_cmd.depth && !dcp->ch_cb.depth && !dcp->ch_async.depth && + !dcp->ch_oobcmd.depth && !dcp->ch_oobcb.depth && !dcp->ch_oobasync.depth; +} + +int dcp_peer_modeset_v27(struct apple_dcp *dcp, u32 timing, u32 color, bool vrr) +{ + struct apple_dcp *peer; + int ret = -EOPNOTSUPP; + + if (!peer_scanout || !peer_allocation_probe || + dcp->fw_compat != DCP_FIRMWARE_V_27_0) + return ret; + if (timing > U32_MAX - 10000 || color > U32_MAX - 10000) + return -EINVAL; + if (!device_trylock(dcp->dev)) + return -EBUSY; + { + guard(mutex)(&dcp_peer_lock); + peer = dcp->peer_reserved; + if (!peer) + goto unlock; + if (peer->peer_owner != dcp || !dcp_peer_transport_idle(dcp) || + !dcp_peer_transport_idle(peer) || peer->fixed_phy || + peer->nr_typec_routes || !peer->crtc || + !peer->crtc->base.state || peer->crtc->base.state->active || + !device_trylock(peer->dev)) { + ret = -EBUSY; + goto unlock; + } + } + /* Do not silently select an untiled mode when paired VRR was requested. + * The internal and single-pipe external adaptive-sync paths are unchanged. + */ + if (vrr) { + struct dcp_peer_get_video_req_v27 req = { + .timing_id = cpu_to_le32(timing), .color_id = cpu_to_le32(color), + }; + struct dcp_peer_get_video_resp_v27 resp; + + ret = dcp_peer_video_call(dcp, "A364", &req, sizeof(req), &resp, sizeof(resp)); + if (ret || !(resp.success & 1) || (resp.requires_merge & 1)) { + ret = ret ?: -EINVAL; + goto peer_unlock; + } + } + ret = dcp_peer_install_video_v27(dcp, peer, timing, color, 4); + if (!ret) { + guard(mutex)(&dcp_peer_lock); + dcp->peer_frames = (struct dcp_peer_frames_v27){ .active = true }; + dev_info(dcp->dev, "paired DRM mode ready: timing=%u color=%u\n", timing, color); + } +peer_unlock: + device_unlock(peer->dev); +unlock: + device_unlock(dcp->dev); + return ret; +} + +/* Return1 after a paired stop,0 for a normal CRTC, or an error. */ +int dcp_peer_stop_v27(struct apple_dcp *dcp) +{ + struct apple_dcp *peer; + int ret = 0; + + if (!peer_scanout || !READ_ONCE(dcp->peer_frames.active)) + return 0; + if (!device_trylock(dcp->dev)) + return -EBUSY; + { + guard(mutex)(&dcp_peer_lock); + peer = dcp->peer_reserved; + if (!peer || peer->peer_owner != dcp || !dcp_peer_transport_idle(dcp) || + !dcp_peer_transport_idle(peer) || !device_trylock(peer->dev)) { + ret = -EBUSY; + goto unlock; + } + } + ret = dcp_peer_probe_quiesce(dcp, peer); + device_unlock(peer->dev); + if (!ret) + ret = 1; +unlock: + device_unlock(dcp->dev); + return ret; +} + +struct apple_dcp *dcp_peer_scanout_v27(struct apple_dcp *dcp) +{ + struct apple_dcp *peer; + + if (!peer_scanout || !READ_ONCE(dcp->peer_frames.active)) + return NULL; + guard(mutex)(&dcp_peer_lock); + if (!dcp->peer_frames.active) + return NULL; + peer = dcp->peer_reserved; + if (!peer || peer->peer_owner != dcp || dcp->peer_frames.failed || + !dcp->peer_tiling.modeset_complete || !peer->peer_tiling.modeset_complete || + dcp->crashed || peer->crashed || dcp->peer_unbinding || peer->peer_unbinding) + return ERR_PTR(-EIO); + return peer; +} + +void dcp_peer_swap_armed_v27(struct apple_dcp *dcp, u32 id) +{ + guard(mutex)(&dcp_peer_lock); + dcp_peer_frame_submit_v27(&dcp->peer_frames, id); +} + +void dcp_peer_swap_failed_v27(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp_peer_lock); + dcp->peer_frames.failed = true; + dcp_mode_invalidate(&dcp->mode_state); +} + +bool dcp_peer_swap_complete_v27(struct apple_dcp *dcp, u32 id, ktime_t now, + void (*report)(struct apple_dcp *, u32, ktime_t)) +{ + struct apple_dcp *owner; + u32 watermark; + + /* Independent CRTCs must not serialize through the pair's mutex. */ + if (!peer_scanout || (!READ_ONCE(dcp->peer_frames.active) && + !READ_ONCE(dcp->peer_owner))) + return false; + guard(mutex)(&dcp_peer_lock); + owner = dcp->peer_owner ?: dcp; + if (!peer_scanout || !owner->peer_frames.active) + return false; + if (dcp_peer_frame_complete_v27(&owner->peer_frames, dcp != owner, id, &watermark)) + /* Queue the combined watermark to the primary IOMFB owner. */ + report(owner, watermark, now); + return true; +} + +int dcp_peer_color_v27(struct apple_dcp *dcp, struct iomfb_set_matrix_req *matrix) +{ + struct apple_dcp *peer = dcp_peer_scanout_v27(dcp); + u32 status, identity = 0x10000; + int ret; + + if (!peer) + return 0; + if (IS_ERR(peer)) + return PTR_ERR(peer); + ret = dcp_peer_video_call(peer, "A423", matrix, sizeof(*matrix), &status, 4); + if (ret || status) + return ret ?: -EIO; + ret = dcp_peer_video_call(peer, "A428", &identity, sizeof(identity), &status, 4); + return ret ?: (status ? -EIO : 0); +} diff --git a/drivers/gpu/drm/apple/dcp-peer-swap-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-swap-probe-v27.inc new file mode 100644 index 0000000000000..f6c8ced4317ed --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-swap-probe-v27.inc @@ -0,0 +1,243 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Explicit background-only paired swap diagnostic. No framebuffer DMA or + * DRM events are owned here. Both DRM CRTCs must remain inactive. + */ +struct dcp_peer_background { + struct iomfb_transaction transaction; + struct dcp_peer_video_wait *wait; + struct dcp_swap_submit_req_v27 wire; + bool sent; +}; + +static void dcp_peer_background_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_peer_background *background = cookie; + struct dcp_swap_submit_resp_v27 failed = { .ret = ~0U }; + struct dcp_swap_submit_resp_v27 *response = out ?: &failed; + + if (!dcp_present_submit_h17p(&dcp->present_state_h17p, + background->wire.swap.swap_id, !response->ret)) + WRITE_ONCE(dcp->crashed, true); + dcp_peer_video_done(dcp, response, background->wait); +} + +static void dcp_peer_background_start(struct apple_dcp *dcp, + struct iomfb_transaction *transaction) +{ + struct dcp_peer_background *background = + container_of(transaction, struct dcp_peer_background, transaction); + struct dcp_method_entry method = { .name = "peer_background_test", + .tag = {'A', '4', '0', '8'} }; + struct dcp_swap_submit_resp_v27 failed = { .ret = ~0U }; + + background->sent = true; + if (!dcp_present_begin_h17p(&dcp->present_state_h17p, + background->wire.swap.swap_id)) { + dcp_peer_video_done(dcp, &failed, background->wait); + return; + } + dcp_push(dcp, false, &method, sizeof(background->wire), sizeof(failed), + &background->wire, dcp_peer_background_done, background); +} + +static void dcp_peer_background_release(struct iomfb_transaction *transaction) +{ + struct dcp_peer_background *background = + container_of(transaction, struct dcp_peer_background, transaction); + + if (!background->sent) + kref_put(&background->wait->refs, dcp_peer_video_free); + kfree(background); +} + +static int dcp_peer_queue_background(struct apple_dcp *dcp, + struct dcp_swap_submit_req_v27 *wire, + struct dcp_peer_video_wait *wait) +{ + struct dcp_peer_background *background; + int ret; + + background = kzalloc_obj(*background); + if (!background) + return -ENOMEM; + background->wire = *wire; + background->wait = wait; + kref_get(&wait->refs); + background->transaction.start = dcp_peer_background_start; + background->transaction.release = dcp_peer_background_release; + ret = iomfb_queue(dcp, &background->transaction); + if (ret) + dcp_peer_background_release(&background->transaction); + return ret; +} + +static int dcp_peer_probe_background(struct apple_dcp *dcp, struct apple_dcp *peer) +{ + struct dcp_swap_start_req_h17p start = {}; + struct dcp_swap_start_resp_h17p a, b; + struct dcp_swap_submit_req_v27 *submit; + struct dcp_swap_submit_resp_v27 response; + struct dcp_peer_video_wait *wait, *peer_wait; + unsigned long deadline; + int ret; + + submit = kzalloc(sizeof(*submit), GFP_KERNEL); + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + peer_wait = kzalloc(sizeof(*peer_wait), GFP_KERNEL); + if (!submit || !wait || !peer_wait) { + kfree(submit); + kfree(wait); + kfree(peer_wait); + return -ENOMEM; + } + kref_init(&wait->refs); + init_completion(&wait->done); + wait->size = sizeof(response); + kref_init(&peer_wait->refs); + init_completion(&peer_wait->done); + peer_wait->size = sizeof(response); + ret = dcp_peer_video_call(dcp, "A406", &start, sizeof(start), &a, sizeof(a)); + if (ret || a.ret) { + ret = ret ?: -EIO; + goto out; + } + ret = dcp_peer_video_call(peer, "A406", &start, sizeof(start), &b, sizeof(b)); + if (ret || b.ret || a.swap_id != b.swap_id) { + dev_err(dcp->dev, "peer swap-start mismatch/error: %u/%u result=%d\n", + a.swap_id, ret ? 0 : b.swap_id, ret); + ret = ret ?: -EIO; + goto out; + } + submit->swap.swap_id = a.swap_id; + submit->swap.swap_enabled = submit->swap.swap_completed = IOMFB_SET_BACKGROUND | 7; + submit->swap.bg_color = 0xffff0000; + memset(submit->surf_null, 1, sizeof(submit->surf_null)); + submit->unknown_pointer_null = submit->unknown_output_null = true; + ret = dcp_peer_queue_background(dcp, submit, wait); + if (ret) + goto out; + /* Queue both engines before waiting, with each present tracked by its + * own IOMFB transaction and completion state. */ + submit->swap.bg_color = 0xff0000ff; + ret = dcp_peer_queue_background(peer, submit, peer_wait); + if (ret) + goto out; + if (!wait_for_completion_timeout(&peer_wait->done, msecs_to_jiffies(2000))) { + ret = -ETIMEDOUT; + goto out; + } + memcpy(&response, peer_wait->reply, sizeof(response)); + if (response.ret) { + ret = -EIO; + goto out; + } + if (!wait_for_completion_timeout(&wait->done, msecs_to_jiffies(2000))) { + ret = -ETIMEDOUT; + goto out; + } + memcpy(&response, wait->reply, sizeof(response)); + if (response.ret) { + ret = -EIO; + goto out; + } + deadline = jiffies + msecs_to_jiffies(2000); + while (!READ_ONCE(dcp->have_swap_complete) || !READ_ONCE(peer->have_swap_complete) || + READ_ONCE(dcp->last_swap_id) != a.swap_id || + READ_ONCE(peer->last_swap_id) != a.swap_id) { + if (time_after_eq(jiffies, deadline)) { + ret = -ETIMEDOUT; + goto out; + } + msleep(2); + } + dev_info(dcp->dev, "peer background swap %u completed on both engines (optical unverified)\n", + a.swap_id); +out: + dev_info(dcp->dev, "peer background test result=%d\n", ret); + kref_put(&wait->refs, dcp_peer_video_free); + kref_put(&peer_wait->refs, dcp_peer_video_free); + kfree(submit); + return ret; +} + +static ssize_t peer_swap_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp, *peer; + struct device *peer_dev = NULL; + bool enabled; + int ret; + + if (kstrtobool(buf, &enabled) || !enabled) + return -EINVAL; + if (!peer_allocation_probe || !device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + { + guard(mutex)(&dcp_peer_lock); + peer = dcp ? dcp->peer_reserved : NULL; + if (!peer || peer->peer_owner != dcp || !dcp_peer_video_idle(dcp) || + !dcp_peer_video_idle(peer) || READ_ONCE(dcp->peer_frames.active) || !dcp->peer_tiling.member || + !peer->peer_tiling.member || !dcp->peer_tiling.modeset_complete || + !peer->peer_tiling.modeset_complete || !dcp->crtc || !peer->crtc || + !dcp->crtc->base.state || !peer->crtc->base.state || + dcp->crtc->base.state->active || peer->crtc->base.state->active || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + peer->fw_compat != DCP_FIRMWARE_V_27_0 || + !device_trylock(peer->dev)) { + ret = -EBUSY; + goto out; + } + peer_dev = peer->dev; + } + ret = dcp_peer_probe_background(dcp, peer); +out: + if (peer_dev) + device_unlock(peer_dev); + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_swap_test); + +/* End a diagnostic pair using the firmware's power/element sequence. Keep the + * reservation on failure: a failed ACK is not evidence that DMA has stopped. + */ +static ssize_t peer_stop_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp, *peer; + struct device *peer_dev = NULL; + bool enabled; + int ret; + + if (kstrtobool(buf, &enabled) || !enabled) + return -EINVAL; + if (!peer_allocation_probe || !device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + { + guard(mutex)(&dcp_peer_lock); + peer = dcp ? dcp->peer_reserved : NULL; + if (!peer || peer->peer_owner != dcp || !dcp_peer_video_idle(dcp) || + !dcp_peer_video_idle(peer) || !dcp->crtc || !peer->crtc || + !dcp->crtc->base.state || !peer->crtc->base.state || + dcp->crtc->base.state->active || peer->crtc->base.state->active || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + peer->fw_compat != DCP_FIRMWARE_V_27_0 || + !device_trylock(peer->dev)) { + ret = -EBUSY; + goto out; + } + peer_dev = peer->dev; + } + ret = dcp_peer_probe_quiesce(dcp, peer); + dev_info(dev, "peer stop test result=%d\n", ret); +out: + if (peer_dev) + device_unlock(peer_dev); + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_stop_test); diff --git a/drivers/gpu/drm/apple/dcp-peer-v26_6.h b/drivers/gpu/drm/apple/dcp-peer-v26_6.h new file mode 100644 index 0000000000000..3b88d5e040303 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-v26_6.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_PEER_V26_6_H +#define APPLE_DCP_PEER_V26_6_H + +#include +#include +#include +#include + +struct dcp_peer_tiling { + u32 phase, modeset_state, x, y, width, height, display_id; + bool member, vftg, primary, ready, mode_flag, display_id_valid; + bool modeset_complete; +}; + +static inline bool dcp_peer_tiling_get( + const struct dcp_peer_tiling *own, + const struct dcp_peer_tiling *related, u32 event, u32 param, u32 *value) +{ + const struct dcp_peer_tiling *target = own; + + if (event == 2 && own->member) { + if (!related) + return false; + target = related; + } + switch (param) { + case 1: *value = target->member; return true; + case 2: + if (event == 1 || target->member) + *value = target->phase; + return true; + case 3: return true; + case 4: + if (event != 1 && !target->member) + return true; + if (!target->display_id_valid) + return false; + *value = target->display_id; + return true; + case 5: case 12: *value = target->x; return true; + case 6: + if (event == 1 || target->member) + *value = target->ready; + return true; + case 8: *value = own->vftg; return true; + case 9: *value = own->primary; return true; + case 10: + if (event == 1 || target->member) + *value = target->modeset_state; + return true; + case 11: *value = 0; return true; + case 13: *value = target->y; return true; + case 14: *value = target->mode_flag; return true; + default: return false; + } +} + +static inline bool dcp_peer_tiling_set(struct dcp_peer_tiling *own, + struct dcp_peer_tiling *related, u32 event, u32 param, u32 value) +{ + struct dcp_peer_tiling *target = event == 2 ? related : own; + + if (!target) + return false; + switch (param) { + case 2: target->phase = value; return true; + case 6: target->ready = !!value; return true; + case 10: target->modeset_state = value; return true; + default: return false; + } +} + +/* + * Display allocation hints — 160 bytes, layout shared with fw27. + * The AP fills primary/peer endpoint IDs and extra-pipe count for a sink + * wider than one pipe's maximum. + */ +struct dcp_display_hints { + u8 hints[0x6c]; + u8 padding[4]; + /* The allocation the AP fills in, or clears when it allocates no peer. */ + struct_group_attr(allocation, __packed, + __le32 primary_id, peer_id; + u8 extra_pipes, flags; + u8 allocation_reserved[22]; + ); + __le32 status; + u8 tail[12]; +} __packed; + +static_assert(sizeof(struct dcp_display_hints) == 160); +static_assert(offsetof(struct dcp_display_hints, primary_id) == 0x70); +static_assert(offsetof(struct dcp_display_hints, status) == 0x90); +static_assert(sizeof_field(struct dcp_display_hints, allocation) == 32); + +/* The widest mode one desktop 26.6 DISPEXT pipe drives (firmware: "max width: 6720"). */ +#define DCP_DESKTOP_PIPE_MAX_WIDTH 6720 + +/* + * Peer validity by firmware endpoint (apple,dcp-endpoint, the firmware's + * Disp_Index). Endpoint 0 is the internal display and never a peer; the + * external pipes merge upwards only (the T8132 rule inherited by T6050): + * dcpext0 (1) is the primary, dcpext1 (2) its peer. The other direction + * pairs, but the peer's half of the picture is noise. + */ +static inline bool dcp_peer_index_allowed_26_6(u32 primary, u32 peer) +{ + return primary && peer > primary; +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-peer-v27.h b/drivers/gpu/drm/apple/dcp-peer-v27.h new file mode 100644 index 0000000000000..cea2a4f13e43b --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-v27.h @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DCP_PEER_V27_H +#define APPLE_DCP_PEER_V27_H + +#include +#include +#include + +#include "dcp-peer-v26_6.h" + +struct dcp_dptx_caps { + u32 width, total_rate, active_rate; +}; + +/* Per-engine state used by D114/D115 in a VFTG pair. This is not + * display-wall emulation: membership is set only while a split mode is active. + */ +/* The record is struct dcp_peer_tiling, shared with the desktop 26.6 pair. */ + +static inline bool dcp_peer_tiling_get_v27( + const struct dcp_peer_tiling *own, + const struct dcp_peer_tiling *related, u32 event, u32 param, u32 *value) +{ + const struct dcp_peer_tiling *target = own; + + /* The getter redirects event 2 only for an active pair. */ + if (event == 2 && own->member) { + if (!related) + return false; + target = related; + } + switch (param) { + case 1: *value = target->member; return true; + case 2: + if (event == 1 || target->member) + *value = target->phase; + return true; + case 3: return true; /* The non-wall case preserves the input. */ + case 4: + if (event != 1 && !target->member) + return true; + if (!target->display_id_valid) + return false; + *value = target->display_id; + return true; + case 5: case 12: *value = target->x; return true; + case 6: + if (event == 1 || target->member) + *value = target->ready; + return true; + case 8: *value = own->vftg; return true; + case 9: *value = own->primary; return true; + case 10: + if (event == 1 || target->member) + *value = target->modeset_state; + return true; + case 11: *value = 0; return true; /* This pair is not a display wall. */ + case 13: *value = target->y; return true; + case 14: *value = target->mode_flag; return true; + default: return false; + } +} + +static inline bool dcp_peer_tiling_set_v27(struct dcp_peer_tiling *own, + struct dcp_peer_tiling *related, u32 event, u32 param, u32 value) +{ + /* Unlike the getter, the setter always redirects event 2. */ + struct dcp_peer_tiling *target = event == 2 ? related : own; + + if (!target) + return false; + switch (param) { + case 2: target->phase = value; return true; + case 6: target->ready = !!value; return true; + case 10: target->modeset_state = value; return true; + default: return false; + } +} + +/* AV display hints and display allocation records. */ +struct dcp_display_hints_v27 { + u8 hints[0x6c]; + u8 padding[4]; + __le32 primary_id, peer_id; + u8 extra_pipes, flags; + u8 allocation_reserved[22]; + __le32 status; + u8 tail[12]; +} __packed; + +static_assert(sizeof(struct dcp_display_hints_v27) == 160); +static_assert(offsetof(struct dcp_display_hints_v27, primary_id) == 0x70); +static_assert(offsetof(struct dcp_display_hints_v27, status) == 0x90); + +/* Extra pipe count, as the T602X display crossbar computes it. + * Caps come from the live DPTX service, never from a forced clock limit. + */ +static inline u32 dcp_peer_extra_pipes(const u8 *hints, + const struct dcp_dptx_caps *caps) +{ + u32 width = get_unaligned_le32(hints + 0x38); + u32 total = get_unaligned_le32(hints + 0x40); + u32 active = get_unaligned_le32(hints + 0x44); + u32 extra; + + if (!(hints[0] & 1) || !width || !total || !active || + !caps->width || !caps->total_rate || !caps->active_rate) + return 0; + extra = (width - 1) / caps->width; + extra = max(extra, (total - 1) / caps->total_rate); + return max(extra, (active - 1) / caps->active_rate); +} + +/* Peer validity rule of the T8132 display crossbar, inherited by T6050. + * Endpoint zero is the internal display; a peer must be on the same die. + */ +static inline bool dcp_peer_index_allowed(u32 primary, u32 peer) +{ + static const u8 masks[] = { 0, 0x1c, 0x18, 0x10, 0 }; + + return primary < ARRAY_SIZE(masks) && peer < ARRAY_SIZE(masks) && + (masks[primary] & BIT(peer)); +} + +#endif diff --git a/drivers/gpu/drm/apple/dcp-peer-v27.inc b/drivers/gpu/drm/apple/dcp-peer-v27.inc new file mode 100644 index 0000000000000..9bae0594b5843 --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-v27.inc @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ + +/* Bring-up gate until the framebuffer merge lifecycle is implemented. This + * only reserves an otherwise unrouted DCP; it never replaces a physical route. + */ +static bool peer_allocation_probe; +module_param(peer_allocation_probe, bool, 0444); +MODULE_PARM_DESC(peer_allocation_probe, "Test firmware27 logical peer allocation (merge bring-up)"); + +bool dcp_peer_tiling_state_v27(struct apple_dcp *dcp, bool write, + u32 event, u32 param, u32 *value) +{ + struct apple_dcp *peer; + + guard(mutex)(&dcp_peer_lock); + if (!peer_allocation_probe || dcp->peer_unbinding || dcp->crashed) + return false; + peer = dcp->peer_reserved ?: dcp->peer_owner; + if (!peer || peer->peer_unbinding || peer->crashed) + return false; + /* A resource reservation is not a framebuffer pair. Before the + * split-mode preparation, keep the established untiled callback path; + * zero-initialized fields are not yet valid tiling-manager state. + */ + if (!dcp->peer_tiling.vftg || !peer->peer_tiling.vftg) + return false; + if (write) + return dcp_peer_tiling_set_v27(&dcp->peer_tiling, + &peer->peer_tiling, event, param, *value); + return dcp_peer_tiling_get_v27(&dcp->peer_tiling, &peer->peer_tiling, + event, param, value); +} + +void dcp_peer_display_ids_v27(struct apple_dcp *dcp, u32 own, u32 related) +{ + struct apple_dcp *peer; + + guard(mutex)(&dcp_peer_lock); + peer = dcp->peer_reserved ?: dcp->peer_owner; + if (!peer || own == related || dcp->peer_unbinding || peer->peer_unbinding) + return; + /* D575 carries external display IDs, not crossbar endpoint indices. */ + dcp->peer_tiling.display_id = own; + dcp->peer_tiling.display_id_valid = true; + peer->peer_tiling.display_id = related; + peer->peer_tiling.display_id_valid = true; +} + +static void dcp_peer_release_locked(struct apple_dcp *dcp) +{ + if (dcp->peer_reserved) { + /* A link-allocation release is not a DMA-stop acknowledgement. + * Keep a prepared pair owned until the AP has powered both down. + * Firmware may still need the tiling rendezvous state to stop. + */ + if (dcp->peer_tiling.vftg || dcp->peer_tiling.member || + dcp->peer_reserved->peer_tiling.vftg || + dcp->peer_reserved->peer_tiling.member) { + dcp->peer_tiling.modeset_complete = false; + dcp->peer_reserved->peer_tiling.modeset_complete = false; + return; + } + memset(&dcp->peer_tiling, 0, sizeof(dcp->peer_tiling)); + memset(&dcp->peer_reserved->peer_tiling, 0, + sizeof(dcp->peer_reserved->peer_tiling)); + dcp->peer_reserved->peer_owner = NULL; + dcp->peer_reserved = NULL; + } +} + +int dcp_peer_hints_v27(struct apple_dcp *dcp, struct dptx_port *port, + const void *data, size_t size, void *reply, size_t reply_size) +{ + struct dcp_display_hints_v27 *r = reply; + struct apple_dcp *peer = NULL; + struct drm_crtc *crtc; + u32 extra; + + if (size != sizeof(*r) || reply_size != sizeof(*r)) + return -EINVAL; + memcpy(reply, data, sizeof(*r)); + /* The first word is hint flags, not a return code. The completion + * status has a field of its own, leaving the request hints unchanged. + */ + r->status = 0; + if (!peer_allocation_probe) + return 0; + if (!of_device_is_compatible(dcp->dev->of_node, "apple,t6050-dcpext") || + port->unit || !dcp->crtc) + return 0; + + extra = dcp_peer_extra_pipes(r->hints, &port->caps); + memset(&r->primary_id, 0, 32); + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_unbinding || dcp->crashed) + return -ENODEV; + if (!extra || !(r->hints[0] & 1)) { + dcp_peer_release_locked(dcp); + memset(&r->primary_id, 0, 32); + return 0; + } + /* This candidate implements a non-tiled, two-pipe reservation. Tiled + * panels and more than one peer require the full display allocator. + */ + if (extra != 1 || (r->hints[0] & 2) || dcp->peer_owner) + return 0; + peer = dcp->peer_reserved; + if (!peer) { + drm_for_each_crtc(crtc, dcp->crtc->base.dev) { + struct apple_dcp *p = platform_get_drvdata(to_apple_crtc(crtc)->dcp); + + /* A dedicated, unrouted peer cannot race Type-C's allocator + * or a fixed connector's HPD handler in this bring-up step. + */ + if (!p || !p->active || p->crashed || p->peer_unbinding || + p->peer_owner || p->peer_reserved || p->fixed_phy || + p->nr_typec_routes || p->active_typec_route || + (p->connector && p->connector->connected) || + !dcp_peer_index_allowed(dcp->index, p->index) || + p->dptx_die != dcp->dptx_die || + !p->crtc || p->crtc->base.dev != dcp->crtc->base.dev || + p->fw_compat != DCP_FIRMWARE_V_27_0 || + !p->dptxport[0].enabled || + p->dptxport[0].caps.width < port->caps.width || + p->dptxport[0].caps.total_rate < port->caps.total_rate || + p->dptxport[0].caps.active_rate < port->caps.active_rate || + p->dptxport[0].connected || p->dptxport[1].connected || + !of_device_is_compatible(p->dev->of_node, "apple,t6050-dcpext")) + continue; + if (!peer || p->index < peer->index) + peer = p; + } + if (!peer) { + dev_info(dcp->dev, "no eligible idle peer for %u extra pipe\n", extra); + return 0; + } + dcp->peer_reserved = peer; + peer->peer_owner = dcp; + dev_info(dcp->dev, "reserved peer DCP%u for %ux%u, total %u active %u\n", + peer->index, get_unaligned_le32(r->hints + 0x38), + get_unaligned_le32(r->hints + 0x3c), + get_unaligned_le32(r->hints + 0x40), + get_unaligned_le32(r->hints + 0x44)); + } + /* The display allocation update writes primary/peer endpoint IDs. + * The extra pipe count supplies byte8. This is full dual-pipe, so the + * T6050 limited-dual-pipe flag (byte9 bit1) remains clear. + */ + r->primary_id = cpu_to_le32(dcp->index); + r->peer_id = cpu_to_le32(peer->index); + r->extra_pipes = extra; + r->flags &= ~3; + return 0; +} diff --git a/drivers/gpu/drm/apple/dcp-peer-video-probe-v27.inc b/drivers/gpu/drm/apple/dcp-peer-video-probe-v27.inc new file mode 100644 index 0000000000000..24c985a91195f --- /dev/null +++ b/drivers/gpu/drm/apple/dcp-peer-video-probe-v27.inc @@ -0,0 +1,250 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Opt-in A364/A365 bring-up. Installs derived modes without selecting them; + * paired power, modeset and swap ownership must be implemented separately. + */ +#include "iomfb-peer-v27.h" + +struct dcp_peer_video_wait { + struct kref refs; + struct completion done; + size_t size; + u8 reply[sizeof(struct dcp_peer_get_video_resp_v27)]; +}; + +static void dcp_peer_video_free(struct kref *ref) +{ + kfree(container_of(ref, struct dcp_peer_video_wait, refs)); +} + +static void dcp_peer_video_done(struct apple_dcp *dcp, void *out, void *cookie) +{ + struct dcp_peer_video_wait *wait = cookie; + + if (wait->size) + memcpy(wait->reply, out, wait->size); + complete(&wait->done); + kref_put(&wait->refs, dcp_peer_video_free); +} + +static int dcp_peer_video_call_stream(struct apple_dcp *dcp, bool independent, + const char *tag, + void *in, size_t in_size, void *out, size_t out_size) +{ + struct dcp_method_entry method = { .name = "peer_video_test" }; + struct dcp_peer_video_wait *wait; + int ret = 0; + + if (out_size > sizeof(wait->reply)) + return -EINVAL; + /* An unrelated AP thread gets its own firmware stream. A369/A367/etc. + * must not be mistaken for nested callbacks of the pending A413. + * Stream1 (the driver's OOB context) is independent of stream0. + */ + if (independent && (dcp->ch_oobcmd.depth || dcp->ch_oobcb.depth)) + return -EBUSY; + wait = kzalloc(sizeof(*wait), GFP_KERNEL); + if (!wait) + return -ENOMEM; + memcpy(method.tag, tag, sizeof(method.tag)); + wait->size = out_size; + init_completion(&wait->done); + kref_init(&wait->refs); + kref_get(&wait->refs); + dcp_push(dcp, independent, &method, in_size, out_size, in, + dcp_peer_video_done, wait); + if (!wait_for_completion_timeout(&wait->done, + msecs_to_jiffies(!memcmp(tag, "A413", 4) ? 8500 : 2000))) + ret = -ETIMEDOUT; + else if (out_size) + memcpy(out, wait->reply, out_size); + /* A late ACK owns only the heap cookie, never a caller's stack. */ + kref_put(&wait->refs, dcp_peer_video_free); + return ret; +} + +static int dcp_peer_video_call(struct apple_dcp *dcp, const char *tag, + void *in, size_t in_size, void *out, size_t out_size) +{ + return dcp_peer_video_call_stream(dcp, false, tag, in, in_size, out, out_size); +} + +#include "dcp-peer-modeset-probe-v27.inc" + +static bool dcp_peer_video_idle(struct apple_dcp *dcp) +{ + return dcp->active && !dcp->crashed && !dcp->peer_unbinding && + !READ_ONCE(dcp->mode_state.changing) && !dcp->ch_cmd.depth && !dcp->ch_cb.depth && + !dcp->ch_async.depth && !dcp->ch_oobcmd.depth && + !dcp->ch_oobcb.depth && !dcp->ch_oobasync.depth; +} + +/* Shared by the explicit diagnostic and normal DRM mode selection. + * Install4 chooses a pair only when firmware's A364 requires one. + */ +static int dcp_peer_install_video_v27(struct apple_dcp *dcp, struct apple_dcp *peer, + u32 timing, u32 color, u32 install) +{ + struct device *dev = dcp->dev; + struct dcp_peer_get_video_req_v27 req = {}; + struct dcp_peer_get_video_resp_v27 resp; + struct dcp_peer_insert_video_req_v27 insert = {}; + struct dcp_peer_insert_video_resp_v27 inserted; + __le32 enabled = cpu_to_le32(1); + int ret; + + req.timing_id = cpu_to_le32(timing); + req.color_id = cpu_to_le32(color); + ret = dcp_peer_video_call(dcp, "A364", &req, sizeof(req), &resp, sizeof(resp)); + if (ret) + return ret; + dev_info(dev, "peer video timing=%u color=%u success=%u merge=%u timing=%*ph color=%*ph\n", + timing, color, resp.success, resp.requires_merge, + 64, resp.video.timing, 32, resp.video.color); + dev_info(dev, "peer timing tail=%*ph\n", 16, resp.video.timing + 64); + if (!(resp.success & 1)) { + ret = -ENOENT; + return ret; + } + if (!install) + return ret; + /* Mode3 explicitly tests a reserved pair using an existing lower-rate + * timing. It does not advertise a mode, raise a limit, or add scanout. + * Keep this diagnostic independent of production mode selection. + */ + if (!(resp.requires_merge & 1) && install != 3) { + if (install == 4 && (dcp->peer_tiling.vftg || peer->peer_tiling.vftg)) { + u8 power[16] = {1}, response[8]; + + ret = dcp_peer_probe_quiesce(dcp, peer); + if (ret) + return ret; + /* A mode-only DRM commit does not call CRTC poweron again. */ + ret = dcp_peer_video_call(dcp, "A484", power, sizeof(power), + response, sizeof(response)); + if (ret || get_unaligned_le32(response + 4)) + return ret ?: -EIO; + } + return -EOPNOTSUPP; + } + ret = dcp_peer_probe_quiesce(dcp, peer); + if (ret) + return ret; + insert.video = resp.video; + dcp_peer_split_timing_v27(&insert.video); + insert.timing_id = cpu_to_le32(timing + 10000); + insert.color_id = cpu_to_le32(color + 10000); + ret = dcp_peer_probe_prepare_vftg(dcp, peer, &insert); + if (ret) + return ret; + ret = dcp_peer_video_call(peer, "A366", &enabled, sizeof(enabled), NULL, 0); + if (ret) + return ret; + ret = dcp_peer_video_call(dcp, "A365", &insert, sizeof(insert), + &inserted, sizeof(inserted)); + if (!ret && !(inserted.success & 1)) + ret = -EIO; + if (ret) + return ret; + /* The first call's in/out timing and color go to the peer. */ + insert.video = inserted.video; + ret = dcp_peer_video_call(peer, "A365", &insert, sizeof(insert), + &inserted, sizeof(inserted)); + if (!ret && !(inserted.success & 1)) + ret = -EIO; + dev_info(dev, "peer derived mode timing=%u color=%u width=%u height=%u clock=%u result=%d (not selected)\n", + timing + 10000, color + 10000, + get_unaligned_le32(insert.video.timing + 8), + get_unaligned_le32(insert.video.timing + 0x28), + get_unaligned_le32(insert.video.timing + 0x44), ret); + if (!ret && install >= 2) + ret = dcp_peer_probe_modeset(dcp, peer, &insert, + resp.requires_merge & 1); + return ret; +} + +static ssize_t peer_video_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp, *peer; + struct device *peer_dev = NULL; + u32 timing, color, install; + char extra; + int ret; + + if (sscanf(buf, "%u %u %u %c", &timing, &color, &install, &extra) != 3 || + install > 3 || timing > U32_MAX - 10000 || color > U32_MAX - 10000) + return -EINVAL; + if (!peer_allocation_probe) + return -EOPNOTSUPP; + /* Trylocks avoid the sysfs-removal/device-lock inversion. Hold both + * devices across RPC waits so neither devm-backed DCP can be unbound. + */ + if (!device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + { + guard(mutex)(&dcp_peer_lock); + peer = dcp ? dcp->peer_reserved : NULL; + if (!peer || peer->peer_owner != dcp || !dcp_peer_video_idle(dcp) || + !dcp_peer_video_idle(peer) || !dcp->crtc || !peer->crtc || + !dcp->crtc->base.state || !peer->crtc->base.state || + (install && (dcp->crtc->base.state->active || + peer->crtc->base.state->active)) || peer->fixed_phy || + peer->nr_typec_routes || peer->peer_tiling.member || + dcp->peer_tiling.member || + dcp->fw_compat != DCP_FIRMWARE_V_27_0 || + peer->fw_compat != DCP_FIRMWARE_V_27_0 || + (peer->connector && peer->connector->connected)) { + ret = -EBUSY; + goto unlock; + } + if (!device_trylock(peer->dev)) { + ret = -EBUSY; + goto unlock; + } + peer_dev = peer->dev; + } + ret = dcp_peer_install_video_v27(dcp, peer, timing, color, install); +unlock: + if (peer_dev) + device_unlock(peer_dev); + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_video_test); + +static ssize_t peer_link_test_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct apple_dcp *dcp; + bool enabled; + int ret; + + if (kstrtobool(buf, &enabled)) + return -EINVAL; + if (!peer_allocation_probe || !device_trylock(dev)) + return -EBUSY; + dcp = dev_get_drvdata(dev); + if (!dcp || !dcp->active || dcp->crashed || !dcp->fixed_phy || + dcp->nr_typec_routes || dcp->connector_type != DRM_MODE_CONNECTOR_HDMIA) { + ret = -EOPNOTSUPP; + goto unlock; + } + { + guard(mutex)(&dcp_peer_lock); + if (dcp->peer_tiling.vftg || dcp->peer_tiling.member) { + ret = -EBUSY; + goto unlock; + } + } + /* Use the existing disconnect/connect lifecycle, serialized with HPD. + * This diagnostic models link re-enumeration, not a physical cable pull. + */ + ret = dcp_typec_test_link(to_platform_device(dev), enabled); +unlock: + device_unlock(dev); + return ret ? ret : count; +} +static DEVICE_ATTR_WO(peer_link_test); diff --git a/drivers/gpu/drm/apple/dcp.c b/drivers/gpu/drm/apple/dcp.c index e84af350c7e08..3f641380c17e4 100644 --- a/drivers/gpu/drm/apple/dcp.c +++ b/drivers/gpu/drm/apple/dcp.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +56,7 @@ #include "dptx-service.h" #include "iomfb.h" #include "ibootep.h" +#include "iomfb_internal.h" #include "parser.h" #include "trace.h" @@ -208,6 +210,32 @@ static bool unstable_edid = true; module_param(unstable_edid, bool, 0644); MODULE_PARM_DESC(unstable_edid, "Enable unstable EDID retrival support"); +/* The firmware 27 pair shares dcp_peer_lock and dcp_peer_forget(). */ +static DEFINE_MUTEX(dcp_peer_lock); + +static void dcp_peer_forget(struct apple_dcp *dcp) +{ + guard(mutex)(&dcp_peer_lock); + dcp->peer_unbinding = true; + if (dcp->peer_reserved) { + dcp->peer_reserved->peer_owner = NULL; + memset(&dcp->peer_reserved->peer_tiling, 0, + sizeof(dcp->peer_reserved->peer_tiling)); + dcp->peer_reserved = NULL; + memset(&dcp->peer_tiling, 0, sizeof(dcp->peer_tiling)); + } + if (dcp->peer_owner) { + memset(&dcp->peer_owner->peer_tiling, 0, + sizeof(dcp->peer_owner->peer_tiling)); + memset(&dcp->peer_tiling, 0, sizeof(dcp->peer_tiling)); + dcp->peer_owner->peer_reserved = NULL; + dcp->peer_owner = NULL; + } +} + +#include "dcp-peer-v27.inc" +#include "dcp-peer-probe-v27.inc" + static void disconnected_hpd_event(struct apple_connector *connector); bool dcp_is_external_h17p(struct apple_dcp *dcp) @@ -322,7 +350,7 @@ void dcp_drm_crtc_page_flip(struct apple_dcp *dcp, ktime_t now) spin_unlock_irqrestore(&crtc->base.dev->event_lock, flags); } -static bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) +bool dcp_firmware_metadata_enabled(struct apple_dcp *dcp) { return dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; } @@ -639,6 +667,8 @@ static struct apple_dcp_afkep *dcp_afkep(struct apple_dcp *dcp, u8 endpoint) return dcp->dcpavservep; if (dcp->dptxep && endpoint == dcp->dptxep->endpoint) return dcp->dptxep; + if (endpoint >= 0x20 && endpoint < 0x40 && dcp->plain_afk[endpoint - 0x20]) + return dcp->plain_afk[endpoint - 0x20]; switch (endpoint) { case AV_ENDPOINT: return dcp->avep; @@ -772,6 +802,7 @@ static void dcp_rtk_crashed(void *cookie, const void *crashlog, size_t crashlog_ return; } dcp->crashed = true; + dcp_peer_forget(dcp); dev_err(dcp->dev, "DCP has crashed\n"); if (dcp->fw_compat == DCP_FIRMWARE_H17P && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { @@ -2420,6 +2451,43 @@ unsigned long* dcp_get_iomfb_surfaces(struct platform_device *pdev) return dcp->iomfb_surfaces; } +/* + * Firmware 27.3 expects the endpoints macOS starts on the internal display + * processor: disp0, the DPTX controller and port, and AFK endpoints for + * services Linux does not use. + */ +static void dcp_start_fw_27_3_endpoints(struct apple_dcp *dcp) +{ + static const struct apple_epic_service_ops no_services[] = { {} }; + static const u8 plain[] = { 0x21, 0x22, 0x25, 0x28, 0x29, 0x2a, 0x2b, 0x31 }; + struct apple_dcp_afkep *ep; + int k, ret; + + ret = ibootep_init(dcp); + if (ret) + dev_warn(dcp->dev, "disp0 endpoint: %d\n", ret); + ret = dptx_controller_init(dcp); + if (ret) + dev_warn(dcp->dev, "DPTX controller endpoint: %d\n", ret); + ret = dptxep_init(dcp); + if (ret) + dev_warn(dcp->dev, "DPTX endpoint: %d\n", ret); + + for (k = 0; k < ARRAY_SIZE(plain); k++) { + if (dcp_afkep(dcp, plain[k])) + continue; + ep = afk_init(dcp, plain[k], no_services); + if (IS_ERR_OR_NULL(ep)) { + dev_warn(dcp->dev, "endpoint %#x: %ld\n", plain[k], PTR_ERR(ep)); + continue; + } + dcp->plain_afk[plain[k] - 0x20] = ep; + ret = afk_start(ep); + if (ret) + dev_warn(dcp->dev, "endpoint %#x start: %d\n", plain[k], ret); + } +} + int dcp_start(struct platform_device *pdev) { struct apple_dcp *dcp = platform_get_drvdata(pdev); @@ -2429,6 +2497,7 @@ int dcp_start(struct platform_device *pdev) init_completion(&dcp->start_done); if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) return iomfb_v26_6_start(dcp); + init_completion(&dcp->typec_display_ready); /* * The T6030 firmware session is adopted for the internal panel. @@ -2470,7 +2539,9 @@ int dcp_start(struct platform_device *pdev) ret); } - if (dcp->phy && dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { + if ((dcp->phy || (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + dcp->hw.num_dptx_ports)) && + dcp->fw_compat >= DCP_FIRMWARE_V_13_5) { ret = ibootep_init(dcp); if (ret) dev_warn(dcp->dev, "Failed to start IBOOT endpoint: %d\n", @@ -2515,6 +2586,8 @@ int dcp_start(struct platform_device *pdev) } else if (dcp->phy) { dev_warn(dcp->dev, "OS firmware incompatible with dptxport EP\n"); } + if (dcp->hw.fw_27_3) + dcp_start_fw_27_3_endpoints(dcp); ret = iomfb_start_rtkit(dcp); if (ret) { dev_err(dcp->dev, "Failed to start IOMFB endpoint: %d\n", ret); @@ -2533,6 +2606,56 @@ int dcp_start(struct platform_device *pdev) return ret; } +/* + * A native T6030 external processor: its DP/HDMI audio endpoint (EP 0x29) + * on the driver's opt-in (hdmi_audio), as the other DCPs start it, for a + * processor whose port@0 links an audio device; once per session, and after + * the first display's EDID was read: with the endpoint's interfaces + * published, an EDID copy (dcpavserv_copy_edid()) has the firmware rebuild + * them and re-plug the sink, so the endpoint starts only once the first + * EDID is cached. + */ +static void dcp_external_audio_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, external_audio_work); + + /* Queued after an EDID came through the running session: RTKit is up. */ + if (!IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) || !hdmi_audio || !dcp->external_native || + dcp->external_audio_started || READ_ONCE(dcp->crashed) || !dcp->rtk) + return; + dcp->external_audio_started = true; + if (!apple_rtkit_has_endpoint(dcp->rtk, AV_ENDPOINT)) { + dev_info(dcp->dev, "no AV audio endpoint\n"); + return; + } +#if IS_ENABLED(CONFIG_DRM_APPLE_AUDIO) + { + int ret = avep_init(dcp); + + if (ret) + dev_warn(dcp->dev, "AV audio endpoint not started: %d\n", ret); + } +#endif +} + +void dcp_external_audio_start(struct apple_dcp *dcp) +{ + if (!dcp->external_native || dcp->external_audio_started) + return; + queue_work(system_freezable_wq, &dcp->external_audio_work); +} + +/* Probe: the audio work, disabled until a native external pipe's bind. */ +static void dcp_external_audio_init(struct apple_dcp *dcp) +{ + INIT_WORK(&dcp->external_audio_work, dcp_external_audio_work); + /* + * A native external pipe's bind enables it (its first EDID queues + * it, through the connector); unbind and suspend disable it again. + */ + disable_work(&dcp->external_audio_work); +} + static void _dcp_poweroff(struct apple_dcp *dcp) { switch (dcp->fw_compat) { @@ -2554,6 +2677,9 @@ static void _dcp_poweroff(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_poweroff_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_poweroff_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -2645,6 +2771,9 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_sleep_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_sleep_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -2655,8 +2784,10 @@ static void __maybe_unused dcp_sleep(struct apple_dcp *dcp) static bool dcp_uses_soft_dpms(struct apple_dcp *dcp) { - return (dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || - (dcp->fw_compat == DCP_FIRMWARE_H17P && + /* The 14.7 panel has its own known-level blank and restore hooks. */ + return (!iomfb_v14_7_panel_nits(dcp) && + dcp_backlight_active(dcp) && dcp_has_panel(dcp)) || + ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && dcp->connector_type == DRM_MODE_CONNECTOR_eDP); } @@ -2704,10 +2835,13 @@ void dcp_poweron(struct platform_device *pdev) } if (dcp_uses_soft_dpms(dcp)) { - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && !READ_ONCE(dcp->pipe_enabled_h17p)) { - iomfb_poweron_h17p(dcp); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + iomfb_poweron_v27(dcp); + else + iomfb_poweron_h17p(dcp); if (!READ_ONCE(dcp->pipe_enabled_h17p)) { dev_err(dcp->dev, "initial display pipe enable failed\n"); WRITE_ONCE(dcp->crashed, true); @@ -2795,6 +2929,9 @@ void dcp_poweron(struct platform_device *pdev) case DCP_FIRMWARE_H17P: iomfb_poweron_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_poweron_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -3309,6 +3446,23 @@ static int dcp_get_disp_regs(struct apple_dcp *dcp) } dcp->nr_disp_registers = count; + for (i = 0; i < count; i++) + dcp->disp_firmware_base[i] = dcp->disp_registers[i]->start; + if (of_property_present(dcp->dev->of_node, "apple,firmware-physical-regs")) { + if (of_property_count_u64_elems(dcp->dev->of_node, + "apple,firmware-physical-regs") != count) + return -EINVAL; + ret = of_property_read_u64_array(dcp->dev->of_node, + "apple,firmware-physical-regs", + dcp->disp_firmware_base, count); + if (ret) + return ret; + for (i = 0; i < count; i++) + if (!IS_ALIGNED(dcp->disp_firmware_base[i], 4) || + resource_size(dcp->disp_registers[i]) > + U64_MAX - dcp->disp_firmware_base[i]) + return -EINVAL; + } return 0; } @@ -3364,6 +3518,13 @@ static enum dcp_firmware_version dcp_check_firmware_version(struct device *dev) char fw_str[DCP_FW_VERSION_STR_LEN]; int ret; + if (hw->firmware_compat == DCP_FIRMWARE_V_27_0) { + ret = dcp_read_fw_version(dev, "apple,firmware-compat", compat_str); + if (ret < 0 || strcmp(compat_str, "27.0.0")) + return DCP_FIRMWARE_UNKNOWN; + return DCP_FIRMWARE_V_27_0; + } + /* * SoCs introduced with H17-generation firmware pin its interface; the * loader may not recognise their boot firmware version. @@ -3543,6 +3704,43 @@ VISIBLE_IF_KUNIT void dcp_enable_typec_work(struct apple_dcp *dcp) } } +static int dcp_firmware_metadata_v27(struct apple_dcp *dcp) +{ + struct device *dev = dcp->dev; + struct device_node *fb; + struct resource resource; + int clocks, index, ret; + + index = of_property_match_string(dev->of_node, "memory-region-names", + "framebuffer"); + if (index < 0) + return -EINVAL; + fb = of_parse_phandle(dev->of_node, "memory-region", index); + if (!fb) + return -EINVAL; + ret = of_address_to_resource(fb, 0, &resource); + of_node_put(fb); + if (ret) + return ret; + dcp->dfb_region_size = resource_size(&resource); + if (!dcp->dfb_region_size) + return -EINVAL; + + clocks = of_property_count_u32_elems(dev->of_node, "apple,clock-ids"); + if (clocks == -ENODATA && + of_property_present(dev->of_node, "apple,clock-ids")) + clocks = 0; + if (clocks < 0 || clocks > 64) + return -EINVAL; + ret = devm_clk_bulk_get_all(dev, &dcp->firmware_clocks); + if (ret < 0) + return ret; + if (ret != clocks) + return -EINVAL; + dcp->num_firmware_clocks = clocks; + return 0; +} + static bool dcp_external_is_idle(struct apple_dcp *dcp) { u32 control = readl_relaxed(dcp->coproc_reg + APPLE_DCP_COPROC_CPU_CONTROL); @@ -3562,6 +3760,8 @@ static bool dcp_external_is_idle(struct apple_dcp *dcp) */ static bool dcp_show_notch(struct apple_dcp *dcp) { + if (dcp->hw.show_notch) + return true; if (dcp->fw_compat != DCP_FIRMWARE_V_14_7 && dcp->fw_compat != DCP_FIRMWARE_V_26_6) return show_notch; @@ -3640,6 +3840,14 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat u32 cpu_ctrl; int ret; + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + INIT_LIST_HEAD(&dcp->iomfb_boolean_properties); + ret = devm_add_action_or_reset(dev, + iomfb_firmware_state_cleanup_v27, dcp); + if (ret) + return ret; + } + if (READ_ONCE(dcp->quiescing)) return dev_err_probe(dev, -EBUSY, "Live DCP session cannot be rebound; reboot required\n"); @@ -3650,8 +3858,10 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat dcp->vblank_works_disabled = false; } - if (dcp->external_prepared) + if (dcp->external_prepared) { + av_audiosrv_reports_restart(dcp); return 0; + } /* A timed-out prior session may still scan these mappings. */ mutex_lock(&dcp->swapped_out_fbs_lock); @@ -3661,7 +3871,8 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat return dev_err_probe(dev, ret, "Previous scanout has not been stopped\n"); - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(42)); + ret = dma_set_mask_and_coherent(dev, + DMA_BIT_MASK(dcp->hw.firmware_dma_bits ?: 42)); if (ret) return ret; @@ -3736,8 +3947,10 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat if (dcp->fw_compat == DCP_FIRMWARE_V_26_6) { ret = iomfb_v26_6_bind(dcp); - if (!ret) + if (!ret) { enable_work(&dcp->dimensions_wq); + av_audiosrv_reports_restart(dcp); + } return ret; } @@ -3746,19 +3959,24 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat if (dcp->external_native) { WRITE_ONCE(dcp->external_detached, false); enable_work(&dcp->vblank_wq); + enable_work(&dcp->external_audio_work); enable_delayed_work(&dcp->swap_watchdog_wq); enable_delayed_work(&dcp->external_retry_wq); enable_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); + av_audiosrv_reports_restart(dcp); return 0; } - if (dcp->external) + if (dcp->external) { + av_audiosrv_reports_restart(dcp); return 0; + } ret = iomfb_v14_7_bind(dcp); /* Balances the disable at probe, as below. */ if (!ret) { enable_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); + av_audiosrv_reports_restart(dcp); } return ret; } @@ -3779,7 +3997,17 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat goto err_piodma; } - dcp->clk = devm_clk_get(dev, NULL); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + ret = dcp_firmware_metadata_v27(dcp); + if (ret) + goto err_piodma; + } + /* + */ + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0 && !dcp->num_firmware_clocks) + dcp->clk = devm_clk_get_optional(dev, NULL); + else + dcp->clk = devm_clk_get(dev, NULL); if (IS_ERR(dcp->clk)) { ret = dev_err_probe(dev, PTR_ERR(dcp->clk), "Unable to find clock\n"); @@ -3865,6 +4093,7 @@ static int dcp_comp_bind_pipe(struct device *dev, struct device *main, void *dat enable_work(&dcp->dimensions_wq); schedule_work(&dcp->dimensions_wq); dcp_enable_typec_work(dcp); + av_audiosrv_reports_restart(dcp); return 0; err_piodma: @@ -3901,7 +4130,18 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v { struct apple_dcp *dcp = dev_get_drvdata(dev); - if (!dcp || READ_ONCE(dcp->ext_dead)) + if (!dcp) + return; + dcp_peer_forget(dcp); + /* + * Stop the native initializer before looking up its owner, including + * the dead-session path. Owner allocation may have landed in an open + * bind devres group; finish its RPC work while endpoint RX is alive. + */ + if (dcp->external_native) + disable_work_sync(&dcp->external_audio_work); + av_audiosrv_quiesce(dcp); + if (READ_ONCE(dcp->ext_dead)) return; disable_work_sync(&dcp->dimensions_wq); @@ -3997,7 +4237,7 @@ VISIBLE_IF_KUNIT void dcp_comp_unbind(struct device *dev, struct device *main, v } } - if (dcp->avep) { + if (!IS_ERR_OR_NULL(dcp->avep)) { av_service_disconnect(dcp); afk_shutdown(dcp->avep); dcp->avep = NULL; @@ -4162,6 +4402,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) dcp->ext_failsafe = of_device_is_compatible(dev->of_node, "apple,t8122-dcpext"); dcp->external = of_device_is_compatible(dev->of_node, "apple,t6030-dcpext"); dcp->external_native = dcp->external && dcp_t6030_ext_native(dev->of_node); + dcp_external_audio_init(dcp); dcp->dev = dev; /* * Type-C and Thunderbolt routes can be activated as soon as they are @@ -4181,6 +4422,7 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) spin_lock_init(&dcp->mode_state.lock); spin_lock_init(&dcp->metadata.lock); mutex_init(&dcp->dcpavserv.lock); + INIT_WORK(&dcp->dcpavserv.close_wq, dcpavserv_close_work); dcp->hw = *(struct apple_dcp_hw_data *)of_device_get_match_data(dev); if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) { @@ -4457,14 +4699,18 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) ret = dcp_comp_bind(dev, NULL, NULL); if (ret) { dcp_disable_typec_work(dcp, true); + av_audiosrv_quiesce(dcp); return ret; } WRITE_ONCE(dcp->external_prepared, true); } if (dcp->external_native) { ret = iomfb_v14_7_external_prepare(dcp); - if (ret) + if (ret) { + disable_work_sync(&dcp->external_audio_work); + av_audiosrv_quiesce(dcp); return ret; + } if (routes_failed) iomfb_v14_7_external_refuse(dcp, "its Type-C display routes did not register"); } @@ -4473,6 +4719,12 @@ static int dcp_platform_probe_pipe(struct platform_device *pdev) if (ret && dcp->external_prepared) { dcp_disable_typec_work(dcp, true); } + /* component_add failure does not unbind. Stop owner work first. */ + if (ret) { + if (dcp->external_native) + disable_work_sync(&dcp->external_audio_work); + av_audiosrv_quiesce(dcp); + } if (ret && dcp->fw_compat == DCP_FIRMWARE_V_26_6) iomfb_v26_6_unbind(dcp); /* A failed bind run from here may already have started RTKit. */ @@ -4594,11 +4846,21 @@ static void dcp_platform_remove(struct platform_device *pdev) struct apple_dcp *dcp = platform_get_drvdata(pdev); if (dcp && dcp->external && !dcp->external_native) { - /* Enabled for the external start at probe. */ + /* Enabled for the external start at probe. No component unbind. */ disable_delayed_work_sync(&dcp->typec_reconnect_wq); + av_audiosrv_quiesce(dcp); iomfb_v14_7_remove(dcp); return; } + if (dcp && dcp->external_native) + disable_work_sync(&dcp->external_audio_work); + /* + * Finish audio RPCs before component_del's unbind flushes RX, and + * before iomfb_v14_7_remove() clears v14->dcp. Unbind repeats the + * helper; the second call does not disable the works again. + */ + if (dcp) + av_audiosrv_quiesce(dcp); component_del(&pdev->dev, &dcp_comp_ops); if (dcp && dcp->fw_compat == DCP_FIRMWARE_V_26_6) iomfb_v26_6_unbind(dcp); @@ -4845,6 +5107,8 @@ static int dcp_platform_suspend(struct device *dev) */ dcp_disable_typec_work(dcp, false); cancel_delayed_work_sync(&dcp->swap_watchdog_wq); + if (dcp->external_native) + disable_work_sync(&dcp->external_audio_work); if (dcp->avep) av_service_disconnect(dcp); @@ -4883,6 +5147,8 @@ static int dcp_platform_resume(struct device *dev) /* Observe future edges before sampling any edges lost in sleep. */ if (dcp->hdmi_hpd_irq) dcp_fabric_run_resume(&dcp_resume_ops, dcp); + if (dcp->external_native) + enable_work(&dcp->external_audio_work); if (dcp->avep) av_service_connect(dcp); @@ -4978,7 +5244,27 @@ static const struct apple_dcp_hw_data apple_dcp_hw_dcpext = { .num_dptx_ports = 2, }; +/* T6050 (J714c): firmware 27, the session the boot loader leaves running. */ +static const struct apple_dcp_hw_data apple_dcp_hw_t6050 = { + .firmware_dma_bits = 41, + .num_dptx_ports = 0, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_V_27_0, + .fw_27_3 = true, + .swap_no_timestamps = true, + .show_notch = true, +}; + +static const struct apple_dcp_hw_data apple_dcp_hw_t6050_dcpext = { + .firmware_dma_bits = 41, + .num_dptx_ports = 2, + .adopt_live_session = true, + .firmware_compat = DCP_FIRMWARE_V_27_0, +}; + static const struct of_device_id of_match[] = { + { .compatible = "apple,t6050-dcp", .data = &apple_dcp_hw_t6050 }, + { .compatible = "apple,t6050-dcpext", .data = &apple_dcp_hw_t6050_dcpext }, { .compatible = "apple,t6020-dcp", .data = &apple_dcp_hw_t6020, }, { .compatible = "apple,t6020-dcpext", .data = &apple_dcp_hw_t6020_dcpext, }, { .compatible = "apple,t8112-dcp", .data = &apple_dcp_hw_t8112, }, @@ -5003,6 +5289,7 @@ static struct platform_driver apple_platform_driver = { .shutdown = dcp_platform_shutdown, .driver = { .name = "apple-dcp", + .dev_groups = dcp_peer_probe_groups, .of_match_table = of_match, .pm = pm_sleep_ptr(&dcp_platform_pm_ops), }, @@ -5017,3 +5304,7 @@ void dcp_unregister(void) { platform_driver_unregister(&apple_platform_driver); } + +#if IS_ENABLED(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) +#include "dcp_external_audio_test.c" +#endif diff --git a/drivers/gpu/drm/apple/dcp.h b/drivers/gpu/drm/apple/dcp.h index baff04ee201e7..8fa39ca85a565 100644 --- a/drivers/gpu/drm/apple/dcp.h +++ b/drivers/gpu/drm/apple/dcp.h @@ -58,6 +58,19 @@ void dcp_poweroff(struct platform_device *pdev); int dcp_typec_prepare_route(struct apple_dcp *dcp); void dcp_poweron(struct platform_device *pdev); void dcp_external_recover(struct platform_device *pdev); +int dcp_peer_hints_v27(struct apple_dcp *dcp, struct dptx_port *port, + const void *data, size_t size, void *reply, size_t reply_size); +bool dcp_peer_tiling_state_v27(struct apple_dcp *dcp, bool write, + u32 event, u32 param, u32 *value); +void dcp_peer_display_ids_v27(struct apple_dcp *dcp, u32 own, u32 related); +int dcp_peer_modeset_v27(struct apple_dcp *dcp, u32 timing, u32 color, bool vrr); +int dcp_peer_color_v27(struct apple_dcp *dcp, struct iomfb_set_matrix_req *matrix); +int dcp_peer_stop_v27(struct apple_dcp *dcp); +struct apple_dcp *dcp_peer_scanout_v27(struct apple_dcp *dcp); +void dcp_peer_swap_armed_v27(struct apple_dcp *dcp, u32 id); +void dcp_peer_swap_failed_v27(struct apple_dcp *dcp); +bool dcp_peer_swap_complete_v27(struct apple_dcp *dcp, u32 id, ktime_t now, + void (*report)(struct apple_dcp *, u32, ktime_t)); int dcp_set_crc(struct drm_crtc *crtc, bool enabled); int dcp_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *commit); int dcp_get_connector_type(struct platform_device *pdev); diff --git a/drivers/gpu/drm/apple/dcp_backlight.c b/drivers/gpu/drm/apple/dcp_backlight.c index 730058ecd3773..4f39639fd5921 100644 --- a/drivers/gpu/drm/apple/dcp_backlight.c +++ b/drivers/gpu/drm/apple/dcp_backlight.c @@ -93,10 +93,16 @@ static u32 brightness_part2[] = { #define DCP_BL_FADE_MS 250U #define DCP_BL_FADE_STEP_MS 16U +static bool dcp_backlight_presents_nits(struct apple_dcp *dcp) +{ + return ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || + iomfb_v14_7_panel_nits(dcp); +} + bool dcp_backlight_active(struct apple_dcp *dcp) { - return dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && + return dcp_backlight_presents_nits(dcp) && READ_ONCE(dcp->backlight.state.ready); } @@ -144,8 +150,9 @@ int dcp_backlight_configure(struct apple_dcp *dcp, u32 maximum, unsigned long flags; int ret; - if (dcp->fw_compat != DCP_FIRMWARE_H17P || !dcp_has_panel(dcp) || - !dcp->crtc || !kick) + if ((dcp->fw_compat != DCP_FIRMWARE_H17P && dcp->fw_compat != DCP_FIRMWARE_V_27_0 && + !iomfb_is_m4(dcp) && !iomfb_v14_7_panel_nits(dcp)) || + !dcp_has_panel(dcp) || !dcp->crtc || !kick) return -EINVAL; mutex_lock(&dcp->bl_register_mutex); @@ -444,8 +451,7 @@ int dcp_backlight_update(struct apple_dcp *dcp) dcp_backlight_kick(dcp); return 0; } - if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) + if (dcp_backlight_presents_nits(dcp)) return -ENODATA; /* @@ -537,9 +543,7 @@ int dcp_backlight_register(struct apple_dcp *dcp) .scale = BACKLIGHT_SCALE_LINEAR, }; props.max_brightness = min(dcp->brightness.maximum, MAX_BRIGHTNESS_PART2 - 1); - if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !dcp_backlight_active(dcp)) + if (dcp_backlight_presents_nits(dcp) && !dcp_backlight_active(dcp)) return -ENODATA; if (dcp_backlight_active(dcp)) { spin_lock_irqsave(&dcp->backlight.lock, flags); diff --git a/drivers/gpu/drm/apple/dcp_external_announce_test.c b/drivers/gpu/drm/apple/dcp_external_announce_test.c index 13dae8d2d8942..bd0af843698f9 100644 --- a/drivers/gpu/drm/apple/dcp_external_announce_test.c +++ b/drivers/gpu/drm/apple/dcp_external_announce_test.c @@ -108,11 +108,15 @@ static struct announce_test *announce_test_init(struct kunit *test) INIT_DELAYED_WORK(&dcp->announce_wq, dcp_announce_work); INIT_DELAYED_WORK(&dcp->typec_reconnect_wq, announce_test_noop); INIT_DELAYED_WORK(&dcp->placeholder_edid_wq, announce_test_noop); + INIT_DELAYED_WORK(&dcp->typec_relink_wq, announce_test_noop); + INIT_DELAYED_WORK(&dcp->hdmi_settle_wq, announce_test_noop); + INIT_DELAYED_WORK(&dcp->hdmi_recheck_wq, announce_test_noop); INIT_DELAYED_WORK(&dcp->bl_fallback_wq, announce_test_noop); INIT_DELAYED_WORK(&dcp->backlight.fade_work, announce_test_noop); INIT_WORK(&dcp->external_stop_work, announce_test_noop); INIT_DELAYED_WORK(&dcp->external_stop_retry_wq, announce_test_noop); INIT_WORK(&dcp->vblank_wq, announce_test_noop); + INIT_WORK(&dcp->dimensions_wq, announce_test_noop); INIT_WORK(&t->con.hotplug_wq, announce_test_event); mutex_init(&t->drm.mode_config.mutex); diff --git a/drivers/gpu/drm/apple/dcp_external_audio_test.c b/drivers/gpu/drm/apple/dcp_external_audio_test.c new file mode 100644 index 0000000000000..c1c32e2c32d3c --- /dev/null +++ b/drivers/gpu/drm/apple/dcp_external_audio_test.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by dcp.c: the native external pipe's audio work across probe, + * bind, sleep and unbind. The work function is swapped for a counter, so a + * start that queued it shows. + */ +#include + +static atomic_t dcp_ext_audio_runs; + +static void dcp_ext_audio_count(struct work_struct *work) +{ + atomic_inc(&dcp_ext_audio_runs); +} + +static unsigned int dcp_ext_audio_start_runs(struct apple_dcp *dcp) +{ + atomic_set(&dcp_ext_audio_runs, 0); + dcp_external_audio_start(dcp); + flush_work(&dcp->external_audio_work); + return atomic_read(&dcp_ext_audio_runs); +} + +static void dcp_ext_audio_stop(void *data) +{ + struct apple_dcp *dcp = data; + + disable_work_sync(&dcp->external_audio_work); +} + +static struct apple_dcp *dcp_ext_audio_probe(struct kunit *test, bool native) +{ + struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, dcp); + dcp->fw_compat = DCP_FIRMWARE_V_14_7; + dcp->external = native; + dcp->external_native = native; + dcp_external_audio_init(dcp); + dcp->external_audio_work.func = dcp_ext_audio_count; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, dcp_ext_audio_stop, dcp), 0); + return dcp; +} + +/* + * Probe leaves the work disabled: a first EDID before bind queues nothing, + * and bind's single enable is balanced. Suspend and resume, unbind and a + * second bind keep it so. + */ +static void dcp_ext_audio_lifecycle(struct kunit *test) +{ + struct apple_dcp *dcp = dcp_ext_audio_probe(test, true); + + KUNIT_EXPECT_EQ(test, dcp_ext_audio_start_runs(dcp), 0U); + /* bind */ + KUNIT_EXPECT_TRUE(test, enable_work(&dcp->external_audio_work)); + KUNIT_EXPECT_EQ(test, dcp_ext_audio_start_runs(dcp), 1U); + /* suspend, then resume */ + disable_work_sync(&dcp->external_audio_work); + KUNIT_EXPECT_EQ(test, dcp_ext_audio_start_runs(dcp), 0U); + KUNIT_EXPECT_TRUE(test, enable_work(&dcp->external_audio_work)); + /* unbind, then bind again */ + disable_work_sync(&dcp->external_audio_work); + KUNIT_EXPECT_TRUE(test, enable_work(&dcp->external_audio_work)); + KUNIT_EXPECT_EQ(test, dcp_ext_audio_start_runs(dcp), 1U); +} + +/* Only a native external pipe starts it, and only once per session. */ +static void dcp_ext_audio_start_gates(struct kunit *test) +{ + struct apple_dcp *other = dcp_ext_audio_probe(test, false); + struct apple_dcp *dcp = dcp_ext_audio_probe(test, true); + + KUNIT_EXPECT_TRUE(test, enable_work(&other->external_audio_work)); + KUNIT_EXPECT_EQ(test, dcp_ext_audio_start_runs(other), 0U); + KUNIT_EXPECT_TRUE(test, enable_work(&dcp->external_audio_work)); + dcp->external_audio_started = true; + KUNIT_EXPECT_EQ(test, dcp_ext_audio_start_runs(dcp), 0U); +} + +static struct kunit_case dcp_ext_audio_cases[] = { + KUNIT_CASE(dcp_ext_audio_lifecycle), + KUNIT_CASE(dcp_ext_audio_start_gates), + {} +}; + +static struct kunit_suite dcp_ext_audio_suite = { + .name = "apple-dcp-external-audio-work", + .test_cases = dcp_ext_audio_cases, +}; + +kunit_test_suite(dcp_ext_audio_suite); diff --git a/drivers/gpu/drm/apple/dptx-drive-v27.inc b/drivers/gpu/drm/apple/dptx-drive-v27.inc new file mode 100644 index 0000000000000..9b8c3e6af1fa2 --- /dev/null +++ b/drivers/gpu/drm/apple/dptx-drive-v27.inc @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ +struct dptxport_drive_v27 { + __le32 status_or_count; + u8 reserved0[12]; + __le32 count_or_status; + u8 reserved1[12]; + struct { + __le32 type; + __le32 voltage; + __le32 pre; + } lane[]; +} __packed; + +static int dptxport_call_drive_v27(struct apple_epic_service *service, bool set, + const void *data, size_t data_size, + void *reply_, size_t reply_size) +{ + const struct dptxport_drive_v27 *request = data; + struct dptxport_drive_v27 *reply = reply_; + struct dptx_port *dptx = service->cookie; + union phy_configure_opts opts = {}; + u32 count; + size_t size; + int ret; + + if (data_size < sizeof(*request) || reply_size < sizeof(*reply)) + return -EINVAL; + count = le32_to_cpu(set ? request->count_or_status : request->status_or_count); + if (count > 4 || count > dptx->lane_count) + return -EINVAL; + size = sizeof(*request) + count * sizeof(request->lane[0]); + if ((set && data_size < size) || (!set && reply_size < size)) + return -EINVAL; + + if (set) { + for (u32 i = 0; i < count; i++) { + + if (le32_to_cpu(request->lane[i].type) || + le32_to_cpu(request->lane[i].voltage) > 3 || + le32_to_cpu(request->lane[i].pre) > 3 || + le32_to_cpu(request->lane[i].voltage) + + le32_to_cpu(request->lane[i].pre) > 3) + return -EINVAL; + opts.dp.voltage[i] = le32_to_cpu(request->lane[i].voltage); + opts.dp.pre[i] = le32_to_cpu(request->lane[i].pre); + } + opts.dp.lanes = count; + opts.dp.link_rate = dptx->phy_ops.dp.link_rate; + opts.dp.set_voltages = 1; + if (count && !service->ep->dcp->dptx_tunnel) { + if (!dptx->atcphy) + return -ENODEV; + ret = phy_configure(dptx->atcphy, &opts); + if (ret) + return ret; + } + for (u32 i = 0; i < count; i++) { + dptx->drive_voltage[i] = opts.dp.voltage[i]; + dptx->drive_pre[i] = opts.dp.pre[i]; + } + } + + memset(reply, 0, reply_size); + if (set) { + reply->count_or_status = cpu_to_le32(count); + } else { + + reply->status_or_count = cpu_to_le32(count); + } + + if (reply_size >= size) { + for (u32 i = 0; i < count; i++) { + reply->lane[i].voltage = cpu_to_le32(dptx->drive_voltage[i]); + reply->lane[i].pre = cpu_to_le32(dptx->drive_pre[i]); + } + } + return 0; +} diff --git a/drivers/gpu/drm/apple/dptxep.c b/drivers/gpu/drm/apple/dptxep.c index d61b28e04546a..d07a549c0aa0b 100644 --- a/drivers/gpu/drm/apple/dptxep.c +++ b/drivers/gpu/drm/apple/dptxep.c @@ -86,6 +86,8 @@ struct dptxport_apcall_set_tiled { __le32 retcode; }; +#include "dptx-drive-v27.inc" + /* * SetTiledDisplayHints payload, as DCP sends it for an LG UltraFine 5K * (offsets in bytes): tile location (column, row) at 0x08/0x0c, tiles @@ -200,6 +202,7 @@ int dptxport_request_display(struct apple_epic_service *service) int dptxport_release_display(struct apple_epic_service *service) { dcp_180hz_invalidate(service->ep->dcp); + dcpavserv_v14_close(service->ep->dcp); return afk_service_call(service, 0, 7, NULL, 0, 16, NULL, 0, 16); } @@ -210,6 +213,9 @@ int dptxport_set_hpd_timeout(struct apple_epic_service *service, bool hpd, size_t received; int ret; + /* The 14.7 external role completes an HPD low only once its AV client closed. */ + if (!hpd) + dcpavserv_v14_close(service->ep->dcp); memset(&cmd, 0, sizeof(cmd)); if (hpd) @@ -266,6 +272,11 @@ dptxport_call_get_max_drive_settings(struct apple_epic_service *service, reply->max_drive_settings[0] = cpu_to_le32(0x3); reply->max_drive_settings[1] = cpu_to_le32(0x3); + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) { + /* Protocol-27 drive settings: type, voltage, pre-emphasis. */ + reply->max_drive_settings[0] = cpu_to_le32(0); + memcpy(reply->_unk1, &reply->max_drive_settings[1], sizeof(__le32)); + } return 0; } @@ -675,8 +686,11 @@ dptxport_call_activate(struct apple_epic_service *service, * (DPTX_INACTIVE=0) from here; waking it earlier hangs the machine. * Activate always replies success to DCP. */ - if (dptx->atcphy && !dcp->phy_managed_by_typec) - phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (dptx->atcphy && !dcp->phy_managed_by_typec) { + ret = phy_set_mode_ext(dptx->atcphy, PHY_MODE_DP, dcp->index); + if (ret) + return ret; + } if (dcp->dptx_tunnel) { ret = dcp_tunnel_dpin_activate(dcp, dptx->unit, true); /* T6030 14.7: report a failed tunnel activation to DCP. */ @@ -845,6 +859,9 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, case DPTX_APCALL_SET_TILED_DISPLAY_HINTS: { int ret; + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dcp_peer_hints_v27(service->ep->dcp, dptx, data, + data_size, reply, reply_size); memcpy(reply, data, min(reply_size, data_size)); ret = dptxport_call_set_tiled_display_hint(service, data, data_size, reply, @@ -855,9 +872,15 @@ static int dptxport_call(struct apple_epic_service *service, u32 idx, return ret; } case DPTX_APCALL_GET_DRIVE_SETTINGS: + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dptxport_call_drive_v27(service, false, data, data_size, + reply, reply_size); return dptxport_call_get_drive_settings(service, data, data_size, reply, reply_size); case DPTX_APCALL_SET_DRIVE_SETTINGS: + if (service->ep->dcp->fw_compat == DCP_FIRMWARE_V_27_0) + return dptxport_call_drive_v27(service, true, data, data_size, + reply, reply_size); return dptxport_call_set_drive_settings(service, data, data_size, reply, reply_size); case DPTX_APCALL_ACTIVATE: @@ -927,6 +950,21 @@ static int dptxport_prepare(struct apple_epic_service *service, const char *name return dptx_service_bind(service, &dcp->dptxport[port], unit); } +static void dptxport_init_props(struct apple_epic_service *service, s64 unit, + const void *props, size_t size) +{ + struct apple_dcp *dcp = service->ep->dcp; + struct dcp_parse_ctx ctx; + struct dcp_dptx_caps caps; + + if (unit < 0 || unit >= ARRAY_SIZE(dcp->dptxport) || + parse(props, size, &ctx) || parse_dptx_caps(&ctx, &caps)) + return; + dcp->dptxport[unit].caps = caps; + dev_info(dcp->dev, "DPTX%lld caps: width=%u total=%u active=%u\n", + unit, caps.width, caps.total_rate, caps.active_rate); +} + static void dptxport_init(struct apple_epic_service *service, const char *name, const char *class, s64 unit) { @@ -1059,6 +1097,7 @@ static const struct apple_epic_service_ops dptxep_ops[] = { .prepare = dptxport_prepare, .teardown = dptxport_teardown, .init = dptxport_init, + .init_props = dptxport_init_props, .call = dptxport_call, }, { @@ -1112,7 +1151,8 @@ int dptx_controller_init(struct apple_dcp *dcp) int dptxep_init(struct apple_dcp *dcp) { struct apple_dcp_afkep *ep; - bool wire_v2 = dcp->hw.iomfb_method_profile == + bool wire_v2 = dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; int ret; u32 port; diff --git a/drivers/gpu/drm/apple/dptxep.h b/drivers/gpu/drm/apple/dptxep.h index 211e72d9855ea..11cd860f9ee34 100644 --- a/drivers/gpu/drm/apple/dptxep.h +++ b/drivers/gpu/drm/apple/dptxep.h @@ -3,6 +3,7 @@ #include #include +#include "dcp-peer-v27.h" enum dptx_apcall { DPTX_APCALL_ACTIVATE = 0, @@ -49,6 +50,7 @@ enum dptx_link_rate { struct apple_epic_service; struct dptx_port { + struct dcp_dptx_caps caps; bool enabled, connected; /* A published session closed; unlike initial discovery, this is terminal. */ bool closed; @@ -63,6 +65,7 @@ struct dptx_port { u32 link_rate, pending_link_rate; u32 drive_settings[2]; u32 trained_voltage[4], trained_pre[4]; + u32 drive_voltage[4], drive_pre[4]; /* * The sink's tile topology as DCP reports it in SetTiledDisplayHints: * tiles across and down, and this stream's tile. Valid once diff --git a/drivers/gpu/drm/apple/epic/dpavservep-v14-test.c b/drivers/gpu/drm/apple/epic/dpavservep-v14-test.c new file mode 100644 index 0000000000000..1912f87454e5d --- /dev/null +++ b/drivers/gpu/drm/apple/epic/dpavservep-v14-test.c @@ -0,0 +1,508 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by dpavservep.c: the AV client of the DCP 14.7 external role. + * The production open, close, Terminate, Close work and EDID copy run on a + * fake service instance; only the client calls and the EDID read are + * stubbed. The Close work is disabled for the workqueue and run here, in + * test context, where the stubs apply; the Open stub can act while the Open + * is on its way, to run each interleaving. + */ +#include +#include +#include + +#define AV14_OPEN 4 +#define AV14_CLOSE 5 + +enum av14_meanwhile { + AV14_NOTHING, + AV14_HPD_LOW, /* dcpavserv_v14_close(), as before an HPD low */ + AV14_TERMINATE, /* the firmware's Terminate report */ + AV14_REPLACED, /* teardown, and a successor in the same slot */ + AV14_COPY, /* another EDID copy */ +}; + +struct av14_test { + struct apple_dcp *dcp; + struct apple_dcp_afkep ep; + struct apple_epic_service svc; + /* Client calls, as group << 16 | command, and the epoch each went to */ + u32 calls[8]; + u64 call_epochs[8]; + unsigned int ncalls; + int call_ret; + u32 open_status; + u64 open_opened; + u32 close_status; + enum av14_meanwhile during_open; + unsigned int reads; + bool read_valid; +}; + +static struct av14_test *av14_current; + +static void av14_report(struct av14_test *t, u16 group, u32 command, u32 magic, size_t size); +static void av14_replace(struct av14_test *t); + +static int av14_call(struct apple_epic_service *service, u16 group, u32 command, + void *reply, size_t size) +{ + struct av14_test *t = av14_current; + enum av14_meanwhile meanwhile = AV14_NOTHING; + + if (t->ncalls < ARRAY_SIZE(t->calls)) { + t->calls[t->ncalls] = (u32)group << 16 | command; + t->call_epochs[t->ncalls] = t->dcp->dcpavserv.epoch; + } + t->ncalls++; + if (command == AV14_OPEN) { + meanwhile = t->during_open; + t->during_open = AV14_NOTHING; + } + switch (meanwhile) { + case AV14_HPD_LOW: + dcpavserv_v14_close(t->dcp); + break; + case AV14_TERMINATE: + av14_report(t, 2, 1, EPIC_SERVICE_CALL_MAGIC, sizeof(struct epic_service_call)); + break; + case AV14_REPLACED: + av14_replace(t); + break; + case AV14_COPY: { + u64 epoch; + const struct drm_edid *edid = dcpavserv_copy_edid(t->dcp, 0, &epoch); + + if (!IS_ERR_OR_NULL(edid)) + drm_edid_free(edid); + break; + } + case AV14_NOTHING: + break; + } + if (t->call_ret) + return t->call_ret; + memset(reply, 0, size); + if (command == AV14_OPEN && size >= 24) { + put_unaligned_le32(t->open_status, reply); + put_unaligned_le64(t->open_opened, reply + 16); + } else if (command == AV14_CLOSE && size >= 4) { + put_unaligned_le32(t->close_status, reply); + } + return 0; +} + +/* One EDID base block: valid, or with its checksum off by one */ +static const struct drm_edid *av14_read(struct apple_epic_service *service) +{ + u8 block[EDID_LENGTH] = { 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00 }; + unsigned int i; + u8 sum = 0; + + av14_current->reads++; + block[18] = 1; /* EDID 1.3 */ + block[19] = 3; + for (i = 0; i < EDID_LENGTH - 1; i++) + sum += block[i]; + block[EDID_LENGTH - 1] = 0x100 - sum + (av14_current->read_valid ? 0 : 1); + return drm_edid_alloc(block, sizeof(block)); +} + +static void av14_stop(void *data) +{ + struct apple_dcp *dcp = data; + + cancel_work_sync(&dcp->dcpavserv.close_wq); +} + +/* Publish the instance in the slot, as dcpavserv_init() does */ +static void av14_publish(struct av14_test *t) +{ + t->svc.enabled = true; + t->svc.torndown = false; + guard(mutex)(&t->dcp->dcpavserv.lock); + t->dcp->dcpavserv.service = afk_service_get(&t->svc); + t->dcp->dcpavserv.enabled = true; + t->dcp->dcpavserv.epoch++; + t->dcp->dcpavserv.opened = NULL; + t->dcp->dcpavserv.opened_epoch = 0; +} + +/* The instance torn down (production teardown) and a successor in its slot */ +static void av14_replace(struct av14_test *t) +{ + dcpavserv_teardown(&t->svc); + av14_publish(t); +} + +/* A published AV instance, on the 14.7 external role or (!native) not */ +static struct av14_test *av14_setup(struct kunit *test, bool native) +{ + struct av14_test *t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + struct device *dev; + + KUNIT_ASSERT_NOT_NULL(test, t); + t->dcp = kunit_kzalloc(test, sizeof(*t->dcp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, t->dcp); + dev = kunit_device_register(test, native ? "dcpavserv-v14" : "dcpavserv-v14-other"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + t->dcp->dev = dev; + t->dcp->fw_compat = DCP_FIRMWARE_V_14_7; + t->dcp->external = native; + t->dcp->external_native = native; + mutex_init(&t->dcp->hpd_mutex); + mutex_init(&t->dcp->dcpavserv.lock); + INIT_WORK(&t->dcp->dcpavserv.close_wq, dcpavserv_close_work); + /* Queued work runs here, in test context: av14_run_close_work() */ + disable_work(&t->dcp->dcpavserv.close_wq); + t->ep.dcp = t->dcp; + spin_lock_init(&t->svc.lock); + t->svc.ep = &t->ep; + av14_publish(t); + t->open_opened = 1; + t->read_valid = true; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, av14_stop, t->dcp), 0); + av14_current = t; + kunit_activate_static_stub(test, dcpavserv_v14_call, av14_call); + kunit_activate_static_stub(test, dcpavserv_read_edid_legacy, av14_read); + return t; +} + +static int av14_copy(struct kunit *test, struct av14_test *t) +{ + const struct drm_edid *edid; + u64 epoch; + + edid = dcpavserv_copy_edid(t->dcp, 0, &epoch); + if (IS_ERR(edid)) + return PTR_ERR(edid); + KUNIT_EXPECT_NOT_NULL(test, edid); + drm_edid_free(edid); + return 0; +} + +static void av14_run_close_work(struct av14_test *t) +{ + dcpavserv_close_work(&t->dcp->dcpavserv.close_wq); +} + +static void av14_report(struct av14_test *t, u16 group, u32 command, u32 magic, size_t size) +{ + struct epic_service_call call = { + .group = cpu_to_le16(group), + .command = cpu_to_le32(command), + .magic = cpu_to_le32(magic), + }; + + dcpavserv_report(&t->svc, EPIC_SUBTYPE_STD_SERVICE, &call, size); +} + +static void av14_terminate(struct av14_test *t) +{ + av14_report(t, 2, 1, EPIC_SERVICE_CALL_MAGIC, sizeof(struct epic_service_call)); +} + +/* Each instance is opened once (DCPAV Open, group 0), before its first EDID read. */ +static void av14_open_once(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_EXPECT_EQ(test, t->ncalls, 1U); + KUNIT_EXPECT_EQ(test, t->calls[0], (u32)AV14_OPEN); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->dcpavserv.opened, &t->svc); + KUNIT_EXPECT_EQ(test, t->dcp->dcpavserv.opened_epoch, t->dcp->dcpavserv.epoch); + KUNIT_EXPECT_FALSE(test, t->dcp->dcpavserv.opening); + KUNIT_EXPECT_EQ(test, t->reads, 2U); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* A refused open is not recorded: the read goes on, the next copy opens again. */ +static void av14_open_refused(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + t->open_status = 0xe00002bc; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + t->open_status = 0; + t->open_opened = 0; /* answered, but not opened */ + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + t->call_ret = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + KUNIT_EXPECT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_EQ(test, t->reads, 3U); + KUNIT_EXPECT_FALSE(test, t->dcp->dcpavserv.opening); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* Not the 14.7 external role (as the T8122 dcpext): no client calls, EDID as read. */ +static void av14_other_role(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, false); + + t->read_valid = false; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + dcpavserv_v14_close(t->dcp); + av14_terminate(t); + KUNIT_EXPECT_EQ(test, t->ncalls, 0U); + KUNIT_EXPECT_EQ(test, t->reads, 1U); +} + +/* The 14.7 external role's EDID is checked block by block before it is published. */ +static void av14_edid_checked(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + t->read_valid = false; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), -EPROTO); + t->read_valid = true; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); +} + +/* Close (group 0, command 5) goes only to the instance that was opened, once. */ +static void av14_close_opened_only(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + dcpavserv_v14_close(t->dcp); + KUNIT_EXPECT_EQ(test, t->ncalls, 0U); + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + dcpavserv_v14_close(t->dcp); + dcpavserv_v14_close(t->dcp); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->calls[1], (u32)AV14_CLOSE); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + /* A Close the firmware refuses still leaves nothing recorded open */ + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + t->close_status = 0xe00002bc; + dcpavserv_v14_close(t->dcp); + KUNIT_EXPECT_EQ(test, t->ncalls, 4U); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* + * Terminate (report group 2, command 1) on the opened instance queues its + * Close; the instance gets no new use, before or after that Close, until a + * successor is published. Other reports change nothing. + */ +static void av14_terminate_opened(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + u64 epoch = t->dcp->dcpavserv.epoch; + + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + av14_report(t, 2, 2, EPIC_SERVICE_CALL_MAGIC, sizeof(struct epic_service_call)); + av14_report(t, 1, 1, EPIC_SERVICE_CALL_MAGIC, sizeof(struct epic_service_call)); + av14_report(t, 2, 1, 0, sizeof(struct epic_service_call)); + av14_report(t, 2, 1, EPIC_SERVICE_CALL_MAGIC, sizeof(struct epic_service_call) - 1); + KUNIT_EXPECT_EQ(test, t->dcp->dcpavserv.terminated_epoch, 0ULL); + + av14_terminate(t); + KUNIT_EXPECT_EQ(test, t->dcp->dcpavserv.terminated_epoch, epoch); + KUNIT_EXPECT_EQ(test, t->dcp->dcpavserv.close_epoch, epoch); + /* While its Close is queued: no EDID, no new open */ + KUNIT_EXPECT_EQ(test, av14_copy(test, t), -ENODEV); + av14_run_close_work(t); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->calls[1], (u32)AV14_CLOSE); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + /* Closed: still no new use of the terminated instance */ + KUNIT_EXPECT_EQ(test, av14_copy(test, t), -ENODEV); + av14_run_close_work(t); + KUNIT_EXPECT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->reads, 1U); + /* Its successor, in the same slot, is opened and read */ + av14_replace(t); + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_EXPECT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* + * A Close (as before an HPD low) after a Terminate does not end the + * Terminate: whether it comes after the queued Close ran or before it, no + * EDID copy opens or reads the terminated instance again. The successor, a + * new epoch in the same slot, is opened. + */ +static void av14_terminate_then_close(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + u64 epoch; + int round; + + for (round = 0; round < 2; round++) { + unsigned int calls = t->ncalls, reads = t->reads; + + epoch = t->dcp->dcpavserv.epoch; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + av14_terminate(t); + if (!round) + av14_run_close_work(t); + dcpavserv_v14_close(t->dcp); + if (round) + av14_run_close_work(t); + /* One Open, one Close, in this instance */ + KUNIT_ASSERT_EQ(test, t->ncalls, calls + 2); + KUNIT_EXPECT_EQ(test, t->calls[calls], (u32)AV14_OPEN); + KUNIT_EXPECT_EQ(test, t->calls[calls + 1], (u32)AV14_CLOSE); + KUNIT_EXPECT_EQ(test, t->dcp->dcpavserv.terminated_epoch, epoch); + /* Not opened or read again */ + KUNIT_EXPECT_EQ(test, av14_copy(test, t), -ENODEV); + KUNIT_EXPECT_EQ(test, av14_copy(test, t), -ENODEV); + KUNIT_EXPECT_EQ(test, t->ncalls, calls + 2); + KUNIT_EXPECT_EQ(test, t->reads, reads + 1); + /* Its successor in the same slot: a new epoch, opened */ + av14_replace(t); + KUNIT_EXPECT_NE(test, t->dcp->dcpavserv.epoch, epoch); + } + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_EXPECT_EQ(test, t->calls[4], (u32)AV14_OPEN); + KUNIT_EXPECT_EQ(test, t->call_epochs[4], t->dcp->dcpavserv.epoch); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* An HPD low's Close while the Open is on its way: its opener closes the client. */ +static void av14_open_vs_close(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + t->during_open = AV14_HPD_LOW; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), -ENODEV); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->calls[0], (u32)AV14_OPEN); + KUNIT_EXPECT_EQ(test, t->calls[1], (u32)AV14_CLOSE); + KUNIT_EXPECT_EQ(test, t->call_epochs[1], t->call_epochs[0]); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + KUNIT_EXPECT_FALSE(test, t->dcp->dcpavserv.opening); + KUNIT_EXPECT_EQ(test, t->reads, 0U); + /* The next EDID copy opens the instance again */ + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_EXPECT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->dcpavserv.opened, &t->svc); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* A Terminate while the Open is on its way: closed by its opener, no new use. */ +static void av14_open_vs_terminate(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + t->during_open = AV14_TERMINATE; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), -ENODEV); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->calls[1], (u32)AV14_CLOSE); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + /* No Close work was queued for it, and running one sends nothing */ + av14_run_close_work(t); + KUNIT_EXPECT_EQ(test, av14_copy(test, t), -ENODEV); + KUNIT_EXPECT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->reads, 0U); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* + * The instance torn down and its slot reused while the Open is on its way: + * the old open went with the old instance; nothing is closed, and the + * successor is opened by the next copy. + */ +static void av14_open_vs_replaced(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + t->during_open = AV14_REPLACED; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), -ENODEV); + KUNIT_EXPECT_EQ(test, t->ncalls, 1U); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_EQ(test, t->calls[1], (u32)AV14_OPEN); + KUNIT_EXPECT_NE(test, t->call_epochs[1], t->call_epochs[0]); + KUNIT_EXPECT_EQ(test, t->dcp->dcpavserv.opened_epoch, t->dcp->dcpavserv.epoch); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* + * An old instance's Close work, queued at its Terminate, runs after its + * successor took the same slot and was opened: the successor stays open. + */ +static void av14_old_close_work_vs_successor(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + av14_terminate(t); + av14_replace(t); + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_ASSERT_EQ(test, t->ncalls, 2U); + av14_run_close_work(t); + KUNIT_EXPECT_EQ(test, t->ncalls, 2U); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->dcpavserv.opened, &t->svc); + KUNIT_EXPECT_EQ(test, t->dcp->dcpavserv.opened_epoch, t->dcp->dcpavserv.epoch); + /* A Terminate of the successor closes the successor */ + av14_terminate(t); + av14_run_close_work(t); + KUNIT_ASSERT_EQ(test, t->ncalls, 3U); + KUNIT_EXPECT_EQ(test, t->calls[2], (u32)AV14_CLOSE); + KUNIT_EXPECT_EQ(test, t->call_epochs[2], t->dcp->dcpavserv.epoch); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* Another EDID copy while the Open is on its way does not open the instance again. */ +static void av14_open_vs_copy(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + t->during_open = AV14_COPY; + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + KUNIT_EXPECT_EQ(test, t->ncalls, 1U); + KUNIT_EXPECT_EQ(test, t->reads, 2U); + KUNIT_EXPECT_PTR_EQ(test, t->dcp->dcpavserv.opened, &t->svc); + KUNIT_EXPECT_FALSE(test, t->dcp->dcpavserv.opening); + KUNIT_EXPECT_EQ(test, t->svc.users, 1U); +} + +/* Detach drops the open and waits for the Close work; the slot's pin goes. */ +static void av14_detach(struct kunit *test) +{ + struct av14_test *t = av14_setup(test, true); + + KUNIT_EXPECT_EQ(test, av14_copy(test, t), 0); + av14_terminate(t); + dpavservep_detach(t->dcp); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.opened); + KUNIT_EXPECT_NULL(test, t->dcp->dcpavserv.service); + /* The Close work for the detached instance sends nothing */ + av14_run_close_work(t); + KUNIT_EXPECT_EQ(test, t->ncalls, 1U); + KUNIT_EXPECT_EQ(test, t->svc.users, 0U); + KUNIT_EXPECT_TRUE(test, t->svc.torndown); +} + +static struct kunit_case av14_cases[] = { + KUNIT_CASE(av14_open_once), + KUNIT_CASE(av14_open_refused), + KUNIT_CASE(av14_other_role), + KUNIT_CASE(av14_edid_checked), + KUNIT_CASE(av14_close_opened_only), + KUNIT_CASE(av14_terminate_opened), + KUNIT_CASE(av14_terminate_then_close), + KUNIT_CASE(av14_open_vs_close), + KUNIT_CASE(av14_open_vs_terminate), + KUNIT_CASE(av14_open_vs_replaced), + KUNIT_CASE(av14_old_close_work_vs_successor), + KUNIT_CASE(av14_open_vs_copy), + KUNIT_CASE(av14_detach), + {} +}; + +static struct kunit_suite av14_suite = { + .name = "apple-dcpavserv-v14", + .test_cases = av14_cases, +}; + +kunit_test_suite(av14_suite); diff --git a/drivers/gpu/drm/apple/epic/dpavservep.c b/drivers/gpu/drm/apple/epic/dpavservep.c index bec8f711fbd14..35c1064da5ecc 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.c +++ b/drivers/gpu/drm/apple/epic/dpavservep.c @@ -7,10 +7,12 @@ #define EDID_EXT_BLOCK_COUNT_OFFSET 0x7E #include +#include #include #include #include +#include #include "../afk.h" #include "../dcp.h" @@ -121,6 +123,9 @@ static void dcpavserv_init(struct apple_epic_service *service, const char *name, if (dcp->dcpavserv.service) { dcp->dcpavserv.enabled = true; dcp->dcpavserv.epoch++; + /* A new instance: the host is no client of it yet. */ + dcp->dcpavserv.opened = NULL; + dcp->dcpavserv.opened_epoch = 0; service->cookie = &dcp->dcpavserv; complete(&dcp->dcpavserv.enable_completion); published = true; @@ -168,7 +173,9 @@ static void dcpavserv_teardown(struct apple_epic_service *service) dcp->dcpavserv.enabled = false; dcp->dcpavserv.service = NULL; dcp->dcpavserv.epoch++; + /* A removed instance took its open with it. */ dcp->dcpavserv.opened = NULL; + dcp->dcpavserv.opened_epoch = 0; dcpavserv_invalidate_edid(dcp); reinit_completion(&dcp->dcpavserv.enable_completion); owned = true; @@ -189,12 +196,14 @@ void dpavservep_detach(struct apple_dcp *dcp) /* Also reject an in-flight prepare/OPEN with no published owner yet. */ dcp->dcpavserv.epoch++; dcp->dcpavserv.opened = NULL; + dcp->dcpavserv.opened_epoch = 0; reinit_completion(&dcp->dcpavserv.enable_completion); if (service) { afk_service_retire(service); dcpavserv_invalidate_edid(dcp); } mutex_unlock(&dcp->dcpavserv.lock); + cancel_work_sync(&dcp->dcpavserv.close_wq); if (service) afk_service_put(service); } @@ -296,10 +305,180 @@ static int parse_report(struct apple_epic_service *service, enum epic_subtype ty return 0; } +/* + * The 14.7 external role: the host opens the published AV instance as a + * client (DCPAV Open, command 4, a zero 32-byte body; the reply carries a + * status at +0 and the opened Boolean at +16) to read the sink's EDID + * through it. An open client is one the firmware waits for, up to 10 s, + * before it removes the instance: it holds the reply to the sink's HPD low + * until the instance's device has closed, and with it the sink's release and + * the next sink. So the client is closed (DCPAV Close, command 5, a zero + * 16-byte body, status at +0) before every HPD low and when the firmware + * terminates the instance (report group 2, command 1). Close goes only to the + * instance that was opened; a removed instance took its open with it. + * Neither call is made under dcpavserv.lock, which never waits for firmware. + * + * Each instance is named by its epoch, which changes even when its slot is + * reused for the next one. A Close or a Terminate that comes while an Open + * is on its way cancels that Open's publication (close_gen): its opener + * then closes the client itself, if the instance is still the current one. + * Work queued for an instance acts on that instance only. + */ +static int dcpavserv_v14_call(struct apple_epic_service *service, u16 group, u32 command, + void *reply, size_t size) +{ + KUNIT_STATIC_STUB_REDIRECT(dcpavserv_v14_call, service, group, command, reply, size); + return afk_service_call(service, group, command, NULL, 0, size, reply, size, 0); +} + +/* Close the client of @service, pinned by the caller; drops that pin. */ +static void dcpavserv_v14_send_close(struct apple_dcp *dcp, struct apple_epic_service *service) +{ + u8 reply[16] = {}; + int ret; + + ret = dcpavserv_v14_call(service, 0, 5, reply, sizeof(reply)); + afk_service_put(service); + /* Removed while the Close was on its way: nothing is left open. */ + if (ret == -ENODEV) + return; + if (!ret && get_unaligned_le32(reply)) + ret = -EIO; + if (ret) + dev_err(dcp->dev, "DCPAVSERV: AV client close failed: %d\n", ret); + else + dev_info(dcp->dev, "DCPAVSERV: AV client closed\n"); +} + +/* + * The open client of instance @epoch, pinned, if that instance is still the + * published one; the client is no longer recorded open either way. + */ +static struct apple_epic_service *dcpavserv_v14_take_opened(struct apple_dcp *dcp, u64 epoch) +{ + struct apple_epic_service *service = NULL; + + lockdep_assert_held(&dcp->dcpavserv.lock); + if (!dcp->dcpavserv.opened || dcp->dcpavserv.opened_epoch != epoch) + return NULL; + if (dcp->dcpavserv.epoch == epoch && dcp->dcpavserv.opened == dcp->dcpavserv.service) + service = afk_service_get(dcp->dcpavserv.service); + dcp->dcpavserv.opened = NULL; + dcp->dcpavserv.opened_epoch = 0; + return service; +} + +/* + * Open the client of @service, instance @epoch, pinned by the caller, for an + * Open admitted at close generation @gen. 0 once it is recorded open; + * -ECANCELED when a Close or a Terminate came meanwhile (the client opened + * is closed again here) or the instance went; otherwise the Open's error. + */ +static int dcpavserv_v14_open(struct apple_dcp *dcp, struct apple_epic_service *service, + u64 epoch, u64 gen) +{ + struct apple_epic_service *close = NULL; + u8 reply[32] = {}; + bool still; + int ret; + + ret = dcpavserv_v14_call(service, 0, 4, reply, sizeof(reply)); + if (!ret && (get_unaligned_le32(reply) || !get_unaligned_le64(reply + 16))) { + dev_err(dcp->dev, "DCPAVSERV: open answered %#x, opened %llu\n", + get_unaligned_le32(reply), get_unaligned_le64(reply + 16)); + ret = -EIO; + } + mutex_lock(&dcp->dcpavserv.lock); + dcp->dcpavserv.opening = false; + still = dcp->dcpavserv.service == service && dcp->dcpavserv.epoch == epoch; + if (!ret && still && dcp->dcpavserv.close_gen == gen) { + dcp->dcpavserv.opened = service; + dcp->dcpavserv.opened_epoch = epoch; + } else if (!ret) { + /* An instance gone meanwhile took its open with it. */ + if (still) + close = afk_service_get(service); + ret = -ECANCELED; + } + mutex_unlock(&dcp->dcpavserv.lock); + if (close) + dcpavserv_v14_send_close(dcp, close); + else if (!ret) + dev_info(dcp->dev, "DCPAVSERV: AV client open\n"); + return ret; +} + +void dcpavserv_v14_close(struct apple_dcp *dcp) +{ + struct apple_epic_service *service; + + if (!dcp_is_v14_7_external(dcp)) + return; + mutex_lock(&dcp->dcpavserv.lock); + /* An Open on its way is closed again by its opener. */ + dcp->dcpavserv.close_gen++; + service = dcpavserv_v14_take_opened(dcp, dcp->dcpavserv.epoch); + /* Any older instance's open went with that instance. */ + dcp->dcpavserv.opened = NULL; + dcp->dcpavserv.opened_epoch = 0; + mutex_unlock(&dcp->dcpavserv.lock); + if (service) + dcpavserv_v14_send_close(dcp, service); +} + +/* The Close a Terminate queued: for that instance only, if it is still published. */ +void dcpavserv_close_work(struct work_struct *work) +{ + struct dcpavserv *av = container_of(work, struct dcpavserv, close_wq); + struct apple_dcp *dcp = container_of(av, struct apple_dcp, dcpavserv); + struct apple_epic_service *service; + + mutex_lock(&av->lock); + service = dcpavserv_v14_take_opened(dcp, av->close_epoch); + mutex_unlock(&av->lock); + if (service) + dcpavserv_v14_send_close(dcp, service); +} + +/* + * Terminate on the published instance: no new use of it, an Open on its way + * cancelled, and the open client's Close, queued for that instance. + */ +static void dcpavserv_v14_report(struct apple_epic_service *service, enum epic_subtype type, + const void *data, size_t size) +{ + struct apple_dcp *dcp = service->ep->dcp; + const struct epic_service_call *call = data; + bool close = false; + u64 epoch; + + if (type != EPIC_SUBTYPE_STD_SERVICE || size < sizeof(*call) || + le32_to_cpu(call->magic) != EPIC_SERVICE_CALL_MAGIC || + le16_to_cpu(call->group) != 2 || le32_to_cpu(call->command) != 1) + return; + mutex_lock(&dcp->dcpavserv.lock); + epoch = dcp->dcpavserv.epoch; + if (dcp->dcpavserv.service == service && dcp->dcpavserv.terminated_epoch != epoch) { + dcp->dcpavserv.terminated_epoch = epoch; + dcp->dcpavserv.close_gen++; + if (dcp->dcpavserv.opened == service && dcp->dcpavserv.opened_epoch == epoch) { + dcp->dcpavserv.close_epoch = epoch; + close = true; + } + } + mutex_unlock(&dcp->dcpavserv.lock); + if (!close) + return; + dev_info(dcp->dev, "DCPAVSERV: AV instance terminated: closing the client\n"); + schedule_work(&dcp->dcpavserv.close_wq); +} + static int dcpavserv_report(struct apple_epic_service *service, enum epic_subtype type, const void *data, size_t data_size) { + if (dcp_is_v14_7_external(service->ep->dcp)) + dcpavserv_v14_report(service, type, data, data_size); return parse_report(service, type, data, data_size); } @@ -320,6 +499,7 @@ static const struct drm_edid *dcpavserv_read_edid_legacy(struct apple_epic_servi u64 data_size; int ret; + KUNIT_STATIC_STUB_REDIRECT(dcpavserv_read_edid_legacy, service); memset(&cmd, 0, sizeof(cmd)); cmd.max_size = cpu_to_le64(EDID_BUF_SIZE); resp = kzalloc(resp_size, GFP_KERNEL); @@ -405,8 +585,11 @@ bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch) const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, u64 *epoch) { + bool v14_7_ext = dcp_is_v14_7_external(dcp); struct apple_epic_service *service; const struct drm_edid *edid; + bool open = false; + u64 gen = 0; scoped_guard(mutex, &dcp->hpd_mutex) { if (!dcpavserv_edid_current(dcp, generation)) @@ -415,13 +598,27 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation mutex_lock(&dcp->dcpavserv.lock); service = dcp->dcpavserv.enabled ? afk_service_get(dcp->dcpavserv.service) : NULL; *epoch = dcp->dcpavserv.epoch; + if (service && v14_7_ext) { + /* A terminated instance gets no new use; its successor is published. */ + if (dcp->dcpavserv.terminated_epoch == *epoch) { + afk_service_put(service); + service = NULL; + } else if (dcp->dcpavserv.opened_epoch != *epoch && !dcp->dcpavserv.opening) { + dcp->dcpavserv.opening = true; + gen = dcp->dcpavserv.close_gen; + open = true; + } + } mutex_unlock(&dcp->dcpavserv.lock); if (!service) return ERR_PTR(-ENODEV); /* * The 14.7 firmware of an external processor answers EDID requests - * once its service has been opened; open each announced instance once. + * through an instance its client has opened (DCPAV Open): open each + * announced instance once, and close it again before the sink's + * release (dcpavserv_v14_close()). An open the firmware refuses is + * reported; the read that follows validates its own result. * * Only on Aurora's DCPLink session (T6030). The T8122 dcpext, which * Linux starts and drives through the IOMFB template, reads the EDID @@ -429,19 +626,26 @@ const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation * open call and drops the display link instead, about 50 ms after it, * which brings the display back 4 s later: hotplug in a loop. */ - if (dcp->fw_compat == DCP_FIRMWARE_V_14_7 && - READ_ONCE(dcp->dcpavserv.opened) != service) { - int ret = afk_service_call(service, 4, 6, NULL, 0, 32, NULL, 0, 32); + if (open) { + int ret = dcpavserv_v14_open(dcp, service, *epoch, gen); - if (ret) { + /* Closed, terminated or gone meanwhile: no read through it. */ + if (ret == -ECANCELED) { afk_service_put(service); - return ERR_PTR(ret); + return ERR_PTR(-ENODEV); } - WRITE_ONCE(dcp->dcpavserv.opened, service); + if (ret) + dev_warn(dcp->dev, "DCPAVSERV: AV client not opened: %d\n", ret); } edid = service->ep->wire_v2 ? dcpavserv_read_edid_inline(service) : dcpavserv_read_edid_legacy(service); + /* The 14.7 external role: every block's header and checksum. */ + if (v14_7_ext && !IS_ERR(edid) && !drm_edid_valid(edid)) { + dev_warn(dcp->dev, "DCPAVSERV: the sink's EDID is not valid\n"); + drm_edid_free(edid); + edid = ERR_PTR(-EPROTO); + } mutex_lock(&dcp->dcpavserv.lock); if (dcp->dcpavserv.service != service || !dcpavserv_edid_service_current(dcp, *epoch)) { @@ -496,7 +700,8 @@ int dpavservep_init(struct apple_dcp *dcp) if (IS_ERR(ep)) return PTR_ERR(ep); /* External H17P uses AFKV2 service calls with host-owned OOB buffers. */ - ep->wire_v2 = dcp->hw.iomfb_method_profile == + ep->wire_v2 = dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL; ep->inline_v2 = false; dcp->dcpavservep = ep; @@ -527,3 +732,7 @@ int dpavservep_init(struct apple_dcp *dcp) return 0; } + +#if IS_ENABLED(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) +#include "dpavservep-v14-test.c" +#endif diff --git a/drivers/gpu/drm/apple/epic/dpavservep.h b/drivers/gpu/drm/apple/epic/dpavservep.h index 001e211feeb6b..bf1762963c98a 100644 --- a/drivers/gpu/drm/apple/epic/dpavservep.h +++ b/drivers/gpu/drm/apple/epic/dpavservep.h @@ -7,6 +7,7 @@ #include #include #include +#include struct drm_edid; struct apple_epic_service; @@ -22,10 +23,34 @@ struct dcpavserv { struct completion enable_completion; u32 unit; struct apple_epic_service *service; - /* The service instance last opened; compared only, never used. */ + /* + * The 14.7 external role's AV client, per instance: an instance is + * named by the epoch it was published in, so a slot reused for its + * successor is not mistaken for it. + * + * opened/opened_epoch: the instance the host opened as a client (DCPAV + * Open), closed again before the sink's HPD low or at its Terminate; + * the pointer is compared, and pinned before use. + * opening: an Open is on its way. close_gen: bumped by every Close and + * Terminate; an Open that sees it change was cancelled, and its opener + * closes what it opened. + * terminated_epoch: Terminate reported; that instance gets no new use. + * close_epoch: the instance the Close work was queued for. + */ struct apple_epic_service *opened; + u64 opened_epoch; + bool opening; + u64 close_gen; + u64 terminated_epoch; + u64 close_epoch; + struct work_struct close_wq; }; +/* The 14.7 external role: close the AV client before the sink is released. */ +void dcpavserv_v14_close(struct apple_dcp *dcp); +/* Its Terminate work; set up once with the DCP (dcp_platform_probe_pipe()). */ +void dcpavserv_close_work(struct work_struct *work); + bool dcpavserv_edid_current(struct apple_dcp *dcp, u64 generation); bool dcpavserv_edid_service_current(struct apple_dcp *dcp, u64 epoch); const struct drm_edid *dcpavserv_copy_edid(struct apple_dcp *dcp, u64 generation, diff --git a/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc b/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc new file mode 100644 index 0000000000000..b7b617b37994d --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-boolean-properties-v27.inc @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +struct dcp_boolean_property_v27 { + struct list_head node; + char name[64]; + bool value; +}; + +static bool trampoline_set_boolean_property_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + struct dcp_boolean_property_v27 *property; + char name[64]; + + trace_iomfb_callback(dcp, tag, __func__); + + memcpy(name, request, sizeof(name)); + name[sizeof(name) - 1] = '\0'; + put_unaligned_le32(0, out); + + list_for_each_entry(property, &dcp->iomfb_boolean_properties, node) + if (!strcmp(property->name, name)) + goto update; + if (dcp->iomfb_boolean_property_count >= 64) + return true; + property = kzalloc(sizeof(*property), GFP_KERNEL); + if (!property) + return true; + memcpy(property->name, name, sizeof(name)); + dcp->iomfb_boolean_property_count++; + list_add_tail(&property->node, &dcp->iomfb_boolean_properties); +update: + property->value = request[64] & 1; + put_unaligned_le32(1, out); + return true; +} + +void iomfb_firmware_state_cleanup_v27(void *data) +{ + struct apple_dcp *dcp = data; + struct dcp_boolean_property_v27 *property, *next; + + external_counts_cleanup_v27(dcp); + list_for_each_entry_safe(property, next, &dcp->iomfb_boolean_properties, node) { + list_del(&property->node); + kfree(property); + dcp->iomfb_boolean_property_count--; + } +} diff --git a/drivers/gpu/drm/apple/iomfb-dfb.h b/drivers/gpu/drm/apple/iomfb-dfb.h new file mode 100644 index 0000000000000..30e84ab8366c3 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-dfb.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +#ifndef __APPLE_IOMFB_DFB_H__ +#define __APPLE_IOMFB_DFB_H__ + +#include +#include +#include +#include +#include + +static inline u64 apple_dcp_dfb_round_size(u64 size) +{ + unsigned int bits = fls64(size); + u64 unit, half, tail; + + if (bits <= 24) + return size; + unit = 1ULL << (bits - 24); + half = unit >> 1; + tail = size & (unit - 1); + size &= ~(unit - 1); + if (tail > half || (tail == half && (size & unit))) + size += unit; + return size; +} + +static inline u64 apple_dcp_dfb_plane_size(u32 width, u32 height, u32 bpe, + u32 compression) +{ + u64 stride, data, cols, rows, tw = 16, th = 16; + unsigned int order; + + if (!compression) { + stride = ALIGN((u64)width * bpe, 128); + return ALIGN(apple_dcp_dfb_round_size(stride * height), 0x4000); + } + if (compression == 2 && bpe > 4) { + th = 8; + if (bpe == 16) + tw = 8; + } + cols = DIV_ROUND_UP(width, tw); + rows = DIV_ROUND_UP(height, th); + if (compression == 3) { + stride = tw * th * cols * bpe; + data = ALIGN(rows * stride, 128); + return data + 8 * roundup_pow_of_two(cols) * roundup_pow_of_two(rows); + } + order = ilog2(0x4000 / bpe); + stride = ALIGN((u64)width * bpe, (u64)bpe << (order / 2)); + data = ALIGN(ALIGN((u64)height, 1ULL << ((order + 1) / 2)) * stride, 128); + return data + ALIGN(2 * roundup_pow_of_two(cols) * roundup_pow_of_two(rows), 128); +} + +static inline int apple_dcp_dfb_allocation(u32 width, u32 height, u32 format, + u32 compression, u64 region_size, + u32 *surface_format, u32 *allocation) +{ + u32 bpe; + u64 size; + bool alpha_plane = false; + + if (!width || !height || width > 16384 || height > 16384 || !region_size) + return -EINVAL; + if (compression != 0 && compression != 2 && compression != 3) + return -EOPNOTSUPP; + switch (format) { + case 0x62336138: + if (compression == 2) { + format = 0x77343061; + bpe = 8; + } else { + bpe = 4; + alpha_plane = true; + } + break; + case 0x77343061: + bpe = 8; + break; + case 0x42475241: + case 0x77333072: + case 0x6c313072: + bpe = 4; + break; + default: + return -EOPNOTSUPP; + } + size = apple_dcp_dfb_plane_size(width, height, bpe, compression); + if (alpha_plane) + size += apple_dcp_dfb_plane_size(width, height, 1, compression); + if (size > U32_MAX) + return -EOVERFLOW; + if (compression) + size = ALIGN(apple_dcp_dfb_round_size(size), 0x4000); + if (size > region_size) + return -ENOSPC; + if (!size || size > U32_MAX) + return -EOVERFLOW; + *surface_format = format; + *allocation = size; + return 0; +} +#endif diff --git a/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc b/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc new file mode 100644 index 0000000000000..1b3710c1c7627 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-display-identity-v27.inc @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +struct dcp_display_product_request_v27 { + u8 vendor[10]; + __le32 vendor_id; + __le32 vendor_length; + __le32 product_id; + u8 vendor_id_null, product_id_null; +} __packed; + +struct dcp_display_product_reply_v27 { + u8 vendor[10]; + __le32 vendor_id; + __le32 product_id; + u8 success, padding; +} __packed; + +static_assert(sizeof(struct dcp_display_product_request_v27) == 24); +static_assert(sizeof(struct dcp_display_product_reply_v27) == 20); +static_assert(offsetof(struct dcp_display_product_reply_v27, product_id) == 14); + +struct dcp_display_identity_callback_v27 { + void *output; + enum dcp_context_id context; +}; + +static void display_identity_done_v27(struct apple_dcp *dcp, void *out, void *cookie) +{ + const struct dcp_display_product_reply_v27 *reply = out; + struct dcp_display_identity_callback_v27 *cb = cookie; + bool vendor_matches = reply && (!reply->vendor[0] || + (!memcmp(reply->vendor, "APP", 3) && !reply->vendor[3])); + bool display_x1 = reply && (reply->success & 1) && vendor_matches && + (le32_to_cpu(reply->product_id) >> 1) == 0x5721; + + put_unaligned_le32(display_x1, cb->output); + dcp_ack(dcp, cb->context); + kfree(cb); +} + +static bool trampoline_display_identity_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + static const struct dcp_method_entry method = { + .name = "display_product_id", .tag = {'A', '3', '7', '5'}, + }; + static const enum dcp_context_id contexts[] = { + DCP_CONTEXT_CB, DCP_CONTEXT_ASYNC, DCP_CONTEXT_OOBCB, DCP_CONTEXT_OOBASYNC, + }; + struct dcp_channel *channels[] = { + &dcp->ch_cb, &dcp->ch_async, &dcp->ch_oobcb, &dcp->ch_oobasync, + }; + struct dcp_display_product_request_v27 request = { + .vendor_length = cpu_to_le32(10), + }; + struct dcp_display_identity_callback_v27 *cb; + int context = -1; + + trace_iomfb_callback(dcp, tag, __func__); + + for (unsigned int i = 0; i < ARRAY_SIZE(channels); i++) { + struct dcp_channel *ch = channels[i]; + + if (ch->depth && ch->output[ch->depth - 1] == out) { + context = contexts[i]; + break; + } + } + if (WARN_ON(context < 0)) + return true; + cb = kzalloc(sizeof(*cb), GFP_KERNEL); + if (!cb) + return true; + cb->output = out; + cb->context = context; + if (dcp_push_nested(dcp, context, &method, sizeof(request), + sizeof(struct dcp_display_product_reply_v27), &request, + display_identity_done_v27, cb)) { + kfree(cb); + return true; + } + return false; +} diff --git a/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc b/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc new file mode 100644 index 0000000000000..ab5602821d3ca --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-external-counts-v27.inc @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright 2026 Aurora Silicon */ + +static DEFINE_SPINLOCK(external_counts_lock_v27); +static u32 external_pipe_count_v27; +static u32 external_apple_display_count_v27; +static u32 external_suppression_count_v27; + +static bool trampoline_update_external_counts_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + bool plug = request[0] & 1; + u32 apple_display = request[1] & 1; + u32 pipes = get_unaligned_le32(request + 2); + unsigned long flags; + + trace_iomfb_callback(dcp, tag, __func__); + + if (dcp->connector_type == DRM_MODE_CONNECTOR_eDP) + return true; + spin_lock_irqsave(&external_counts_lock_v27, flags); + if ((plug && (pipes > U32_MAX - external_pipe_count_v27 || + apple_display > U32_MAX - external_apple_display_count_v27)) || + (!plug && (pipes > dcp->external_pipes_v27 || + apple_display > dcp->external_displays_v27))) { + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + dev_warn_ratelimited(dcp->dev, "invalid external display count update\n"); + return true; + } + dcp->external_pipes_v27 += plug ? pipes : -pipes; + dcp->external_displays_v27 += plug ? apple_display : -apple_display; + external_pipe_count_v27 += plug ? pipes : -pipes; + external_apple_display_count_v27 += plug ? apple_display : -apple_display; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; +} + +static bool trampoline_get_external_counts_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + unsigned long flags; + u32 counts[3]; + + trace_iomfb_callback(dcp, tag, __func__); + spin_lock_irqsave(&external_counts_lock_v27, flags); + counts[0] = external_pipe_count_v27; + counts[1] = external_apple_display_count_v27; + counts[2] = external_suppression_count_v27; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + + for (unsigned int i = 0; i < ARRAY_SIZE(counts); i++) + if (!(request[12 + i] & 1)) + put_unaligned_le32(counts[i], (u8 *)out + 4 * i); + return true; +} + +static bool trampoline_apply_suppression_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *request = in; + unsigned long flags; + + trace_iomfb_callback(dcp, tag, __func__); + spin_lock_irqsave(&external_counts_lock_v27, flags); + if (((request[0] & 1) && external_suppression_count_v27 == U32_MAX) || + (!(request[0] & 1) && !dcp->external_suppression_v27)) { + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; + } + dcp->external_suppression_v27 += (request[0] & 1) ? 1U : -1U; + external_suppression_count_v27 += (request[0] & 1) ? 1U : -1U; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); + return true; +} + +static void external_counts_cleanup_v27(struct apple_dcp *dcp) +{ + unsigned long flags; + + spin_lock_irqsave(&external_counts_lock_v27, flags); + external_pipe_count_v27 -= dcp->external_pipes_v27; + external_apple_display_count_v27 -= dcp->external_displays_v27; + external_suppression_count_v27 -= dcp->external_suppression_v27; + dcp->external_pipes_v27 = 0; + dcp->external_displays_v27 = 0; + dcp->external_suppression_v27 = 0; + spin_unlock_irqrestore(&external_counts_lock_v27, flags); +} diff --git a/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc b/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc new file mode 100644 index 0000000000000..c2a681328c102 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-peer-runtime-v27.inc @@ -0,0 +1,125 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include "iomfb-peer-v27.h" + +static void dcp_peer_swap_error(struct apple_dcp *owner, const char *where, u32 status) +{ + lockdep_assert_held(&owner->iomfb.lock); + dev_err(owner->dev, "paired swap %s failed: %#x\n", where, status); + dcp_peer_swap_failed_v27(owner); + /* One engine may already have accepted this frame. A failed peer ACK + * is not a DMA-retirement witness for either scanout. The clean queue + * retains its active transaction and next-scanout references on crash. */ + WRITE_ONCE(owner->crashed, true); + dcp_swap_watchdog_complete(owner); + dcp_present_failed(owner); + schedule_work(&owner->vblank_wq); +} + +static void dcp_peer_submit_error_on_owner(struct apple_dcp *owner, void *data, void *cookie) +{ + u32 *status = data; + dcp_peer_swap_error(owner, "peer submit", *status); +} + +static void dcp_peer_submitted(struct apple_dcp *dcp, void *data, void *cookie) +{ + struct apple_dcp *owner = cookie; + struct dcp_swap_submit_resp_v27 *resp = data; + u32 status = resp ? resp->ret : ~0U; + u32 id; + + if (owner != dcp) { + if (status && iomfb_queue_callback(owner, dcp_peer_submit_error_on_owner, + &status, sizeof(status), NULL)) { + WRITE_ONCE(owner->crashed, true); + schedule_work(&owner->vblank_wq); + } + return; + } + lockdep_assert_held(&owner->iomfb.lock); + if (READ_ONCE(owner->peer_frames.failed)) + return; + id = owner->swap.h17p.swap.swap_id; + if (status) { + dcp_peer_swap_error(owner, "primary submit", status); + return; + } + if (dcp_present_submit(owner, id, true)) { + dcp_peer_swap_error(owner, "submit state", -EPROTO); + return; + } + /* Per-endpoint receive ordering handles the primary ACK before its + * completion. The paired watermark still requires both completions. */ + owner->swap_start = ktime_get(); + owner->swap_submit_timestamp = arch_timer_read_counter(); + dcp_swap_watchdog_arm(owner); +} + +static void dcp_peer_started_on_owner(struct apple_dcp *owner, void *data, void *cookie) +{ + struct dcp_swap_start_resp_h17p *resp = data; + struct apple_dcp *peer = dcp_peer_scanout_v27(owner); + u32 id = owner->swap.h17p.swap.swap_id; + + lockdep_assert_held(&owner->iomfb.lock); + if (IS_ERR_OR_NULL(peer) || peer != cookie || resp->ret || resp->swap_id != id) { + dcp_peer_swap_error(owner, "start/ID", resp->ret ?: -EPROTO); + return; + } + /* Keep the logical atomic input intact. Both split wire records are + * primary-owned; dcp_push copies the peer's record into its own queue. */ + iomfb_serialize_present_h17p(&owner->present_h17p, &owner->swap.h17p); + if (!iomfb_apply_backlight_h17p(owner, &owner->swap.h17p, &owner->present_h17p)) { + dcp_peer_swap_error(owner, "surface preparation", -EINVAL); + return; + } + iomfb_serialize_present_v27(&owner->present_v27, &owner->swap.h17p); + owner->present_v27.swap.brightness_update = owner->present_h17p.swap[0x354] & 1; + owner->present_v27.swap.brightness_nits = get_unaligned_le64(owner->present_h17p.swap + 0x35e); + dcp_peer_split_swap_v27(&owner->present_v27, &owner->peer_present_v27, + owner->peer_tiling.width); + if (dcp_present_retires(owner)) + dcp_arm_retained_framebuffers(owner, id); + dcp_peer_swap_armed_v27(owner, id); + trace_iomfb_swap_submit(owner, id); + trace_iomfb_swap_submit(peer, id); + dcp_push(owner, false, &dcp_methods[dcpep_swap_submit], sizeof(owner->present_v27), + sizeof(struct dcp_swap_submit_resp_v27), &owner->present_v27, + dcp_peer_submitted, owner); + if (READ_ONCE(owner->crashed)) + return; + dcp_push(peer, false, &dcp_methods[dcpep_swap_submit], sizeof(owner->peer_present_v27), + sizeof(struct dcp_swap_submit_resp_v27), &owner->peer_present_v27, + dcp_peer_submitted, owner); + if (READ_ONCE(peer->crashed)) + dcp_peer_swap_error(owner, "peer publication", -EIO); +} + +static void dcp_peer_started(struct apple_dcp *peer, void *data, void *cookie) +{ + struct apple_dcp *owner = cookie; + struct dcp_swap_start_resp_h17p failed = { .ret = ~0U }; + + if (iomfb_queue_callback(owner, dcp_peer_started_on_owner, + data ?: &failed, sizeof(failed), peer)) { + WRITE_ONCE(owner->crashed, true); + schedule_work(&owner->vblank_wq); + } +} + +static bool dcp_peer_start_second(struct apple_dcp *owner, + struct dcp_swap_start_resp_h17p *resp) +{ + struct dcp_swap_start_req_h17p req = {}; + struct apple_dcp *peer = dcp_peer_scanout_v27(owner); + + if (!peer) + return false; + if (IS_ERR(peer) || resp->ret) { + dcp_peer_swap_error(owner, "primary start", resp->ret ?: -EIO); + return true; + } + owner->swap.h17p.swap.swap_id = resp->swap_id; + dcp_swap_start(peer, false, &req, dcp_peer_started, owner); + return true; +} diff --git a/drivers/gpu/drm/apple/iomfb-peer-v27.h b/drivers/gpu/drm/apple/iomfb-peer-v27.h new file mode 100644 index 0000000000000..ea3b586e037c9 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb-peer-v27.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_IOMFB_PEER_V27_H +#define APPLE_IOMFB_PEER_V27_H + +#include +#include +#include +#include +#include "iomfb_plane.h" +#include "iomfb_v27.h" + +/* A364 and A365 share the video-data prefix, but not the final + * flags: A364 has a requires-merge boolean before three pointer-null flags; + * A365 has only two pointer-null flags and padding. + */ +struct dcp_peer_video_data_v27 { + u8 timing[0x50]; + u8 color[0x20]; +} __packed; + +struct dcp_peer_get_video_req_v27 { + struct dcp_peer_video_data_v27 video; + __le32 timing_id, color_id; + u8 requires_merge, timing_null, color_null, requires_merge_null; +} __packed; + +struct dcp_peer_get_video_resp_v27 { + struct dcp_peer_video_data_v27 video; + u8 requires_merge, success, padding[2]; +} __packed; + +struct dcp_peer_insert_video_req_v27 { + struct dcp_peer_video_data_v27 video; + __le32 timing_id, color_id; + u8 timing_null, color_null, padding[2]; +} __packed; + +struct dcp_peer_insert_video_resp_v27 { + struct dcp_peer_video_data_v27 video; + u8 success, padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_peer_get_video_req_v27) == 0x7c); +static_assert(sizeof(struct dcp_peer_get_video_resp_v27) == 0x74); +static_assert(sizeof(struct dcp_peer_insert_video_req_v27) == 0x7c); +static_assert(sizeof(struct dcp_peer_insert_video_resp_v27) == 0x74); + +/* The digital-out mode setter. + * Keep vertical timings, color data and all unmodified fields intact. + * The signed horizontal offset uses division toward zero. + */ +static inline void dcp_peer_split_timing_v27(struct dcp_peer_video_data_v27 *v) +{ + static const u8 half_fields[] = { 4, 8, 12, 16, 20, 28, 68 }; + + for (unsigned int i = 0; i < sizeof(half_fields); i++) { + u8 *field = v->timing + half_fields[i]; + + put_unaligned_le32(get_unaligned_le32(field) >> 1, field); + } + put_unaligned_le32((s32)get_unaligned_le32(v->timing + 24) / 2, + v->timing + 24); +} + +/* VFTG splits a surface at the half-panel boundary. + * Source and destination use pixels here, as in the existing IOMFB plane + * path. Use rational arithmetic in the kernel rather than floating point; + * round a shared source boundary once so neither half loses/duplicates it. + * DRM has already clipped negative/offscreen destination coordinates. + */ +static inline bool dcp_peer_clip_surface_v27(const struct dcp_rect *src, + const struct dcp_rect *dst, u32 tile_x, u32 tile_width, + struct dcp_rect *tile_src, struct dcp_rect *tile_dst) +{ + u64 left = max_t(u64, dst->x, tile_x); + u64 right = min_t(u64, (u64)dst->x + dst->w, + (u64)tile_x + tile_width); + u64 first, last; + + memset(tile_src, 0, sizeof(*tile_src)); + memset(tile_dst, 0, sizeof(*tile_dst)); + if (!src->w || !src->h || !dst->w || !dst->h || left >= right) + return false; + first = div_u64((left - dst->x) * src->w + dst->w / 2, dst->w); + last = div_u64((right - dst->x) * src->w + dst->w / 2, dst->w); + if (first >= last || (u64)src->x + last > U32_MAX) + return false; + *tile_src = *src; + tile_src->x += first; + tile_src->w = last - first; + *tile_dst = *dst; + tile_dst->x = left - tile_x; + tile_dst->w = right - left; + return true; +} + + +/* Clone descriptors only: both DCPs use the same display DMA domain/IOVA. */ +static inline void dcp_peer_split_swap_v27(struct dcp_swap_submit_req_v27 *left, + struct dcp_swap_submit_req_v27 *right, + u32 tile_width) +{ + *right = *left; + for (unsigned int i = 0; i < ARRAY_SIZE(left->surf_null); i++) { + struct dcp_rect src = left->swap.src_rect[i]; + struct dcp_rect dst = left->swap.dst_rect[i]; + bool present = !left->surf_null[i]; + struct dcp_swap_submit_req_v27 *halves[] = {left, right}; + + for (unsigned int side = 0; side < 2; side++) { + struct dcp_swap_submit_req_v27 *req = halves[side]; + struct dcp_rect clipped_src, clipped_dst; + + if (present && dcp_peer_clip_surface_v27(&src, &dst, + side * tile_width, tile_width, &clipped_src, &clipped_dst)) { + req->swap.src_rect[i] = clipped_src; + req->swap.dst_rect[i] = clipped_dst; + } else { + req->surf_null[i] = true; + req->surf_iova[i] = 0; + req->swap.surf_ids[i] = 0; + memset(&req->swap.src_rect[i], 0, sizeof(req->swap.src_rect[i])); + memset(&req->swap.dst_rect[i], 0, sizeof(req->swap.dst_rect[i])); + } + } + } +} + +#endif diff --git a/drivers/gpu/drm/apple/iomfb.c b/drivers/gpu/drm/apple/iomfb.c index f8e068fff0b65..fb9f42b8581d1 100644 --- a/drivers/gpu/drm/apple/iomfb.c +++ b/drivers/gpu/drm/apple/iomfb.c @@ -48,6 +48,10 @@ static int dcp_tx_offset(enum dcp_context_id id) case DCP_CONTEXT_OOBCB: case DCP_CONTEXT_OOBCMD: return 0x08000; + case DCP_CONTEXT_ASYNC_CB: + return 0x20000; + case DCP_CONTEXT_OOBASYNC_CB: + return 0x28000; default: return -EINVAL; } @@ -56,8 +60,10 @@ static int dcp_tx_offset(enum dcp_context_id id) static int dcp_channel_offset(enum dcp_context_id id) { switch (id) { + case DCP_CONTEXT_ASYNC_CB: case DCP_CONTEXT_ASYNC: return 0x40000; + case DCP_CONTEXT_OOBASYNC_CB: case DCP_CONTEXT_OOBASYNC: return 0x48000; case DCP_CONTEXT_CB: @@ -127,8 +133,10 @@ static struct dcp_channel *dcp_get_channel(struct apple_dcp *dcp, return &dcp->ch_oobcb; case DCP_CONTEXT_OOBCMD: return &dcp->ch_oobcmd; + case DCP_CONTEXT_ASYNC_CB: case DCP_CONTEXT_ASYNC: return &dcp->ch_async; + case DCP_CONTEXT_OOBASYNC_CB: case DCP_CONTEXT_OOBASYNC: return &dcp->ch_oobasync; default: @@ -164,7 +172,7 @@ static u8 dcp_pop_depth(u8 *depth) /* Older firmware and the H17G method profile retain their existing transport. */ static bool iomfb_uses_queue(struct apple_dcp *dcp) { - return dcp->fw_compat == DCP_FIRMWARE_H17P && + return (dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G; } @@ -199,7 +207,8 @@ static int iomfb_enqueue_opaque_x(struct apple_dcp *dcp) if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17P_EXTERNAL) return 0; - if (atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + atomic_read(&dcp->iomfb.opaque_x_state) != IOMFB_OPAQUE_X_NEEDED) return 0; if (dcp->iomfb.queued >= 32) return -EBUSY; @@ -243,7 +252,8 @@ static void iomfb_queue_advance(struct apple_dcp *dcp) /* The active operation may still be visible to firmware. */ return; } - if (!iomfb_channels_idle(dcp) || dcp->present_state_h17p.pending) + if (!iomfb_channels_idle(dcp) || dcp->present_state_h17p.pending || + READ_ONCE(dcp->iomfb.callback_count)) return; if (transaction) { @@ -289,7 +299,8 @@ static void iomfb_queue_work(struct work_struct *work) goto unlock; } if (dcp->iomfb.active || !iomfb_channels_idle(dcp) || - dcp->present_state_h17p.pending || list_empty(&dcp->iomfb.pending)) + dcp->present_state_h17p.pending || READ_ONCE(dcp->iomfb.callback_count) || + list_empty(&dcp->iomfb.pending)) goto unlock; transaction = list_first_entry(&dcp->iomfb.pending, @@ -382,12 +393,92 @@ static void iomfb_backlight_retry(struct work_struct *work) schedule_work(&dcp->bl_update_wq); } +/* A peer endpoint may finish a dependency while this endpoint has an active + * transaction. Continue on its serialized owner, not by taking another + * endpoint's IOMFB mutex from a receive callback. */ +struct iomfb_callback { + struct list_head link; + dcp_callback_t callback; + void *cookie; + u8 data[]; +}; + +int iomfb_queue_callback(struct apple_dcp *dcp, dcp_callback_t callback, + const void *data, size_t size, void *cookie) +{ + struct iomfb_callback *item; + unsigned long flags; + int ret = 0; + + /* This queue carries control replies and completion watermarks only. */ + if (!callback || size > 64 || (size && !data)) + return -EINVAL; + item = kzalloc(struct_size(item, data, size), GFP_KERNEL); + if (!item) + return -ENOMEM; + item->callback = callback; + item->cookie = cookie; + if (size) + memcpy(item->data, data, size); + spin_lock_irqsave(&dcp->iomfb.callback_lock, flags); + if (READ_ONCE(dcp->iomfb.stopped) || READ_ONCE(dcp->crashed)) { + ret = -ESHUTDOWN; + } else if (dcp->iomfb.callback_count >= 64) { + ret = -EBUSY; + } else { + list_add_tail(&item->link, &dcp->iomfb.callbacks); + dcp->iomfb.callback_count++; + /* Stop takes this lock before cancelling work, closing enqueue/stop. */ + schedule_work(&dcp->iomfb.callback_work); + } + spin_unlock_irqrestore(&dcp->iomfb.callback_lock, flags); + if (ret) + kfree(item); + return ret; +} + +static struct iomfb_callback *iomfb_pop_callback(struct apple_dcp *dcp) +{ + struct iomfb_callback *item = NULL; + unsigned long flags; + + spin_lock_irqsave(&dcp->iomfb.callback_lock, flags); + if (!list_empty(&dcp->iomfb.callbacks)) { + item = list_first_entry(&dcp->iomfb.callbacks, struct iomfb_callback, link); + list_del(&item->link); + dcp->iomfb.callback_count--; + } + spin_unlock_irqrestore(&dcp->iomfb.callback_lock, flags); + return item; +} + +static void iomfb_callback_work(struct work_struct *work) +{ + struct apple_dcp *dcp = container_of(work, struct apple_dcp, iomfb.callback_work); + struct iomfb_callback *item; + + mutex_lock(&dcp->iomfb.lock); + while ((item = iomfb_pop_callback(dcp))) { + if (!dcp->iomfb.stopped && !READ_ONCE(dcp->crashed)) { + WRITE_ONCE(dcp->iomfb.owner, current); + item->callback(dcp, item->data, item->cookie); + WRITE_ONCE(dcp->iomfb.owner, NULL); + } + kfree(item); + } + iomfb_queue_advance(dcp); + mutex_unlock(&dcp->iomfb.lock); +} + void iomfb_queue_init(struct apple_dcp *dcp) { mutex_init(&dcp->iomfb.lock); INIT_LIST_HEAD(&dcp->iomfb.pending); INIT_LIST_HEAD(&dcp->iomfb.dropped); INIT_WORK(&dcp->iomfb.work, iomfb_queue_work); + spin_lock_init(&dcp->iomfb.callback_lock); + INIT_LIST_HEAD(&dcp->iomfb.callbacks); + INIT_WORK(&dcp->iomfb.callback_work, iomfb_callback_work); INIT_DELAYED_WORK(&dcp->iomfb.timeout, iomfb_queue_timeout); INIT_DELAYED_WORK(&dcp->iomfb.backlight_retry, iomfb_backlight_retry); atomic_set(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING); @@ -395,14 +486,22 @@ void iomfb_queue_init(struct apple_dcp *dcp) void iomfb_queue_stop(struct apple_dcp *dcp) { + struct iomfb_callback *item; + unsigned long flags; + if (!iomfb_uses_queue(dcp)) return; mutex_lock(&dcp->iomfb.lock); - dcp->iomfb.stopped = true; + spin_lock_irqsave(&dcp->iomfb.callback_lock, flags); + WRITE_ONCE(dcp->iomfb.stopped, true); + spin_unlock_irqrestore(&dcp->iomfb.callback_lock, flags); iomfb_discard_pending(dcp); mutex_unlock(&dcp->iomfb.lock); cancel_work_sync(&dcp->iomfb.work); + cancel_work_sync(&dcp->iomfb.callback_work); + while ((item = iomfb_pop_callback(dcp))) + kfree(item); cancel_delayed_work_sync(&dcp->iomfb.timeout); cancel_delayed_work_sync(&dcp->iomfb.backlight_retry); } @@ -423,7 +522,7 @@ bool iomfb_queue_drain(struct apple_dcp *dcp, unsigned long timeout) mutex_lock(&dcp->iomfb.lock); idle = READ_ONCE(dcp->crashed) || dcp->iomfb.stopped || (!dcp->iomfb.active && list_empty(&dcp->iomfb.pending) && - !dcp->present_state_h17p.pending); + !dcp->present_state_h17p.pending && !READ_ONCE(dcp->iomfb.callback_count)); mutex_unlock(&dcp->iomfb.lock); if (idle) return true; @@ -619,10 +718,38 @@ static int iomfb_queue_command(struct apple_dcp *dcp, bool oob, return ret; } +/* + * Firmware 27.3 numbers these calls 50 or 51 higher than firmware 27.0, and + * set_create_DFB and vi_set_temperature_hint one higher (macOS 27.3's + * IOMobileGraphicsFamily-DCP). + */ +static void iomfb_tag_27_3(char *tag) +{ + static const char map[][2][5] = { + { "A389", "A390" }, { "A390", "A391" }, { "A401", "A451" }, + { "A406", "A456" }, { "A408", "A458" }, { "A411", "A461" }, + { "A412", "A462" }, { "A413", "A463" }, { "A414", "A464" }, + { "A415", "A465" }, { "A423", "A473" }, { "A427", "A477" }, + { "A428", "A478" }, { "A450", "A500" }, { "A453", "A503" }, + { "A454", "A504" }, { "A458", "A508" }, { "A466", "A516" }, + { "A467", "A517" }, { "A475", "A526" }, { "A478", "A529" }, + { "A479", "A530" }, { "A483", "A534" }, { "A484", "A535" }, + { "A485", "A536" }, { "A487", "A538" }, { "A489", "A540" }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(map); i++) { + if (!memcmp(tag, map[i][0], 4)) { + memcpy(tag, map[i][1], 4); + return; + } + } +} + /* Call a DCP function given by a tag */ -void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *call, - u32 in_len, u32 out_len, void *data, dcp_callback_t cb, - void *cookie) +static void dcp_push_context(struct apple_dcp *dcp, enum dcp_context_id context, + const struct dcp_method_entry *call, u32 in_len, + u32 out_len, void *data, dcp_callback_t cb, void *cookie) { struct dcp_method_entry resolved = *call; @@ -634,23 +761,16 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca } #endif - /* Reply chains stay on the serialized receiver; callers enqueue copies. */ - if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { - if (iomfb_queue_command(dcp, oob, call, in_len, out_len, - data, cb, cookie)) - WRITE_ONCE(dcp->crashed, true); - return; - } - - if (dcp->fw_compat == DCP_FIRMWARE_H17P && READ_ONCE(dcp->crashed)) + if (dcp_fail_stop(dcp) && READ_ONCE(dcp->crashed)) return; if (dcp->hw.iomfb_method_profile == DCP_IOMFB_METHODS_H17G && call->tag_h17g[0]) memcpy(resolved.tag, call->tag_h17g, sizeof(resolved.tag)); + if (dcp->hw.fw_27_3) + iomfb_tag_27_3(resolved.tag); call = &resolved; - enum dcp_context_id context = dcp_call_context(dcp, oob); struct dcp_channel *ch = dcp_get_channel(dcp, context); struct dcp_packet_header header = { @@ -704,6 +824,54 @@ void dcp_push(struct apple_dcp *dcp, bool oob, const struct dcp_method_entry *ca dcpep_msg(context, data_len, offset)); } +void dcp_push(struct apple_dcp *dcp, bool oob, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie) +{ + /* Reply chains stay on the serialized receiver; callers enqueue copies. */ + if (iomfb_uses_queue(dcp) && READ_ONCE(dcp->iomfb.owner) != current) { + if (iomfb_queue_command(dcp, oob, call, in_len, out_len, + data, cb, cookie)) + WRITE_ONCE(dcp->crashed, true); + return; + } + + dcp_push_context(dcp, dcp_call_context(dcp, oob), call, + in_len, out_len, data, cb, cookie); +} + +/* A nested query must return on the firmware-initiated stream it answers. */ +int dcp_push_nested(struct apple_dcp *dcp, enum dcp_context_id origin, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie) +{ + enum dcp_context_id context; + struct dcp_channel *ch; + + if (READ_ONCE(dcp->iomfb.owner) != current || READ_ONCE(dcp->crashed)) + return -ESHUTDOWN; + switch (origin) { + case DCP_CONTEXT_CB: + case DCP_CONTEXT_OOBCB: + context = origin; + break; + case DCP_CONTEXT_ASYNC: + context = DCP_CONTEXT_ASYNC_CB; + break; + case DCP_CONTEXT_OOBASYNC: + context = DCP_CONTEXT_OOBASYNC_CB; + break; + default: + return -EINVAL; + } + ch = dcp_get_channel(dcp, context); + if (!ch || !ch->depth || ch->depth >= DCP_MAX_CALL_DEPTH || + (u64)sizeof(struct dcp_packet_header) + in_len + out_len > 0x8000) + return -EINVAL; + dcp_push_context(dcp, context, call, in_len, out_len, data, cb, cookie); + return READ_ONCE(dcp->crashed) ? -EPROTO : 0; +} + /* Parse a callback tag "D123" into the ID 123. Returns -EINVAL on failure. */ int dcp_parse_tag(char tag[4]) { @@ -1264,7 +1432,9 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, u8 depth; bool handled, zero_reply; - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) && (length < sizeof(*hdr) || !ch || ch->depth >= DCP_MAX_CALL_DEPTH || offset >= 0x8000 || length > 0x8000 - offset)) { dev_err(dev, "invalid IOMFB callback envelope\n"); @@ -1277,13 +1447,15 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, handled = tag >= 0 && tag < IOMFB_MAX_CB && dcp->cb_handlers && dcp->cb_handlers[tag]; - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && (tag < 0 || tag >= IOMFB_MAX_CB || (u64)sizeof(*hdr) + hdr->in_len + hdr->out_len != length || (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && handled && - !iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len)))) { - dev_err(dev, "unsupported IOMFB callback %d (%u, %u)\n", + !(dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? + iomfb_v27_callback_size_valid(tag, hdr->in_len, hdr->out_len) : + iomfb_check_callback_h17p(dcp, tag, hdr->in_len, hdr->out_len))))) { + dev_err(dev, "unqualified IOMFB callback %d (%u, %u)\n", tag, hdr->in_len, hdr->out_len); dcp->crashed = true; return; @@ -1313,7 +1485,9 @@ static void dcpep_handle_cb(struct apple_dcp *dcp, enum dcp_context_id context, * waits for the ack forever and the outer call never returns. * On H17P answer it with a zeroed output instead. */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P) { + if (dcp->fw_compat == DCP_FIRMWARE_H17P || + dcp->fw_compat == DCP_FIRMWARE_V_27_0 || + dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) { if (hdr->out_len) memset(out, 0, hdr->out_len); depth = dcp_push_depth(&ch->depth); @@ -1379,7 +1553,7 @@ static void dcpep_handle_ack(struct apple_dcp *dcp, enum dcp_context_id context) * its output cannot be trusted. The tag is left out of the check: it * has only been seen preserved on 13.5 firmware. */ - if (dcp->fw_compat != DCP_FIRMWARE_H17P) { + if ((dcp->fw_compat != DCP_FIRMWARE_H17P && dcp->fw_compat != DCP_FIRMWARE_V_27_0)) { record = out - sent->in_len - sizeof(*record); if (record->in_len != sent->in_len || record->out_len != sent->out_len) { @@ -1865,6 +2039,8 @@ int dcp_get_modes(struct drm_connector *connector) apple_connector->drm_edid = edid; apple_connector->drm_edid_generation = edid_generation; apple_connector->drm_edid_service_epoch = service_epoch; + /* A native T6030 pipe: its audio, after the first EDID. */ + dcp_external_audio_start(dcp); } } if (nr_modes && apple_connector->drm_edid) { @@ -2093,6 +2269,9 @@ int dcp_crtc_atomic_modeset(struct drm_crtc *crtc, case DCP_FIRMWARE_H17P: ret = iomfb_modeset_h17p(dcp, crtc_state); break; + case DCP_FIRMWARE_V_27_0: + ret = iomfb_modeset_v27(dcp, crtc_state); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); @@ -2371,8 +2550,9 @@ bool iomfb_present_complete_h17p(struct apple_dcp *dcp) if (transaction->brightness_only) return false; } - atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, - IOMFB_OPAQUE_X_NEEDED); + if (dcp->fw_compat != DCP_FIRMWARE_V_27_0) + atomic_cmpxchg(&dcp->iomfb.opaque_x_state, IOMFB_OPAQUE_X_WAITING, + IOMFB_OPAQUE_X_NEEDED); iomfb_scanout_complete_h17p(dcp); return true; } @@ -2452,7 +2632,10 @@ static void iomfb_backlight_start(struct apple_dcp *dcp, return; dcp->swap.h17p = scanout->request; - iomfb_present_backlight_h17p(dcp); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + iomfb_present_backlight_v27(dcp); + else + iomfb_present_backlight_h17p(dcp); } static void iomfb_backlight_release(struct iomfb_transaction *transaction) @@ -2510,7 +2693,10 @@ static void iomfb_atomic_start(struct apple_dcp *dcp, return; } dcp->iomfb.next_scanout = atomic->scanout; - iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); + if (dcp->fw_compat == DCP_FIRMWARE_V_27_0) + iomfb_flush_v27(dcp, atomic->crtc, atomic->state); + else + iomfb_flush_h17p(dcp, atomic->crtc, atomic->state); } static void iomfb_atomic_release(struct iomfb_transaction *transaction) @@ -2618,6 +2804,9 @@ void dcp_flush(struct drm_crtc *crtc, struct drm_atomic_commit *commit) case DCP_FIRMWARE_H17P: iomfb_flush_h17p(dcp, crtc, commit); break; + case DCP_FIRMWARE_V_27_0: + iomfb_flush_v27(dcp, crtc, commit); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -2639,6 +2828,9 @@ static void iomfb_start(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_start_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_start_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; @@ -2714,7 +2906,7 @@ int iomfb_start_rtkit(struct apple_dcp *dcp) * has nothing answering it on the AP side, and without this one the * display DART faults. */ - if (dcp->fw_compat == DCP_FIRMWARE_H17P && + if ((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) { ret = apple_rtkit_start_ep(dcp->rtk, REMOTE_ALLOC_ENDPOINT); if (ret) @@ -2753,6 +2945,9 @@ void iomfb_shutdown(struct apple_dcp *dcp) case DCP_FIRMWARE_H17P: iomfb_shutdown_h17p(dcp); break; + case DCP_FIRMWARE_V_27_0: + iomfb_shutdown_v27(dcp); + break; default: WARN_ONCE(true, "Unexpected firmware version: %u\n", dcp->fw_compat); break; diff --git a/drivers/gpu/drm/apple/iomfb.h b/drivers/gpu/drm/apple/iomfb.h index 25c2e0254183f..ccc913bb4a128 100644 --- a/drivers/gpu/drm/apple/iomfb.h +++ b/drivers/gpu/drm/apple/iomfb.h @@ -4,6 +4,8 @@ #ifndef __APPLE_DCPEP_H__ #define __APPLE_DCPEP_H__ +#include +#include #include #include "version_utils.h" @@ -15,6 +17,7 @@ enum dcp_context_id { /* Callback */ DCP_CONTEXT_CB = 0, + DCP_CONTEXT_ASYNC_CB = 1, /* Command */ DCP_CONTEXT_CMD = 2, @@ -24,6 +27,7 @@ enum dcp_context_id { /* Out-of-band callback */ DCP_CONTEXT_OOBCB = 4, + DCP_CONTEXT_OOBASYNC_CB = 5, /* Out-of-band command */ DCP_CONTEXT_OOBCMD = 6, @@ -184,6 +188,8 @@ enum dcpep_method { * H17P pipe-configuration calls made before the first swap. Most * names are unknown; they are identified by method number. */ + dcpep_pipe_query_353, + dcpep_pipe_set_352, dcpep_pipe_cfg_415, dcpep_pipe_cfg_031, dcpep_pipe_cfg_414, @@ -406,4 +412,42 @@ struct dcpep_get_tiling_state_resp { u32 ret; } __packed; +/* + * A name and a serialized dictionary: the firmware's analytics event + * (D113 on 27.x, D114 on desktop 26.6). + */ +struct dcp_dict_passthrough_req { + u8 key[0x40]; + u8 value[0x1000]; + u8 value_null; + u8 padding[3]; +} __packed; + +struct dcp_dict_passthrough_resp { + u8 value[0x1000]; + u8 ret; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_dict_passthrough_req) == 0x1044); +static_assert(sizeof(struct dcp_dict_passthrough_resp) == 0x1004); + +/* + * The analytics event passes its dictionary by pointer, so the firmware + * deserializes the reply's copy when the call returns, and stops if that + * fails. Return the dictionary as it came, and report the event not sent. + * A zeroed reply is not a dictionary. + */ +static inline void +dcp_dict_passthrough_reply(const struct dcp_dict_passthrough_req *req, + struct dcp_dict_passthrough_resp *resp) +{ + if (req->value_null) + memset(resp->value, 0, sizeof(resp->value)); + else + memcpy(resp->value, req->value, sizeof(resp->value)); + resp->ret = false; + memset(resp->padding, 0, sizeof(resp->padding)); +} + #endif diff --git a/drivers/gpu/drm/apple/iomfb_internal.h b/drivers/gpu/drm/apple/iomfb_internal.h index bd55bfd475f3e..5d32a1c034e91 100644 --- a/drivers/gpu/drm/apple/iomfb_internal.h +++ b/drivers/gpu/drm/apple/iomfb_internal.h @@ -112,3 +112,12 @@ void dcp_ack(struct apple_dcp *dcp, enum dcp_context_id context); /* The user may own drm_display_mode, so we need to search for our copy */ bool lookup_mode(struct apple_dcp *dcp, const struct drm_display_mode *mode, struct dcp_display_mode *out); + +u8 dcp_crtc_eotf(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit); +int dcp_crtc_check_hdr(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit); +void dcp_hdr_metadata_forget(struct apple_dcp *dcp); +int dcp_push_nested(struct apple_dcp *dcp, enum dcp_context_id origin, + const struct dcp_method_entry *call, u32 in_len, u32 out_len, + void *data, dcp_callback_t cb, void *cookie); diff --git a/drivers/gpu/drm/apple/iomfb_plane.h b/drivers/gpu/drm/apple/iomfb_plane.h index 73f104059d29c..31a2ddc9525bb 100644 --- a/drivers/gpu/drm/apple/iomfb_plane.h +++ b/drivers/gpu/drm/apple/iomfb_plane.h @@ -15,6 +15,7 @@ #define DCP_FORMAT_BGRA fourcc_code('A', 'R', 'G', 'B') #define DCP_FORMAT_RGBA fourcc_code('A', 'B', 'G', 'R') +#define DCP_FORMAT_AhGR fourcc_code('A', 'h', 'G', 'R') /* 16-bit half float ABGR */ #define DCP_FORMAT_W30R fourcc_code('r', '0', '3', 'w') // wide gamut packed 10-bit RGB without alpha #define DCP_FORMAT_L10R fourcc_code('r', '0', '1', 'l') // full range packed 10-bit RGB with alpha @@ -34,6 +35,14 @@ #define DCP_FORMAT_XF22 fourcc_code('2', '2', 'f', 'x') // P210 full range 2 plane 10-bit YCbCR #define DCP_FORMAT_XF44 fourcc_code('4', '4', 'f', 'x') // P410 full range 2 plane 10-bit YCbCR +#define DCP_FORMAT_P420 fourcc_code('0', '2', '4', 'p') /* P030 video range 2 plane 10-bit YCbCr, 3x1 blocks */ +#define DCP_FORMAT_P422 fourcc_code('2', '2', '4', 'p') /* P230 video range 2 plane 10-bit YCbCr, 3x1 blocks */ +#define DCP_FORMAT_P444 fourcc_code('4', '4', '4', 'p') /* P430 video range 2 plane 10-bit YCbCr, 3x1 blocks */ + +#define DCP_FORMAT_PF20 fourcc_code('0', '2', 'f', 'p') /* P030 full range 2 plane 10-bit YCbCr, 3x1 blocks */ +#define DCP_FORMAT_PF22 fourcc_code('2', '2', 'f', 'p') /* P230 full range 2 plane 10-bit YCbCr, 3x1 blocks */ +#define DCP_FORMAT_PF44 fourcc_code('4', '4', 'f', 'p') /* P430 full range 2 plane 10-bit YCbCr, 3x1 blocks */ + enum dcp_colorspace { DCP_COLORSPACE_BG_SRGB = 0, DCP_COLORSPACE_BT601 = 1, @@ -49,6 +58,24 @@ enum dcp_xfer_func { DCP_XFER_FUNC_HDR = 16, }; +enum dcp_addr_fmt { + DCP_ADDR_FMT_LINEAR = 0, + /* Twiddled layout supported by AGX */ + DCP_ADDR_FMT_GPU_TWIDDLED = 2, + /* Tiled layout supported by AGX */ + DCP_ADDR_FMT_GPU_TILED = 3, + /* Tiled layout used for buffers shared between hardware blocks */ + DCP_ADDR_FMT_INTERCHANGE_TILED = 5, +}; + +enum dcp_compression_type { + DCP_COMPRESSION_NONE = 0, + /* Optional for GPU_TWIDDLED and GPU_TILED */ + DCP_COMPRESSION_AGX = 2, + /* Always required for INTERCHANGE_TILED */ + DCP_COMPRESSION_INTERCHANGE = 3, +}; + struct dcp_rect { u32 x; u32 y; @@ -68,7 +95,7 @@ struct dcp_plane_info { u8 tile_w; u8 tile_h; u8 unk[0xd]; - u8 unk2; + u8 address_fmt; u8 unk3[0x26]; } __packed; @@ -94,6 +121,18 @@ struct dcp_compression_info { u8 pad2; } __packed; +struct dcp_image_layout { + u32 tile_w; + u32 tile_h; + u32 width_tiles; + u32 height_tiles; + u32 tile_bytes; + /* For compressed layouts */ + u32 meta_offset; + u32 meta_size; + u32 meta_tile_bytes; +}; + /* Information describing a surface */ struct dcp_surface { u8 is_tiled; diff --git a/drivers/gpu/drm/apple/iomfb_template.c b/drivers/gpu/drm/apple/iomfb_template.c index ad3f1a5291bae..4806edfcb4f44 100644 --- a/drivers/gpu/drm/apple/iomfb_template.c +++ b/drivers/gpu/drm/apple/iomfb_template.c @@ -30,6 +30,7 @@ #include "dcp-internal.h" #include "iomfb.h" #include "iomfb_internal.h" +#include "iomfb-dfb.h" #include "parser.h" #include "dcp-property.h" #include "trace.h" @@ -65,6 +66,9 @@ IOMFB_THUNK_INOUT(get_color_remap_mode); IOMFB_THUNK_INOUT(last_client_close); IOMFB_THUNK_INOUT(abort_swaps_dcp); +/* The queued swap of this firmware's layout. */ +#define DCP_FW_SWAP(dcp) DCP_FW_UNION((dcp)->swap) + #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, struct dcp_swap_submit_req_h17p *request, @@ -79,10 +83,56 @@ static void dcp_swap_submit(struct apple_dcp *dcp, bool oob, cb(dcp, NULL, cookie); return; } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + iomfb_serialize_present_v27(&dcp->present_v27, request); + dcp->present_v27.swap.brightness_update = + dcp->present_h17p.swap[0x354] & 1; + dcp->present_v27.swap.brightness_nits = + get_unaligned_le64(dcp->present_h17p.swap + 0x35e); + data = &dcp->present_v27; +#else data = &dcp->present_h17p; +#endif } - dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], sizeof(*request), +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + /* + * Firmware 27.3's swap record is 0x830 bytes: 0x40 more before swap_id, + * which moves to 0xd8, the rest of the firmware-27.0 record after it, + * and the kernel's fields from 0x720, of which the firmware reads the + * mask of the mapped surfaces at 0x728. The brightness block moves + * 0x60, to 0x3b4: the firmware reads its own fields at 0x394..0x3b3. + * The surfaces and the rest of the request follow the record. + */ + if (dcp->hw.fw_27_3 && data == &dcp->present_v27) { + const u8 *v = data; + u8 *w = dcp->present_27_3; + u32 enabled = get_unaligned_le32(v + offsetof(struct dcp_swap_v27, + swap_enabled)); + + memset(w, 0, 0x830); + memcpy(w, v, 0x98); + memcpy(w + 0xd8, v + 0x98, + offsetof(struct dcp_swap_v27, brightness_update) - 0x98); + memcpy(w + 0x3b4, v + offsetof(struct dcp_swap_v27, brightness_update), + offsetof(struct dcp_swap_v27, reserved_366) - + offsetof(struct dcp_swap_v27, brightness_update)); + put_unaligned_le32(enabled & 0xf, w + 0x728); + memcpy(w + 0x830, v + sizeof(struct dcp_swap_v27), + sizeof(struct dcp_swap_submit_req_v27) - + sizeof(struct dcp_swap_v27)); + dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], + sizeof(dcp->present_27_3), + sizeof(struct dcp_swap_submit_resp_h17p), w, cb, cookie); + return; + } +#endif + dcp_push(dcp, oob, &dcp_methods[dcpep_swap_submit], +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + sizeof(struct dcp_swap_submit_req_v27), +#else + sizeof(*request), +#endif sizeof(struct dcp_swap_submit_resp_h17p), (void *)data, cb, cookie); } #else @@ -116,8 +166,8 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, * is no vblank and no swap completion. Keep the mode the bootloader * programmed, which is the panel's native mode, instead of requesting one. */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -#define DCP_INHERIT_BOOT_MODE true +#if DCP_FW_H17(DCP_FW_VER) +#define DCP_INHERIT_BOOT_MODE (DCP_FW_VER < DCP_FW_VERSION(27, 0, 0)) #else #define DCP_INHERIT_BOOT_MODE false #endif @@ -129,8 +179,8 @@ DCP_THUNK_INOUT(dcp_set_power_state, dcpep_set_power_state, * straight to set_power_state, whose number and sizes are known. Power-off * still sends its clear swap first. */ -#if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) -#define DCP_HAS_CLIENT_TEARDOWN false +#if DCP_FW_H17(DCP_FW_VER) +#define DCP_HAS_CLIENT_TEARDOWN ((DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0))) #else #define DCP_HAS_CLIENT_TEARDOWN true #endif @@ -246,25 +296,27 @@ static u32 dcpep_cb_zero(struct apple_dcp *dcp) return 0; } -static void dcpep_cb_swap_complete(struct apple_dcp *dcp, - struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +static void dcp_report_swap_complete(struct apple_dcp *dcp, u32 id, ktime_t now) { - ktime_t now = ktime_get(); bool retire = true; -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +#if DCP_FW_DESKTOP_26(DCP_FW_VER) + if (dcp_peer_swap_complete_26_6(dcp, id)) + return; +#endif +#if DCP_FW_H17(DCP_FW_VER) if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G && - !dcp_present_complete_h17p(&dcp->present_state_h17p, resp->swap_id)) { + !dcp_present_complete_h17p(&dcp->present_state_h17p, id)) { /* * Abort semantics are not captured on H17P. If an aborted * present completes after all, its framebuffers were already * unarmed and its event signalled; record it and carry on. */ if (dcp_present_was_aborted_h17p(&dcp->present_state_h17p, - resp->swap_id)) { + id)) { dev_warn_ratelimited(dcp->dev, "completion for aborted present %u ignored\n", - resp->swap_id); + id); return; } /* @@ -273,37 +325,73 @@ static void dcpep_cb_swap_complete(struct apple_dcp *dcp, * in flight now is a different one. */ if (dcp_present_was_retired_h17p(&dcp->present_state_h17p, - resp->swap_id)) { + id)) { dev_warn_ratelimited(dcp->dev, "completion for retired present %u ignored\n", - resp->swap_id); + id); return; } - dev_err(dcp->dev, "unexpected present completion %u\n", resp->swap_id); + dev_err(dcp->dev, "unexpected present completion %u\n", id); dcp->crashed = true; return; } if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) retire = iomfb_present_complete_h17p(dcp); #endif - trace_iomfb_swap_complete(dcp, resp->swap_id); - dcp->last_swap_id = resp->swap_id; +#if DCP_FW_DESKTOP_26(DCP_FW_VER) + /* Late or duplicate completions cannot retire the replacement's flip. */ + if (!dcp_desktop_present_complete(dcp, id)) + return; + dcp_backlight_desktop_finish(dcp, id, false, true); +#endif + dcp->last_swap_id = id; + dcp->have_swap_complete = true; dcp_swap_watchdog_complete(dcp); if (retire) { - dcp_release_retained_framebuffers(dcp, resp->swap_id); + dcp_release_retained_framebuffers(dcp, id); dcp_drm_crtc_page_flip(dcp, now); } if (READ_ONCE(dcp->crc_enabled)) { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { - iomfb_queue_crc_h17p(dcp, resp->swap_id); + iomfb_queue_crc_h17p(dcp, id); return; } #endif u32 crc32 = 0; - drm_crtc_add_crc_entry(&dcp->crtc->base, true, resp->swap_id, &crc32); + drm_crtc_add_crc_entry(&dcp->crtc->base, true, id, &crc32); + } +} + +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) +struct dcp_peer_completion { u32 id; ktime_t time; }; +static void dcp_peer_report_on_owner(struct apple_dcp *owner, void *data, void *cookie) +{ + struct dcp_peer_completion *completion = data; + dcp_report_swap_complete(owner, completion->id, completion->time); +} +static void dcp_peer_queue_completion(struct apple_dcp *owner, u32 id, ktime_t time) +{ + struct dcp_peer_completion completion = { id, time }; + if (iomfb_queue_callback(owner, dcp_peer_report_on_owner, &completion, + sizeof(completion), NULL)) { + WRITE_ONCE(owner->crashed, true); + schedule_work(&owner->vblank_wq); } } +#endif + +static void dcpep_cb_swap_complete(struct apple_dcp *dcp, + struct DCP_FW_NAME(dc_swap_complete_resp) *resp) +{ + ktime_t now = ktime_get(); + trace_iomfb_swap_complete(dcp, resp->swap_id); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp_peer_swap_complete_v27(dcp, resp->swap_id, now, dcp_peer_queue_completion)) + return; +#endif + dcp_report_swap_complete(dcp, resp->swap_id, now); +} /* special */ static void complete_vi_set_temperature_hint(struct apple_dcp *dcp, void *out, void *cookie) @@ -345,7 +433,7 @@ static_assert(sizeof(struct dcp_h17p_dfb_surface_req) == 0x230); DCP_THUNK_INOUT(dcp_register_dfb_surface, dcpep_register_dfb_surface, struct dcp_h17p_dfb_surface_req, u32); -static void dcp_ack_dfb_surface(struct apple_dcp *dcp) +static void __maybe_unused dcp_ack_dfb_surface(struct apple_dcp *dcp) { struct dcp_channel *ch = &dcp->ch_cb; u8 *succ = ch->output[ch->depth - 1]; @@ -360,7 +448,7 @@ static void dcp_ack_dfb_surface(struct apple_dcp *dcp) * default surface ID. The bytes at +0x29..+0x2c are an opaque word whose * meaning is not known. */ -static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) +static void __maybe_unused dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) { s[0x002] = 0x01; memcpy(s + 0x00b, "8a3b", 4); @@ -388,7 +476,15 @@ static void dcp_fill_dfb_surface(struct apple_dcp *dcp, u8 *s) */ static void complete_dfb_surface(struct apple_dcp *dcp, void *out, void *cookie) { +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + struct dcp_channel *ch = &dcp->ch_cb; + + /* update_dfb() reports success in bit 0 of its reply */ + *(u8 *)ch->output[ch->depth - 1] = out && (*(u8 *)out & 1); + dcp_ack(dcp, DCP_CONTEXT_CB); +#else dcp_ack_dfb_surface(dcp); +#endif } static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, @@ -408,12 +504,45 @@ static bool iomfbep_cb_create_dfb_surface(struct apple_dcp *dcp, int tag, if (!req) return true; + +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + { + struct dcp_dfb_dimensions_h17p dim; + struct dcp_surface *surface = (void *)req->surface; + u32 format, allocation; + int ret = -EINVAL; + + memcpy(&dim, in, sizeof(dim)); + if (dcp->dfb_format_valid && dcp->dfb_compression_valid) + ret = apple_dcp_dfb_allocation(dim.width, dim.height, + dcp->dfb_format, dcp->dfb_compression, + dcp->dfb_region_size, &format, &allocation); + if (ret) { + dev_err(dcp->dev, "unsupported DFB layout %ux%u format=%#x compression=%u (%d)\n", + dim.width, dim.height, dcp->dfb_format, + dcp->dfb_compression, ret); + *(u8 *)out = false; + kfree(req); + return true; + } + + surface->is_premultiplied = 1; + surface->format = format; + surface->pix_size = 1; + surface->pel_w = 1; + surface->pel_h = 1; + surface->buf_size = allocation; + surface->surface_id = 5; + surface->has_comp = 1; + surface->has_planes = 1; + } +#else dcp_fill_dfb_surface(dcp, req->surface); +#endif dcp_register_dfb_surface(dcp, false, req, complete_dfb_surface, NULL); kfree(req); - /* deferred ACK: the A472 completion reports success for us */ return false; } #endif @@ -470,8 +599,21 @@ static bool iomfbep_cb_match_backlight_service(struct apple_dcp *dcp, int tag, v static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *prop) { switch (prop->id) { +#if !DCP_FW_DESKTOP_26(DCP_FW_VER) + /* The desktop 26.x firmware publishes no panel level here. */ + #if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + case 20: +#else case IOMFB_PROPERTY_NITS_V13_3: + /* + * 13.x firmware publishes another property under the 14.7 ID, with + * value 1, right before the panel level. Read as nits it is 0, and a + * backlight registered from it starts at the lowest level. + */ +#if DCP_FW_VERSION(14, 7, 0) <= DCP_FW_VER case IOMFB_PROPERTY_NITS_V14_7: +#endif +#endif { #if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER /* The H17P takeover level comes from the powerlog hint interface. */ @@ -487,6 +629,7 @@ static void iomfb_cb_pr_publish(struct apple_dcp *dcp, struct iomfb_property *pr } break; } +#endif default: dev_dbg(dcp->dev, "pr_publish: id: %d = %u\n", prop->id, prop->value); } @@ -898,6 +1041,15 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d * dma_map_resource() them instead fails and hands the firmware * DMA_MAPPING_ERROR as a device address. */ + dma_addr_t dva = dcp->fw_compat == DCP_FIRMWARE_V_27_0 ? + dcp->disp_firmware_base[req->index] : rsrc->start; +#elif DCP_FW_DESKTOP_26(DCP_FW_VER) + /* + * The desktop 26.x firmware takes the register windows' + * physical addresses as well. It reaches them on its + * DART-bypass stream; a mapping in its DMA stream would not + * affect those accesses. + */ dma_addr_t dva = rsrc->start; #elif DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) dma_addr_t dva = dcpep_map_reg_dva(dcp, req->index, rsrc); @@ -905,7 +1057,12 @@ static struct DCP_FW_NAME(dcp_map_reg_resp) dcpep_cb_map_reg(struct apple_dcp *d #endif return (struct DCP_FW_NAME(dcp_map_reg_resp)){ - .addr = rsrc->start, + .addr = +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + dcp->disp_firmware_base[req->index], +#else + rsrc->start, +#endif .length = resource_size(rsrc), #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) .dva = dva, @@ -1089,6 +1246,17 @@ static bool dcpep_process_chunks(struct apple_dcp *dcp, dev_warn(dcp->dev, "TimingElements without valid modes!\n"); dcp_180hz_notify(dcp); dcp_firmware_metadata_timing(dcp); +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp->peer_reserved) + for (unsigned int i = 0; i < dcp->nr_modes; i++) { + struct dcp_display_mode *m = &dcp->modes[i]; + + if ((u64)m->mode.clock * 1000 > dcp->dptxport[0].caps.total_rate) + dev_info(dcp->dev, "peer mode timing=%u color=%u " DRM_MODE_FMT "\n", + m->timing_mode_id, m->color_mode_id, + DRM_MODE_ARG(&m->mode)); + } +#endif } else if (!strcmp(req->key, "DisplayAttributes")) { struct dcp_parse_ctx metadata_ctx = ctx; @@ -1220,6 +1388,27 @@ static void boot_1_11(struct apple_dcp *dcp, void *out, void *cookie) static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) { +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + struct dcp_dfb_dimensions_h17p dim; + + if (!out) + return; + memcpy(&dim, out, sizeof(dim)); + + if (dim.width > 16384 || dim.height > 16384 + ) { + struct dcp_channel *ch = &dcp->ch_cb; + + dev_err(dcp->dev, "invalid default framebuffer dimensions %ux%u\n", + dim.width, dim.height); + *(u8 *)ch->output[ch->depth - 1] = false; + dcp_ack(dcp, DCP_CONTEXT_CB); + return; + } + + dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); + +#else struct dcp_dfb_dimensions_h17p dim; /* @@ -1240,12 +1429,25 @@ static void boot_1_10(struct apple_dcp *dcp, void *out, void *cookie) dim.width, dim.height); dcp_set_dfb_dimensions(dcp, false, &dim, boot_1_11, NULL); + +#endif } static void boot_1_9(struct apple_dcp *dcp, void *out, void *cookie) { u32 zero = 0; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + struct dcp_dfb_dimensions_h17p dim; + + if (!out) + return; + memcpy(&dim, out, sizeof(dim)); + if (dim.width && dim.height) { + boot_2(dcp, out, cookie); + return; + } +#endif dcp_get_dfb_state(dcp, false, &zero, boot_1_10, NULL); } @@ -1253,6 +1455,12 @@ static void boot_1_8(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_dfb_layout_req_h17p req = {}; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out) + return; + dcp->dfb_format_valid = *((u8 *)out + 0x28) & 1; + memcpy(&dcp->dfb_format, out, sizeof(dcp->dfb_format)); +#endif dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); } @@ -1260,6 +1468,12 @@ static void boot_1_7(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_dfb_info_h17p info = {}; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out) + return; + dcp->dfb_compression_valid = *((u8 *)out + 4) & 1; + memcpy(&dcp->dfb_compression, out, sizeof(dcp->dfb_compression)); +#endif dcp_get_dfb_info(dcp, false, &info, boot_1_8, NULL); } @@ -1267,6 +1481,16 @@ static void boot_1_6(struct apple_dcp *dcp, void *out, void *cookie) { u64 zero = 0; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out) + return; + if (!(*(u8 *)out & 1)) { + struct dcp_dfb_layout_req_h17p req = {}; + + dcp_get_dfb_layout(dcp, false, &req, boot_1_9, NULL); + return; + } +#endif dcp_get_dfb_compression_info(dcp, false, &zero, boot_1_7, NULL); } #endif @@ -1433,7 +1657,8 @@ static void release_swap_cookie(struct kref *ref) */ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { - struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_UNION(dcp->swap); +#if DCP_FW_VER < DCP_FW_VERSION(27, 0, 0) + struct DCP_FW_NAME(dcp_swap_submit_req) *req = &DCP_FW_SWAP(dcp); unsigned int i; /* @@ -1482,6 +1707,7 @@ static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) memmove(r + off - 2, r + off, n - off); memset(r + n - 2, 0, 2); } +#endif } #else static void dcp_h17p_prepare_swap(struct apple_dcp *dcp) { } @@ -1673,13 +1899,26 @@ static void dcp_on_set_power_state(struct apple_dcp *dcp, void *out, void *cooki .unklong = 1, }; +#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER + /* + * setPowerState(state, b1, b2, &out, b5, b6) as macOS 27.3's api power + * path calls it with a client: b1 and b5 set, b2 and b6 clear. + */ + if (dcp->hw.fw_27_3) + req.unkint = 0x00010001; +#endif dcp_set_power_state(dcp, DCP_POWER_OOB, &req, dcp_on_final, cookie); } static void dcp_on_set_parameter(struct apple_dcp *dcp, void *out, void *cookie) { struct dcp_set_parameter_dcp param = { - .param = IOMFBPARAM_ADAPTIVE_SYNC, + .param = +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + 15, +#else + IOMFBPARAM_ADAPTIVE_SYNC, +#endif .value = { 0 }, #if DCP_FW_VER >= DCP_FW_VERSION(13, 2, 0) .count = 3, @@ -1788,9 +2027,14 @@ static void last_client_closed_poff(struct apple_dcp *dcp, void *out, void *cook static void aborted_swaps_dcp_poff(struct apple_dcp *dcp, void *out, void *cookie) { +#if DCP_FW_VER >= DCP_FW_VERSION(14, 7, 0) + /* iomfb_last_client_close times out on 14.7 */ + last_client_closed_poff(dcp, NULL, cookie); +#else struct iomfb_last_client_close_req last_client_req = {}; iomfb_last_client_close(dcp, false, &last_client_req, last_client_closed_poff, cookie); +#endif } void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) @@ -1812,6 +2056,25 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) return; #endif +#if DCP_FW_DESKTOP_26(DCP_FW_VER) + /* A merged pair stops as a pair; its clear swap would not complete. */ + ret = dcp_peer_stop_26_6(dcp); + if (ret < 0) + dev_err(dcp->dev, "paired power-off failed: %d\n", ret); + if (ret > 0) + goto peer_stopped; +#endif + +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + ret = dcp_peer_stop_v27(dcp); + if (ret < 0) { + dev_err(dcp->dev, "paired power-off failed: %d\n", ret); + return; + } + if (ret > 0) + goto close_client; +#endif + cookie = kzalloc(sizeof(*cookie), GFP_KERNEL); if (!cookie) return; @@ -1848,6 +2111,12 @@ void DCP_FW_NAME(iomfb_poweroff)(struct apple_dcp *dcp) dev_dbg(dcp->dev, "%s: clear swap submitted: %u\n", __func__, swap_id); +#if DCP_FW_DESKTOP_26(DCP_FW_VER) +peer_stopped: +#endif +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) +close_client: +#endif poff_cookie = kzalloc(sizeof(*poff_cookie), GFP_KERNEL); if (!poff_cookie) return; @@ -1910,7 +2179,15 @@ void DCP_FW_NAME(iomfb_sleep)(struct apple_dcp *dcp) struct dcp_wait_cookie *cookie; -#if DCP_FW_VERSION(26, 0, 0) <= DCP_FW_VER +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + ret = dcp_peer_stop_v27(dcp); + if (ret < 0) { + dev_err(dcp->dev, "paired sleep failed: %d\n", ret); + return; + } +#endif + +#if DCP_FW_H17(DCP_FW_VER) if (dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) return; #endif @@ -2017,6 +2294,12 @@ static void dcpep_cb_hotplug(struct apple_dcp *dcp, u64 *connected) action |= DCP_HOTPLUG_NOTIFY; } dcp_handle_hotplug_actions(dcp, action); + /* Fresh modes and positive firmware HPD, not lane-count negotiation, + * make IOMFB ready for the resume modeset. */ + if (*connected && dcp->fw_compat == DCP_FIRMWARE_V_27_0 && + dcp_is_typec_output(dcp)) + complete_all(&dcp->typec_display_ready); + } static void @@ -2186,6 +2469,10 @@ static void dcp_swapped(struct apple_dcp *dcp, void *data, void *cookie) dcp_swap_watchdog_arm(dcp); } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) +#include "iomfb-peer-runtime-v27.inc" +#endif + static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) { struct DCP_FW_NAME(dcp_swap_start_resp) *resp = data; @@ -2201,7 +2488,11 @@ static void dcp_swap_started(struct apple_dcp *dcp, void *data, void *cookie) dcp_drm_crtc_vblank(dcp->crtc); return; } - DCP_FW_UNION(dcp->swap).swap.swap_id = resp->swap_id; +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp_peer_start_second(dcp, resp)) + return; +#endif + DCP_FW_SWAP(dcp).swap.swap_id = resp->swap_id; if (dcp_present_retires(dcp)) dcp_arm_retained_framebuffers(dcp, resp->swap_id); @@ -2285,6 +2576,16 @@ static void complete_set_digital_out_mode(struct apple_dcp *dcp, void *data, static void dcp_on_set_adaptive_sync(struct apple_dcp *dcp, void *out, void *cookie) { +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (!out || get_unaligned_le32(out)) { + struct dcp_wait_cookie *wait = cookie; + + wait->status = -EOPNOTSUPP; + complete(&wait->done); + kref_put(&wait->refcount, release_wait_cookie); + return; + } +#endif dcp_set_digital_out_mode(dcp, false, &dcp->mode, complete_set_digital_out_mode, cookie); } @@ -2293,7 +2594,12 @@ static void dcp_set_adaptive_sync(struct apple_dcp *dcp, u32 min_vrr, void *cookie) { struct dcp_set_parameter_dcp param = { - .param = IOMFBPARAM_ADAPTIVE_SYNC, + .param = +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + 15, +#else + IOMFBPARAM_ADAPTIVE_SYNC, +#endif .value = { min_vrr, /* minRR, 16.16 fixed-point Hz */ 0, /* mediaTargetRate */ @@ -2369,7 +2675,17 @@ int DCP_FW_NAME(iomfb_modeset)(struct apple_dcp *dcp, DCP_IOMFB_METHODS_H17P_EXTERNAL) { dev_dbg(dcp->dev, "inheriting the bootloader's display mode\n"); complete_set_digital_out_mode(dcp, NULL, cookie); - } else if (mode->vrr && + } else +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if ((ret = dcp_peer_modeset_v27(dcp, mode->timing_mode_id, + mode->color_mode_id, crtc_state->vrr_enabled)) != -EOPNOTSUPP) { + cookie->status = ret; + if (!ret) + crtc_state->color_mgmt_changed = true; + complete_set_digital_out_mode(dcp, NULL, cookie); + } else +#endif + if (mode->vrr && dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17P_EXTERNAL) dcp_set_adaptive_sync(dcp, @@ -2529,7 +2845,8 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru req->clear = 1; } - if (has_surface && (dcp->use_timestamps || dcp->vrr_enabled)) { + if (has_surface && (dcp->use_timestamps || dcp->vrr_enabled) && + !dcp->hw.swap_no_timestamps) { u64 submit_timestamp = dcp->swap_submit_timestamp; u64 timestamp = arch_timer_read_counter(); @@ -2583,6 +2900,18 @@ void DCP_FW_NAME(iomfb_flush)(struct apple_dcp *dcp, struct drm_crtc *crtc, stru mat.matrix[0] = mat.matrix[4] = mat.matrix[8] = 1LLU << 32; } +#if (DCP_FW_H17(DCP_FW_VER) && DCP_FW_VER >= DCP_FW_VERSION(27, 0, 0)) + if (dcp_peer_color_v27(dcp, &mat)) { + dcp_peer_swap_error(dcp, "color matrix", -EIO); + return; + } +#endif +#if DCP_FW_DESKTOP_26(DCP_FW_VER) + dcp->desktop->gamma_pending = dcp_gamma_update(dcp, crtc_state); + dcp->desktop->matrix = mat; + if (dcp_vrr_update(dcp, crtc_state, dcp_matrix_then_swap)) + return; +#endif #if DCP_FW_VER >= DCP_FW_VERSION(26, 0, 0) iomfb_set_matrix(dcp, false, &mat, dcp_pre_swap_a428, NULL); #else diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.c b/drivers/gpu/drm/apple/iomfb_v14_7.c index 743c14cbbf63b..474841ff8d0ae 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.c +++ b/drivers/gpu/drm/apple/iomfb_v14_7.c @@ -31,6 +31,7 @@ #include #include #include +#include #include "afk.h" #include "dcp.h" @@ -42,6 +43,7 @@ #include "iomfb_v14_7_board.h" #include "iomfb_v14_7_link.h" #include "iomfb_v14_7_swap.h" +#include "av.h" #include "parser.h" #define A(n) DCP_V14_TAG('A', n) @@ -76,7 +78,16 @@ static int dcp_v14_pmp_wait_ready(struct device_node *entry, #define DCP_V14_RTKIT_RETRIES 4 #define DCP_V14_MAX_PROPERTIES 256 #define DCP_V14_MAX_RAW 32 +/* + * The size of one raw property and of all of them. An external processor + * publishes an HDMI 2.1 (FRL) sink's TimingElements as one chunked transfer + * of more than 1 MiB (1.38 MB for a 4K120 TV), so it keeps up to 8 MiB per + * property and 24 MiB in all; the panel's properties stay small. + */ +#define DCP_V14_MAX_RAW_SIZE SZ_1M #define DCP_V14_MAX_RAW_BYTES SZ_8M +#define DCP_V14_EXT_MAX_RAW_SIZE SZ_8M +#define DCP_V14_EXT_MAX_RAW_BYTES (24 * SZ_1M) #define DCP_V14_MAX_BUFFERS 64 #define DCP_V14_MAX_BUFFER SZ_16M #define DCP_V14_MAX_BUFFER_BYTES SZ_64M @@ -150,11 +161,30 @@ static const struct dcp_v14_board dcp_v14_board_j516c = { .log_bringup = true, }; +/* + * J514C (MacBook Pro 14", M3 Max): the T6031 DCP running the J516C image + * (the DCP image is the SoC's; its UUID is checked at probe), in the J514S + * chassis, so the J514 panel: the board file names the J514S panel values, + * not yet read from a J514C, and the panel size comes from the boot + * framebuffer. Every panel timing and the display clock are logged. + */ +static const struct dcp_v14_board dcp_v14_board_j514c = { + .name = "J514C", + .debugfs = "dcp-j514c", + .dcp_compatible = "apple,t6031-dcp", + .machine = "apple,j514c", + .firmware_uuid = DCP_V14_J516C_FIRMWARE_UUID, + .handoff = "apple,t6031-handoff", + .promotion = true, + .log_bringup = true, +}; + static const struct dcp_v14_board *const dcp_v14_boards[] = { &dcp_v14_board_t6030, &dcp_v14_board_j613, &dcp_v14_board_j615, &dcp_v14_board_j516c, + &dcp_v14_board_j514c, }; struct dcp_v14_property { @@ -364,13 +394,23 @@ static struct dcp_v14_property *dcp_v14_property(struct apple_dcp_v14 *v14, u32 return p; } +static u32 dcp_v14_raw_limit(const struct apple_dcp_v14 *v14) +{ + return v14->external ? DCP_V14_EXT_MAX_RAW_SIZE : DCP_V14_MAX_RAW_SIZE; +} + +static u32 dcp_v14_raw_total(const struct apple_dcp_v14 *v14) +{ + return v14->external ? DCP_V14_EXT_MAX_RAW_BYTES : DCP_V14_MAX_RAW_BYTES; +} + static int dcp_v14_raw_property(struct apple_dcp_v14 *v14, u32 service, const u8 *key, const void *data, u32 size) { u32 i, old_size = 0; void *copy; - if (!memchr(key, 0, 64) || size > SZ_1M) + if (!memchr(key, 0, 64) || size > dcp_v14_raw_limit(v14)) return -EINVAL; for (i = 0; i < v14->raw_count; i++) if (v14->raw[i].service == service && !strcmp(v14->raw[i].key, key)) @@ -379,7 +419,7 @@ static int dcp_v14_raw_property(struct apple_dcp_v14 *v14, u32 service, const u8 return -ENOSPC; if (i < v14->raw_count) old_size = v14->raw[i].size; - if (v14->raw_bytes - old_size + size > DCP_V14_MAX_RAW_BYTES) + if (v14->raw_bytes - old_size + size > dcp_v14_raw_total(v14)) return -ENOSPC; copy = kvmemdup(data, size ?: 1, GFP_KERNEL); if (!copy) @@ -817,7 +857,8 @@ static int dcp_v14_external_callback(struct apple_dcp_v14 *v14, u32 tag, const u if (tag == D(127) && SHAPE(4, 4)) { count = get_unaligned_le32(in); ret = -EINVAL; - if (!v14->chunk && count && count <= 0x100001) { + /* The announced count includes one byte that is never sent. */ + if (!v14->chunk && count && count <= dcp_v14_raw_limit(v14) + 1) { v14->chunk_size = count - 1; v14->chunk_offset = 0; v14->chunk = kvzalloc(max_t(u32, 1, v14->chunk_size), GFP_KERNEL); @@ -983,7 +1024,7 @@ static int dcp_v14_callback(void *cookie, u32 tag, const void *input, u32 in_siz /* Chunked property transfer: start, chunk, end. */ if (tag == D(127) && SHAPE(4, 4)) { count = get_unaligned_le32(in); - if (v14->chunk || !count || count > 0x100001) + if (v14->chunk || !count || count > dcp_v14_raw_limit(v14) + 1) return -EINVAL; v14->chunk_size = count - 1; v14->chunk_offset = 0; @@ -1617,25 +1658,17 @@ enum { static void dcpext_bringup(struct work_struct *work); /* - * The bounded translator (900-config-tooling) is optional at link time; - * without it, refuse a tuple that does not fit before using the unbounded one. + * Translate one iommu-addresses tuple, refusing a truncated one. + * of_translate_dma_region_checked() is an ordinary exported symbol and + * DRM_APPLE depends on OF_ADDRESS, so it is always linked: call it directly. + * The former symbol_get() could never resolve it -- symbol_get() takes only + * GPL-only symbols -- so the bounded check was silently skipped. */ static const __be32 *dcpext_translate_region(struct device_node *np, const __be32 *maps, size_t cells, phys_addr_t *iova, size_t *length) { - typeof(&of_translate_dma_region_checked) checked; - const __be32 *next; - - checked = symbol_get(of_translate_dma_region_checked); - if (checked) { - next = checked(np, maps, cells, iova, length); - symbol_put(of_translate_dma_region_checked); - return next; - } - if (cells < (size_t)(of_n_addr_cells(np) + of_n_size_cells(np))) - return NULL; - return of_translate_dma_region(np, maps, iova, length); + return of_translate_dma_region_checked(np, maps, cells, iova, length); } /* @@ -2843,15 +2876,45 @@ bool iomfb_v14_7_external_showing(struct apple_dcp *dcp, return live; } +bool iomfb_v14_7_external_powered(struct apple_dcp *dcp) +{ + struct apple_dcp_v14 *v14 = dcp->v14; + + return dcp->external && v14 && READ_ONCE(v14->powered) && !READ_ONCE(v14->failed); +} + +/* The sink's audio around an external mode set. */ +static void dcp_v14_audio_off(struct apple_dcp *dcp) +{ + KUNIT_STATIC_STUB_REDIRECT(dcp_v14_audio_off, dcp); + av_service_disconnect(dcp); +} + +static void dcp_v14_audio_on(struct apple_dcp *dcp) +{ + KUNIT_STATIC_STUB_REDIRECT(dcp_v14_audio_on, dcp); + av_service_connect(dcp); +} + /* An external display: power down, set the new mode, power up. */ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state *crtc_state) { struct apple_dcp_v14 *v14 = dcp->v14; struct dcp_display_mode mode = {}; + bool audio_off = false; int ret = 0; if (!v14 || READ_ONCE(v14->failed) || !v14->opened) return -EIO; + /* + * The sink's audio stops before the display is powered down for the + * mode change (as dcp_poweroff() has it), outside the RPC lock; it is + * connected again once the display is on, below. + */ + if (READ_ONCE(v14->powered) && dcp->avep) { + dcp_v14_audio_off(dcp); + audio_off = true; + } ret = dcp_v14_external_settle(dcp, v14); if (ret) goto out; @@ -2872,6 +2935,8 @@ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state DRM_MODE_ARG(&crtc_state->mode), mode.timing_mode_id, mode.color_mode_id); atomic_set(&dcp->external_retries, 0); dcp_mode_set_valid(&dcp->mode_state, true); + if (dcp->avep) + dcp_v14_audio_on(dcp); return 0; } if (v14->powered && dcp_v14_set_power(v14, false)) @@ -2895,13 +2960,23 @@ static int dcp_v14_external_modeset(struct apple_dcp *dcp, struct drm_crtc_state dev_info(dcp->dev, "display mode " DRM_MODE_FMT " (timing %u, color %u): %d\n", DRM_MODE_ARG(&crtc_state->mode), mode.timing_mode_id, mode.color_mode_id, ret); if (ret) { - /* A mode that is gone, or a description that never settled. */ + /* + * A mode that is gone, or a description that never settled: + * nothing was powered down, and the display goes on in its old + * mode. So does its audio; no plug event would bring it back + * if the retries fail too. + */ + if (audio_off && iomfb_v14_7_external_powered(dcp)) + dcp_v14_audio_on(dcp); dcp_external_retry(dcp, "display mode not set", ret, 500); return ret; } atomic_set(&dcp->external_retries, 0); scoped_guard(spinlock_irqsave, &dcp->mode_state.lock) dcp->mode_state.valid = true; + /* A472 on: the sink's audio again. */ + if (dcp->avep) + dcp_v14_audio_on(dcp); return 0; } @@ -2961,6 +3036,9 @@ void iomfb_v14_7_poweroff(struct apple_dcp *dcp) /* An external display is powered down; the next enable sets its mode again. */ if (dcp->external) { + /* Audio stops with the display, as dcp_poweroff() has it. */ + if (dcp->avep) + av_service_disconnect(dcp); if (v14 && v14->opened && !READ_ONCE(v14->failed)) { mutex_lock(&v14->lock); if (v14->powered && dcp_v14_set_power(v14, false)) @@ -2974,3 +3052,7 @@ void iomfb_v14_7_poweroff(struct apple_dcp *dcp) if (v14 && v14->started) dcp_v14_present(v14, NULL, 0, 0, 0, NULL); } + +#if IS_ENABLED(CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST) +#include "iomfb_v14_7_external_audio_test.c" +#endif diff --git a/drivers/gpu/drm/apple/iomfb_v14_7.h b/drivers/gpu/drm/apple/iomfb_v14_7.h index f6fe9e199b014..4c3d16043748c 100644 --- a/drivers/gpu/drm/apple/iomfb_v14_7.h +++ b/drivers/gpu/drm/apple/iomfb_v14_7.h @@ -47,6 +47,8 @@ int iomfb_v14_7_external_open(struct apple_dcp *dcp); const char *iomfb_v14_7_external_busy(struct apple_dcp *dcp); /* The external session can no longer be used until reboot. */ bool iomfb_v14_7_external_failed(struct apple_dcp *dcp); +/* The attached display is powered (A472 on) in the mode last set. */ +bool iomfb_v14_7_external_powered(struct apple_dcp *dcp); /* The firmware runs this mode, set last, and shows swaps in it. */ bool iomfb_v14_7_external_showing(struct apple_dcp *dcp, const struct drm_display_mode *mode); @@ -73,4 +75,13 @@ void iomfb_v14_7_flush(struct apple_dcp *dcp, struct drm_crtc *crtc, void iomfb_v14_7_poweron(struct apple_dcp *dcp); void iomfb_v14_7_poweroff(struct apple_dcp *dcp); +/* + * The 14.7 nits backlight policy (T6030 board record) is not in this tree; + * the firmware-27 port's callers see no 14.7 panel. + */ +static inline bool iomfb_v14_7_panel_nits(struct apple_dcp *dcp) +{ + return false; +} + #endif /* __APPLE_IOMFB_V14_7_H__ */ diff --git a/drivers/gpu/drm/apple/iomfb_v14_7_external_audio_test.c b/drivers/gpu/drm/apple/iomfb_v14_7_external_audio_test.c new file mode 100644 index 0000000000000..a20810109fca2 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v14_7_external_audio_test.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * The sink's audio around a failed external mode set, included at the end + * of iomfb_v14_7.c so that the production dcp_v14_external_modeset() runs. + * Only the audio service is stubbed. The mode lookup fails on an empty + * catalog after a complete description, before anything is powered down. + */ +#include +#include +#include + +struct v14_ext_audio { + struct apple_dcp *dcp; + struct apple_dcp_v14 *v14; + unsigned int offs, ons; +}; + +static struct v14_ext_audio *v14_ext_audio_current; + +static void v14_ext_audio_off(struct apple_dcp *dcp) +{ + v14_ext_audio_current->offs++; +} + +static void v14_ext_audio_on(struct apple_dcp *dcp) +{ + v14_ext_audio_current->ons++; +} + +static void v14_ext_audio_retry(struct work_struct *work) +{ +} + +static void v14_ext_audio_stop(void *data) +{ + struct apple_dcp *dcp = data; + + cancel_delayed_work_sync(&dcp->external_retry_wq); +} + +/* A started, described native external session, its display @powered. */ +static struct v14_ext_audio *v14_ext_audio_setup(struct kunit *test, bool powered) +{ + struct v14_ext_audio *s = kunit_kzalloc(test, sizeof(*s), GFP_KERNEL); + struct device *dev; + + KUNIT_ASSERT_NOT_NULL(test, s); + dev = kunit_device_register(test, "dcp-v14-ext-audio"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + s->dcp = devm_kzalloc(dev, sizeof(*s->dcp), GFP_KERNEL); + s->v14 = devm_kzalloc(dev, sizeof(*s->v14), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, s->dcp); + KUNIT_ASSERT_NOT_NULL(test, s->v14); + s->dcp->dev = dev; + s->dcp->fw_compat = DCP_FIRMWARE_V_14_7; + s->dcp->external = true; + s->dcp->external_native = true; + /* Never dereferenced: the audio service is stubbed. */ + s->dcp->avep = (struct apple_dcp_afkep *)s; + mutex_init(&s->dcp->modes_lock); + INIT_DELAYED_WORK(&s->dcp->external_retry_wq, v14_ext_audio_retry); + s->dcp->v14 = s->v14; + s->v14->dev = dev; + s->v14->dcp = s->dcp; + s->v14->external = true; + s->v14->started = true; + s->v14->opened = true; + s->v14->powered = powered; + s->v14->described = DCP_V14_DESC_ALL; + mutex_init(&s->v14->lock); + init_waitqueue_head(&s->v14->described_wait); + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, v14_ext_audio_stop, s->dcp), 0); + v14_ext_audio_current = s; + kunit_activate_static_stub(test, dcp_v14_audio_off, v14_ext_audio_off); + kunit_activate_static_stub(test, dcp_v14_audio_on, v14_ext_audio_on); + return s; +} + +/* The mode is gone, the display stays on in its old mode: so does its audio. */ +static void v14_ext_audio_failed_modeset_keeps_audio(struct kunit *test) +{ + struct v14_ext_audio *s = v14_ext_audio_setup(test, true); + struct drm_crtc_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_EXPECT_EQ(test, dcp_v14_external_modeset(s->dcp, state), -EINVAL); + KUNIT_EXPECT_EQ(test, s->offs, 1U); + KUNIT_EXPECT_EQ(test, s->ons, 1U); + KUNIT_EXPECT_TRUE(test, s->v14->powered); + KUNIT_EXPECT_FALSE(test, s->v14->failed); + /* The mode set is still retried. */ + KUNIT_EXPECT_TRUE(test, delayed_work_pending(&s->dcp->external_retry_wq)); + KUNIT_EXPECT_FALSE(test, mutex_is_locked(&s->v14->lock)); +} + +/* A display that was off has no audio to stop, and none comes back. */ +static void v14_ext_audio_failed_modeset_display_off(struct kunit *test) +{ + struct v14_ext_audio *s = v14_ext_audio_setup(test, false); + struct drm_crtc_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, state); + KUNIT_EXPECT_EQ(test, dcp_v14_external_modeset(s->dcp, state), -EINVAL); + KUNIT_EXPECT_EQ(test, s->offs, 0U); + KUNIT_EXPECT_EQ(test, s->ons, 0U); +} + +/* + * Out of retries: the display stays on in its old mode, and so does its + * audio. Nothing else would connect the sink again. + */ +static void v14_ext_audio_no_retry_left(struct kunit *test) +{ + struct v14_ext_audio *s = v14_ext_audio_setup(test, true); + struct drm_crtc_state *state = kunit_kzalloc(test, sizeof(*state), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, state); + /* Past any retry budget (DCP_EXTERNAL_RETRIES in dcp.c). */ + atomic_set(&s->dcp->external_retries, 100); + KUNIT_EXPECT_EQ(test, dcp_v14_external_modeset(s->dcp, state), -EINVAL); + KUNIT_EXPECT_FALSE(test, delayed_work_pending(&s->dcp->external_retry_wq)); + KUNIT_EXPECT_EQ(test, s->offs, 1U); + KUNIT_EXPECT_EQ(test, s->ons, 1U); +} + +static struct kunit_case v14_ext_audio_cases[] = { + KUNIT_CASE(v14_ext_audio_failed_modeset_keeps_audio), + KUNIT_CASE(v14_ext_audio_failed_modeset_display_off), + KUNIT_CASE(v14_ext_audio_no_retry_left), + {} +}; + +static struct kunit_suite v14_ext_audio_suite = { + .name = "dcp_v14_external_audio", + .test_cases = v14_ext_audio_cases, +}; + +kunit_test_suite(v14_ext_audio_suite); diff --git a/drivers/gpu/drm/apple/iomfb_v27.c b/drivers/gpu/drm/apple/iomfb_v27.c new file mode 100644 index 0000000000000..061bba87e6748 --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v27.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright The Asahi Linux Contributors */ +/* Copyright 2026 Aurora Silicon */ + +#include +#include + +#include "dcp-link.h" +#include "iomfb_v27.h" +#include "iomfb_v12_3.h" +#include "iomfb_v13_3.h" +#include "version_utils.h" +#include "dcp-internal.h" + +void iomfb_serialize_present_v27(struct dcp_swap_submit_req_v27 *wire, + const struct dcp_swap_submit_req_h17p *request) +{ + memset(wire, 0, sizeof(*wire)); + memcpy(wire->swap.timestamp, request->swap.timestamp, + sizeof(wire->swap.timestamp)); + wire->swap.unk_38 = request->swap.unk_38; + /* The common geometry runs from swap_id through bg_color. */ + memcpy(&wire->swap.swap_id, &request->swap.swap_id, + offsetof(struct dcp_swap_v27, reserved_158) - + offsetof(struct dcp_swap_v27, swap_id)); + memcpy(wire->surf, request->surf, sizeof(wire->surf)); + memcpy(wire->surf_iova, request->surf_iova, + sizeof(*request) - + offsetof(struct dcp_swap_submit_req_h17p, surf_iova)); +} + +struct dcp_h17p_hotplug_request { + __le64 connected; + u8 tiled_display[0x4c]; + u8 tiled_display_null; + u8 padding[3]; +} __packed; + +struct dcp_h17p_swap_info_request { + u8 swap_info[0xe0]; + u8 swap_info_null; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_h17p_hotplug_request) == 0x58); +static_assert(sizeof(struct dcp_h17p_swap_info_request) == 0xe4); + +static_assert(sizeof(struct dc_swap_complete_resp_h17p) == 0x730); +static_assert(offsetof(struct dc_swap_complete_resp_h17p, swap_id) == 0); +static_assert(offsetof(struct dcp_swap_v27, brightness_update) == 0x354); +static_assert(offsetof(struct dcp_swap_v27, brightness_nits) == 0x35e); +static_assert(sizeof(struct dcp_swap_v27) == 0x6e0); + +static const struct dcp_method_entry dcp_methods[dcpep_num_methods] = { + IOMFB_METHOD("A000", dcpep_late_init_signal), + IOMFB_METHOD("A027", dcpep_setup_video_limits), + IOMFB_METHOD("A136", iomfbep_a131_pmu_service_matched), + IOMFB_METHOD("A137", iomfbep_a132_backlight_service_matched), + IOMFB_METHOD("A389", dcpep_set_create_dfb), + IOMFB_METHOD("A390", iomfbep_a358_vi_set_temperature_hint), + IOMFB_METHOD("A401", dcpep_start_signal), + IOMFB_METHOD("A406", dcpep_swap_start), + IOMFB_METHOD("A408", dcpep_swap_submit), + IOMFB_METHOD("A411", dcpep_set_display_device), + IOMFB_METHOD("A412", dcpep_is_main_display), + IOMFB_METHOD("A413", dcpep_set_digital_out_mode), + IOMFB_METHOD("A423", iomfbep_set_matrix), + IOMFB_METHOD("A427", iomfbep_get_color_remap_mode), + IOMFB_METHOD("A450", dcpep_set_parameter_dcp), + IOMFB_METHOD("A454", dcpep_create_default_fb), + IOMFB_METHOD("A037", dcpep_get_dfb_compression_info), + IOMFB_METHOD("A036", dcpep_get_dfb_info), + IOMFB_METHOD("A103", dcpep_get_dfb_layout), + IOMFB_METHOD("A453", dcpep_get_dfb_state), + IOMFB_METHOD("A104", dcpep_set_dfb_dimensions), + IOMFB_METHOD("A105", dcpep_commit_dfb_info), + IOMFB_METHOD("A035", dcpep_dfb_query), + IOMFB_METHOD("A384", dcpep_dfb_ready_query), + IOMFB_METHOD("A353", dcpep_pipe_query_353), + IOMFB_METHOD("A352", dcpep_pipe_set_352), + IOMFB_METHOD("A415", dcpep_pipe_cfg_415), + IOMFB_METHOD("A033", dcpep_pipe_cfg_031), + IOMFB_METHOD("A414", dcpep_pipe_cfg_414), + IOMFB_METHOD("A489", dcpep_pipe_query_478), + IOMFB_METHOD("A485", dcpep_pipe_query_474), + IOMFB_METHOD("A487", dcpep_pipe_set_476), + IOMFB_METHOD("A428", dcpep_pipe_set_428), + IOMFB_METHOD("A458", dcpep_enable_disable_video_power_savings), + IOMFB_METHOD("A466", dcpep_first_client_open), + IOMFB_METHOD("A467", iomfbep_last_client_close), + IOMFB_METHOD("A475", dcpep_set_display_refresh_properties), + IOMFB_METHOD("A478", dcpep_flush_supports_power), + IOMFB_METHOD("A479", iomfbep_abort_swaps_dcp), + IOMFB_METHOD("A484", dcpep_set_power_state), + IOMFB_METHOD("A483", dcpep_register_dfb_surface), +}; + +#define iomfb_modeset_h17p iomfb_modeset_v27 +#define iomfb_flush_h17p iomfb_flush_v27 +#define iomfb_poweron_h17p iomfb_poweron_v27 +#define iomfb_poweroff_h17p iomfb_poweroff_v27 +#define iomfb_sleep_h17p iomfb_sleep_v27 +#define iomfb_shutdown_h17p iomfb_shutdown_v27 +#define iomfb_present_backlight_h17p iomfb_present_backlight_v27 +#define DCP_FW h17p +#define DCP_FW_VER DCP_FW_VERSION(27, 0, 0) + +#include "iomfb_template.c" + +static bool trampoline_rt_bandwidth_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + u64 scratch = dcp->disp_firmware_base[dcp->disp_bw_scratch_index] + + dcp->disp_bw_scratch_offset; + u64 clock_request = 0; + + trace_iomfb_callback(dcp, tag, __func__); + if (apple_dcp_h17p_rt_bw_encode_reply(in, out, scratch, clock_request)) + dev_warn(dcp->dev, "D003 bandwidth request is malformed\n"); + return true; +} + +static bool trampoline_get_frequency_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct apple_dcp_h17p_clock_request *request = in; + u32 index = le32_to_cpu(request->clock_id); + u64 rate = 0; + + if (index < dcp->num_firmware_clocks) + rate = clk_get_rate(dcp->firmware_clocks[index].clk); + else + dev_err(dcp->dev, "DCP requested undeclared provider clock index %u\n", + index); + *(__le64 *)out = cpu_to_le64(rate); + return true; +} + +static bool trampoline_hotplug_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_hotplug_request *request = in; + + /* The opt-in peer allocator needs the firmware's tiling metadata to verify + * the VFTG relationship before implementing its framebuffer lifecycle. + * Keep both halves: the endpoint IDs are at the end of this payload. + */ + if (dcp->peer_reserved || dcp->peer_owner) { + dev_info(dcp->dev, "peer HPD connected=%llu null=%u head=%*ph\n", + le64_to_cpu(request->connected), request->tiled_display_null, + 40, request->tiled_display); + dev_info(dcp->dev, "peer HPD tail=%*ph\n", 36, + request->tiled_display + 40); + } + if (!(request->tiled_display_null & 1) && request->connected && + !(get_unaligned_le32(request->tiled_display + 0xc) & BIT(3))) + dcp_peer_display_ids_v27(dcp, + get_unaligned_le32(request->tiled_display + 0x3b), + get_unaligned_le32(request->tiled_display + 0x47)); + if (!(request->tiled_display_null & 1)) + memcpy(out, request->tiled_display, + sizeof(request->tiled_display)); + return trampoline_hotplug(dcp, tag, out, in); +} + +static bool trampoline_swap_info_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_swap_info_request *request = in; + + trace_iomfb_callback(dcp, tag, __func__); + if (!(request->swap_info_null & 1)) + memcpy(out, request->swap_info, sizeof(request->swap_info)); + return true; +} + +static bool iomfbep_cb_video_range_h17p(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + __le32 *resp = out; + + resp[0] = cpu_to_le32(0x435b0000); + resp[1] = 0; + return true; +} + +struct dcp_h17p_get_data_prop_req { + char obj[4]; + char key[0x40]; + __le32 length; + u8 data_null; + u8 length_null; + u8 padding[2]; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_req) == 0x4c); + +struct dcp_h17p_get_data_prop_resp { + u8 data[0xc00]; + __le32 length; +} __packed; +static_assert(sizeof(struct dcp_h17p_get_data_prop_resp) == 0xc04); + +static bool trampoline_get_data_prop_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_h17p_get_data_prop_req *req = in; + struct dcp_h17p_get_data_prop_resp *resp = out; + const struct property *prop; + u32 cap = min_t(u32, le32_to_cpu(req->length), sizeof(resp->data)); + + if (memcmp(req->obj, "VORP", sizeof(req->obj)) || + strnlen(req->key, sizeof(req->key)) == sizeof(req->key)) + return true; + + prop = of_find_property(dcp->dev->of_node, req->key, NULL); + if (!prop || prop->length < 0 || prop->length > cap) + return true; + memcpy(resp->data, prop->value, prop->length); + resp->length = cpu_to_le32(prop->length); + return true; +} + +struct dcp_allocate_buffer_req_v27 { + __le32 options; + __le64 size; + __le32 alignment; + u8 dva_null; + u8 capacity_null; + u8 padding[2]; +} __packed; + +struct dcp_allocate_buffer_resp_v27 { + __le64 dva; + __le64 capacity; + __le32 mem_desc_id; +} __packed; + +struct dcp_buffer_lifecycle_req_v27 { + __le32 mem_desc_id; + __le32 options; +} __packed; + +static_assert(sizeof(struct dcp_allocate_buffer_req_v27) == 0x14); +static_assert(sizeof(struct dcp_allocate_buffer_resp_v27) == 0x14); +static_assert(offsetof(struct dcp_allocate_buffer_resp_v27, mem_desc_id) == 0x10); +static_assert(sizeof(struct dcp_buffer_lifecycle_req_v27) == 8); + +static bool trampoline_allocate_buffer_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_allocate_buffer_req_v27 *req = in; + struct dcp_allocate_buffer_resp_v27 *resp = out; + struct dcp_allocate_buffer_req allocation = { + .unk0 = le32_to_cpu(req->options), + .size = le64_to_cpu(req->size), + .unk2 = le32_to_cpu(req->alignment), + }; + struct dcp_allocate_buffer_resp buffer; + + if (!allocation.size || allocation.size > SIZE_MAX - SZ_16K || + allocation.unk2 > PAGE_SIZE || + (allocation.unk2 && !is_power_of_2(allocation.unk2))) + return true; + buffer = dcpep_cb_allocate_buffer(dcp, &allocation); + if (!req->dva_null) + resp->dva = cpu_to_le64(buffer.dva); + if (!req->capacity_null) + resp->capacity = cpu_to_le64(ALIGN(buffer.dva_size, SZ_16K)); + resp->mem_desc_id = cpu_to_le32(buffer.mem_desc_id); + return true; +} + +static bool trampoline_buffer_lifecycle_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcp_buffer_lifecycle_req_v27 *req = in; + __le32 *status = out; + u32 id = le32_to_cpu(req->mem_desc_id); + + if (!id || id >= DCP_MAX_MAPPINGS || + !test_bit(id, dcp->memdesc_map) || !dcp->memdesc[id].buf) { + *status = cpu_to_le32(14); + return true; + } + if (tag == 452) + dma_wmb(); + else + dma_rmb(); + *status = 0; + return true; +} + +static bool trampoline_ideal_screen_space_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const char *gamut; + u32 space = 1; + + if (!of_property_read_string(dcp->dev->of_node, + "apple,artwork-display-gamut", &gamut)) { + if (!strcmp(gamut, "P3")) + space = 2; + else if (!strcmp(gamut, "AdobeRGB")) + space = 3; + } + *(__le32 *)out = cpu_to_le32(space); + return true; +} + +static bool trampoline_tiling_state_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcpep_get_tiling_state_req *req = in; + struct dcpep_get_tiling_state_resp *resp = out; + + resp->value = req->value; + resp->ret = 0; + if (!(req->value_null & 1) && + dcp_peer_tiling_state_v27(dcp, false, req->event, req->param, + &resp->value)) { + resp->ret = 1; + return true; + } + if (!req->value_null && (req->param == 1 || req->param == 11) && + dcp->connector && !dcp->connector->base.has_tile) { + resp->value = 0; + resp->ret = 1; + } + return true; +} + +/* D115: same16-byte input as D114, boolean/padded4 output. */ +static bool trampoline_set_tiling_state_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const struct dcpep_get_tiling_state_req *req = in; + u32 value = req->value; + + *(__le32 *)out = cpu_to_le32(!(req->value_null & 1) && + dcp_peer_tiling_state_v27(dcp, true, req->event, req->param, &value)); + return true; +} + +#include "iomfb-display-identity-v27.inc" +#include "iomfb-external-counts-v27.inc" +#include "iomfb-boolean-properties-v27.inc" + +#define DCP_ANALYTICS_EVENT_SIZE 0x1044 +#define DCP_ANALYTICS_NAME_SIZE 0x40 +#define DCP_ANALYTICS_DICT_SIZE (DCP_ANALYTICS_EVENT_SIZE - DCP_ANALYTICS_NAME_SIZE) +#define DCP_ANALYTICS_FULL_DUMPS 16 + +/* + * D113: the firmware reports an analytics event, its name in the first 0x40 + * bytes and then the event's serialized payload dictionary. The dictionary is + * an in/out argument: the 0x1004-byte reply area is the dictionary the + * firmware reads back once the call returns. Hand it back unchanged. + * + * Replying with zeroes there made every DCP crash on its first analytics + * event, which the firmware sends about an hour after it starts; all DCPs + * start together at boot, so all of them went down within seconds. + */ +static bool dcpep_cb_core_analytics_v27(struct apple_dcp *dcp, int tag, + void *out, void *in) +{ + const u8 *event = in; + unsigned int len = DCP_ANALYTICS_EVENT_SIZE; + char name[128]; + + static_assert(DCP_ANALYTICS_DICT_SIZE == 0x1004); + + strscpy(name, event, min(sizeof(name), (size_t)DCP_ANALYTICS_NAME_SIZE)); + dev_info_ratelimited(dcp->dev, "CoreAnalytics event %s, reply area %16ph\n", + name, out); + if (dcp->analytics_dumps < DCP_ANALYTICS_FULL_DUMPS) { + dcp->analytics_dumps++; + while (len && !event[len - 1]) + len--; + print_hex_dump(KERN_INFO, "dcp analytics: ", DUMP_PREFIX_OFFSET, + 32, 1, event, len, true); + } + /* The same record and answer as desktop 26.6's analytics event. */ + dcp_dict_passthrough_reply(in, out); + return true; +} + +static const iomfb_cb_handler cb_handlers[IOMFB_MAX_CB] = { + [0] = dcpep_cb_d000_h17p, + [1] = trampoline_true, + [2] = trampoline_nop, + [3] = trampoline_rt_bandwidth_h17p, + [6] = trampoline_set_frame_sync_props_h17p, + [100] = iomfbep_cb_match_pmu_service, + [101] = trampoline_zero, + [102] = trampoline_nop, + [104] = trampoline_nop, + [106] = trampoline_nop, + [107] = trampoline_true, + [108] = trampoline_true, + [109] = trampoline_true, + [110] = trampoline_true, + [111] = trampoline_create_backlight_service, + [112] = trampoline_true, + [113] = dcpep_cb_core_analytics_v27, + [116] = trampoline_nop, + [117] = trampoline_zero, + [118] = trampoline_zero, + [119] = trampoline_zero, + [114] = trampoline_tiling_state_v27, + [115] = trampoline_set_tiling_state_v27, + [120] = dcpep_cb_boot_1, + [121] = trampoline_false, + [122] = trampoline_false, + [123] = trampoline_zero, + [124] = trampoline_read_edt_data, + [126] = trampoline_prop_start, + [127] = trampoline_prop_chunk, + [128] = trampoline_prop_end, + [129] = trampoline_allocate_bandwidth, + [130] = trampoline_update_external_counts_v27, + [131] = trampoline_get_external_counts_v27, + [132] = trampoline_apply_suppression_v27, + [133] = trampoline_display_identity_v27, + [201] = trampoline_map_piodma, + [202] = trampoline_unmap_piodma, + [206] = iomfbep_cb_match_pmu_service_2, + [207] = iomfbep_cb_match_backlight_service, + [204] = trampoline_ideal_screen_space_v27, + [208] = trampoline_nop, + [209] = trampoline_get_time, + [300] = trampoline_pr_publish, + [400] = trampoline_get_data_prop_v27, + [401] = trampoline_get_uint_prop, + [404] = trampoline_nop, + [406] = trampoline_set_fx_prop, + [408] = trampoline_get_frequency_v27, + [411] = trampoline_map_reg, + [413] = trampoline_true, + [414] = trampoline_sr_set_property_int, + [415] = trampoline_true, + [451] = trampoline_allocate_buffer_v27, + [452] = trampoline_buffer_lifecycle_v27, + [453] = trampoline_buffer_lifecycle_v27, + [454] = trampoline_release_mem_desc, + [552] = trampoline_true, + [561] = trampoline_true, + [563] = trampoline_true, + [565] = trampoline_true, + [567] = trampoline_true, + [569] = trampoline_set_boolean_property_v27, + [572] = trampoline_zero, + [574] = iomfbep_cb_video_range_h17p, + [575] = trampoline_hotplug_h17p, + [576] = trampoline_nop, + [586] = iomfbep_cb_create_dfb_surface, + [587] = trampoline_nop, + [588] = trampoline_nop, + [589] = trampoline_swap_info_h17p, + + [592] = trampoline_nop, + [594] = trampoline_swap_complete, + [596] = trampoline_swap_complete_intent_gated, + [597] = trampoline_abort_swap_ap_gated, + [598] = trampoline_enable_backlight_message_ap_gated, + [599] = trampoline_nop, + [601] = trampoline_false, + [602] = trampoline_false, + [603] = trampoline_nop, +}; + +static const struct { + u16 input, output; +} callback_sizes[IOMFB_MAX_CB] = { + [0] = { 0x0, 0x4 }, + [1] = { 0x0, 0x4 }, + [2] = { 0x0, 0x0 }, + [3] = { 0x4, 0x14 }, + [6] = { 0x54, 0x50 }, + [300] = { 0x10, 0x0 }, + [100] = { 0x0, 0x0 }, + [101] = { 0x0, 0x4 }, + [102] = { 0x44, 0x0 }, + [104] = { 0x44, 0x0 }, + [106] = { 0x40, 0x0 }, + [107] = { 0x0, 0x4 }, + [108] = { 0x0, 0x4 }, + [109] = { 0x0, 0x4 }, + [110] = { 0x0, 0x4 }, + [111] = { 0x0, 0x4 }, + [112] = { 0x0, 0x4 }, + [113] = { 0x1044, 0x1004 }, + [116] = { 0xc, 0x0 }, + [117] = { 0x14, 0x4 }, + [118] = { 0x3e88, 0x4 }, + [119] = { 0x4, 0x4 }, + [114] = { 0x10, 0x8 }, + [115] = { 0x10, 0x4 }, + [120] = { 0x0, 0x4 }, + [121] = { 0x0, 0x4 }, + [122] = { 0x0, 0x4 }, + [123] = { 0x0, 0x4 }, + [124] = { 0x64, 0x24 }, + [126] = { 0x4, 0x4 }, + [127] = { 0x1008, 0x4 }, + [128] = { 0x40, 0x4 }, + [129] = { 0x1c, 0x14 }, + [130] = { 0x8, 0x0 }, + [131] = { 0x10, 0xc }, + [132] = { 0x4, 0x0 }, + [133] = { 0x0, 0x4 }, + [201] = { 0xc, 0x10 }, + [202] = { 0x18, 0x0 }, + [206] = { 0x0, 0x4 }, + [207] = { 0x0, 0x4 }, + [204] = { 0x0, 0x4 }, + [208] = { 0x4, 0x0 }, + [209] = { 0x0, 0x8 }, + [400] = { 0x4c, 0xc04 }, + [401] = { 0x50, 0xc }, + [404] = { 0x48, 0x0 }, + [406] = { 0x48, 0x0 }, + [408] = { 0x8, 0x8 }, + [411] = { 0x10, 0x1c }, + [413] = { 0x1048, 0x4 }, + [414] = { 0x50, 0x4 }, + [415] = { 0x4c, 0x4 }, + [451] = { 0x14, 0x14 }, + [452] = { 0x8, 0x4 }, + [453] = { 0x8, 0x4 }, + [454] = { 0x4, 0x4 }, + [552] = { 0x1044, 0x4 }, + [561] = { 0x1044, 0x4 }, + [563] = { 0x4c, 0x4 }, + [565] = { 0x48, 0x4 }, + [567] = { 0x80, 0x4 }, + [569] = { 0x44, 0x4 }, + [572] = { 0x4, 0x4 }, + [574] = { 0x4, 0x8 }, + [575] = { 0x58, 0x4c }, + [576] = { 0x4, 0x0 }, + [586] = { 0x8, 0x4 }, + [587] = { 0x0, 0x0 }, + [588] = { 0x18, 0x0 }, + [589] = { 0xe4, 0xe0 }, + [592] = { 0x0, 0x0 }, + [594] = { 0x730, 0x0 }, + [596] = { 0x14, 0x0 }, + [597] = { 0x4, 0x0 }, + [598] = { 0x4, 0x0 }, + [599] = { 0x4, 0x0 }, + [601] = { 0x0, 0x4 }, + [602] = { 0x0, 0x4 }, + [603] = { 0x0, 0x0 }, +}; + +bool iomfb_v27_callback_size_valid(unsigned int tag, u32 input, u32 output) +{ + if (tag >= ARRAY_SIZE(callback_sizes) || !cb_handlers[tag]) + return false; + return input >= callback_sizes[tag].input && + output >= callback_sizes[tag].output; +} + +void iomfb_start_v27(struct apple_dcp *dcp) +{ + dcp->cb_handlers = cb_handlers; + dcp_start_signal(dcp, false, dcp_started, NULL); +} + +#undef DCP_FW_VER +#undef DCP_FW diff --git a/drivers/gpu/drm/apple/iomfb_v27.h b/drivers/gpu/drm/apple/iomfb_v27.h new file mode 100644 index 0000000000000..ac1ded5eb047c --- /dev/null +++ b/drivers/gpu/drm/apple/iomfb_v27.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright The Asahi Linux Contributors */ +/* Copyright 2026 Aurora Silicon */ + +#ifndef APPLE_IOMFB_V27_H +#define APPLE_IOMFB_V27_H + +#include "iomfb_h17p.h" + +struct dcp_swap_v27 { + + u64 timestamp[7]; + u64 unk_38; + + u64 debug_info[2]; + + u64 event_wait[SWAP_SURFACES]; + u64 event_signal_on_glass; + u64 event_signal[SWAP_SURFACES]; + + u32 swap_id; + + u32 surf_ids[SWAP_SURFACES]; + struct dcp_rect src_rect[SWAP_SURFACES]; + u32 surf_flags[SWAP_SURFACES]; + u32 surf_unk[SWAP_SURFACES]; + struct dcp_rect dst_rect[SWAP_SURFACES]; + u32 swap_enabled; + u32 swap_completed; + + u32 bg_color; + u8 reserved_158[0x1fc]; + + u8 brightness_update; + u8 reserved_355[9]; + u64 brightness_nits; + u8 reserved_366[0x37a]; +} __packed; + +struct dcp_surface_v27 { + struct dcp_surface base; + u8 padding[47]; +} __packed; + +struct dcp_swap_submit_req_v27 { + struct dcp_swap_v27 swap; + struct dcp_surface_v27 surf[SWAP_SURFACES]; + u64 surf_iova[SWAP_SURFACES]; + u64 unk_u64_a[SWAP_SURFACES]; + + u8 unkbool; + u64 unkdouble; + u64 unkU64; + u8 unkbool2; + u32 clear; + u32 unkU32Ptr; + u8 swap_null; + u8 surf_null[SWAP_SURFACES]; + u8 unkoutbool_null; + u8 unknown_pointer_null; + u8 unknown_output_null; + u8 padding[2]; +} __packed; + +struct dcp_swap_submit_resp_v27 { + u8 unkoutbool; + u32 unkU32out; + u32 ret; + u8 padding[3]; +} __packed; + +static_assert(sizeof(struct dcp_swap_v27) == 0x6e0); +static_assert(sizeof(struct dcp_surface_v27) == 0x22c); +static_assert(sizeof(struct dcp_swap_submit_req_v27) == 0xff4); +static_assert(offsetof(struct dcp_swap_submit_req_v27, surf) == 0x6e0); +static_assert(offsetof(struct dcp_swap_submit_req_v27, surf_iova) == 0xf90); +static_assert(offsetof(struct dcp_swap_submit_req_v27, swap_null) == 0xfea); +static_assert(offsetof(struct dcp_swap_v27, brightness_update) == 0x354); +static_assert(offsetof(struct dcp_swap_v27, brightness_nits) == 0x35e); + +void iomfb_serialize_present_v27(struct dcp_swap_submit_req_v27 *wire, + const struct dcp_swap_submit_req_h17p *request); +bool iomfb_v27_callback_size_valid(unsigned int tag, u32 input, u32 output); +void iomfb_firmware_state_cleanup_v27(void *data); +int iomfb_modeset_v27(struct apple_dcp *dcp, struct drm_crtc_state *state); +void iomfb_flush_v27(struct apple_dcp *dcp, struct drm_crtc *crtc, + struct drm_atomic_commit *commit); +void iomfb_start_v27(struct apple_dcp *dcp); +void iomfb_shutdown_v27(struct apple_dcp *dcp); +void iomfb_poweron_v27(struct apple_dcp *dcp); +void iomfb_poweroff_v27(struct apple_dcp *dcp); +void iomfb_sleep_v27(struct apple_dcp *dcp); +void iomfb_present_backlight_v27(struct apple_dcp *dcp); + +#endif diff --git a/drivers/gpu/drm/apple/parser-test.c b/drivers/gpu/drm/apple/parser-test.c index d7cdc1544b3fc..9cf57af1b3c94 100644 --- a/drivers/gpu/drm/apple/parser-test.c +++ b/drivers/gpu/drm/apple/parser-test.c @@ -1077,7 +1077,9 @@ static void parser_optional_flag_extent(struct kunit *test) static void parser_attribute_deferred(struct kunit *test) { struct apple_dcp *dcp = kunit_kzalloc(test, sizeof(*dcp), GFP_KERNEL); - struct apple_connector *connector = kunit_kzalloc(test, sizeof(*connector), GFP_KERNEL); + struct parser_catalog_observer *observer = + kunit_kzalloc(test, sizeof(*observer), GFP_KERNEL); + struct apple_connector *connector = observer ? &observer->connector : NULL; struct drm_device *dev = kunit_kzalloc(test, sizeof(*dev), GFP_KERNEL); struct parser_blob *blob = kunit_kzalloc(test, sizeof(*blob), GFP_KERNEL); struct dcp_parse_ctx ctx; @@ -1088,6 +1090,8 @@ static void parser_attribute_deferred(struct kunit *test) KUNIT_ASSERT_NOT_NULL(test, dev); KUNIT_ASSERT_NOT_NULL(test, blob); dcp_modes_init(dcp); + observer->dcp = dcp; + INIT_WORK(&connector->hotplug_wq, parser_catalog_ready); mutex_init(&dev->mode_config.mutex); connector->base.dev = dev; dcp->connector = connector; @@ -1109,6 +1113,8 @@ static void parser_attribute_deferred(struct kunit *test) KUNIT_EXPECT_EQ(test, dcp->modes[0].mode.width_mm, 520); mutex_unlock(&dev->mode_config.mutex); flush_work(&dcp->dimensions_wq); + flush_work(&connector->hotplug_wq); + KUNIT_EXPECT_EQ(test, observer->notifications, 1U); KUNIT_EXPECT_EQ(test, connector->base.display_info.width_mm, 520U); KUNIT_EXPECT_EQ(test, connector->base.display_info.height_mm, 290U); dcp_modes_release(dcp); diff --git a/drivers/gpu/drm/apple/parser.c b/drivers/gpu/drm/apple/parser.c index 0de67aa8fe710..34271cf1c095b 100644 --- a/drivers/gpu/drm/apple/parser.c +++ b/drivers/gpu/drm/apple/parser.c @@ -1438,12 +1438,57 @@ int parse_display_metadata(struct dcp_parse_ctx *handle, struct dcp_display_meta return 0; } +int parse_dptx_caps(struct dcp_parse_ctx *handle, struct dcp_dptx_caps *caps) +{ + struct iterator it; + struct dcp_dptx_caps parsed = {}; + int ret = 0; + + ret = iterator_begin(handle, &it, true); + if (ret) + return ret; + for (; it.idx < it.len; ++it.idx) { + u32 *field = NULL; + char *key = parse_string(it.handle); + s64 value; + + if (IS_ERR(key)) + return PTR_ERR(key); + if (!strcmp(key, "MaxPixelWidth")) + field = &parsed.width; + else if (!strcmp(key, "MaxTotalPixelRate")) + field = &parsed.total_rate; + else if (!strcmp(key, "MaxActivePixelRate")) + field = &parsed.active_rate; + kfree(key); + if (!field) { + ret = skip(it.handle); + if (ret) + return ret; + if (it.handle->pos > it.handle->len) + return -EINVAL; + continue; + } + if (*field) + return -EINVAL; + ret = parse_int_bound(it.handle, &value, 1, U32_MAX); + if (ret) + return ret; + *field = value; + } + if (!parsed.width || !parsed.total_rate || !parsed.active_rate) + return -ENOENT; + *caps = parsed; + return 0; +} + int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, const char **class, s64 *unit, bool h17p_keys) { int ret = 0; struct iterator it; bool parsed_unit = false; + bool explicit_unit = false; bool parsed_name = false; bool parsed_class = false; @@ -1491,10 +1536,12 @@ int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, else parsed_class = true; } else if (!strcmp(key, "EPICUnit") || - (h17p_keys && !strcmp(key, "interface-id"))) { + (h17p_keys && !explicit_unit && !strcmp(key, "interface-id"))) { ret = parse_int(it.handle, unit); - if (!ret) + if (!ret) { parsed_unit = true; + explicit_unit |= !strcmp(key, "EPICUnit"); + } } else { ret = skip(it.handle); } @@ -1776,14 +1823,19 @@ static int parse_mode_in_avep_element(struct dcp_parse_ctx *handle, return ret; } } else if (consume_string(it.handle, "ElementData")) { + size_t size = DCP_SOUND_COOKIE_SIZE; const u8 *blob; - ret = parse_blob(it.handle, sizeof(*cookie), &blob); +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + if (dcp_parse_v14_7_audio(handle)) + size = DCP_SOUND_COOKIE_SIZE_V14_7; +#endif + ret = parse_blob(it.handle, size, &blob); if (ret) return ret; if (cookie) - memcpy(cookie, blob, sizeof(*cookie)); + memcpy(cookie, blob, size); } else { ret = skip_pair(it.handle); if (ret) diff --git a/drivers/gpu/drm/apple/parser.h b/drivers/gpu/drm/apple/parser.h index 0557cc09e63e4..52d43bd317072 100644 --- a/drivers/gpu/drm/apple/parser.h +++ b/drivers/gpu/drm/apple/parser.h @@ -13,6 +13,7 @@ struct apple_dcp; struct apple_connector; struct apple_dcp_typec_route; +struct dcp_dptx_caps; struct dcp_parse_ctx { struct apple_dcp *dcp; @@ -21,8 +22,26 @@ struct dcp_parse_ctx { /* External H17P candidates require separate publication/lookup proof. */ bool automatic_1080p180_rgb8; bool ambiguous_180hz; +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + /* + * T6030 external processor audio elements (DCP firmware 14.7, external + * role): an element's ElementData, the link data the audio service + * takes, is 32 bytes. + */ + bool v14_7_audio; +#endif }; +/* The T6030 external processor's audio elements (DRM_APPLE_DCP_T6030_AUDIO only). */ +static inline bool dcp_parse_v14_7_audio(const struct dcp_parse_ctx *ctx) +{ +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) + return ctx->v14_7_audio; +#else + return false; +#endif +} + enum dcp_color_eotf { DCP_EOTF_SDR_GAMMA = 0, // "SDR gamma" DCP_EOTF_HDR_GAMMA = 1, // "HDR gamma" @@ -171,6 +190,7 @@ int parse_display_attributes(struct dcp_parse_ctx *handle, int *width_mm, int *height_mm, bool *backlight_control); int parse_epic_service_init(struct dcp_parse_ctx *handle, const char **name, const char **class, s64 *unit, bool h17p_keys); +int parse_dptx_caps(struct dcp_parse_ctx *handle, struct dcp_dptx_caps *caps); struct dcp_sound_format_mask { u64 formats; /* SNDRV_PCM_FMTBIT_* */ @@ -178,8 +198,20 @@ struct dcp_sound_format_mask { unsigned int nchans; }; +/* + * An audio element's ElementData, the link data of the audio service's link + * methods: 24 bytes up to DCP firmware 13.5, 32 on the 14.7 external role. + */ +#define DCP_SOUND_COOKIE_SIZE 24 +#if IS_ENABLED(CONFIG_DRM_APPLE_DCP_T6030_AUDIO) +#define DCP_SOUND_COOKIE_SIZE_V14_7 32 +#define DCP_SOUND_COOKIE_MAX DCP_SOUND_COOKIE_SIZE_V14_7 +#else +#define DCP_SOUND_COOKIE_MAX DCP_SOUND_COOKIE_SIZE +#endif + struct dcp_sound_cookie { - u8 data[24]; + u8 data[DCP_SOUND_COOKIE_MAX]; }; struct snd_pcm_chmap_elem; diff --git a/drivers/gpu/drm/apple/plane.c b/drivers/gpu/drm/apple/plane.c index aa5bbc9551d2c..cde215f62b143 100644 --- a/drivers/gpu/drm/apple/plane.c +++ b/drivers/gpu/drm/apple/plane.c @@ -8,6 +8,7 @@ #include "dcp-internal.h" #include "iomfb_internal.h" +#include "iomfb_plane.h" #include #include #include @@ -18,14 +19,206 @@ #include #include +#include +#include +#include + +#define APPLE_GPU_CACHELINE 16 #define FRAC_16_16(mult, div) (((mult) << 16) / (div)) +struct apple_plane_layout { + struct dcp_image_layout image; + u32 width; + u32 height; + u32 offset; + u32 stride; + u32 size; + u32 tile_size; +}; + +static int apple_plane_ix_bpp(const struct drm_format_info *fmt, u32 plane, + u32 *bpp) +{ + u32 bw = drm_format_info_block_width(fmt, plane); + u32 bh = drm_format_info_block_height(fmt, plane); + u32 bits = fmt->char_per_block[plane] * 8; + u32 block_pixels; + + /* + * P030/P230 pack three pixels per block. Neither an integer bpp + * (which would be zero) nor a truncated bpp describes their tiles. + * Keep these formats linear until their interchange packing is known. + */ + if (!bw || !bh || check_mul_overflow(bw, bh, &block_pixels) || !bits || + bits % block_pixels) + return -EINVAL; + *bpp = bits / block_pixels; + + return 0; +} + +static bool apple_plane_format_mod_supported(struct drm_plane *plane, + u32 format, u64 modifier) +{ + const struct drm_format_info *fmt = drm_format_info(format); + u32 bpp; + unsigned int i; + + if (!fmt) + return false; + + /* The callback replaces DRM's generic modifier-list validation. */ + for (i = 0; i < plane->modifier_count; i++) + if (modifier == plane->modifiers[i]) + break; + if (i == plane->modifier_count) + return false; + + switch (modifier) { + case DRM_FORMAT_MOD_LINEAR: + case DRM_FORMAT_MOD_INVALID: + return true; + case DRM_FORMAT_MOD_APPLE_INTERCHANGE: + for (u32 i = 0; i < fmt->num_planes; i++) + if (apple_plane_ix_bpp(fmt, i, &bpp)) + return false; + return true; + default: + return false; + } +} + +static int apple_get_ix_layout(u32 width, u32 height, + const struct drm_format_info *fmt, u32 plane, + struct apple_plane_layout *out) +{ + struct dcp_image_layout *layout = &out->image; + u32 bpp, meta_tile_w, meta_tile_h, tiles, bytes; + int ret; + + ret = apple_plane_ix_bpp(fmt, plane, &bpp); + if (ret) + return ret; + + /* + * Compression subtiles are always 16x16 with 8 bytes of metadata, + * but IOMFB requires values relative to the image tile dimensions. + */ + if (fmt->is_yuv && plane == 0) { + layout->tile_w = 32; + layout->tile_h = 32; + layout->meta_tile_bytes = 32; + } else { + layout->tile_w = 16; + layout->tile_h = 16; + layout->meta_tile_bytes = 8; + } + + layout->width_tiles = (width - 1) / layout->tile_w + 1; + layout->height_tiles = (height - 1) / layout->tile_h + 1; + layout->tile_bytes = layout->tile_w * layout->tile_h * bpp / 8; + if (layout->tile_bytes > U16_MAX || + check_mul_overflow(layout->width_tiles, layout->tile_bytes, + &out->stride) || + check_mul_overflow(out->stride, layout->height_tiles, &bytes) || + check_add_overflow(bytes, APPLE_GPU_CACHELINE - 1, &bytes)) + return -EINVAL; + layout->meta_offset = bytes & ~(APPLE_GPU_CACHELINE - 1); + + /* The rounded dimensions and all wire-format sizes must fit u32. */ + if (width > BIT(31) || height > BIT(31)) + return -EINVAL; + meta_tile_w = DIV_ROUND_UP(roundup_pow_of_two(width), layout->tile_w); + meta_tile_h = DIV_ROUND_UP(roundup_pow_of_two(height), layout->tile_h); + if (check_mul_overflow(meta_tile_w, meta_tile_h, &tiles) || + check_mul_overflow(tiles, layout->meta_tile_bytes, &bytes) || + check_add_overflow(bytes, APPLE_GPU_CACHELINE - 1, &bytes)) + return -EINVAL; + layout->meta_size = bytes & ~(APPLE_GPU_CACHELINE - 1); + if (check_add_overflow(layout->meta_offset, layout->meta_size, + &out->size)) + return -EINVAL; + out->tile_size = layout->tile_bytes; + + return 0; +} + +/* Use exactly the same geometry for admission and surface construction. */ +static int apple_plane_get_fb_layout(struct drm_framebuffer *fb, + struct apple_plane_layout *planes, + u32 *buf_size) +{ + const struct drm_format_info *fmt = fb->format; + struct drm_gem_dma_object *first; + u32 end; + int ret; + + if (!fb->width || !fb->height || !fmt->hsub || !fmt->vsub || + !fmt->num_planes || fmt->num_planes > DCP_SURF_MAX_PLANES) + return -EINVAL; + first = drm_fb_dma_get_gem_obj(fb, 0); + if (!first) + return -EINVAL; + + *buf_size = 0; + for (u32 i = 0; i < fmt->num_planes; i++) { + struct apple_plane_layout *p = &planes[i]; + struct drm_gem_dma_object *obj = drm_fb_dma_get_gem_obj(fb, i); + + /* DCP currently uses one GEM object for the whole surface. */ + if (obj != first || fb->offsets[i] < fb->offsets[0]) + return -EINVAL; + *p = (struct apple_plane_layout){ + .width = i ? DIV_ROUND_UP_ULL(fb->width, fmt->hsub) : + fb->width, + .height = i ? DIV_ROUND_UP_ULL(fb->height, fmt->vsub) : + fb->height, + .offset = fb->offsets[i] - fb->offsets[0], + }; + if (!p->width || !p->height) + return -EINVAL; + + switch (fb->modifier) { + case DRM_FORMAT_MOD_APPLE_INTERCHANGE: + ret = apple_get_ix_layout(p->width, p->height, fmt, i, + p); + if (ret) + return ret; + break; + case DRM_FORMAT_MOD_INVALID: + case DRM_FORMAT_MOD_LINEAR: + if (!fb->pitches[i] || fb->pitches[i] & 63 || + check_mul_overflow(p->height, fb->pitches[i], + &p->size)) + return -EINVAL; + p->image.tile_w = drm_format_info_block_width(fmt, i); + p->image.tile_h = drm_format_info_block_height(fmt, i); + p->tile_size = p->image.tile_w * p->image.tile_h; + p->stride = fb->pitches[i]; + break; + default: + return -EINVAL; + } + + if (check_add_overflow(p->offset, p->size, &end) || + (u64)fb->offsets[i] + p->size > obj->base.size || + check_add_overflow(*buf_size, p->size, buf_size)) + return -EINVAL; + } + if ((u64)fb->offsets[0] + *buf_size > first->base.size) + return -EINVAL; + + return 0; +} + static int apple_plane_atomic_check(struct drm_plane *plane, struct drm_atomic_commit *commit) { struct drm_plane_state *new_plane_state; struct drm_crtc_state *crtc_state; struct drm_rect *dst; + struct apple_plane_layout layouts[DCP_SURF_MAX_PLANES]; + u32 buf_size; int ret; new_plane_state = drm_atomic_get_new_plane_state(commit, plane); @@ -56,6 +249,10 @@ static int apple_plane_atomic_check(struct drm_plane *plane, if (ret < 0) return ret; + ret = apple_plane_get_fb_layout(new_plane_state->fb, layouts, &buf_size); + if (ret) + return ret; + if (!new_plane_state->visible) return 0; @@ -84,29 +281,6 @@ static int apple_plane_atomic_check(struct drm_plane *plane, return -EINVAL; } - /* - * Pitches have to be 64-byte aligned. - */ - for (u32 i = 0; i < new_plane_state->fb->format->num_planes; i++) - if (new_plane_state->fb->pitches[i] & 63) - return -EINVAL; - - /* - * FIXME: dcp can currently only use multi-planar buffers using the same - * object for all planes. It has a mandatory iommu so it should - * be no problem to map multiple objects "linearly" into DCP - * virtual address space and calculate the offsets accordingly. - * Or maybe it can accept multiple BOs via the per plane field - * `base`. - */ - if (new_plane_state->fb->format->num_planes > 1) { - const struct drm_gem_object *first = new_plane_state->fb->obj[0]; - for (u32 i = 1; i < new_plane_state->fb->format->num_planes; i++) - if (new_plane_state->fb->obj[i] != NULL && - new_plane_state->fb->obj[i] != first) - return -EINVAL; - } - return 0; } @@ -147,6 +321,9 @@ static u32 drm_format_to_dcp(u32 drm, enum drm_color_range range) case DRM_FORMAT_ARGB2101010: return DCP_FORMAT_L10R; + case DRM_FORMAT_ABGR16161616F: + return DCP_FORMAT_AhGR; + /* semi planar YCbCr formats, limited and full range */ case DRM_FORMAT_NV12: return fr ? DCP_FORMAT_420F : DCP_FORMAT_420V; @@ -160,12 +337,21 @@ static u32 drm_format_to_dcp(u32 drm, enum drm_color_range range) return fr ? DCP_FORMAT_XF20 : DCP_FORMAT_X420; case DRM_FORMAT_P210: return fr ? DCP_FORMAT_XF22 : DCP_FORMAT_X422; + /* semi planar 10-bit YCbCr formats, 3x1 blocks, limited and full range */ + case DRM_FORMAT_P030: + return fr ? DCP_FORMAT_PF20 : DCP_FORMAT_P420; + case DRM_FORMAT_P230: + return fr ? DCP_FORMAT_PF22 : DCP_FORMAT_P422; /* - * TODO: missing DRM fourcc for P410 + * TODO: missing DRM fourccs for P410 / P430 */ #if defined(DRM_FORMAT_P410) case DRM_FORMAT_P410: return fr ? DCP_FORMAT_XF44 : DCP_FORMAT_X444; +#endif +#if defined(DRM_FORMAT_P430) + case DRM_FORMAT_P430: + return fr ? DCP_FORMAT_PF44 : DCP_FORMAT_P444; #endif } @@ -213,6 +399,8 @@ static void apple_plane_atomic_update(struct drm_plane *plane, struct drm_plane_state *base = drm_atomic_get_new_plane_state(commit, plane); struct apple_plane_state *new_state; struct drm_gem_dma_object *obj; + struct apple_plane_layout layouts[DCP_SURF_MAX_PLANES]; + u32 buf_size; bool is_premultiplied = false; if (!base) @@ -227,6 +415,12 @@ static void apple_plane_atomic_update(struct drm_plane *plane, struct drm_framebuffer *fb = base->fb; const struct drm_format_info *fmt = fb->format; + + if (WARN_ON_ONCE(apple_plane_get_fb_layout(fb, layouts, &buf_size))) { + memset(&new_state->surf, 0, sizeof(new_state->surf)); + return; + } + /* * DCP doesn't support XBGR8 / XRGB8 / XBGR2101010 natively. Blending as * pre-multiplied alpha with a black background can be used as @@ -250,8 +444,8 @@ static void apple_plane_atomic_update(struct drm_plane *plane, .stride = fb->pitches[0], .width = fb->width, .height = fb->height, - .buf_size = fb->height * fb->pitches[0], .surface_id = plane->base.id, + .buf_size = buf_size, /* Only used for compressed or multiplanar surfaces */ .pix_size = 1, @@ -259,30 +453,51 @@ static void apple_plane_atomic_update(struct drm_plane *plane, .pel_h = 1, .has_comp = 1, .has_planes = 1, + .has_compr_info = 1, }; /* Populate plane information for planar formats */ struct dcp_surface *surf = &new_state->surf; for (int i = 0; fb->format->num_planes && i < fb->format->num_planes; i++) { - u32 width = drm_format_info_plane_width(fb->format, fb->width, i); - u32 height = drm_format_info_plane_height(fb->format, fb->height, i); - u32 bh = drm_format_info_block_height(fb->format, i); - u32 bw = drm_format_info_block_width(fb->format, i); + const struct apple_plane_layout *p = &layouts[i]; + const struct dcp_image_layout *layout = &p->image; surf->planes[i] = (struct dcp_plane_info){ - .width = width, - .height = height, - .base = fb->offsets[i] - fb->offsets[0], - .offset = fb->offsets[i] - fb->offsets[0], - .stride = fb->pitches[i], - .size = height * fb->pitches[i], - .tile_size = bw * bh, - .tile_w = bw, - .tile_h = bh, + .width = p->width, + .height = p->height, + .base = p->offset, + .offset = p->offset, + .tile_w = layout->tile_w, + .tile_h = layout->tile_h, + .stride = p->stride, + .size = p->size, + .tile_size = p->tile_size, }; - if (i > 0) - surf->buf_size += surf->planes[i].size; + switch (fb->modifier) { + case DRM_FORMAT_MOD_APPLE_INTERCHANGE: + surf->planes[i].address_fmt = DCP_ADDR_FMT_INTERCHANGE_TILED; + + surf->compression_info[i] = (struct dcp_compression_info) { + .tile_w = layout->tile_w, + .tile_h = layout->tile_h, + .data_offset = surf->planes[i].offset, + .metadata_offset = surf->planes[i].offset + layout->meta_offset, + .meta_bytes = layout->meta_tile_bytes, + .tiles_w = layout->width_tiles, + .tiles_h = layout->height_tiles, + .tile_bytes = layout->tile_bytes, + .row_stride = p->stride, + .compression_type = DCP_COMPRESSION_INTERCHANGE, + }; + break; + case DRM_FORMAT_MOD_INVALID: + case DRM_FORMAT_MOD_LINEAR: + surf->planes[i].address_fmt = DCP_ADDR_FMT_LINEAR; + break; + default: + break; + } } /* the obvious helper call drm_fb_dma_get_gem_addr() adjusts @@ -400,6 +615,7 @@ static const struct drm_plane_funcs apple_plane_funcs = { .reset = apple_plane_reset, .atomic_duplicate_state = apple_plane_duplicate_state, .atomic_destroy_state = apple_plane_destroy_state, + .format_mod_supported = apple_plane_format_mod_supported, }; /* @@ -414,6 +630,7 @@ static const struct drm_plane_funcs apple_plane_funcs = { * advertise formats without alpha. */ static const u32 dcp_primary_formats[] = { + DRM_FORMAT_ABGR16161616F, DRM_FORMAT_XRGB2101010, DRM_FORMAT_ARGB2101010, DRM_FORMAT_XRGB8888, @@ -427,6 +644,11 @@ static const u32 dcp_primary_formats[] = { DRM_FORMAT_P210, #if defined(DRM_FORMAT_P410) DRM_FORMAT_P410, +#endif + DRM_FORMAT_P030, + DRM_FORMAT_P230, +#if defined(DRM_FORMAT_P430) + DRM_FORMAT_P430, #endif }; @@ -448,6 +670,7 @@ static const u32 dcp_primary_formats_h17p[] = { }; static const u32 dcp_overlay_formats[] = { + DRM_FORMAT_ABGR16161616F, DRM_FORMAT_ARGB2101010, DRM_FORMAT_ARGB8888, DRM_FORMAT_ABGR8888, @@ -458,6 +681,11 @@ static const u32 dcp_overlay_formats[] = { DRM_FORMAT_P210, #if defined(DRM_FORMAT_P410) DRM_FORMAT_P410, +#endif + DRM_FORMAT_P030, + DRM_FORMAT_P230, +#if defined(DRM_FORMAT_P430) + DRM_FORMAT_P430, #endif }; @@ -492,7 +720,13 @@ static const u32 dcp_overlay_formats_12_x[] = { #endif }; -u64 apple_format_modifiers[] = { +static u64 format_modifiers[] = { + DRM_FORMAT_MOD_APPLE_INTERCHANGE, + DRM_FORMAT_MOD_LINEAR, + DRM_FORMAT_MOD_INVALID +}; + +static u64 format_modifiers_no_ix[] = { DRM_FORMAT_MOD_LINEAR, DRM_FORMAT_MOD_INVALID }; @@ -502,7 +736,8 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, u32 iomfb_surf, bool supports_l10r, bool supports_xrgb2101010, - enum drm_plane_type type) + enum drm_plane_type type, + bool interchange) { struct apple_plane *plane; const u32 *fmts; @@ -522,7 +757,8 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, } plane = drmm_universal_plane_alloc(dev, struct apple_plane, base, possible_crtcs, &apple_plane_funcs, fmts, num_fmts, - apple_format_modifiers, type, NULL); + interchange ? format_modifiers : format_modifiers_no_ix, + type, NULL); break; case DRM_PLANE_TYPE_OVERLAY: case DRM_PLANE_TYPE_CURSOR: @@ -535,7 +771,8 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, } plane = drmm_universal_plane_alloc(dev, struct apple_plane, base, possible_crtcs, &apple_plane_funcs, fmts, num_fmts, - apple_format_modifiers, type, NULL); + interchange ? format_modifiers : format_modifiers_no_ix, + type, NULL); break; default: return ERR_PTR(-EINVAL); @@ -579,7 +816,7 @@ struct drm_plane *apple_plane_init_v14_7(struct drm_device *dev, plane = drmm_universal_plane_alloc(dev, struct apple_plane, base, possible_crtcs, &apple_plane_funcs, dcp_primary_formats_14_7, ARRAY_SIZE(dcp_primary_formats_14_7), - apple_format_modifiers, type, NULL); + format_modifiers_no_ix, type, NULL); if (IS_ERR(plane)) return ERR_CAST(plane); diff --git a/drivers/gpu/drm/apple/plane.h b/drivers/gpu/drm/apple/plane.h index ebb2420bf4041..d7905573276f4 100644 --- a/drivers/gpu/drm/apple/plane.h +++ b/drivers/gpu/drm/apple/plane.h @@ -37,7 +37,8 @@ struct drm_plane *apple_plane_init(struct drm_device *dev, u32 iomfb_surf, bool supports_l10r, bool supports_xrgb2101010, - enum drm_plane_type type); + enum drm_plane_type type, + bool interchange); struct drm_plane *apple_plane_init_v14_7(struct drm_device *dev, unsigned long possible_crtcs, u32 iomfb_surf, diff --git a/drivers/gpu/drm/apple/systemep.c b/drivers/gpu/drm/apple/systemep.c index a32f0a0bcf6a0..2659053e837cf 100644 --- a/drivers/gpu/drm/apple/systemep.c +++ b/drivers/gpu/drm/apple/systemep.c @@ -100,8 +100,9 @@ static int powerlog_report(struct apple_epic_service *service, enum epic_subtype dev_dbg(dcp->dev, "systemep: mNits event: Nits: %u.%03u, iDAC: %u\n", mnits.millinits / 1000, mnits.millinits % 1000, mnits.idac); - if (dcp->fw_compat == DCP_FIRMWARE_H17P && - dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) { + if (((dcp->fw_compat == DCP_FIRMWARE_H17P || dcp->fw_compat == DCP_FIRMWARE_V_27_0) && + dcp->hw.iomfb_method_profile != DCP_IOMFB_METHODS_H17G) || + dcp->fw_compat == DCP_FIRMWARE_V_26_6_DESKTOP) { if (!dcp_has_panel(dcp)) return 0; ret = dcp_backlight_takeover(dcp, mnits.millinits); diff --git a/drivers/gpu/drm/apple/v14-board-test.c b/drivers/gpu/drm/apple/v14-board-test.c index aa95cd2580857..801ed2f623a4c 100644 --- a/drivers/gpu/drm/apple/v14-board-test.c +++ b/drivers/gpu/drm/apple/v14-board-test.c @@ -73,13 +73,39 @@ static void v14_board_j516c(struct kunit *test) KUNIT_EXPECT_EQ(test, b->ctm_get, 0); } -/* A T6031 DCP on another board, or on a T6034, has no record: refused. */ -static void v14_board_t6031_other_boards(struct kunit *test) +/* + * The J514C: the same T6031 DCP and image, the panel from the boot + * framebuffer, the first boots logged. + */ +static void v14_board_j514c(struct kunit *test) { + const struct dcp_v14_board *b; bool known; - KUNIT_EXPECT_NULL(test, board_for("apple,t6031-dcp", j514c, &known)); + b = board_for("apple,t6031-dcp", j514c, &known); + KUNIT_ASSERT_NOT_NULL(test, b); KUNIT_EXPECT_TRUE(test, known); + KUNIT_EXPECT_STREQ(test, b->name, "J514C"); + KUNIT_EXPECT_STREQ(test, b->debugfs, "dcp-j514c"); + KUNIT_EXPECT_STREQ(test, b->machine, "apple,j514c"); + KUNIT_EXPECT_STREQ(test, b->handoff, "apple,t6031-handoff"); + KUNIT_EXPECT_STREQ(test, b->firmware_uuid, J516C_DCP_IMAGE); + KUNIT_EXPECT_EQ(test, b->panel_width, 0); + KUNIT_EXPECT_EQ(test, b->panel_height, 0); + KUNIT_EXPECT_TRUE(test, b->promotion); + KUNIT_EXPECT_TRUE(test, b->log_bringup); + KUNIT_EXPECT_EQ(test, b->ctm_set, 0); + KUNIT_EXPECT_TRUE(test, dcp_v14_board_admits(b, J516C_DCP_IMAGE)); + KUNIT_EXPECT_FALSE(test, dcp_v14_board_admits(b, T6030_DCP_IMAGE)); + /* Not the J516C record. */ + KUNIT_EXPECT_PTR_NE(test, b, board_for("apple,t6031-dcp", j516c, &known)); +} + +/* A T6031 DCP on a T6034 has no record: refused. */ +static void v14_board_t6031_other_boards(struct kunit *test) +{ + bool known; + KUNIT_EXPECT_NULL(test, board_for("apple,t6031-dcp", j516m, &known)); KUNIT_EXPECT_TRUE(test, known); /* Not a 14.x internal DCP at all. */ @@ -150,6 +176,7 @@ static void v14_board_unchanged(struct kunit *test) static struct kunit_case v14_board_cases[] = { KUNIT_CASE(v14_board_j516c), + KUNIT_CASE(v14_board_j514c), KUNIT_CASE(v14_board_t6031_other_boards), KUNIT_CASE(v14_board_j516c_image), KUNIT_CASE(v14_board_unchanged), diff --git a/drivers/gpu/drm/apple/version_utils.h b/drivers/gpu/drm/apple/version_utils.h index 4627fd148e15c..d34eb87784f6f 100644 --- a/drivers/gpu/drm/apple/version_utils.h +++ b/drivers/gpu/drm/apple/version_utils.h @@ -17,6 +17,15 @@ * H17-generation firmware (26.0 and later) keeps the earlier layout and * extends it with its own fields. */ +/* Desktop 26.x firmware flag (no desktop DCP is driven in this tree). */ +#define DCP_FW_DESKTOP (1 << 24) +#define DCP_FW_VERSION_DESKTOP(x, y, z) (DCP_FW_VERSION(x, y, z) | DCP_FW_DESKTOP) +#define DCP_FW_H17(ver) \ + ((ver) >= DCP_FW_VERSION(26, 0, 0) && !((ver) & DCP_FW_DESKTOP)) +#define DCP_FW_DESKTOP_26(ver) \ + (((ver) & DCP_FW_DESKTOP) && \ + ((ver) & ~DCP_FW_DESKTOP) >= DCP_FW_VERSION(26, 0, 0)) + #define DCP_FW_LAYOUT_14_7(ver) \ ((ver) >= DCP_FW_VERSION(14, 7, 0) && (ver) < DCP_FW_VERSION(26, 0, 0)) @@ -38,6 +47,12 @@ enum dcp_firmware_version { DCP_FIRMWARE_H17P, /* J613 macOS 26.6.2 (25G83) IOMFB over DCPLink (iomfb_v26_6.c). */ DCP_FIRMWARE_V_26_6, + /* T8132/J713 retained EP37 session (not driven in this tree). */ + DCP_FIRMWARE_M4, + /* T6041 desktop 26.6 IOMFB (not driven in this tree). */ + DCP_FIRMWARE_V_26_6_DESKTOP, + /* T6050 (J714c) firmware 27 over DCPLink (iomfb_v27.c). */ + DCP_FIRMWARE_V_27_0, }; #endif /*__APPLE_VERSION_UTILS_H__*/ diff --git a/drivers/gpu/drm/asahi/Kconfig b/drivers/gpu/drm/asahi/Kconfig index 4281f72993bae..958600a91b06b 100644 --- a/drivers/gpu/drm/asahi/Kconfig +++ b/drivers/gpu/drm/asahi/Kconfig @@ -35,3 +35,18 @@ config DRM_ASAHI_DEBUG_ALLOCATOR Say N unless you are debugging firmware structures or porting to a new firmware version. + +config DRM_ASAHI_G15 + bool "Native G15 firmware backend (M3 Pro/Max)" + depends on DRM_ASAHI + depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m) + select APPLE_T6030_GPU_PMP_LINK + select CRYPTO_LIB_SHA256 + default n + help + Enable the native G15 backend for explicitly admitted M3 Pro/Max + firmware and boards. Other GPUs keep the maintained backend. + The firmware session is retained until reboot once started. + The native display driver must be built in, or both drivers must + be modules, so a modular display provider finishes initialization + before the GPU module can probe through its direct helper import. diff --git a/drivers/gpu/drm/asahi/Makefile b/drivers/gpu/drm/asahi/Makefile index a3aff32eea160..e1987914e0956 100644 --- a/drivers/gpu/drm/asahi/Makefile +++ b/drivers/gpu/drm/asahi/Makefile @@ -2,3 +2,5 @@ obj-$(CONFIG_DRM_ASAHI) += asahi.o asahi-y := asahi_core.o sysfs.o + +asahi-$(CONFIG_DRM_ASAHI_G15) += g15/glue.o diff --git a/drivers/gpu/drm/asahi/asahi_core.rs b/drivers/gpu/drm/asahi/asahi_core.rs index 4dca72fb85dc0..73a798fc69c00 100644 --- a/drivers/gpu/drm/asahi/asahi_core.rs +++ b/drivers/gpu/drm/asahi/asahi_core.rs @@ -15,6 +15,8 @@ mod float; mod fw; mod gem; mod g17; +#[cfg(CONFIG_DRM_ASAHI_G15)] +mod g15; mod gpu; mod hw; mod initdata; @@ -36,12 +38,21 @@ mod workqueue; use kernel::prelude::*; +#[cfg(CONFIG_DRM_ASAHI_G15)] +type G15Registration = + kernel::driver::Registration>>; + #[pin_data] struct DriverModule { + #[cfg(CONFIG_DRM_ASAHI_G15)] + #[pin] + _g15: G15Registration, // Field drop order unregisters/joins driver producers before queue teardown. #[pin] _driver: kernel::driver::Registration>, _cleanup: cleanup::QueueOwner, + #[cfg(CONFIG_DRM_ASAHI_G15)] + _g15_ready: (), } impl kernel::InPlaceModule for DriverModule { @@ -49,10 +60,15 @@ impl kernel::InPlaceModule for DriverModule { try_pin_init!(Self { // Initialize the queue first, before registration can invoke probe. _cleanup: cleanup::QueueOwner::new()?, + #[cfg(CONFIG_DRM_ASAHI_G15)] + _g15 <- kernel::driver::Registration::new(c"asahi-g15", module), _driver <- kernel::driver::Registration::new( ::NAME, module, ), + // All fallible fields are initialized. This final unit initializer cannot fail. + #[cfg(CONFIG_DRM_ASAHI_G15)] + _g15_ready: g15::activate(), }) } } @@ -87,6 +103,50 @@ module! { // permissions: 0o644, description: "Fully isolate GPU contexts (limits performance)", }, + g15_deadline_ms: u32 { + default: 5000, + description: "G15: host deadline of a published prefix without progress, ms (min 100)", + }, + g15_compute_deadline_ms: u32 { + default: 1000, + description: "G15: no-progress budget of a started compute before the class-2 kill, ms", + }, + g15_repoll_us: u32 { + default: 15, + description: "G15: repoll interval while a ready prefix waits for record consumption, us (5..100)", + }, + g15_resnap_us: u32 { + default: 100, + description: "G15: completion repoll interval, us (25..1000)", + }, + g15_idle_standby_us: u32 { + default: 0, + description: "G15: firmware idle standby time override, us (Globals +0x7a8, 100..100000; 0: the device tree's, else the tuned value)", + }, + g15_idle_min_conf: u32 { + default: 500, + description: "G15: firmware idle minimum confidence, permille (Globals +0x7c4)", + }, + g15_recovery: u32 { + default: 1, + description: "G15: answer firmware halts with the halt/resume handshake (0: fail-stop)", + }, + g15_stream_gate: u32 { + default: 1, + description: "G15: admit only known CDM/VDM stream records (0: bring-up only)", + }, + g15_pb_max_blocks: u32 { + default: 3072, + description: "G15: parameter-buffer growth maximum given to the firmware, blocks of 128 KiB (1024..4095 in effect; the blocks above it, up to 4095, answer partial renders; the 512 MiB heap is allocated either way)", + }, + g15_early_tiling: u32 { + default: 0, + description: "G15: advertise FRAGMENT_BARRIERS, mapped to early tiling (slower on 2419)", + }, + g15_accept_text_prefix: u64 { + default: 0, + description: "G15: admit an unpinned GPU firmware build whose TEXT SHA-256 starts with these 8 bytes (bring-up only)", + }, stats_export: u8 { default: 1, // permissions: 0, diff --git a/drivers/gpu/drm/asahi/driver.rs b/drivers/gpu/drm/asahi/driver.rs index 510a70ba4f0bd..e71531c9a5b1f 100644 --- a/drivers/gpu/drm/asahi/driver.rs +++ b/drivers/gpu/drm/asahi/driver.rs @@ -146,6 +146,10 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t8140"), ProbeConfig::G17(&hw::t8140::CONFIG) ), + ( + of::DeviceId::new(c"apple,agx-t6050"), + ProbeConfig::G17(&hw::t6050::CONFIG) + ), ( of::DeviceId::new(c"apple,agx-t8103"), ProbeConfig::Legacy(&hw::t8103::HWCONFIG) @@ -182,10 +186,12 @@ kernel::of_device_table!( of::DeviceId::new(c"apple,agx-t8122"), ProbeConfig::M3(crate::m3::Chip::T8122) ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] ( of::DeviceId::new(c"apple,agx-t6030"), ProbeConfig::M3(crate::m3::Chip::T6030) ), + #[cfg(not(CONFIG_DRM_ASAHI_G15))] ( of::DeviceId::new(c"apple,agx-t6031"), ProbeConfig::M3(crate::m3::Chip::T6031) @@ -305,6 +311,33 @@ impl platform::Driver for AsahiPlatformDriver { } impl AsahiPlatformDriver { + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn probe_g15<'bound>( + pdev: &'bound platform::Device>, + cfg: &'static crate::g15::Config, + ) -> Result> { + // SAFETY: Admission checks the SoC and the UAT output width before DMA starts. + unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(cfg.uat_oas)?)? }; + let resources = regs::Resources::new(pdev)?; + let drm = drm::UnregisteredDevice::::new(pdev, Ok(()))?; + let gpu = crate::g15::Gpu::new(pdev, &drm, cfg)?; + let data = try_pin_init!(AsahiDrmRegistrationData { + gpu: gpu as Arc, + resources, + }); + // SAFETY: The registration is stored and unregistered by platform data destruction. + // G15 retains its firmware-visible state independently until reboot, even on error. + let reg = unsafe { drm::Registration::new(pdev.as_ref(), drm, data, 0)? }; + Ok(AsahiPlatformDriverData { + _device: reg.device().into(), + _reg: reg, + g17: None, + raw_dev: core::ptr::null_mut(), + raw_stats: core::ptr::null_mut(), + stats: None, + }) + } + fn probe_g17<'bound>( pdev: &'bound platform::Device>, cfg: &'static g17::Config, diff --git a/drivers/gpu/drm/asahi/file.rs b/drivers/gpu/drm/asahi/file.rs index 49e5166092fd2..608864949c976 100644 --- a/drivers/gpu/drm/asahi/file.rs +++ b/drivers/gpu/drm/asahi/file.rs @@ -17,7 +17,6 @@ use crate::{ }, gem, mmu, - module_parameters, queue, util::{ align, @@ -252,7 +251,7 @@ impl drm::file::DriverFile for File { device.registration_data_with(|reg_data| { let gpu = ®_data.gpu; let id = gpu.ids().file.next(); - let user_range = gpu.uat_geometry().user_usable_range(); + let user_range = gpu.user_range(); mod_dev_dbg!(device, "[File {}]: DRM device opened\n", id); Ok(KBox::pin_init(File::new(id, user_range), GFP_KERNEL)?) @@ -374,17 +373,7 @@ impl File { gpu.get_params(&mut params)?; - if gpu.supports_vm_status() { - params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 - | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; - } - if gpu.queue_limits().is_some() { - params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; - } - - if *module_parameters::fault_control.value() == 0xb { - params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; - } + params.features |= gpu.feature_flags(); let size = core::mem::size_of::().min(data.size.try_into()?); @@ -577,7 +566,7 @@ impl File { /// IOCTL: vm_destroy: Destroy a `Vm`. pub(crate) fn vm_destroy( _device: &AsahiDrmDevice, - _reg_data: &driver::AsahiDrmRegistrationData<'_>, + reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_vm_destroy, file: &DrmFile, ) -> Result { @@ -585,6 +574,8 @@ impl File { if vm.is_none() { Err(ENOENT) } else { + drop(vm); + reg_data.gpu.vm_closed(); Ok(0) } } @@ -592,7 +583,7 @@ impl File { /// IOCTL: gem_create: Create a new GEM object. pub(crate) fn gem_create( device: &AsahiDrmDevice, - _reg_data: &driver::AsahiDrmRegistrationData<'_>, + reg_data: &driver::AsahiDrmRegistrationData<'_>, data: &mut uapi::drm_asahi_gem_create, file: &DrmFile, ) -> Result { @@ -629,6 +620,9 @@ impl File { let gem = gem::new_object(device, data.size.try_into()?, data.flags, resv_obj, reclaim_cpu)?; + if reg_data.gpu.prepare_new_objects() { + gem::prepare_new_object(&gem)?; + } let handle = gem.create_handle(file)?; data.handle = handle; @@ -704,7 +698,13 @@ impl File { return Err(EINVAL); } - if reg_data.gpu.as_any().is::() { + if let Some(limit) = reg_data.gpu.bind_batch_limit() { + if data.num_binds == 0 || data.num_binds > limit || data.userptr == 0 + || data.stride % 8 != 0 || data.stride as usize > kernel::page::PAGE_SIZE { + return Err(EINVAL); + } + } + if reg_data.gpu.as_any().is::() || reg_data.gpu.tracked_submission() { return Self::vm_bind_g17(data, file); } @@ -1284,7 +1284,7 @@ impl File { .reader() .read_all(bytes, GFP_KERNEL) }; - let guarded = gpu.as_any().is::(); + let guarded = gpu.as_any().is::() || gpu.tracked_submission(); // Once G17 output syncobjs are resolved, its queue must install a fence // on every failure. Take the fallible userspace snapshot before that point. if guarded { diff --git a/drivers/gpu/drm/asahi/g15/SOURCES.json b/drivers/gpu/drm/asahi/g15/SOURCES.json new file mode 100644 index 0000000000000..13deae1310736 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/SOURCES.json @@ -0,0 +1,229 @@ +{ + "schema": 1, + "source_pins": { + "target": "10ff0fe408d83936d6c7783b0382a3092e520ea5", + "pr148": "ee4f0f3ee2958ddc0243bb5e71a0bea88215d01b", + "sf": "65cd54ba0da8f93e4530f851382ccb5187ab794c" + }, + "scope": "Public native G15 subtree only. No private notes, whole-driver replacement, global UAPI selector, display/PMGR/memory-floor policy or branding.", + "source_carries": [ + { + "source": "04795f0d9600a3a695b3bd96fdb023720bea2883", + "commit": "54f7d6904f8d935781b79b7175b556b1ceeb9619", + "author": "Eryk Wieliczko", + "note": "G15 subtree snapshot including the preceding firmware/engine/recovery commits a8b817ef0614,39e7cf0bfdaa,01812cc4894f; no shared APIs or display/PMP policy imported." + }, + { + "source": "d8422e044e37f5d366422b2f37db8571fa64d11b", + "commit": "41275bfd985b7ccbcad6cb6650d94696fb0b29c3", + "author": "Naeem Malik", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "7c2857681b843ddab2c66abc8c04d12746efcba4", + "commit": "1f28f12c5e11608a8d7694d3b8db8a867f058f45", + "author": "Naeem Malik", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "0c98b7e731e8c42189593d7b9b9835785e4f6b07", + "commit": "bcb43bb61412df2808079f05fa684938c5e26528", + "author": "Naeem Malik", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "d3283ae4798ebd8c5e0de418dcbe9adb5b09327d", + "commit": "9538af3d5d3911ddd4762e7cc3b026180e193a88", + "author": "Naeem Malik", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "2a2def4ae6625d4a09d0cc944fc72b6682a2cf0c", + "commit": "e3d7591ae5f8135a4a3915c76cc849fcc1c2c996", + "author": "Naeem Malik", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "b091ba71dfe11dc66c83542131420027f1a13281", + "commit": "4d38ccc69d4781336ce5263f32365436abe35c2c", + "author": "Naeem Malik", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "ad229034eb4300fcbfe40c658d22f38f34454438", + "commit": "113194ca413fc0b05c2ffd5e7d26ab4c5224baf5", + "author": "DjDeveloperr", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "fa62b3345de5d8887201d3750e4f8ecc83a20778", + "commit": "369360537e628e7b04ebf24ac1d2a90786cf6c89", + "author": "DjDeveloperr", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "0da467881effc823ee51616f361c2e446b540cf5", + "commit": "2371dbf829c408917d6b4c52095931c0166a5331", + "author": "DjDeveloperr", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "897412529e3941f99e159a2585f2574179b38307", + "commit": "d9f1f541cd7ab5a5f01b60d9523c6e51db1113b1", + "author": "DjDeveloperr", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "90dc887b72d791809cfc8e2b92fcf42a186cf716", + "commit": "b58306ce0518bf0982166e3acfcce8661eb8d852", + "author": "DjDeveloperr", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "ee4f0f3ee2958ddc0243bb5e71a0bea88215d01b", + "commit": "2cdad45526fb46273dcd03474c3ab074eb97af7d", + "author": "DjDeveloperr", + "note": "G15 subtree only; shared layers and registration adapted separately." + }, + { + "source": "0fcf28d2cd305c6622475a01b81b334adf65a422", + "commit": "1903ae11e30a93f39fa39c0172cb5b3022e12ccb", + "author": "Chris Kearney", + "note": "G15 subtree only from the public native-profile bridge. Includes existing maintained loader/PMP admission adaptations 9333b0728dc4,674fd94f0b20,19cebe363276,d68cf5c0040f and c70e30b60623; no Rust API or global profile/header switch imported." + }, + { + "source": "16a4b74b0d7912cbb174c0a4c4503776391ccf8a", + "commit": "029d221c3d49284f593198165befb74a892baf39", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "d3b664b79b70f099630f48437af134682a1b11d7", + "commit": "1426152ddedbecc3766942256810560163522980", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "52705c9919c94bcd86a7278ef4b99c150f26c0b5", + "commit": "be6a5b4303fb1403c70c666aa98e06fb24109baa", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "c77b143bc5161fa19354aeb9dd3ae6f323e6d851", + "commit": "2173a9adef3f0eb7eb3111c6faafca4286dd0eee", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "1a49fbfaafff0c9693bfbe3d1847305725d6787b", + "commit": "45953c21d0f3711f629c9f332ca37adaa13416ef", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "ca6e846e41e364f343257de50a0fdce7a2ed4586", + "commit": "a63a953ee1d887bb339bbd4d29121aedf6949eb2", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "ea93a3d4d2ff7c8bc24fe4796b98fc7ba9627072", + "commit": "bf160c16ec91df92f9b1442724010ad4595feb41", + "author": "Chris Kearney", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "186c30e76a428ca80197fa91444fbb44a7712f79", + "commit": "6a1c99e7b960305fae441f20f2bcd95ae4eca80d", + "author": "Chris Kearney", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "20d359950d0c981e762fbb43c315866fc86c271b", + "commit": "1d769b3d9f606505230a5780f14930cf40269025", + "author": "Chris Kearney", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "bec6e4f83afd815b38803e754333b8954ce7bc29", + "commit": "5c597f936e919a31db34aa74a9a7782522f31c2b", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "3be092a24989523cab10253243981ab32c92193a", + "commit": "5c490e10e442cca145bc3a9847bd6d14982cc5b3", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "2c8d031fccc0ce4b125318dc011884de342325c7", + "commit": "fea7b79a3aa5c41799ba8a3410616a7d51d06c9f", + "author": "Chris Kearney", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "d0fc3c53d90f0be24644c0f3d8e3ba46bf9a0e0d", + "commit": "4b7929dfe546b9ae829458051350d07b1d1d3fe1", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "e2a97e967bea7b45c02801fa9afc4ae450be5a40", + "commit": "77528f1df378237950a3a6f85c9021b0129a4523", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "1a16318bc89c0658152405a3f587436b06c15157", + "commit": "44fdda4028002d93514879505903cf6cc8e02240", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "dba2a110de00743eeecd41edf23e54ede4491d5e", + "commit": "571e1d0c9da42d15fd7b4a023fb84cefde0904c0", + "author": "Naeem Malik", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "3d247356aa8a4ee5f50b24d3d6fd4afc89cf5c6f", + "commit": "f3893274d2a6d559be1b9d4e99c4f1294d2f3bbe", + "author": "Chris Kearney", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "f3668549340d6cb9bb84f2bc5719b78abdf4b5bb", + "commit": "bc29e6b988e2d597b0c5338d8891dfd473901644", + "author": "Chris Kearney", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "6fef6f3820ba34a2c2cbd0b0ee95ec473def4ffc", + "commit": "83aa75f592b015fb940109ee7578c674999348c6", + "author": "Chris Kearney", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "d5317ea1542d42382a7d6e3dbcdc3d611d5c5237", + "commit": "1d1702b49ef6a72062da37c1fa98f6cb72fb8fdf", + "author": "Chris Kearney", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + }, + { + "source": "9e67aed169c3cc6a0388d0184d8db76936299385", + "commit": "c739064a9d1a7c388ae8bb74dde4ee099b374b68", + "author": "Chris Kearney", + "note": "G15 subtree hunks only; asahi-m3 path becomes maintained asahi/g15; tests carried in the dedicated test adaptation." + } + ], + "adaptation_notes": { + "registration": "Separate no-alias G15 adapter in the maintained asahi module; retained failed probe and manual-unbind suppression. Existing normal/G17 paths kept.", + "drm": "Uses target RegistrationData and current DRM/GEM/GPUVM APIs; no Uninit conversion or forged registered context.", + "mmio": "G15 retained Mapping owns its register windows; target regs::Resources remains borrowed by RegistrationData.", + "sg_table": "Target held_sg_table permanently pins backing and owns SG lifetime inside GEM; G15 HeldPages retains a GEM reference, not the removed owned-SG wrapper.", + "page_tables": "Only adopted G15 page tables enable point-of-coherency publication; normal/G17 default remains false.", + "wire": "Native per-device wire facade is separately supplied; canonical normal/G17 UAPI unchanged." + } +} diff --git a/drivers/gpu/drm/asahi/g15/adt.rs b/drivers/gpu/drm/asahi/g15/adt.rs new file mode 100644 index 0000000000000..aece4f083c0ad --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/adt.rs @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The GPU regions, firmware segments and performance states: from the boot loader's ADT +//! export (T6030), or from the GPU device tree node the boot loader fills (T6031). +//! +//! # T6030: the ADT export +//! +//! The GPU node that the PMP/ADT handoff (`drivers/soc/apple/j514s-pmp-handoff.c`) synthesizes +//! has only `compatible = "apple,agx-t6030"`, `reg`/`reg-names` (`asc`, `sgx`), `mboxes`/ +//! `mbox-names` (`mbox`), `power-domains` and `apple,pmp`; it has no memory regions, operating +//! points or firmware-compat. Regions, segments and performance states come from the ADT: +//! `/reserved-memory` child `compatible = "phram"`, `label = "adt"` (16 KiB aligned, at most +//! 2 MiB; copied, never parsed in place). Properties: `/arm-io/sgx` `compatible == "gpu,t6030"`, +//! `{gpu-region,gfx-shared-region,gfx-handoff,gfx-shared-l2-region}-{base,size}`, +//! `perf-states`/`perf-states-sram` (14 x {u32 Hz, u32 mV}) and the fixed constants checked by +//! `collect_perf`; `/arm-io/gfx-asc` `segment-ranges` (2 x 32 bytes) and `segment-names` +//! (`__TEXT;__DATA`); `/arm-io/gfx-asc/iop-gfx-nub` `segment-ranges` (identical, length +//! checked too) and `pre-loaded == 1`; `/chosen` `dram-base`/`dram-size`. +//! Admission: every region page-aligned, inside DRAM and disjoint from the others; +//! TEXT at [`super::hw::UPPER`], DATA right after it. Firmware stays outside System RAM. +//! The maintained native-GPU profile also admits the four UAT regions as reserved RAM, +//! only with their exact enabled reserved-memory owners and every page PageReserved. +//! +//! # T6031: the device tree node +//! +//! The boot loader fills the kernel's GPU node from the machine's ADT: `memory-region` entries +//! `ttbs`, `pagetables`, `handoff` and `pagetables-l2` (the UAT regions), `fw-text` and +//! `fw-data` (the loaded segments, no-map), `apple,firmware-segment-vas` (their two VAs) and +//! the `operating-points-v2` table (one `opp-hz` per performance state, `opp-microvolt` with +//! one cell per cluster). The same admission applies, without the DRAM bounds, and the four UAT +//! regions may lie in System RAM: they are reserved-memory nodes without `no-map`, which the +//! kernel maps, so the driver reaches them through its own mapping ([`super::regs::Mapping`]). + +use core::ops::Range; +use kernel::{c_str, device, prelude::*}; + +use super::boot::glue; +use super::hw; + +/// Number of ADT performance states of the board-image layout. +pub(crate) const PERF_STATES: usize = 14; + +/// What the driver takes from the ADT. +pub(crate) struct GpuAdt { + /// TTBAT page. + pub(crate) ttbs: Range, + /// Firmware kernel root and its tables. + pub(crate) tables: Range, + /// PPL handoff page. + pub(crate) handoff: Range, + /// Shared L2 page: the table of root entry 2 (the host window). + pub(crate) shared_l2: Range, + /// Loaded firmware TEXT and DATA, physical. + pub(crate) text: Range, + pub(crate) data: Range, + /// Performance states: MHz, core mV, SRAM mV. + pub(crate) perf_mhz: [u32; PERF_STATES], + pub(crate) perf_mv: [u32; PERF_STATES], + pub(crate) sram_mv: [u32; PERF_STATES], + /// Per-cluster core mV of each state, rounded up from the device tree's microvolts, how + /// many voltage cells the node has (1 on the ADT path), and the state's power in mW (0 on + /// the ADT path). + pub(crate) cluster_mv: [[u32; CLUSTERS]; PERF_STATES], + pub(crate) cluster_cells: usize, + pub(crate) perf_mw: [u32; PERF_STATES], +} + +/// Voltage cells the layout keeps per state. +pub(crate) const CLUSTERS: usize = 4; + +/// A private copy of the ADT. +struct Adt<'a> { + data: &'a [u8], +} + +fn le32(b: &[u8], off: usize) -> Option { + Some(u32::from_le_bytes(b.get(off..off + 4)?.try_into().ok()?)) +} + +fn le64(b: &[u8], off: usize) -> Option { + Some(u64::from_le_bytes(b.get(off..off + 8)?.try_into().ok()?)) +} + +/// `strncmp(a, b, n) == 0` for a byte field `a` and a string `b`, both treated as +/// NUL-terminated (bytes past the end of either count as NUL). +fn strncmp_eq(a: &[u8], b: &str, n: usize) -> bool { + let b = b.as_bytes(); + for i in 0..n { + let x = a.get(i).copied().unwrap_or(0); + let y = b.get(i).copied().unwrap_or(0); + if x != y { + return false; + } + if x == 0 { + return true; + } + } + true +} + +impl Adt<'_> { + /// Returns the end of the node at `off`, or `None` on a malformed node. + fn node_end(&self, mut off: usize, depth: u32) -> Option { + let size = self.data.len(); + if depth > 32 || off > size || size - off < 8 { + return None; + } + let props = le32(self.data, off)?; + let children = le32(self.data, off + 4)?; + if props > 4096 || children > 1024 { + return None; + } + off += 8; + for _ in 0..props { + if size - off < 36 { + return None; + } + let len = (le32(self.data, off + 32)? & 0x7fffffff) as usize; + off += 36; + if len > size - off { + return None; + } + off += len.next_multiple_of(4); + if off > size { + return None; + } + } + for _ in 0..children { + off = self.node_end(off, depth + 1)?; + } + Some(off) + } + + /// The value of property `name` of `node`. + fn prop(&self, node: usize, name: &str) -> Option<&[u8]> { + let props = le32(self.data, node)?; + let mut off = node + 8; + for _ in 0..props { + let len = (le32(self.data, off + 32)? & 0x7fffffff) as usize; + if strncmp_eq(self.data.get(off..off + 32)?, name, 32) { + return self.data.get(off + 36..off + 36 + len); + } + off += 36 + len.next_multiple_of(4); + } + None + } + + /// The child named `name` of `node`. + fn child(&self, node: usize, name: &str) -> Option { + let props = le32(self.data, node)?; + let children = le32(self.data, node + 4)?; + let mut off = node + 8; + for _ in 0..props { + off += 36 + ((le32(self.data, off + 32)? & 0x7fffffff) as usize).next_multiple_of(4); + } + for _ in 0..children { + if let Some(n) = self.prop(off, "name") { + if !n.is_empty() && strncmp_eq(n, name, n.len()) { + return Some(off); + } + } + off = self.node_end(off, 1)?; + } + None + } + + fn u64(&self, node: usize, name: &str) -> Option { + let p = self.prop(node, name)?; + if p.len() != 8 { + return None; + } + le64(p, 0) + } + + fn u32(&self, node: usize, name: &str) -> Option { + let p = self.prop(node, name)?; + if p.len() != 4 { + return None; + } + le32(p, 0) + } +} + +/// This machine's GPU performance states (per-chip binning): ADT sgx `perf-states` and +/// `perf-states-sram`, (frequency Hz, millivolt) pairs. The board-image layout implements one +/// table shape; another shape is refused. Values the board-image layout takes as given are +/// compared, not trusted. +fn collect_perf(dev: &device::Device, adt: &Adt<'_>, sgx: usize, out: &mut GpuAdt) -> Result { + const FIXED: [(&str, u32); 5] = [ + ("perf-state-count", PERF_STATES as u32), + ("gpu-num-perf-states", PERF_STATES as u32 - 1), + ("perf-state-table-count", 1), + ("gpu-power-sample-period", 8), + ("gpu-pwr-sample-period-aic-clks", 200000), + ]; + for (name, value) in FIXED { + if adt.u32(sgx, name) != Some(value) { + dev_err!(dev, "ADT sgx {} is not the supported {}\n", name, value); + return Err(EINVAL); + } + } + let (Some(core), Some(sram)) = ( + adt.prop(sgx, "perf-states"), + adt.prop(sgx, "perf-states-sram"), + ) else { + dev_err!(dev, "ADT perf-state tables missing or of another shape\n"); + return Err(EINVAL); + }; + if core.len() != 8 * PERF_STATES || sram.len() != core.len() { + dev_err!(dev, "ADT perf-state tables missing or of another shape\n"); + return Err(EINVAL); + } + for i in 0..PERF_STATES { + let hz = le32(core, 8 * i).ok_or(EINVAL)?; + if le32(sram, 8 * i) != Some(hz) + || hz % 1000000 != 0 + || hz > 3000000000 + || ((i == 0) != (hz == 0)) + { + dev_err!(dev, "ADT perf-state {} ({} Hz) not usable\n", i, hz); + return Err(EINVAL); + } + out.perf_mhz[i] = hz / 1000000; + out.perf_mv[i] = le32(core, 8 * i + 4).ok_or(EINVAL)?; + out.sram_mv[i] = le32(sram, 8 * i + 4).ok_or(EINVAL)?; + if !(100..=1100).contains(&out.perf_mv[i]) || !(100..=1100).contains(&out.sram_mv[i]) { + dev_err!( + dev, + "ADT perf-state {} voltage {}/{} mV out of range\n", + i, + out.perf_mv[i], + out.sram_mv[i] + ); + return Err(EINVAL); + } + } + Ok(()) +} + +/// Region names of the admission messages. +const REGION_NAMES: [&str; 6] = ["ttbs", "tables", "handoff", "shared-l2", "text", "data"]; + +/// Parses the private ADT copy. +fn parse(dev: &device::Device, adt: &Adt<'_>) -> Result { + const PROPS: [(&str, &str); 4] = [ + ("gpu-region-base", "gpu-region-size"), + ("gfx-shared-region-base", "gfx-shared-region-size"), + ("gfx-handoff-base", "gfx-handoff-size"), + ("gfx-shared-l2-region-base", "gfx-shared-l2-region-size"), + ]; + if adt.node_end(0, 0).is_none() { + return Err(EINVAL); + } + let armio = adt.child(0, "arm-io").ok_or(EINVAL)?; + let sgx = adt.child(armio, "sgx").ok_or(EINVAL)?; + let asc = adt.child(armio, "gfx-asc").ok_or(EINVAL)?; + let nub = adt.child(asc, "iop-gfx-nub").ok_or(EINVAL)?; + let compatible = adt.prop(sgx, "compatible").ok_or(EINVAL)?; + let nul = compatible.iter().position(|&b| b == 0).ok_or(EINVAL)?; + if &compatible[..nul] != b"gpu,t6030" { + return Err(EINVAL); + } + let mut region: [Range; 6] = Default::default(); + for (i, (base_key, size_key)) in PROPS.iter().enumerate() { + let base = adt.u64(sgx, base_key).ok_or(EINVAL)?; + let size = adt.u64(sgx, size_key).ok_or(EINVAL)?; + region[i] = base..base.checked_add(size).ok_or(EINVAL)?; + } + let ranges = adt.prop(asc, "segment-ranges").ok_or(EINVAL)?; + let nub_ranges = adt.prop(nub, "segment-ranges").ok_or(EINVAL)?; + let names = adt.prop(asc, "segment-names").ok_or(EINVAL)?; + // Both range tables must be exactly 2 x 32 bytes and identical. + if ranges.len() != 64 || nub_ranges.len() != 64 || ranges != nub_ranges { + return Err(EINVAL); + } + if !strncmp_eq(names, "__TEXT;__DATA", names.len()) { + return Err(EINVAL); + } + let preloaded = adt.prop(nub, "pre-loaded").ok_or(EINVAL)?; + if preloaded.len() != 4 || le32(preloaded, 0) != Some(1) { + return Err(EINVAL); + } + let mut va = [0u64; 2]; + for i in 0..2 { + let r = &ranges[32 * i..32 * i + 32]; + let pa = le64(r, 0).ok_or(EINVAL)?; + va[i] = le64(r, 8).ok_or(EINVAL)?; + if le64(r, 0x10) != Some(pa) || le32(r, 0x1c) != Some(if i != 0 { 0 } else { 1 }) { + return Err(EINVAL); + } + let size = le32(r, 0x18).ok_or(EINVAL)? as u64; + region[4 + i] = pa..pa.checked_add(size).ok_or(EINVAL)?; + } + // Firmware ABI: TEXT at the base of the upper half, DATA right after it. + let text_size = region[4].end - region[4].start; + if va[0] != hw::UPPER || va[1] != va[0] + text_size { + return Err(EINVAL); + } + let chosen = adt.child(0, "chosen"); + let dram = chosen + .and_then(|chosen| Some((adt.u64(chosen, "dram-base")?, adt.u64(chosen, "dram-size")?))); + let Some((dram_base, dram_size)) = + dram.filter(|&(base, size)| size != 0 && base.checked_add(size).is_some()) + else { + dev_err!(dev, "ADT DRAM bounds unreadable\n"); + return Err(EINVAL); + }; + // The maintained loader deliberately retains the UAT's linear mapping. + // Cross-check every ADT range against its exact enabled FDT reservation; + // admit_regions still rejects holes, allocator-owned pages and overlaps. + for (i, r) in region[..4].iter().enumerate() { + if !glue::reserved_uat(i as u32, r.start, r.end - r.start) { + dev_err!(dev, "GPU UAT {} has no matching boot reservation\n", REGION_NAMES[i]); + return Err(EINVAL); + } + } + admit_regions(dev, ®ion, Some(dram_base..dram_base + dram_size), true)?; + let [ttbs, tables, handoff, shared_l2, text, data] = region; + let mut out = GpuAdt { + ttbs, + tables, + handoff, + shared_l2, + text, + data, + perf_mhz: [0; PERF_STATES], + perf_mv: [0; PERF_STATES], + sram_mv: [0; PERF_STATES], + cluster_mv: [[0; CLUSTERS]; PERF_STATES], + cluster_cells: 1, + perf_mw: [0; PERF_STATES], + }; + collect_perf(dev, adt, sgx, &mut out)?; + for i in 0..PERF_STATES { + out.cluster_mv[i][0] = out.perf_mv[i]; + } + Ok(out) +} + +/// Structural admission of the six regions (same order as [`REGION_NAMES`]): page-aligned, +/// addressable, the fixed UAT sizes, disjoint from each other, and either outside System RAM +/// (the T6030 carveouts) or, with `reserved_ram` and for the four UAT regions only, lying +/// entirely in System RAM as reserved memory the GPU node names (the T6031 device tree). The +/// firmware segments are always outside System RAM. +fn admit_regions( + dev: &device::Device, + region: &[Range; 6], + dram: Option>, + reserved_ram: bool, +) -> Result { + let fixed = [ + hw::TTBS_SIZE, + hw::TABLES_SIZE, + hw::HANDOFF_SIZE, + hw::SHARED_L2_SIZE, + ]; + for i in 0..region.len() { + let r = ®ion[i]; + let size = r.end - r.start; + let end = r.end.wrapping_sub(1); + let outside_dram = match &dram { + Some(dram) => r.start < dram.start || end > dram.end - 1, + None => false, + }; + let placed = if reserved_ram && i < 4 { + glue::ram_disjoint(r.start, size) || glue::ram_within(r.start, size) + } else { + glue::ram_disjoint(r.start, size) + }; + if size == 0 + || (r.start | size) & (hw::PAGE - 1) != 0 + || end > (hw::PA_MASK | (hw::PAGE - 1)) + || outside_dram + || (i < 4 && size != fixed[i]) + || !placed + { + dev_err!( + dev, + "GPU region {} ({}) [mem {:#x}-{:#x}] not admissible\n", + i, + REGION_NAMES[i], + r.start, + end + ); + return Err(EINVAL); + } + for j in 0..i { + if r.start < region[j].end && region[j].start < r.end { + dev_err!(dev, "GPU regions {} and {} overlap\n", j, i); + return Err(EINVAL); + } + } + } + Ok(()) +} + +/// The GPU node's regions, segments and operating points (T6031). +fn from_device_tree(dev: &device::Device) -> Result { + const NAMES: [&CStr; 6] = [ + c_str!("ttbs"), + c_str!("pagetables"), + c_str!("handoff"), + c_str!("pagetables-l2"), + c_str!("fw-text"), + c_str!("fw-data"), + ]; + let node = dev.of_node().ok_or(ENODEV)?; + let mut region: [Range; 6] = Default::default(); + for (i, name) in NAMES.iter().enumerate() { + let r = node + .reserved_mem_region_to_resource_byname(name) + .inspect_err(|_| dev_err!(dev, "GPU node has no {} memory region\n", REGION_NAMES[i]))?; + let start = r.start() as u64; + let size = r.size() as u64; + region[i] = start..start.checked_add(size).ok_or(EINVAL)?; + } + // Firmware ABI: TEXT at the base of the upper half, DATA right after it. + let vas: KVec = node + .get_property(c_str!("apple,firmware-segment-vas")) + .inspect_err(|_| dev_err!(dev, "GPU node has no apple,firmware-segment-vas\n"))?; + let text_size = region[4].end - region[4].start; + if vas.len() != 2 || vas[0] != hw::UPPER || vas[1] != vas[0] + text_size { + dev_err!(dev, "GPU firmware segment VAs are not the firmware ABI's\n"); + return Err(EINVAL); + } + admit_regions(dev, ®ion, None, true)?; + let [ttbs, tables, handoff, shared_l2, text, data] = region; + let mut out = GpuAdt { + ttbs, + tables, + handoff, + shared_l2, + text, + data, + perf_mhz: [0; PERF_STATES], + perf_mv: [0; PERF_STATES], + sram_mv: [0; PERF_STATES], + cluster_mv: [[0; CLUSTERS]; PERF_STATES], + cluster_cells: 0, + perf_mw: [0; PERF_STATES], + }; + // Operating points: MHz, the highest cluster voltage of each and the clusters' voltages + // (rounded up) and power for the hardware record; the SRAM voltage is not in the device + // tree. + let opps = node + .parse_phandle(c_str!("operating-points-v2"), 0) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "GPU node has no operating points\n"))?; + let mut count = 0; + for opp in opps.children() { + let hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + let uw: Option = opp.get_opt_property(c_str!("opp-microwatt"))?; + let mv = uv.iter().max().copied().unwrap_or(0) / 1000; + if uv.is_empty() || uv.len() > CLUSTERS || (count > 0 && uv.len() != out.cluster_cells) { + dev_err!(dev, "GPU operating point {} has {} voltage cells\n", count, uv.len()); + return Err(EINVAL); + } + if count < PERF_STATES { + out.perf_mhz[count] = u32::try_from(hz / 1_000_000).map_err(|_| { + dev_err!(dev, "GPU operating point {} is {} Hz\n", count, hz); + EINVAL + })?; + out.perf_mv[count] = mv; + out.sram_mv[count] = mv; + for (c, v) in uv.iter().enumerate() { + out.cluster_mv[count][c] = v.div_ceil(1000); + } + out.cluster_cells = uv.len(); + out.perf_mw[count] = uw.unwrap_or(0) / 1000; + } + count += 1; + } + if count != PERF_STATES { + dev_err!( + dev, + "GPU node has {} operating points, not {}\n", + count, + PERF_STATES + ); + return Err(EINVAL); + } + Ok(out) +} + +/// Reads and admits the GPU's regions, segments and performance states (read-only), from the +/// source the SoC configuration names. +pub(crate) fn read(dev: &device::Device, cfg: &super::Config) -> Result { + if cfg.platform == hw::Platform::DeviceTree { + return from_device_tree(dev); + } + let (start, size) = glue::adt_phram()?; + if size == 0 || size > 0x200000 || start & 0x3fff != 0 { + return Err(EINVAL); + } + let copy = super::identity::copy_physical(start, usize::try_from(size)?)?; + parse(dev, &Adt { data: © }) +} diff --git a/drivers/gpu/drm/asahi/g15/board.rs b/drivers/gpu/drm/asahi/g15/board.rs new file mode 100644 index 0000000000000..da0a7408575bf --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board.rs @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Board-data templates of the supported firmware build. +//! +//! Sparse 16-byte-line images (HardwareData 35,332 B, Globals 15,868 B, Power 2,292 B), each +//! with its SHA-256: [`expand`] expands and verifies them, [`from_adt`] rebuilds DVFS tables A/B +//! from the ADT performance states (`DVFS_A` {0,1,2,3,5,7,9,11,13}, `DVFS_B` +//! {0,1,2,3,4,6,8,10,12}; MHz at +0xb5c, +0xf9c, +0x1350, core mV +0xb9c, SRAM mV +0xd9c, top +//! MHz +0x4dc4). The tables in [`data`] are generated, and every expanded image is checked +//! against its hash at boot. +//! +//! A SoC whose board data comes from its device tree ([`super::hw::Platform::DeviceTree`]) gets +//! its hardware record from [`hardware_record`] ([`record`]) and its Globals and Power from +//! [`device_tree`]: the G15S templates with the GPU node's power words and the hardware +//! record's maximum power ([`words`]). + +pub(crate) mod blob; +pub(crate) mod data; +pub(crate) mod record; +pub(crate) mod template; +pub(crate) mod words; + +use kernel::{c_str, device, prelude::*}; + +use super::adt::{self, GpuAdt}; +use super::boot::glue; +use super::fw::initdata::{globals, hwdata, power}; +use super::hw::{self, va}; +pub(crate) use template::{Line, Template}; + +static_assert!(words::GLOBALS_SIZE == globals::SIZE); +static_assert!(words::POWER_SIZE == power::SIZE); +static_assert!(words::HWDATA_A == hwdata::HWDATA_A); +static_assert!(words::HWDATA_A_MAX_POWER == hwdata::MAX_POWER_A); +static_assert!(words::TOP_STATE as usize == DVFS_STATES - 1); +static_assert!(record::HWDATA_A == hwdata::HWDATA_A); +static_assert!(record::MAX_POWER == hwdata::HWDATA_A + hwdata::MAX_POWER_A); +static_assert!(record::IOMAPS == hwdata::IOMAPS); +static_assert!(record::PERF_STATES == adt::PERF_STATES); +static_assert!(record::CLUSTERS == adt::CLUSTERS); + +/// Expands a template into `out` and verifies its hash. +pub(crate) fn expand(template: &Template, out: &mut [u8]) -> Result { + template::fill(template, out).map_err(|_| EINVAL)?; + if glue::sha256(&out[..template.size]) != template.sha256 { + return Err(EINVAL); + } + Ok(()) +} + +/// The GPU node's power words ([`words::PROPS`] and [`words::ZONES`]). +fn power_words(dev: &device::Device) -> Result { + let node = dev.of_node().ok_or(ENODEV)?; + let mut values = [0u32; words::PROPS.len()]; + for (value, name) in values.iter_mut().zip(words::PROPS.iter()) { + *value = node + .get_property(name) + .inspect_err(|_| dev_err!(dev, "GPU node has no u32 {}\n", name))?; + } + let zones: KVec = node.get_opt_property(words::ZONES)?.unwrap_or_default(); + words::PowerWords::new(values, &zones).map_err(|why| { + dev_err!(dev, "GPU node power words: {}\n", why); + EINVAL + }) +} + +/// A CS or AFR operating-point table of the GPU node (`name`): MHz and one voltage cell per +/// state, at most `max` states. +fn aux_opps( + dev: &device::Device, + node: &kernel::of::Node, + name: &CStr, + out: &mut [record::Opp], +) -> Result { + let table = node + .parse_phandle(name, 0) + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "GPU node has no {}\n", name))?; + let mut n = 0; + for opp in table.children() { + let hz: u64 = opp.get_property(c_str!("opp-hz"))?; + let uv: KVec = opp.get_property(c_str!("opp-microvolt"))?; + if uv.len() != 1 || n >= out.len() { + dev_err!( + dev, + "GPU {} table: state {} has {} cells\n", + name, + n, + uv.len() + ); + return Err(EINVAL); + } + let mhz = u32::try_from(hz / 1_000_000).map_err(|_| { + dev_err!(dev, "GPU {} table: state {} is {} Hz\n", name, n, hz); + EINVAL + })?; + out[n] = record::Opp { + mhz, + mv: [uv[0].div_ceil(1000), 0, 0, 0], + cells: 1, + mw: 0, + }; + n += 1; + } + if n == 0 { + dev_err!(dev, "GPU {} table is empty\n", name); + return Err(EINVAL); + } + Ok(n) +} + +/// The hardware record of a device-tree SoC ([`record`]): built from the GPU node's properties +/// and operating-point tables, the layout's IOMapping table and the ADT regions, with the +/// words the device tree does not carry copied from `file`, the machine's own record. +pub(crate) fn hardware_record( + dev: &device::Device, + cfg: &super::Config, + adt: &GpuAdt, + file: &[u8], +) -> Result> { + let layout = cfg.layout; + let node = dev.of_node().ok_or(ENODEV)?; + let mut props = [0u32; record::PROPS.len()]; + for (value, (name, default)) in props.iter_mut().zip(record::PROPS.iter()) { + *value = match node.get_opt_property(name)? { + Some(v) => v, + None => default + .ok_or(EINVAL) + .inspect_err(|_| dev_err!(dev, "GPU node has no u32 {}\n", name))?, + }; + } + let zones: KVec = node.get_opt_property(words::ZONES)?.unwrap_or_default(); + let mut main = [record::Opp::default(); record::PERF_STATES]; + for (i, o) in main.iter_mut().enumerate() { + *o = record::Opp { + mhz: adt.perf_mhz[i], + mv: adt.cluster_mv[i], + cells: adt.cluster_cells, + mw: adt.perf_mw[i], + }; + } + let mut cs = [record::Opp::default(); 16]; + let mut afr = [record::Opp::default(); 8]; + let ncs = aux_opps(dev, &node, c_str!("apple,cs-opp"), &mut cs)?; + let nafr = aux_opps(dev, &node, c_str!("apple,afr-opp"), &mut afr)?; + if layout.record_slots.len() != layout.iomaps.len() { + return Err(EINVAL); + } + let mut slots = [record::IoSlot::default(); record::IOMAP_SLOTS]; + for (k, (map, &(total, element, writable))) in + layout.iomaps.iter().zip(layout.record_slots).enumerate() + { + slots[k] = record::IoSlot { + slot: map.slot, + pa: map.pa + map.delta, + va: map.va + map.delta, + total, + element, + writable, + }; + } + let max_power_mw = u32::from_le_bytes( + file.get(record::MAX_POWER..record::MAX_POWER + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + ); + let inputs = record::Inputs { + props, + zones: &zones, + main: &main, + cs: &cs[..ncs], + afr: &afr[..nafr], + slots: &slots[..layout.iomaps.len()], + empty_writable: layout.record_empty_writable, + ttbat: adt.ttbs.start, + usc_base: hw::client::COMPACT_WINDOW.start, + user_end: hw::USER_RANGE.end, + timestamps: va::TIMESTAMPS, + max_power_mw, + }; + let mut out = KVVec::from_elem(0u8, record::SIZE, GFP_KERNEL)?; + record::from_file(&mut out, file) + .and_then(|()| record::build(&mut out, &inputs)) + .map_err(|why| { + dev_err!(dev, "hardware record from the device tree: {}\n", why); + EINVAL + })?; + Ok(out) +} + +/// Globals and Power of a device-tree SoC: the verified G15S templates with the GPU node's +/// power words and the maximum power of `hardware`, this machine's hardware record. Returns the +/// two images and that maximum power in mW. +pub(crate) fn device_tree( + dev: &device::Device, + hardware: &[u8], +) -> Result<(KVVec, KVVec, u32)> { + let w = power_words(dev)?; + let at = hwdata::HWDATA_A + hwdata::MAX_POWER_A; + let max_power_mw = u32::from_le_bytes( + hardware + .get(at..at + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + ); + let mut glob = KVVec::from_elem(0u8, globals::SIZE, GFP_KERNEL)?; + let mut pwr = KVVec::from_elem(0u8, power::SIZE, GFP_KERNEL)?; + if expand(&data::GLOBALS, &mut glob).is_err() || expand(&data::POWER, &mut pwr).is_err() { + dev_err!(dev, "board data template hash mismatch\n"); + return Err(EINVAL); + } + words::globals(&mut glob, max_power_mw, &w) + .and_then(|()| words::power(&mut pwr, max_power_mw, &w)) + .map_err(|why| { + dev_err!(dev, "board data from the device tree: {}\n", why); + EINVAL + })?; + Ok((glob, pwr, max_power_mw)) +} + +/// DVFS states of the firmware tables. +pub(crate) const DVFS_STATES: usize = 9; +const DVFS_A: [usize; DVFS_STATES] = [0, 1, 2, 3, 5, 7, 9, 11, 13]; +const DVFS_B: [usize; DVFS_STATES] = [0, 1, 2, 3, 4, 6, 8, 10, 12]; + +/// Table A's frequency of each firmware DVFS state, MHz: the ADT (or device-tree) operating +/// points the firmware's state indices name. +pub(crate) fn table_a_mhz(adt: &GpuAdt) -> [u32; DVFS_STATES] { + DVFS_A.map(|i| adt.perf_mhz[i]) +} + +fn put32(hw: &mut [u8], offset: usize, value: u32) -> Result { + hw.get_mut(offset..offset + 4) + .ok_or(EINVAL)? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) +} + +/// The machine-specific part of HardwareData, rebuilt from this machine's ADT performance +/// states over the template: the firmware's two DVFS tables take the three shared low states, +/// then per voltage level the higher (table A) and the lower (table B) frequency of the ADT +/// pair. Table A carries the core and SRAM voltages (eight identical lanes per state) and its top +/// frequency. With the reference machine's ADT this reproduces the template exactly. Returns the +/// top frequency in kHz. +pub(crate) fn from_adt(dev: &device::Device, adt: &GpuAdt, hw: &mut [u8]) -> Result { + for s in 1..DVFS_STATES { + let (a, pa) = (DVFS_A[s], DVFS_A[s - 1]); + let (b, pb) = (DVFS_B[s], DVFS_B[s - 1]); + if adt.perf_mhz[a] <= adt.perf_mhz[pa] + || adt.perf_mhz[b] <= adt.perf_mhz[pb] + || adt.perf_mv[a] < adt.perf_mv[pa] + || adt.sram_mv[a] < adt.sram_mv[pa] + { + dev_err!(dev, "ADT perf-states do not form ordered DVFS tables\n"); + return Err(EINVAL); + } + } + for s in 0..DVFS_STATES { + let (a, b) = (DVFS_A[s], DVFS_B[s]); + put32(hw, 0xb5c + 4 * s, adt.perf_mhz[a])?; + put32(hw, 0xf9c + 4 * s, adt.perf_mhz[b])?; + put32(hw, 0x1350 + 4 * s, adt.perf_mhz[b])?; + for k in 0..8 { + put32(hw, 0xb9c + 0x20 * s + 4 * k, adt.perf_mv[a])?; + put32(hw, 0xd9c + 0x20 * s + 4 * k, adt.sram_mv[a])?; + } + } + let top = adt.perf_mhz[DVFS_A[DVFS_STATES - 1]]; + put32(hw, 0x4dc4, top)?; + Ok(top * 1000) +} diff --git a/drivers/gpu/drm/asahi/g15/board/blob.rs b/drivers/gpu/drm/asahi/g15/board/blob.rs new file mode 100644 index 0000000000000..04cff8bbaa3d9 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board/blob.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Board images from a firmware file. +//! +//! A SoC whose board images are not built-in templates names a firmware file +//! ([`super::super::hw::Board::Blob`]), built into the kernel with `CONFIG_EXTRA_FIRMWARE` or +//! loaded from `/lib/firmware`, and pins its SHA-256. The file holds the three images as the +//! firmware of that machine accepted them; the build then writes the pointers and policy words +//! it owns into them (TTBAT, IOMapping VAs, the timestamp arena, the idle policy). +//! +//! Layout, little-endian: the magic `T6031BRD`, u32 format version (1), u32 image count (3), +//! then per image `u32 kind, u32 size` with kinds 0 (HardwareData), 1 (Globals) and +//! 2 (Power), then the images' bytes in that order. Sizes must match the SoC's layout. + +use kernel::{device, firmware::Firmware, prelude::*}; + +use super::super::boot::glue; +use super::super::identity::Hex; + +const MAGIC: &[u8; 8] = b"T6031BRD"; +const VERSION: u32 = 1; +const IMAGES: usize = 3; + +/// The three images. +pub(crate) struct Images { + pub(crate) hardware: KVVec, + pub(crate) globals: KVVec, + pub(crate) power: KVVec, +} + +fn le32(b: &[u8], off: usize) -> Result { + Ok(u32::from_le_bytes( + b.get(off..off + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +fn copy(b: &[u8], range: core::ops::Range) -> Result> { + let src = b.get(range).ok_or(EINVAL)?; + let mut out = KVVec::from_elem(0u8, src.len(), GFP_KERNEL)?; + out.copy_from_slice(src); + Ok(out) +} + +/// Requests `name`, verifies its digest against `sha256` and splits it into the images, each +/// of the size `sizes` gives for its kind. +pub(crate) fn load( + dev: &device::Device, + name: &CStr, + sha256: &[u8; 32], + sizes: [usize; IMAGES], +) -> Result { + let fw = Firmware::request(name, dev).inspect_err(|e| { + dev_err!(dev, "board images {}: not available ({:?})\n", name, e) + })?; + let data = fw.data(); + let digest = glue::sha256(data); + if digest != *sha256 { + dev_err!( + dev, + "board images {}: sha256 {} is not the pinned one\n", + name, + Hex(&digest) + ); + return Err(EINVAL); + } + if data.len() < 16 || &data[..8] != MAGIC || le32(data, 8)? != VERSION { + dev_err!(dev, "board images {}: unknown header\n", name); + return Err(EINVAL); + } + if le32(data, 12)? as usize != IMAGES { + dev_err!(dev, "board images {}: not {} images\n", name, IMAGES); + return Err(EINVAL); + } + let mut at = 16 + IMAGES * 8; + let mut images: [Option>; IMAGES] = [None, None, None]; + for i in 0..IMAGES { + let kind = le32(data, 16 + i * 8)? as usize; + let size = le32(data, 16 + i * 8 + 4)? as usize; + if kind != i || size != sizes[i] { + dev_err!( + dev, + "board images {}: image {} is kind {} of {} bytes, need {}\n", + name, + i, + kind, + size, + sizes[i] + ); + return Err(EINVAL); + } + images[i] = Some(copy(data, at..at + size)?); + at += size; + } + if at != data.len() { + dev_err!(dev, "board images {}: trailing bytes\n", name); + return Err(EINVAL); + } + let [hardware, globals, power] = images; + dev_info!( + dev, + "board images {}: sha256 {} accepted\n", + name, + Hex(&digest) + ); + Ok(Images { + hardware: hardware.ok_or(EINVAL)?, + globals: globals.ok_or(EINVAL)?, + power: power.ok_or(EINVAL)?, + }) +} diff --git a/drivers/gpu/drm/asahi/g15/board/data.rs b/drivers/gpu/drm/asahi/g15/board/data.rs new file mode 100644 index 0000000000000..15dc6fc86ade7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board/data.rs @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +// Generated tables. Do not edit. + +//! G15S initial board data (hardware, globals, power) of the supported firmware build: +//! configuration and calibration templates. Sparse: only nonzero 16-byte lines are listed; +//! everything else is zero. Verified by SHA-256 at boot ([`super::expand`]). + +use super::{Line, Template}; + +/// Hardware template (35332 bytes, 207 lines). +#[rustfmt::skip] +pub(crate) static HARDWARE: Template = Template { + size: 35332, + sha256: [ + 0x3b, 0xef, 0x05, 0x37, 0xfb, 0x82, 0x89, 0xe6, + 0x8d, 0xec, 0x7c, 0x78, 0x9f, 0xdc, 0x93, 0xad, + 0x01, 0x50, 0x95, 0x61, 0x48, 0x7c, 0xbf, 0xfb, + 0xca, 0xfb, 0xb0, 0xe3, 0x3f, 0x50, 0x1b, 0x38, + ], + lines: &[ + Line { offset: 0x0, data: [0x00, 0x00, 0x00, 0x00, 0x6f, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0xff, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x10, data: [0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00] }, + Line { offset: 0x20, data: [0x00, 0x80, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x20, 0xfc, 0xff, 0xff] }, + Line { offset: 0x640, data: [0x00, 0x00, 0xd0, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x650, data: [0x00, 0x40, 0x14, 0x00, 0x00, 0x40, 0x14, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x660, data: [0x00, 0x10, 0x10, 0x0e, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x670, data: [0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x680, data: [0x00, 0x40, 0x01, 0x51, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x690, data: [0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x6a0, data: [0x00, 0x00, 0x00, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x6b0, data: [0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x720, data: [0x00, 0xc0, 0x2b, 0x50, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x730, data: [0x00, 0x10, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x760, data: [0x00, 0x80, 0xe0, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x770, data: [0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x7a0, data: [0x00, 0x00, 0x00, 0x20, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x7b0, data: [0x00, 0xa0, 0x12, 0x00, 0x00, 0x50, 0x09, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x7c0, data: [0x00, 0xc0, 0x04, 0x51, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x7d0, data: [0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x880, data: [0x00, 0x00, 0x3d, 0x50, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x890, data: [0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x8a0, data: [0x00, 0x00, 0x3c, 0x50, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x8b0, data: [0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x8c0, data: [0x00, 0x80, 0x3d, 0x50, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x8d0, data: [0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x920, data: [0x00, 0x00, 0x00, 0x93, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x930, data: [0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x960, data: [0x00, 0xc0, 0x45, 0x09, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x970, data: [0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x980, data: [0x00, 0x00, 0x28, 0x50, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x990, data: [0x00, 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x9e0, data: [0x00, 0xc0, 0xe5, 0x90, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x9f0, data: [0x00, 0x40, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xa20, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x60, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0xa30, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0xa40, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xa50, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xa60, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x5d, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0xa70, data: [0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0xa80, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xa90, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00] }, + Line { offset: 0xab0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00] }, + Line { offset: 0xac0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0xad0, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xae0, data: [0x00, 0x00, 0x00, 0x00, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xb20, data: [0x14, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xb40, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00] }, + Line { offset: 0xb50, data: [0x04, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xb60, data: [0x52, 0x01, 0x00, 0x00, 0x6a, 0x02, 0x00, 0x00, 0x1c, 0x03, 0x00, 0x00, 0x9c, 0x03, 0x00, 0x00] }, + Line { offset: 0xb70, data: [0x20, 0x04, 0x00, 0x00, 0x9e, 0x04, 0x00, 0x00, 0x20, 0x05, 0x00, 0x00, 0x64, 0x05, 0x00, 0x00] }, + Line { offset: 0xb90, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00] }, + Line { offset: 0xba0, data: [0x7d, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00] }, + Line { offset: 0xbb0, data: [0x7d, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00] }, + Line { offset: 0xbc0, data: [0x8f, 0x02, 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00] }, + Line { offset: 0xbd0, data: [0x8f, 0x02, 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00, 0x8f, 0x02, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00] }, + Line { offset: 0xbe0, data: [0xb7, 0x02, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00] }, + Line { offset: 0xbf0, data: [0xb7, 0x02, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0xb7, 0x02, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00] }, + Line { offset: 0xc00, data: [0xbc, 0x02, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00] }, + Line { offset: 0xc10, data: [0xbc, 0x02, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00, 0xdf, 0x02, 0x00, 0x00] }, + Line { offset: 0xc20, data: [0xdf, 0x02, 0x00, 0x00, 0xdf, 0x02, 0x00, 0x00, 0xdf, 0x02, 0x00, 0x00, 0xdf, 0x02, 0x00, 0x00] }, + Line { offset: 0xc30, data: [0xdf, 0x02, 0x00, 0x00, 0xdf, 0x02, 0x00, 0x00, 0xdf, 0x02, 0x00, 0x00, 0x1b, 0x03, 0x00, 0x00] }, + Line { offset: 0xc40, data: [0x1b, 0x03, 0x00, 0x00, 0x1b, 0x03, 0x00, 0x00, 0x1b, 0x03, 0x00, 0x00, 0x1b, 0x03, 0x00, 0x00] }, + Line { offset: 0xc50, data: [0x1b, 0x03, 0x00, 0x00, 0x1b, 0x03, 0x00, 0x00, 0x1b, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00] }, + Line { offset: 0xc60, data: [0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00] }, + Line { offset: 0xc70, data: [0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00] }, + Line { offset: 0xc80, data: [0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00] }, + Line { offset: 0xc90, data: [0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00] }, + Line { offset: 0xca0, data: [0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00] }, + Line { offset: 0xcb0, data: [0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xd90, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xda0, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xdb0, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xdc0, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xdd0, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xde0, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xdf0, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xe00, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xe10, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xe20, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xe30, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xe40, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00] }, + Line { offset: 0xe50, data: [0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x2a, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00] }, + Line { offset: 0xe60, data: [0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00] }, + Line { offset: 0xe70, data: [0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x66, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00] }, + Line { offset: 0xe80, data: [0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00] }, + Line { offset: 0xe90, data: [0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0x9d, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00] }, + Line { offset: 0xea0, data: [0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00] }, + Line { offset: 0xeb0, data: [0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0xbb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xfa0, data: [0x52, 0x01, 0x00, 0x00, 0x6a, 0x02, 0x00, 0x00, 0x1c, 0x03, 0x00, 0x00, 0x40, 0x03, 0x00, 0x00] }, + Line { offset: 0xfb0, data: [0xb8, 0x03, 0x00, 0x00, 0x28, 0x04, 0x00, 0x00, 0x9e, 0x04, 0x00, 0x00, 0xda, 0x04, 0x00, 0x00] }, + Line { offset: 0xfd0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x8f, 0x82, 0x3f] }, + Line { offset: 0xfe0, data: [0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f] }, + Line { offset: 0xff0, data: [0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f] }, + Line { offset: 0x1060, data: [0x0f, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00] }, + Line { offset: 0x1070, data: [0x34, 0x00, 0x00, 0x00, 0x46, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00] }, + Line { offset: 0x10a0, data: [0x00, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x38, 0x00, 0x00, 0x00] }, + Line { offset: 0x10b0, data: [0x44, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x5d, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00] }, + Line { offset: 0x10e0, data: [0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00] }, + Line { offset: 0x10f0, data: [0x07, 0x00, 0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00] }, + Line { offset: 0x1120, data: [0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00] }, + Line { offset: 0x1130, data: [0x06, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00] }, + Line { offset: 0x1340, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00] }, + Line { offset: 0x1350, data: [0x00, 0x00, 0x00, 0x00, 0x52, 0x01, 0x00, 0x00, 0x6a, 0x02, 0x00, 0x00, 0x1c, 0x03, 0x00, 0x00] }, + Line { offset: 0x1360, data: [0x40, 0x03, 0x00, 0x00, 0xb8, 0x03, 0x00, 0x00, 0x28, 0x04, 0x00, 0x00, 0x9e, 0x04, 0x00, 0x00] }, + Line { offset: 0x1370, data: [0xda, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1390, data: [0x7d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x13b0, data: [0x8f, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x13d0, data: [0xb7, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x13f0, data: [0xbc, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1410, data: [0xdf, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1430, data: [0x1b, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1450, data: [0x66, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1470, data: [0x9d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1490, data: [0xbb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1590, data: [0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x15b0, data: [0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x15d0, data: [0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x15f0, data: [0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1610, data: [0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1630, data: [0x2a, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1650, data: [0x66, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1670, data: [0x9d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1690, data: [0xbb, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x17b0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x17c0, data: [0x03, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x17e0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x17f0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0x1800, data: [0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1810, data: [0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x1820, data: [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff] }, + Line { offset: 0x1830, data: [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff] }, + Line { offset: 0x1840, data: [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff] }, + Line { offset: 0x1860, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4580, data: [0x00, 0x00, 0x00, 0x00, 0x40, 0x0d, 0x03, 0x00, 0x40, 0x0d, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4590, data: [0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x45a0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0x45b0, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x45c0, data: [0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00] }, + Line { offset: 0x45d0, data: [0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4600, data: [0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f] }, + Line { offset: 0x4610, data: [0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f, 0x5c, 0x8f, 0x82, 0x3f] }, + Line { offset: 0x4620, data: [0x5c, 0x8f, 0x82, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4bd0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x71, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4be0, data: [0x00, 0x00, 0x00, 0x00, 0x9f, 0x2e, 0x7f, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x54, 0x61, 0x51, 0x3b] }, + Line { offset: 0x4bf0, data: [0x00, 0x00, 0x00, 0x00, 0x86, 0x95, 0xa5, 0x3c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4c00, data: [0x00, 0x7c, 0x36, 0x47, 0xdb, 0x0f, 0xa9, 0x40, 0x00, 0x00, 0x00, 0x00, 0x36, 0x49, 0x8c, 0xbc] }, + Line { offset: 0x4c10, data: [0x08, 0xfd, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4c20, data: [0x00, 0x00, 0x00, 0x00, 0x7c, 0xb6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4c50, data: [0x00, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4c70, data: [0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x3f] }, + Line { offset: 0x4c80, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x42, 0x60, 0x65, 0x3e] }, + Line { offset: 0x4c90, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x47, 0xcd, 0xcc, 0xcc, 0x3d] }, + Line { offset: 0x4ca0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00] }, + Line { offset: 0x4cb0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xb6, 0x00, 0x00] }, + Line { offset: 0x4cc0, data: [0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, 0x0b, 0x00] }, + Line { offset: 0x4d00, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4d10, data: [0x5a, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4d20, data: [0x06, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f] }, + Line { offset: 0x4d30, data: [0x52, 0xb8, 0x7e, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e, 0x0a, 0xd7, 0xa3, 0x3b, 0xd7, 0xa3, 0xe0, 0x3f] }, + Line { offset: 0x4d40, data: [0xd7, 0xa3, 0xe0, 0x3f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbe, 0x42, 0x33, 0x33, 0xbb, 0x40] }, + Line { offset: 0x4d50, data: [0x33, 0x33, 0xbb, 0x40, 0x1b, 0xca, 0xeb, 0x40, 0x64, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00] }, + Line { offset: 0x4d60, data: [0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, 0x00] }, + Line { offset: 0x4d90, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00] }, + Line { offset: 0x4db0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x47, 0x00, 0x00, 0x20, 0x42] }, + Line { offset: 0x4dc0, data: [0x00, 0x00, 0x48, 0x44, 0x64, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4dd0, data: [0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4de0, data: [0x00, 0x00, 0xc8, 0x42, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f] }, + Line { offset: 0x4df0, data: [0xcd, 0xcc, 0x4c, 0x3e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4e20, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x2a, 0x40] }, + Line { offset: 0x4e30, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00] }, + Line { offset: 0x4e50, data: [0x00, 0x00, 0x80, 0x3f, 0x00, 0x00, 0x00, 0x00, 0xf0, 0xa7, 0x46, 0x41, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4e60, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x47, 0x00, 0x00, 0x14, 0x42, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4e70, data: [0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00] }, + Line { offset: 0x4e80, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x4eb0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5160, data: [0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5400, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xb6, 0x00, 0x00, 0x7c, 0xb6, 0x00, 0x00] }, + Line { offset: 0x5410, data: [0x7c, 0xb6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5420, data: [0x00, 0x00, 0x80, 0x3f, 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0xee, 0x02, 0x00] }, + Line { offset: 0x54f0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00] }, + Line { offset: 0x5500, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xcc, 0x4c, 0x3f, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5510, data: [0xcd, 0xcc, 0x4c, 0x3e, 0x00, 0x00, 0x00, 0x00, 0xec, 0x51, 0xb8, 0x3e, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5520, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x47, 0x33, 0x33, 0xb3, 0x3f, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5530, data: [0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00] }, + Line { offset: 0x5540, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x36, 0x47, 0x7c, 0xb6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5550, data: [0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa6, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5570, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5670, data: [0x00, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5680, data: [0x39, 0x8e, 0x63, 0x3f, 0xab, 0xaa, 0x2a, 0x3f, 0x39, 0x8e, 0xe3, 0x3d, 0xab, 0xaa, 0xaa, 0x3e] }, + Line { offset: 0x5690, data: [0xcd, 0xcc, 0xcc, 0xbe, 0xcd, 0xcc, 0x4c, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x47] }, + Line { offset: 0x56a0, data: [0x00, 0x00, 0xa0, 0xc0, 0x00, 0x00, 0x20, 0xc1, 0x00, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00] }, + Line { offset: 0x56b0, data: [0x20, 0x03, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, 0x45] }, + Line { offset: 0x56c0, data: [0x78, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00] }, + Line { offset: 0x56d0, data: [0x00, 0x5e, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xca, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x56e0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5700, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x1f, 0x00, 0x00] }, + Line { offset: 0x5710, data: [0x04, 0x00, 0x00, 0x00, 0xc4, 0x09, 0x00, 0x00, 0x58, 0x02, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00] }, + Line { offset: 0x5720, data: [0x04, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5750, data: [0x00, 0x00, 0x80, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5760, data: [0x28, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5800, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x2a, 0x40, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x5810, data: [0x7d, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x6020, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x8020, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x87c0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x36, 0x6e, 0x01] }, + Line { offset: 0x8810, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0x8820, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x4f, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + ], +}; + +/// Globals template (15868 bytes, 17 lines). +#[rustfmt::skip] +pub(crate) static GLOBALS: Template = Template { + size: 15868, + sha256: [ + 0x95, 0x1b, 0xc7, 0x79, 0x0c, 0x61, 0xe3, 0x5a, + 0x51, 0xfd, 0x33, 0x48, 0x5a, 0x78, 0x45, 0x41, + 0x50, 0xfb, 0xfb, 0x09, 0xf1, 0xa9, 0x56, 0x20, + 0x22, 0xff, 0x4a, 0x58, 0x5e, 0xe5, 0x50, 0x2a, + ], + lines: &[ + Line { offset: 0x20, data: [0xb8, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0x30, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x50, data: [0xff, 0xff, 0x28, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00] }, + Line { offset: 0x70, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0xb6, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00] }, + Line { offset: 0x80, data: [0x20, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x03, 0x00, 0x00] }, + Line { offset: 0x7a0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x02, 0x00, 0x00, 0x00, 0x00, 0x80, 0x3f] }, + Line { offset: 0x7b0, data: [0xcd, 0xcc, 0x4c, 0x3f, 0xcd, 0xcc, 0x4c, 0x3e, 0x66, 0x66, 0x66, 0x3f, 0xcd, 0xcc, 0xcc, 0x3d] }, + Line { offset: 0x7c0, data: [0x00, 0x00, 0x80, 0x3e, 0x33, 0x33, 0x33, 0x3f, 0x66, 0x66, 0x66, 0x3f, 0x06, 0x00, 0x00, 0x00] }, + Line { offset: 0x7d0, data: [0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00] }, + Line { offset: 0x970, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00] }, + Line { offset: 0x980, data: [0x0a, 0x00, 0x00, 0x00, 0xfa, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0x990, data: [0x64, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x9a0, data: [0x02, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0x9b0, data: [0x28, 0x00, 0x00, 0x00, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00] }, + Line { offset: 0x9c0, data: [0x00, 0x00, 0x01, 0x02, 0x01, 0x01, 0x5a, 0x4b, 0x01, 0x01, 0x01, 0x02, 0x5a, 0x4b, 0x01, 0x01] }, + Line { offset: 0x9d0, data: [0x01, 0x01, 0x5a, 0x4b, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xdf0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + ], +}; + +/// Power template (2292 bytes, 7 lines). +#[rustfmt::skip] +pub(crate) static POWER: Template = Template { + size: 2292, + sha256: [ + 0xf8, 0x96, 0x12, 0x0a, 0xb1, 0xad, 0xdc, 0x0c, + 0xf7, 0x50, 0xe0, 0xf2, 0x78, 0x48, 0x4a, 0xde, + 0xb0, 0x14, 0xfc, 0x7a, 0x05, 0x02, 0x60, 0xa4, + 0xa4, 0x12, 0x3d, 0xf2, 0x5c, 0x6d, 0xcb, 0x62, + ], + lines: &[ + Line { offset: 0x10, data: [0x05, 0x00, 0x00, 0x00, 0xec, 0x51, 0xb8, 0x3e, 0x33, 0x33, 0xb3, 0x3f, 0x28, 0x00, 0x00, 0x00] }, + Line { offset: 0x20, data: [0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xa0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf8, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xb0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x42, 0xf0, 0xa7, 0x46, 0x41] }, + Line { offset: 0xc0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + Line { offset: 0xd0, data: [0x7c, 0xb6, 0x00, 0x00, 0xcd, 0xcc, 0xcc, 0x3d, 0x42, 0x60, 0x65, 0x3e, 0x00, 0x00, 0x00, 0x00] }, + Line { offset: 0xe0, data: [0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00] }, + ], +}; diff --git a/drivers/gpu/drm/asahi/g15/board/record.rs b/drivers/gpu/drm/asahi/g15/board/record.rs new file mode 100644 index 0000000000000..959bc6a4ab0a2 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board/record.rs @@ -0,0 +1,1402 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The J516C hardware record (0xb480 bytes) from the device tree. +//! +//! The record is HardwareDataB (identity, IOMapping records, the DVFS tables) at 0, HardwareDataA +//! (the power, performance and thermal controllers) at [`HWDATA_A`], and the G15C tail (sensor +//! keys). Every word is one of: +//! +//! - a device-tree value or a function of one ([`PROPS`], the operating-point tables, the +//! reserved `ttbs` region): the controller gains, time constants and their derived filter +//! words (`(tc - 1) / tc`, `1 / tc`, gain x period, periods x 4, periods x period, clocks), +//! the two voltage-sorted DVFS tables with per-cluster voltages, the CS and AFR tables, the +//! state cap and base state; +//! - a layout constant of the 2419 firmware's G15C record: flags, the IOMapping flag words, +//! the identity words, fixed factors (65536, 95, 100, 1.02, 0.8/0.2, 625, 11000, 125, 5); +//! - a word the device tree does not carry ([`FROM_FILE`]), copied from the machine's board file +//! by [`from_file`] and listed with its rule in the project notes: the maximum power +//! (HardwareDataA +0x6a4; every other maximum-power word derives from it), the MTR sensor +//! masks, the relative-power table (the device tree's operating-point powers are a boot-loader +//! stand-in) and the tail's six sensor rows; +//! - a word the file carries and the firmware was shown not to need ([`UNNEEDED`]): zero. +//! +//! HardwareDataA has the G15S shape of the T6030 templates with insertions ([`a`]): 8 bytes +//! before the fast-die controller (+0x8bc), 16-wide instead of 8-wide coefficient rows (+0x200 +//! twice), 16-byte instead of 8-byte sensor masks (+8 twice), 0x600 and 0x2000 bytes of zero +//! blocks further on. The driver still writes the TTBAT, the IOMapping VAs, the timestamp arena +//! and the initdata timestamp as it does for the templates. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. + +use core::ffi::CStr; + +use super::words::{f32_div, f32_from_u32, f32_mul}; + +pub(crate) const SIZE: usize = 0xb480; +pub(crate) const HWDATA_A: usize = 0x4580; +/// IOMapping records: 32 bytes per slot. +pub(crate) const IOMAPS: usize = 0x640; +pub(crate) const IOMAP_SLOTS: usize = 31; +/// DVFS table entries. +pub(crate) const STATES: usize = 16; +/// Entries of the CS and AFR tables. +const CS_STATES: usize = 16; +const AFR_STATES: usize = 8; +/// Operating points of the layout and voltage lanes per state. +pub(crate) const PERF_STATES: usize = 14; +pub(crate) const CLUSTERS: usize = 4; +const LANES: usize = 8; +/// The AIC timer base clock, kHz. +const BASE_CLOCK_KHZ: u32 = 24000; + +// IEEE single bit patterns of the record's fixed factors. +const F_1: u32 = 0x3f800000; +const F_0_2: u32 = 0x3e4ccccd; +const F_0_8: u32 = 0x3f4ccccd; +const F_0_95: u32 = 0x3f733333; +const F_1_02: u32 = 0x3f828f5c; +const F_5: u32 = 0x40a00000; +const F_95: u32 = 0x42be0000; +const F_100: u32 = 0x42c80000; +const F_8000: u32 = 0x45fa0000; +const F_65536: u32 = 0x47800000; + +/// The GPU node's properties the record takes, each one u32 (gains are IEEE single bits), with +/// the value a node without the property gets (`None`: required). +pub(crate) const PROPS: [(&CStr, Option); 43] = [ + (c"apple,power-sample-period", None), + (c"apple,perf-base-pstate", None), + (c"apple,min-sram-microvolt", None), + (c"apple,csafr-min-sram-microvolt", None), + (c"apple,avg-power-filter-tc-ms", None), + (c"apple,avg-power-ki-only", None), + (c"apple,avg-power-kp", None), + (c"apple,avg-power-min-duty-cycle", None), + (c"apple,avg-power-target-filter-tc", None), + (c"apple,fast-die0-integral-gain", None), + (c"apple,fast-die0-proportional-gain", None), + (c"apple,fast-die0-prop-tgt-delta", Some(0)), + (c"apple,perf-boost-ce-step", None), + (c"apple,perf-boost-min-util", None), + (c"apple,perf-filter-drop-threshold", None), + (c"apple,perf-filter-time-constant", None), + (c"apple,perf-filter-time-constant2", None), + (c"apple,perf-integral-gain", None), + (c"apple,perf-integral-gain2", None), + (c"apple,perf-integral-min-clamp", None), + (c"apple,perf-proportional-gain", None), + (c"apple,perf-proportional-gain2", None), + (c"apple,perf-reset-iters", Some(6)), + (c"apple,perf-tgt-utilization", None), + (c"apple,ppm-filter-time-constant-ms", None), + (c"apple,ppm-ki", None), + (c"apple,ppm-kp", None), + (c"apple,pwr-filter-time-constant", None), + (c"apple,pwr-integral-gain", None), + (c"apple,pwr-integral-min-clamp", None), + (c"apple,pwr-min-duty-cycle", None), + (c"apple,pwr-proportional-gain", None), + (c"apple,pwr-sample-period-aic-clks", None), + (c"apple,se-engagement-criteria", None), + (c"apple,se-filter-time-constant", None), + (c"apple,se-filter-time-constant-1", None), + (c"apple,se-inactive-threshold", None), + (c"apple,se-ki", None), + (c"apple,se-ki-1", None), + (c"apple,se-kp", None), + (c"apple,se-kp-1", None), + (c"apple,se-reset-criteria", None), + (c"apple,se-target", None), +]; +const PERIOD_MS: usize = 0; +const BASE_PSTATE: usize = 1; +const MIN_SRAM_UV: usize = 2; +const CSAFR_MIN_SRAM_UV: usize = 3; +const AVG_TC_MS: usize = 4; +const AVG_KI: usize = 5; +const AVG_KP: usize = 6; +const AVG_MIN_DUTY: usize = 7; +const AVG_TGT_TC: usize = 8; +const FD_KI: usize = 9; +const FD_KP: usize = 10; +const FD_DELTA: usize = 11; +const BOOST_CE: usize = 12; +const BOOST_MIN: usize = 13; +const PERF_DROP: usize = 14; +const PERF_TC: usize = 15; +const PERF_TC2: usize = 16; +const PERF_KI: usize = 17; +const PERF_KI2: usize = 18; +const PERF_MIN_CLAMP: usize = 19; +const PERF_KP: usize = 20; +const PERF_KP2: usize = 21; +const PERF_RESET: usize = 22; +const PERF_TGT: usize = 23; +const PPM_TC_MS: usize = 24; +const PPM_KI: usize = 25; +const PPM_KP: usize = 26; +const PWR_TC: usize = 27; +const PWR_KI: usize = 28; +const PWR_MIN_CLAMP: usize = 29; +const PWR_MIN_DUTY: usize = 30; +const PWR_KP: usize = 31; +const AIC_CLKS: usize = 32; +const SE_CRIT: usize = 33; +const SE_TC: usize = 34; +const SE_TC1: usize = 35; +const SE_INACT: usize = 36; +const SE_KI: usize = 37; +const SE_KI1: usize = 38; +const SE_KP: usize = 39; +const SE_KP1: usize = 40; +const SE_RESET: usize = 41; +const SE_TARGET: usize = 42; + +/// One operating point: MHz, the clusters' voltages in mV (rounded up from microvolts), how +/// many voltage cells the node had, and mW. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Opp { + pub(crate) mhz: u32, + pub(crate) mv: [u32; CLUSTERS], + pub(crate) cells: usize, + pub(crate) mw: u32, +} + +impl Opp { + fn max_mv(&self) -> u32 { + self.mv[..self.cells].iter().copied().max().unwrap_or(0) + } +} + +/// One IOMapping record: slot, physical address and firmware VA (both with the in-page offset), +/// mapped bytes, element bytes, writable. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct IoSlot { + pub(crate) slot: u8, + pub(crate) pa: u64, + pub(crate) va: u64, + pub(crate) total: u32, + pub(crate) element: u32, + pub(crate) writable: bool, +} + +/// Everything the record is built from. +pub(crate) struct Inputs<'a> { + pub(crate) props: [u32; PROPS.len()], + /// `apple,power-zones` cells (triples). + pub(crate) zones: &'a [u32], + pub(crate) main: &'a [Opp], + pub(crate) cs: &'a [Opp], + pub(crate) afr: &'a [Opp], + pub(crate) slots: &'a [IoSlot], + /// Unmapped slots whose flag word the record nevertheless holds as 1. + pub(crate) empty_writable: &'a [u8], + /// The TTBAT page (the `ttbs` region). + pub(crate) ttbat: u64, + /// Firmware VAs the record repeats: the USC base, the page past the user range, the + /// timestamp arena. + pub(crate) usc_base: u64, + pub(crate) user_end: u64, + pub(crate) timestamps: u64, + /// The maximum GPU power in mW ([`MAX_POWER`], a word the device tree does not carry). + pub(crate) max_power_mw: u32, +} + +/// HardwareDataA's maximum-power word (= `fw::initdata::hwdata::MAX_POWER_A`). +pub(crate) const MAX_POWER: usize = HWDATA_A + 0x6a4; + +/// Words the device tree does not carry, copied from the board file. Record offsets. +#[rustfmt::skip] +pub(crate) const FROM_FILE: [usize; 32] = [ + // The relative powers of states 1..7 (state 8 is 100): the device tree's operating-point + // powers are a boot-loader stand-in. + 0x1060, 0x1064, 0x1068, 0x106c, 0x1070, 0x1074, 0x1078, + // HardwareDataA: the maximum power, the MTR fast-die masks (+0x8ac, +0x1490) and the alarm + // mask (+0x24a8); the device tree's apple,fast-die0-sensor-mask is another value. + MAX_POWER, + HWDATA_A + 0x8ac, HWDATA_A + 0x8b0, + HWDATA_A + 0x1490, HWDATA_A + 0x1494, + HWDATA_A + 0x24a8, HWDATA_A + 0x24ac, + // The tail's six sensor rows (key, id, kind; 12 bytes each; the boot loader does not export + // the aon-ptd table). + HWDATA_A + 0x6c58, HWDATA_A + 0x6c5c, HWDATA_A + 0x6c60, + HWDATA_A + 0x6c64, HWDATA_A + 0x6c68, HWDATA_A + 0x6c6c, + HWDATA_A + 0x6c70, HWDATA_A + 0x6c74, HWDATA_A + 0x6c78, + HWDATA_A + 0x6c7c, HWDATA_A + 0x6c80, HWDATA_A + 0x6c84, + HWDATA_A + 0x6c88, HWDATA_A + 0x6c8c, HWDATA_A + 0x6c90, + HWDATA_A + 0x6c94, HWDATA_A + 0x6c98, HWDATA_A + 0x6c9c, +]; + +/// Words the board file carries that the record leaves zero: the firmware ran, passed every +/// check and reached its top state without them (one boot each). The cross-check accepts the +/// file's value there. 27 words in the HardwareDataB +0x38..+0x638 block and two all-ones words +/// after it (zero in the G15S templates); the tail's thirteen readings (a single at +/// HardwareDataA +0x6d34, twelve singles after +0x6d80). +#[rustfmt::skip] +pub(crate) const UNNEEDED: [usize; 42] = [ + 0xe0, 0xe8, 0xf4, 0xf8, 0xfc, 0x100, 0x104, 0x108, 0x10c, 0x2d8, 0x2dc, 0x2e0, 0x2e4, + 0x2e8, 0x2ec, 0x3e0, 0x3e8, 0x3f4, 0x3f8, 0x400, 0x40c, 0x5d8, 0x5dc, 0x5e0, 0x5e4, + 0x5e8, 0x5ec, 0x30d8, 0x30f0, + HWDATA_A + 0x6d34, + HWDATA_A + 0x6d8c, HWDATA_A + 0x6d90, HWDATA_A + 0x6d94, HWDATA_A + 0x6d98, + HWDATA_A + 0x6dcc, HWDATA_A + 0x6dd0, HWDATA_A + 0x6dd4, HWDATA_A + 0x6dd8, + HWDATA_A + 0x6dec, HWDATA_A + 0x6df4, HWDATA_A + 0x6dfc, HWDATA_A + 0x6e04, +]; + +/// The tail's sensor rows: six rows of {key, id, kind} from [`SENSOR_ROWS`]; the keys of the +/// rows whose kind byte is `g`, their presence mask, then the `r` and `f` rows' keys with a +/// count of one each. +const SENSOR_ROWS: usize = HWDATA_A + 0x6c58; +const SENSOR_ROW_COUNT: usize = 6; +const SENSOR_G_KEYS: usize = HWDATA_A + 0x6cdc; +const SENSOR_G_MASK: usize = HWDATA_A + 0x6cfc; +const SENSOR_R_KEY: usize = HWDATA_A + 0x6d04; +const SENSOR_R_COUNT: usize = HWDATA_A + 0x6d0c; +const SENSOR_F_KEY: usize = HWDATA_A + 0x6d14; +const SENSOR_F_COUNT: usize = HWDATA_A + 0x6d18; + +type R = Result; + +fn get32(b: &[u8], at: usize) -> R { + let w = b.get(at..at + 4).ok_or("record too short")?; + Ok(u32::from_le_bytes([w[0], w[1], w[2], w[3]])) +} + +fn put32(b: &mut [u8], at: usize, v: u32) -> R<()> { + b.get_mut(at..at + 4) + .ok_or("record too short")? + .copy_from_slice(&v.to_le_bytes()); + Ok(()) +} + +fn put64(b: &mut [u8], at: usize, v: u64) -> R<()> { + b.get_mut(at..at + 8) + .ok_or("record too short")? + .copy_from_slice(&v.to_le_bytes()); + Ok(()) +} + +/// `1 / n` as a single. +fn inv(n: u32) -> R { + f32_div(F_1, f32_from_u32(n)).ok_or("a time constant is 0") +} + +/// `(n - 1) / n` as a single (the filter's complement, `1 - 1/n`). +fn complement(n: u32) -> R { + if n == 0 { + return Err("a time constant is 0"); + } + f32_div(f32_from_u32(n - 1), f32_from_u32(n)).ok_or("a time constant is 0") +} + +/// `a * b` of device-tree words: refused, not wrapped (and no overflow panic), when it does not +/// fit the record's u32. +fn mul(a: u32, b: u32) -> R { + a.checked_mul(b).ok_or(OVERFLOW) +} + +const OVERFLOW: &str = "a device-tree value overflows the hardware record's u32 arithmetic"; + +/// The G15C HardwareDataA offset of a G15S-layout word. +pub(crate) const fn a(off: usize) -> usize { + let shifted = if off < 0x8bc { + off + } else if off < 0x970 { + off + 8 + } else if off < 0xb70 { + // The coefficient tables, 16 instead of 8 singles per row: not written here. + off + 8 + } else if off < 0x1290 { + off + 0x208 + } else if off < 0x14b4 { + off + 0x210 + } else if off < 0x1aa0 { + off + 0xa10 + } else if off < 0x3aa4 { + off + 0xa18 + } else { + off + 0x2a18 + }; + HWDATA_A + shifted +} + +/// The two voltage-sorted DVFS tables of the operating points: entry 0 is state 0 (off), then +/// per distinct maximum voltage, ascending, the highest-frequency state (table A, `primary`) +/// and the lowest-frequency one (table B, `secondary`). States sharing a voltage must share +/// their cluster voltages; frequencies must rise along both tables and every cluster's voltage +/// along table A. +pub(crate) struct Tables { + pub(crate) len: usize, + pub(crate) primary: [u8; STATES], + pub(crate) secondary: [u8; STATES], +} + +impl Tables { + pub(crate) fn top(&self) -> usize { + self.len - 1 + } +} + +pub(crate) fn tables(main: &[Opp]) -> R { + if main.len() < 2 || main.len() > u8::MAX as usize { + return Err("operating points: fewer than 2 or more than 255"); + } + if main[0].mhz != 0 || main[1..].iter().any(|s| s.mhz == 0) { + return Err("operating points: state 0 is not the only 0 MHz state"); + } + let mut t = Tables { + len: 1, + primary: [0; STATES], + secondary: [0; STATES], + }; + let mut last_mv = 0; + loop { + let Some(mv) = main[1..] + .iter() + .map(Opp::max_mv) + .filter(|&mv| mv > last_mv) + .min() + else { + break; + }; + let group = (1..main.len()).filter(|&i| main[i].max_mv() == mv); + let (mut first, mut hi, mut lo) = (0usize, 0usize, 0usize); + for i in group { + if first == 0 { + first = i; + } + if hi == 0 || main[i].mhz > main[hi].mhz { + hi = i; + } + if lo == 0 || main[i].mhz < main[lo].mhz { + lo = i; + } + if main[i].mv != main[first].mv { + return Err("operating points at one voltage differ per cluster"); + } + } + if t.len >= STATES { + return Err("operating points: more than 16 voltages"); + } + let (pp, ps) = ( + &main[t.primary[t.len - 1] as usize], + &main[t.secondary[t.len - 1] as usize], + ); + if main[hi].mhz <= pp.mhz || main[lo].mhz <= ps.mhz { + return Err("operating points: frequency does not rise with voltage"); + } + if t.len > 1 && (0..main[hi].cells).any(|c| main[hi].mv[c] <= pp.mv[c]) { + return Err("operating points: a cluster's voltage does not rise with the table"); + } + t.primary[t.len] = hi as u8; + t.secondary[t.len] = lo as u8; + t.len += 1; + last_mv = mv; + } + if t.len < 2 { + return Err("operating points: no usable state"); + } + Ok(t) +} + +/// A voltage lane of a state: the cluster's voltage, or cluster 0's past the node's cells. +fn lane(s: &Opp, j: usize) -> u32 { + if j < s.cells { + s.mv[j] + } else { + s.mv[0] + } +} + +fn hwdata_b(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { + let p = &i.props; + put64(out, 0x0, 0x6f_0000_0000)?; + put64(out, 0x8, 0xffc0_0000)?; + put64(out, 0x10, i.usc_base)?; + put64(out, 0x18, i.usc_base)?; + put64(out, 0x20, i.user_end)?; + put64(out, 0x28, i.timestamps)?; + for s in i.slots { + if s.slot as usize >= IOMAP_SLOTS { + return Err("IOMapping slot past the table"); + } + let r = IOMAPS + 32 * s.slot as usize; + put64(out, r, s.pa)?; + put64(out, r + 8, s.va)?; + put32(out, r + 0x10, s.total)?; + put32(out, r + 0x14, s.element)?; + put32(out, r + 0x18, s.writable as u32)?; + } + for &slot in i.empty_writable { + if slot as usize >= IOMAP_SLOTS || i.slots.iter().any(|s| s.slot == slot) { + return Err("an empty-writable slot is mapped or past the table"); + } + put32(out, IOMAPS + 32 * slot as usize + 0x18, 1)?; + } + // Identity and flags. + put32(out, 0xa28, 0x6031)?; + put32(out, 0xa2c, 1)?; + put32(out, 0xa30, 2)?; + for at in [0xa38, 0xa3c, 0xa40, 0xa50, 0xa58, 0xa60] { + put32(out, at, 1)?; + } + put32(out, 0xa68, BASE_CLOCK_KHZ)?; + put32(out, 0xa6c, 1)?; + put32(out, 0xa70, p[PERIOD_MS])?; + put32(out, 0xa78, 1)?; + put64(out, 0xa7c, 0x1_0000_0001)?; + put32(out, 0xa9c, 0x1f)?; + put64(out, 0xabc, 4)?; + put32(out, 0xacc, 1)?; + put32(out, 0xad0, 1)?; + put32(out, 0xae4, 0x31)?; + put32(out, 0xb04, 1)?; + put32(out, 0xb20, 0x28)?; + put32(out, 0xb24, 0xff)?; + put32(out, 0xb40, 1)?; + put64(out, 0xb44, i.ttbat)?; + // GPU core 24 (G15C), revision 4, 40 core slots. + put32(out, 0xb4c, 24)?; + put32(out, 0xb50, 4)?; + put32(out, 0xb54, 40)?; + put32(out, 0xb58, t.top() as u32)?; + let min_sram = p[MIN_SRAM_UV].div_ceil(1000); + let top = &i.main[t.primary[t.top()] as usize]; + let base_index = p[BASE_PSTATE] as usize; + let base = i + .main + .get(base_index) + .ok_or("perf-base-pstate past the states")?; + if t.primary[base_index] as usize != base_index { + return Err("perf-base-pstate is not its own table-A entry"); + } + if top.mhz <= base.mhz { + return Err("the top state is not above the base state"); + } + for k in 0..t.len { + let ps = &i.main[t.primary[k] as usize]; + let ss = &i.main[t.secondary[k] as usize]; + put32(out, 0xb5c + 4 * k, ps.mhz)?; + put32(out, 0xf9c + 4 * k, ss.mhz)?; + put32(out, 0x1350 + 4 * k, ss.mhz)?; + for j in 0..LANES { + let v = lane(ps, j); + put32(out, 0xb9c + 0x20 * k + 4 * j, v)?; + put32(out, 0xd9c + 0x20 * k + 4 * j, v.max(min_sram))?; + } + for j in 0..ss.cells { + put32(out, 0x1390 + 0x20 * k + 4 * j, ss.mv[j])?; + put32(out, 0x1590 + 0x20 * k + 4 * j, ss.mv[j].max(min_sram))?; + } + put32(out, 0xfdc + 4 * k, F_1_02)?; + put32(out, 0x10dc + 4 * k, t.primary[k] as u32)?; + put32(out, 0x111c + 4 * k, t.secondary[k] as u32)?; + if k > 0 { + // Relative power (a FROM_FILE word on the J516C) and relative boost frequency. + if top.mw != 0 { + put32( + out, + 0x105c + 4 * k, + u32::try_from(100 * u64::from(ps.mw) / u64::from(top.mw)) + .map_err(|_| "a relative power does not fit its u32 word")?, + )?; + } + if ps.mhz > base.mhz { + let span = u64::from(top.mhz - base.mhz); + put32( + out, + 0x109c + 4 * k, + (100 * u64::from(ps.mhz - base.mhz) / span) as u32, + )?; + } + } + } + put32(out, 0x134c, t.top() as u32)?; + // The CS and AFR tables: the top state twice, frequencies, voltages, SRAM voltages. + let cs_sram = p[CSAFR_MIN_SRAM_UV] / 1000; + let mut aux = |base: usize, tab: &[Opp], n: usize| -> R { + if tab.is_empty() || tab.len() > n { + return Err("CS/AFR table empty or longer than its layout"); + } + let top = (tab.len() - 1) as u32; + put32(out, base, top)?; + put32(out, base + 4, top)?; + let (fr, vo, sr) = (base + 8, base + 8 + 4 * n, base + 8 + 12 * n); + for (k, s) in tab.iter().enumerate() { + put32(out, fr + 4 * k, s.mhz)?; + for j in 0..s.cells.min(2) { + put32(out, vo + 8 * k + 4 * j, s.mv[j])?; + put32(out, sr + 8 * k + 4 * j, s.mv[j].max(cs_sram))?; + } + } + Ok(sr + 8 * n) + }; + let end = aux(0x115c, i.cs, CS_STATES)?; + if end != 0x12a4 { + return Err("CS table layout"); + } + let end = aux(0x12a4, i.afr, AFR_STATES)?; + if end != 0x134c { + return Err("AFR table layout"); + } + put32(out, 0x17b8, 5)?; + put32(out, 0x17c0, 3)?; + put32(out, 0x17c4, 7)?; + put32(out, 0x17c8, 7)?; + for at in [ + 0x17e8, 0x17fc, 0x1800, 0x1804, 0x1808, 0x1810, 0x1814, 0x1818, 0x1860, + ] { + put32(out, at, 1)?; + } + for at in (0x1820..0x1850).step_by(4) { + put32(out, at, u32::MAX)?; + } + Ok(()) +} + +fn hwdata_a(out: &mut [u8], i: &Inputs<'_>, t: &Tables) -> R<()> { + let p = &i.props; + let period_ms = p[PERIOD_MS]; + if period_ms == 0 { + return Err("the power sample period is 0"); + } + let period_s = f32_div(f32_from_u32(period_ms), f32_from_u32(1000)).ok_or("period")?; + let per_period = + |gain: usize| f32_mul(p[gain], period_s).ok_or("a gain is not a finite normal single"); + let max_scaled = 100 * t.top() as u32; + let base_scaled = mul(100, p[BASE_PSTATE])?; + let max_power = i.max_power_mw; + let max_power_f = f32_from_u32(max_power); + let mut w = |off: usize, v: u32| put32(out, a(off), v); + w(0x4, p[AIC_CLKS])?; + w(0x8, p[AIC_CLKS])?; + w(0x10, 4)?; + w(0x14, F_1)?; + w(0x2c, 1)?; + w(0x30, 1)?; + w(0x44, base_scaled)?; + w(0x48, 1)?; + w(0x4c, max_scaled)?; + w(0x54, 100)?; + for k in 0..t.len { + w(0x80 + 4 * k, F_1_02)?; + } + w(0x658, 625)?; + // Power controller. + w(0x664, complement(p[PWR_TC])?)?; + w(0x66c, inv(p[PWR_TC])?)?; + w(0x674, p[PWR_KI])?; + w(0x67c, f32_from_u32(p[PWR_MIN_CLAMP]))?; + w(0x680, max_power_f)?; + w(0x684, p[PWR_KP])?; + let k = f32_div(f32_from_u32(max_scaled), max_power_f).ok_or("maximum power is 0")?; + w(0x68c, k ^ 0x8000_0000)?; + w(0x690, p[PWR_MIN_DUTY].wrapping_sub(max_scaled))?; + w(0x698, max_scaled)?; + w(0x6a4, max_power)?; + w(0x6d4, max_scaled)?; + // PPM controller. + let ppm_tc = p[PPM_TC_MS] / period_ms; + w(0x6f0, mul(4, ppm_tc)?)?; + w(0x6fc, complement(ppm_tc)?)?; + w(0x704, inv(ppm_tc)?)?; + w(0x70c, per_period(PPM_KI)?)?; + w(0x718, F_65536)?; + w(0x71c, p[PPM_KP])?; + w(0x728, p[PWR_MIN_DUTY])?; + w(0x72c, max_scaled)?; + w(0x73c, max_power)?; + w(0x744, p[PPM_TC_MS])?; + w(0x74c, mul(mul(ppm_tc, period_ms)?, BASE_CLOCK_KHZ)?)?; + // Performance controller. + w(0x788, p[PERF_TGT])?; + w(0x790, p[BOOST_MIN])?; + w(0x794, p[BOOST_CE])?; + w(0x798, p[PERF_RESET])?; + w(0x7a0, 6)?; + w(0x7a4, 1)?; + w(0x7a8, p[PERF_DROP])?; + w(0x7ac, complement(p[PERF_TC])?)?; + w(0x7b0, complement(p[PERF_TC2])?)?; + w(0x7b4, inv(p[PERF_TC])?)?; + w(0x7b8, inv(p[PERF_TC2])?)?; + w(0x7bc, p[PERF_KI])?; + w(0x7c0, p[PERF_KI2])?; + w(0x7c4, f32_from_u32(p[PERF_MIN_CLAMP]))?; + w(0x7c8, F_95)?; + w(0x7cc, p[PERF_KP])?; + w(0x7d0, p[PERF_KP2])?; + let boost = (t.top() as u32) + .checked_sub(p[BASE_PSTATE]) + .ok_or("perf-base-pstate above the top state")?; + w(0x7d4, f32_div(f32_from_u32(boost), F_0_95).ok_or("boost")?)?; + w(0x7d8, base_scaled)?; + w(0x7dc, max_scaled)?; + w(0x7e0, base_scaled)?; + w(0x7ec, p[PERF_TGT])?; + w(0x81c, base_scaled)?; + w(0x838, F_65536)?; + w(0x83c, f32_from_u32(p[PWR_MIN_DUTY]))?; + w(0x840, f32_from_u32(max_scaled))?; + w(0x844, i.main.iter().map(|s| s.mhz).max().unwrap_or(0))?; + w(0x854, p[PWR_MIN_DUTY])?; + w(0x860, F_100)?; + w(0x864, max_scaled)?; + w(0x86c, F_0_8)?; + w(0x870, F_0_2)?; + // Fast-die controller (its sensor mask is a FROM_FILE word). + w(0x8bc, 4)?; + w(0x8d0, F_1)?; + w(0x8d8, per_period(FD_KI)?)?; + w(0x8e4, F_65536)?; + w(0x8e8, p[FD_KP])?; + w(0x8f4, p[PWR_MIN_DUTY])?; + w(0x8f8, max_scaled)?; + w(0x8fc, max_scaled)?; + w(0x900, mul(100, p[FD_DELTA])?)?; + w(0x908, 11000)?; + w(0x938, max_scaled)?; + w(0xbe0, max_scaled)?; + // Power zones and the average-power controller. + w(0xe80, 1)?; + w(0xe84, (i.zones.len() / 3) as u32)?; + for at in [0xe88, 0xe8c, 0xe90] { + w(at, max_power)?; + } + let tgt_tc = p[AVG_TGT_TC]; + w(0xe9c, complement(tgt_tc)?)?; + w(0xea0, inv(tgt_tc)?)?; + w(0xea4, mul(4, tgt_tc)?)?; + w(0xea8, mul(period_ms, tgt_tc)?)?; + w(0xeac, mul(mul(period_ms, tgt_tc)?, BASE_CLOCK_KHZ)?)?; + if i.zones.len() % 3 != 0 || i.zones.len() > 15 { + return Err("the power zones are not up to five u32 triples"); + } + for (z, zone) in i.zones.chunks_exact(3).enumerate() { + let at = 0xeb4 + 0x28 * z; + let (target, offset, tc) = (zone[0], zone[1], zone[2]); + w(at + 0x4, target)?; + w( + at + 0x8, + target + .checked_sub(offset) + .ok_or("a power zone's offset is larger than its target")?, + )?; + w(at + 0xc, mul(4, tc)?)?; + w(at + 0x10, mul(period_ms, tc)?)?; + w(at + 0x14, 1_320_000_000)?; + w(at + 0x1c, complement(tc)?)?; + w(at + 0x20, inv(tc)?)?; + } + let avg_tc = p[AVG_TC_MS] / period_ms; + w(0xf7c, mul(4, avg_tc)?)?; + w(0xf88, complement(avg_tc)?)?; + w(0xf90, inv(avg_tc)?)?; + w(0xf98, per_period(AVG_KI)?)?; + w(0xfa4, F_65536)?; + w(0xfa8, p[AVG_KP])?; + w(0xfb4, p[AVG_MIN_DUTY])?; + w(0xfb8, max_scaled)?; + w(0xfbc, max_scaled)?; + w(0xfc4, max_power_f)?; + w(0xfc8, max_power)?; + w(0xfd0, p[AVG_TC_MS])?; + w(0xfd8, mul(mul(avg_tc, period_ms)?, BASE_CLOCK_KHZ)?)?; + w(0xff8, max_scaled)?; + // The shader-engine controller block at +0x10b4. + const SE: usize = 0x10b4; + let clocks = mul(BASE_CLOCK_KHZ, period_ms)?; + w(SE + 0x40, 50)?; + w(SE + 0x44, 1)?; + w(SE + 0x4c, complement(p[SE_TC])?)?; + w(SE + 0x50, complement(p[SE_TC1])?)?; + w(SE + 0x54, inv(p[SE_TC])?)?; + w(SE + 0x58, inv(p[SE_TC1])?)?; + w(SE + 0x5c, per_period(SE_KI)?)?; + w(SE + 0x60, per_period(SE_KI1)?)?; + w(SE + 0x68, F_65536)?; + w(SE + 0x6c, p[SE_KP])?; + w(SE + 0x70, p[SE_KP1])?; + w(SE + 0x78, 100)?; + w(SE + 0x7c, max_scaled)?; + w(SE + 0x80, 100)?; + w(SE + 0x88, F_8000)?; + w(SE + 0x8c, p[SE_TARGET])?; + w(SE + 0x94, mul(p[SE_TC], period_ms)?)?; + w(SE + 0x98, mul(p[SE_TC1], period_ms)?)?; + w(SE + 0x9c, mul(p[SE_TC], clocks)?)?; + w(SE + 0xa4, mul(p[SE_TC1], clocks)?)?; + w(SE + 0xb0, F_65536)?; + w(SE + 0xd8, 8000)?; + w(SE + 0xdc, 4)?; + w(SE + 0xe0, p[SE_INACT])?; + w(SE + 0xe4, p[SE_CRIT])?; + w(SE + 0xe8, 2)?; + w(SE + 0xec, 4)?; + w(SE + 0xf0, p[SE_RESET])?; + w(SE + 0x11c, F_65536)?; + w(SE + 0x12c, 40)?; + w(SE + 0x130, max_scaled)?; + // Past the second sensor mask (FROM_FILE), the alarm mask (FROM_FILE) and the end flag. + w(0x1290, 125)?; + w(0x1294, 1)?; + w(0x1aa0, 1)?; + w(0x3aa4, F_5)?; + Ok(()) +} + +/// The tail's sensor keys from the six rows (which [`from_file`] fills). +fn tail(out: &mut [u8]) -> R<()> { + let mut g = 0usize; + for row in 0..SENSOR_ROW_COUNT { + let key = get32(out, SENSOR_ROWS + 12 * row)?; + match (key >> 24) as u8 { + b'g' => { + put32(out, SENSOR_G_KEYS + 4 * g, key)?; + g += 1; + } + b'r' => { + put32(out, SENSOR_R_KEY, key)?; + put32(out, SENSOR_R_COUNT, 1)?; + } + b'f' => { + put32(out, SENSOR_F_KEY, key)?; + put32(out, SENSOR_F_COUNT, 1)?; + } + _ => return Err("a sensor row has an unknown kind"), + } + } + if g > 4 { + return Err("more than four g sensor rows"); + } + put32(out, SENSOR_G_MASK, (1u32 << g) - 1)?; + put32(out, HWDATA_A + 0x6d80, 1)?; + put32(out, HWDATA_A + 0x6d84, 1)?; + Ok(()) +} + +/// Copies the [`FROM_FILE`] words of `file` (the board file's record) into `out`. +pub(crate) fn from_file(out: &mut [u8], file: &[u8]) -> R<()> { + for &at in FROM_FILE.iter() { + put32(out, at, get32(file, at)?)?; + } + Ok(()) +} + +/// The first word at which the built record differs from the file's, except the [`UNNEEDED`] +/// words, where the file's value is accepted as long as the built word is zero. +pub(crate) fn cross_check(built: &[u8], file: &[u8]) -> Option<(usize, u32, u32)> { + let word = |s: &[u8], at: usize| -> u32 { + let mut w = [0u8; 4]; + for (i, out) in w.iter_mut().enumerate() { + *out = s.get(at + i).copied().unwrap_or(0); + } + u32::from_le_bytes(w) + }; + (0..built.len().max(file.len())) + .step_by(4) + .map(|at| (at, word(built, at), word(file, at))) + .find(|&(at, b, f)| b != f && !(b == 0 && UNNEEDED.contains(&at))) +} + +/// Builds the record into `out` (zeroed first) from the device-tree inputs; the [`FROM_FILE`] +/// words must have been copied in first ([`from_file`]) or are left as they are. +pub(crate) fn build(out: &mut [u8], i: &Inputs<'_>) -> R<()> { + if out.len() != SIZE { + return Err("record of another size"); + } + let mut keep = [0u32; FROM_FILE.len()]; + for (k, &at) in FROM_FILE.iter().enumerate() { + keep[k] = get32(out, at)?; + } + out.fill(0); + let t = tables(i.main)?; + hwdata_b(out, i, &t)?; + hwdata_a(out, i, &t)?; + // The kept words win over what the rules put there (the relative powers, the masks). + for (k, &at) in FROM_FILE.iter().enumerate() { + put32(out, at, keep[k])?; + } + tail(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn get(b: &[u8], o: usize) -> u32 { + get32(b, o).unwrap() + } + + fn opp(mhz: u32, mv: [u32; 4], mw: u32) -> Opp { + Opp { + mhz, + mv, + cells: 4, + mw, + } + } + + /// Operating points shaped like the J516C's: eight voltages, the upper five with two states. + fn main_opps() -> Vec { + let mut v = vec![opp(0, [125; 4], 0)]; + let groups: [(u32, [u32; 4], &[u32]); 8] = [ + (605, [605, 605, 615, 615], &[338]), + (680, [680, 680, 665, 665], &[618]), + (730, [730, 730, 715, 715], &[796]), + (770, [770, 770, 755, 755], &[832, 924]), + (800, [800, 800, 785, 785], &[952, 1056]), + (860, [855, 860, 845, 845], &[1064, 1182]), + (930, [915, 930, 915, 915], &[1182, 1312]), + (955, [940, 955, 940, 940], &[1242, 1380]), + ]; + for (_, mv, mhz) in groups { + for &m in mhz { + v.push(opp(m, mv, m * 40)); + } + } + v + } + + fn props() -> [u32; PROPS.len()] { + let mut p = [0u32; PROPS.len()]; + p[PERIOD_MS] = 8; + p[BASE_PSTATE] = 1; + p[MIN_SRAM_UV] = 815000; + p[CSAFR_MIN_SRAM_UV] = 815000; + p[AVG_TC_MS] = 32; + p[AVG_KI] = 0x41480000; + p[AVG_KP] = 0x3e99999a; + p[AVG_MIN_DUTY] = 40; + p[AVG_TGT_TC] = 1; + p[FD_KI] = 0x443b8000; + p[FD_KP] = 0x42080000; + p[PERF_TC] = 5; + p[PERF_TC2] = 200; + p[PERF_RESET] = 6; + p[PPM_TC_MS] = 34; + p[PPM_KI] = 0x4198cccd; + p[PPM_KP] = 0x3e99999a; + p[PWR_TC] = 313; + p[PWR_MIN_DUTY] = 40; + p[AIC_CLKS] = 200000; + p[SE_TC] = 9; + p[SE_TC1] = 3; + p[SE_KI] = (-50.0f32).to_bits(); + p[SE_KI1] = (-100.0f32).to_bits(); + p[SE_TARGET] = 1400; + p + } + + fn inputs<'a>(main: &'a [Opp], cs: &'a [Opp], slots: &'a [IoSlot]) -> Inputs<'a> { + Inputs { + props: props(), + zones: &[], + main, + cs, + afr: cs, + slots, + empty_writable: &[4, 6], + ttbat: 0x10bfffb8000, + usc_base: 0x10_0000_0000, + user_end: 0x2ff_ffff_8000, + timestamps: 0xfffffc2ffc000000, + max_power_mw: 54321, + } + } + + #[test] + fn factors() { + assert_eq!(F_1, 1.0f32.to_bits()); + assert_eq!(F_0_2, 0.2f32.to_bits()); + assert_eq!(F_0_8, 0.8f32.to_bits()); + assert_eq!(F_0_95, 0.95f32.to_bits()); + assert_eq!(F_1_02, 1.02f32.to_bits()); + assert_eq!(F_5, 5.0f32.to_bits()); + assert_eq!(F_95, 95.0f32.to_bits()); + assert_eq!(F_100, 100.0f32.to_bits()); + assert_eq!(F_8000, 8000.0f32.to_bits()); + assert_eq!(F_65536, 65536.0f32.to_bits()); + // The filter complement is the correctly rounded (n - 1) / n, also where 1 - 1/n is + // a rounding tie. + assert_eq!(complement(3).unwrap(), (2.0f32 / 3.0).to_bits()); + assert_eq!(complement(313).unwrap(), (312.0f32 / 313.0).to_bits()); + assert_eq!(inv(9).unwrap(), (1.0f32 / 9.0).to_bits()); + assert!(inv(0).is_err()); + } + + #[test] + fn unneeded() { + // No UNNEEDED word is a FROM_FILE word; the cross-check accepts the file's value there + // only against a zero built word, and nothing else. + assert!(UNNEEDED.iter().all(|at| !FROM_FILE.contains(at))); + assert!(UNNEEDED.iter().all(|&at| at % 4 == 0 && at + 4 <= SIZE)); + let mut file = vec![0u8; SIZE]; + for (k, &at) in UNNEEDED.iter().enumerate() { + put32(&mut file, at, 0x2000 + k as u32).unwrap(); + } + let built = vec![0u8; SIZE]; + assert_eq!(cross_check(&built, &file), None); + let mut built = built; + put32(&mut built, UNNEEDED[0], 1).unwrap(); + assert_eq!(cross_check(&built, &file), Some((UNNEEDED[0], 1, 0x2000))); + let mut built = vec![0u8; SIZE]; + put32(&mut built, 0x10, 5).unwrap(); + assert_eq!(cross_check(&built, &file), Some((0x10, 5, 0))); + } + + #[test] + fn layout() { + assert_eq!(a(0x8b8), HWDATA_A + 0x8b8); + assert_eq!(a(0x8bc), HWDATA_A + 0x8c4); + assert_eq!(a(0x938), HWDATA_A + 0x940); + assert_eq!(a(0xbe0), HWDATA_A + 0xde8); + assert_eq!(a(0x1288), HWDATA_A + 0x1490); + assert_eq!(a(0x1290), HWDATA_A + 0x14a0); + assert_eq!(a(0x1a98), HWDATA_A + 0x24a8); + assert_eq!(a(0x1aa0), HWDATA_A + 0x24b8); + assert_eq!(a(0x3aa4), HWDATA_A + 0x64bc); + assert_eq!(MAX_POWER, a(0x6a4)); + // FROM_FILE is sorted, unique and inside the record. + assert!(FROM_FILE.windows(2).all(|w| w[0] < w[1])); + assert!(FROM_FILE.iter().all(|&o| o % 4 == 0 && o + 4 <= SIZE)); + } + + #[test] + fn dvfs_tables() { + let main = main_opps(); + let t = tables(&main).unwrap(); + assert_eq!(t.len, 9); + assert_eq!(&t.primary[..9], &[0, 1, 2, 3, 5, 7, 9, 11, 13]); + assert_eq!(&t.secondary[..9], &[0, 1, 2, 3, 4, 6, 8, 10, 12]); + // One state: nothing to sort. + assert!(tables(&main[..1]).is_err()); + // Frequency falling with voltage. + let mut bad = main.clone(); + bad[2].mhz = 300; + assert!(tables(&bad).is_err()); + // Two states at one voltage with different cluster voltages. + let mut bad = main.clone(); + bad[5].mv[2] = 760; + assert!(tables(&bad).is_err()); + // A cluster whose voltage does not rise. + let mut bad = main.clone(); + bad[13].mv[3] = 915; + assert!(tables(&bad).is_err()); + } + + #[test] + fn record_words() { + let main = main_opps(); + let cs: Vec = [24u32, 480, 624, 810, 1024, 1140, 1236] + .iter() + .zip([635, 635, 680, 720, 790, 855, 875]) + .map(|(&mhz, mv)| Opp { + mhz, + mv: [mv, 0, 0, 0], + cells: 1, + mw: 0, + }) + .collect(); + let slots = [ + IoSlot { + slot: 0, + pa: 0x408d00000, + va: 0xfffffc2068000000, + total: 0x144000, + element: 0x144000, + writable: true, + }, + IoSlot { + slot: 1, + pa: 0x20e101000, + va: 0xfffffc2068149000, + total: 0x4000, + element: 0x4000, + writable: false, + }, + ]; + let i = inputs(&main, &cs, &slots); + let mut r = vec![0u8; SIZE]; + // FROM_FILE words survive the build; the sensor rows feed the tail. + for (k, &at) in FROM_FILE.iter().enumerate() { + put32(&mut r, at, 0x1000 + k as u32).unwrap(); + } + // The file's maximum power is what the input repeats. + put32(&mut r, MAX_POWER, 54321).unwrap(); + let rows = [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"]; + for (k, row) in rows.iter().enumerate() { + put32(&mut r, SENSOR_ROWS + 12 * k, u32::from_le_bytes(**row)).unwrap(); + } + build(&mut r, &i).unwrap(); + let good = r.clone(); + assert_eq!(get(&r, 0x1060), 0x1000); + assert_eq!(get(&r, 0x30f0), 0); + // Identity, IOMapping records, flags. + assert_eq!(get(&r, 0xa28), 0x6031); + assert_eq!(get(&r, 0x640), 0x08d00000); + assert_eq!(get(&r, 0x644), 4); + assert_eq!(get(&r, 0x648), 0x68000000); + assert_eq!(get(&r, 0x658), 1); + assert_eq!(get(&r, 0x660), 0x0e101000); + assert_eq!(get(&r, 0x678), 0); + assert_eq!(get(&r, 0x640 + 32 * 4 + 0x18), 1); + assert_eq!(get(&r, 0x640 + 32 * 5 + 0x18), 0); + assert_eq!(get(&r, 0xb44), 0xfffb8000); + assert_eq!(get(&r, 0xb48), 0x10b); + assert_eq!(get(&r, 0xb58), 8); + // Tables: A frequencies, B frequencies, per-cluster lanes, SRAM floor, index maps. + assert_eq!(get(&r, 0xb5c + 4 * 8), 1380); + assert_eq!(get(&r, 0xf9c + 4 * 8), 1242); + assert_eq!(get(&r, 0x1350 + 4 * 4), 832); + assert_eq!(get(&r, 0xb9c + 0x20 + 4 * 2), 615); + assert_eq!(get(&r, 0xb9c + 0x20 + 4 * 5), 605); + assert_eq!(get(&r, 0xd9c + 0x20 + 4 * 2), 815); + assert_eq!(get(&r, 0x1390 + 0x20 * 8 + 4 * 1), 955); + assert_eq!(get(&r, 0x10dc + 4 * 8), 13); + assert_eq!(get(&r, 0x111c + 4 * 8), 12); + assert_eq!(get(&r, 0x107c), 100); + assert_eq!(get(&r, 0x109c + 4 * 8), 100); + // CS table: top twice, then the arrays with the SRAM floor. + assert_eq!(get(&r, 0x115c), 6); + assert_eq!(get(&r, 0x1160), 6); + assert_eq!(get(&r, 0x1164), 24); + assert_eq!(get(&r, 0x11a4), 635); + assert_eq!(get(&r, 0x1224), 815); + assert_eq!(get(&r, 0x12a4), 6); + assert_eq!(get(&r, 0x12ac), 24); + // HardwareDataA: scaled states, the maximum power and its derivatives, filters, SE. + assert_eq!(get(&r, a(0x4c)), 800); + assert_eq!(get(&r, a(0x680)), 54321.0f32.to_bits()); + assert_eq!(get(&r, a(0x68c)), (-800.0f32 / 54321.0).to_bits()); + assert_eq!(get(&r, a(0x690)), (40i32 - 800) as u32); + assert_eq!(get(&r, MAX_POWER), 54321); + assert_eq!(get(&r, a(0x6fc)), (3.0f32 / 4.0).to_bits()); + assert_eq!(get(&r, a(0x74c)), 4 * 8 * 24000); + assert_eq!(get(&r, a(0x7d4)), (7.0f32 / 0.95).to_bits()); + assert_eq!(get(&r, a(0x844)), 1380); + assert_eq!(get(&r, a(0x8d8)), 0x40c00001); + assert_eq!(get(&r, a(0x8e8)), 0x42080000); + assert_eq!(get(&r, a(0xe88)), 54321); + assert_eq!(get(&r, a(0xf88)), 0x3f400000); + assert_eq!(get(&r, a(0xfd8)), 4 * 8 * 24000); + assert_eq!(get(&r, a(0x10b4 + 0x4c)), (8.0f32 / 9.0).to_bits()); + assert_eq!(get(&r, a(0x10b4 + 0x5c)), (-50.0f32 * 0.008).to_bits()); + assert_eq!(get(&r, a(0x10b4 + 0x9c)), 9 * 192000); + assert_eq!(get(&r, a(0x3aa4)), F_5); + // Tail keys. + assert_eq!(get(&r, SENSOR_G_KEYS), u32::from_le_bytes(*b"RT0g")); + assert_eq!(get(&r, SENSOR_G_KEYS + 12), u32::from_le_bytes(*b"RT3g")); + assert_eq!(get(&r, SENSOR_G_MASK), 0xf); + assert_eq!(get(&r, SENSOR_R_KEY), u32::from_le_bytes(*b"RT0r")); + assert_eq!(get(&r, SENSOR_F_COUNT), 1); + // Refusals. + let mut short = vec![0u8; SIZE - 4]; + assert!(build(&mut short, &i).is_err()); + let mut j = inputs(&main, &cs, &slots); + j.props[PERIOD_MS] = 0; + assert!(build(&mut r, &j).is_err()); + let mut j = inputs(&main, &cs, &slots); + j.empty_writable = &[0]; + assert!(build(&mut r, &j).is_err()); + let mut j = inputs(&main, &cs, &slots); + j.max_power_mw = 0; + assert!(build(&mut r, &j).is_err()); + let mut j = inputs(&main, &cs, &slots); + j.zones = &[100, 101, 1]; + assert!(build(&mut r, &j).is_err()); + // Device-tree values whose arithmetic leaves u32 are refused, not wrapped or panicking: + // each on a copy of the record that built, so that nothing else refuses it first. + let refused = |j: &Inputs<'_>| { + let mut x = good.clone(); + build(&mut x, j) + }; + assert_eq!(refused(&i), Ok(())); + for (prop, value) in [ + (AVG_TGT_TC, 0x4000_0000), + (PPM_TC_MS, u32::MAX), + (AVG_TC_MS, u32::MAX), + (SE_TC, 0x2000_0000), + (SE_TC1, 0x2000_0000), + (FD_DELTA, 0x0300_0000), + ] { + let mut j = inputs(&main, &cs, &slots); + j.props[prop] = value; + assert_eq!(refused(&j), Err(OVERFLOW), "prop {prop} = {value:#x}"); + } + let mut j = inputs(&main, &cs, &slots); + j.zones = &[5000, 200, 0x4000_0000]; + assert_eq!(refused(&j), Err(OVERFLOW)); + let mut skewed = main.clone(); + skewed[1].mw = u32::MAX; + skewed[13].mw = 1; + assert_eq!( + refused(&inputs(&skewed, &cs, &slots)), + Err("a relative power does not fit its u32 word") + ); + // The admission cross-check as initdata::blob() composes it: the FROM_FILE words from + // the file, the rest built, then the first differing word against the file. A file the + // inputs reproduce passes; a rule word the file holds differently is found at its + // offset; a FROM_FILE word follows the file and is never a difference. + let first_difference = super::super::words::first_difference; + let checked = |file: &[u8]| { + let mut built = vec![0u8; SIZE]; + from_file(&mut built, file).unwrap(); + build(&mut built, &i).unwrap(); + (first_difference(&built, file), built) + }; + assert_eq!(checked(&good).0, None); + let mut file = good.clone(); + put32(&mut file, a(0x4c), 801).unwrap(); + assert_eq!(checked(&file).0, Some((a(0x4c), 800, 801))); + let mut file = good.clone(); + assert!(FROM_FILE.contains(&0x1064)); + put32(&mut file, 0x1064, get(&good, 0x1064) ^ 0x5a).unwrap(); + let (difference, built) = checked(&file); + assert_eq!(difference, None); + assert_eq!(get(&built, 0x1064), get(&good, 0x1064) ^ 0x5a); + } + + // --- The fixture: a saved copy of a J516C's device tree and its private board file. --- + + fn be32(b: &[u8], o: usize) -> u32 { + u32::from_be_bytes(b[o..o + 4].try_into().unwrap()) + } + + /// All nodes of a flattened device tree: path -> properties. + fn fdt(dtb: &[u8]) -> Vec<(String, Vec<(String, Vec)>)> { + assert_eq!(be32(dtb, 0), 0xd00dfeed); + let strings = be32(dtb, 12) as usize; + let mut at = be32(dtb, 8) as usize; + let mut path: Vec = Vec::new(); + let mut out: Vec<(String, Vec<(String, Vec)>)> = Vec::new(); + loop { + let token = be32(dtb, at); + at += 4; + match token { + 1 => { + let end = dtb[at..].iter().position(|&b| b == 0).unwrap(); + path.push(String::from_utf8(dtb[at..at + end].to_vec()).unwrap()); + at = (at + end + 1).next_multiple_of(4); + out.push((path.join("/"), Vec::new())); + } + 2 => { + path.pop(); + } + 3 => { + let len = be32(dtb, at) as usize; + let name_off = strings + be32(dtb, at + 4) as usize; + let name_end = dtb[name_off..].iter().position(|&b| b == 0).unwrap(); + let name = + String::from_utf8(dtb[name_off..name_off + name_end].to_vec()).unwrap(); + let value = dtb[at + 8..at + 8 + len].to_vec(); + at = (at + 8 + len).next_multiple_of(4); + let node = path.join("/"); + out.iter_mut() + .rev() + .find(|(p, _)| *p == node) + .unwrap() + .1 + .push((name, value)); + } + 4 => {} + 9 => return out, + _ => panic!("fdt token {token}"), + } + } + } + + fn prop<'a>(props: &'a [(String, Vec)], name: &str) -> Option<&'a [u8]> { + props + .iter() + .find(|(n, _)| n == name) + .map(|(_, v)| v.as_slice()) + } + + fn opp_table( + nodes: &[(String, Vec<(String, Vec)>)], + phandle: u32, + cells: usize, + ) -> Vec { + let (table, _) = nodes + .iter() + .find(|(_, p)| prop(p, "phandle").map(|v| be32(v, 0)) == Some(phandle)) + .unwrap(); + let mut rows: Vec<(String, Opp)> = nodes + .iter() + .filter(|(p, _)| { + p.starts_with(&format!("{table}/")) + && p.matches('/').count() == table.matches('/').count() + 1 + }) + .map(|(p, props)| { + let hz = u64::from_be_bytes(prop(props, "opp-hz").unwrap().try_into().unwrap()); + let uv = prop(props, "opp-microvolt").unwrap(); + assert_eq!(uv.len(), 4 * cells); + let mut mv = [0u32; CLUSTERS]; + for (c, m) in mv.iter_mut().enumerate().take(cells) { + *m = be32(uv, 4 * c).div_ceil(1000); + } + let mw = prop(props, "opp-microwatt").map_or(0, |w| be32(w, 0) / 1000); + ( + p.clone(), + Opp { + mhz: (hz / 1_000_000) as u32, + mv, + cells, + mw, + }, + ) + }) + .collect(); + rows.sort_by(|x, y| x.0.cmp(&y.0)); + rows.into_iter().map(|(_, o)| o).collect() + } + + /// No GPU node property value panics the builder (CONFIG_RUST_OVERFLOW_CHECKS is on in our + /// kernels): each of the properties at the u32 and single-precision boundaries, on a copy + /// of a seed record that builds. A value may be refused; it must not panic. + #[test] + fn device_tree_u32_boundaries_do_not_panic() { + let main = main_opps(); + let cs = [Opp { + mhz: 24, + mv: [635, 0, 0, 0], + cells: 1, + mw: 0, + }]; + let mut seed = vec![0u8; SIZE]; + put32(&mut seed, MAX_POWER, 54321).unwrap(); + for (k, key) in [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"] + .iter() + .enumerate() + { + put32(&mut seed, SENSOR_ROWS + 12 * k, u32::from_le_bytes(**key)).unwrap(); + } + assert_eq!(build(&mut seed, &inputs(&main, &cs, &[])), Ok(())); + for prop in 0..PROPS.len() { + for value in [ + 0, 1, 7, 8, 31, 0x3f80_0000, 0x4000_0000, 0x7f80_0000, 0x7fc0_0000, 0x8000_0000, + u32::MAX, + ] { + let mut i = inputs(&main, &cs, &[]); + i.props[prop] = value; + let mut image = seed.clone(); + let built = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + build(&mut image, &i) + })); + assert!(built.is_ok(), "prop {} value {:#x}", prop, value); + } + } + } + + /// Builder on the saved device tree + the file's FROM_FILE words == the file's hardware + /// record, byte for byte. Needs `G15_J516C_FIXTURES=` with `j516c.dtb` and + /// `t6031-j516c-board.bin` (off the tree): ignored by default, `--ignored` runs it. + #[test] + #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] + fn j516c_fixture() { + let dir = std::env::var_os("G15_J516C_FIXTURES") + .expect("G15_J516C_FIXTURES names the directory with j516c.dtb and the board file"); + let dir = std::path::Path::new(&dir); + let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); + let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); + assert_eq!(&blob[..8], b"T6031BRD"); + let at = 16 + 3 * 8; + let file = &blob[at..at + SIZE]; + + let nodes = fdt(&dtb); + let (_, gpu) = nodes + .iter() + .find(|(_, p)| { + prop(p, "compatible") + .is_some_and(|c| c.split(|&b| b == 0).any(|s| s == b"apple,agx-t6031")) + }) + .expect("a T6031 GPU node"); + let mut props = [0u32; PROPS.len()]; + for (v, (name, default)) in props.iter_mut().zip(PROPS.iter()) { + *v = match prop(gpu, name.to_str().unwrap()) { + Some(b) => be32(b, 0), + None => default.unwrap_or_else(|| panic!("GPU node has no {name:?}")), + }; + } + let zones: Vec = prop(gpu, "apple,power-zones") + .map(|b| b.chunks(4).map(|c| be32(c, 0)).collect()) + .unwrap_or_default(); + let main = opp_table( + &nodes, + be32(prop(gpu, "operating-points-v2").unwrap(), 0), + 4, + ); + let cs = opp_table(&nodes, be32(prop(gpu, "apple,cs-opp").unwrap(), 0), 1); + let afr = opp_table(&nodes, be32(prop(gpu, "apple,afr-opp").unwrap(), 0), 1); + assert_eq!(main.len(), PERF_STATES); + // The ttbs region: the first memory-region phandle's reserved-memory node. + let regions = prop(gpu, "memory-region").unwrap(); + let (_, ttbs) = nodes + .iter() + .find(|(_, p)| prop(p, "phandle").map(|v| be32(v, 0)) == Some(be32(regions, 0))) + .unwrap(); + let reg = prop(ttbs, "reg").unwrap(); + let ttbat = u64::from_be_bytes(reg[..8].try_into().unwrap()); + // The IOMapping slots: the T6031 table's addresses and sizes with the file's VAs (the + // VAs are the driver's; the layout packs them from a base with a page gap). + let slots_table: [(u8, u64, u32, u32, bool); 17] = [ + (0, 0x408d00000, 0x144000, 0x144000, true), + (1, 0x20e100000, 0x4000, 0x4000, false), + (2, 0x292414000, 0x4000, 0x4000, true), + (3, 0x408000000, 0x20000, 0x20000, true), + (7, 0x290444000, 0x4000, 0x4000, false), + (9, 0x408e08000, 0x8000, 0x8000, true), + (11, 0x220000000, 8 * 0xd8000, 0xd8000, true), + (12, 0x29244c000, 0x4000, 0x4000, false), + (18, 0x2903d0000, 0x4000, 0x4000, true), + (19, 0x2903c0000, 0x4000, 0x4000, false), + (20, 0x2903d8000, 0x4000, 0x4000, true), + (21, 0x408eb4000, 0x4000, 0x4000, true), + (24, 0x409000000, 0x8000, 0x8000, true), + (25, 0x3c945c000, 0x4000, 0x4000, true), + (26, 0x290400000, 0x8000, 0x8000, false), + (28, 0x2a4dc0000, 0x24000, 0x24000, false), + (29, 0x408e1c000, 0x4000, 0x4000, false), + ]; + let mut va = 0xfffffc2068000000u64; + let slots: Vec = slots_table + .iter() + .map(|&(slot, pa, total, element, writable)| { + let delta = pa & 0x3fff; + let s = IoSlot { + slot, + pa, + va: va + delta, + total, + element, + writable, + }; + va += (delta + u64::from(total) + 0x3fff) & !0x3fff; + va += 0x4000; + s + }) + .collect(); + let i = Inputs { + props, + zones: &zones, + main: &main, + cs: &cs, + afr: &afr, + slots: &slots, + empty_writable: &[4, 6, 8, 10, 13, 14, 15, 16, 17], + ttbat, + usc_base: 0x10_0000_0000, + user_end: 0x2ff_ffff_8000, + timestamps: 0xfffffc2ffc000000, + max_power_mw: get(file, MAX_POWER), + }; + let mut built = vec![0u8; SIZE]; + from_file(&mut built, file).unwrap(); + build(&mut built, &i).unwrap(); + assert_eq!(cross_check(&built, file), None, "(offset, built, file)"); + for &at in UNNEEDED.iter() { + assert_eq!(get(&built, at), 0); + } + } +} diff --git a/drivers/gpu/drm/asahi/g15/board/template.rs b/drivers/gpu/drm/asahi/g15/board/template.rs new file mode 100644 index 0000000000000..3a53939ae08c0 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board/template.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Sparse board-image templates: 16-byte lines over a zero image. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. +//! The hash check of an expanded template is the kernel side's ([`super::expand`]). + +/// One template line: 16 bytes at `offset`. +pub(crate) struct Line { + pub(crate) offset: u32, + pub(crate) data: [u8; 16], +} + +/// A sparse template. +pub(crate) struct Template { + pub(crate) size: usize, + pub(crate) lines: &'static [Line], + pub(crate) sha256: [u8; 32], +} + +/// Fills the first `template.size` bytes of `out` with the template: zero, then its lines. `Err` +/// when `out` is too short or a line starts past the end. +pub(crate) fn fill(template: &Template, out: &mut [u8]) -> Result<(), ()> { + let out = out.get_mut(..template.size).ok_or(())?; + out.fill(0); + for line in template.lines { + let offset = line.offset as usize; + if offset >= template.size { + return Err(()); + } + let n = 16.min(template.size - offset); + out[offset..offset + n].copy_from_slice(&line.data[..n]); + } + Ok(()) +} diff --git a/drivers/gpu/drm/asahi/g15/board/words.rs b/drivers/gpu/drm/asahi/g15/board/words.rs new file mode 100644 index 0000000000000..4c5082465d428 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/board/words.rs @@ -0,0 +1,653 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The J516C Globals and Power images from the device tree. +//! +//! The 2419 firmware's Globals (15,868 bytes) and Power (2,292 bytes) are, on the J516C, the G15S +//! templates ([`super::data`]) with these words of its own: +//! +//! - The GPU node's power words ([`PROPS`]): `apple,idleoff-standby-timer` (Globals +0x7a8); +//! in Power the average-power controller (+0x10 filter periods = `apple,avg-power-filter-tc-ms` +//! / `apple,power-sample-period`, +0x14 `apple,avg-power-ki-only` x period, +0x18 +//! `apple,avg-power-kp`, +0x1c `apple,avg-power-min-duty-cycle`, +0x20 +//! `apple,avg-power-target-filter-tc`), the power zones (+0xc count, +0x24 up to five +//! {target, target - offset, filter} rows of `apple,power-zones`), the fast-die controller +//! (+0xb8 `apple,fast-die0-proportional-gain`, +0xbc `apple,fast-die0-integral-gain` x period) +//! and the PPM controller (+0xd4 `apple,ppm-kp`, +0xd8 `apple,ppm-ki` x period). A gain x +//! period is the IEEE single product of the gain and `period_ms / 1000`, each rounded to +//! single ([`f32_mul`], [`f32_div`]). +//! - The maximum power: the hardware record's HardwareDataA +0x6a4 word (Globals +0x78, Power +//! +0xd0). The G15S template carries its own machine's there too. +//! - The performance-state cap: 100 x the top firmware DVFS state, 8 (Globals +0x7c, +0x80, +//! +0x8c). +//! - Words the device tree does not carry: command submission enabled at initdata (Globals +//! +0x70 = 1; the G15S template has 0), the start path's +0x980 = 100 (written again at +//! start on every SoC), the soft-fault word (u32 3 at the unaligned +0xdf9 of the G15S image) +//! at +0x1201, where the G15C Globals has it, and the Power +0x244 flag = 1. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. + +use core::ffi::CStr; + +pub(crate) const GLOBALS_SIZE: usize = 15868; +pub(crate) const POWER_SIZE: usize = 2292; + +/// HardwareDataA in the hardware record (= `fw::initdata::hwdata::HWDATA_A`) and its maximum +/// power word (= `fw::initdata::hwdata::MAX_POWER_A`). +pub(crate) const HWDATA_A: usize = 0x4580; +pub(crate) const HWDATA_A_MAX_POWER: usize = 0x6a4; + +/// The top firmware DVFS state (= `board::DVFS_STATES - 1`). +pub(crate) const TOP_STATE: u32 = 8; + +/// Globals words. +mod globals { + pub(super) const COMMAND_SUBMISSION: usize = 0x70; + pub(super) const MAX_POWER: usize = 0x78; + /// The power-interface targets 1 and 2 and the performance-state cap. + pub(super) const STATE_CAP: [usize; 3] = [0x7c, 0x80, 0x8c]; + pub(super) const STANDBY_TIMER: usize = 0x7a8; + pub(super) const UNK_980: usize = 0x980; + /// The soft-fault word: unaligned, in the G15S and the G15C image. + pub(super) const SOFT_FAULT_G15S: usize = 0xdf9; + pub(super) const SOFT_FAULT_G15C: usize = 0x1201; + pub(super) const SOFT_FAULT: u32 = 3; +} + +/// Power words. +mod power { + pub(super) const ZONE_COUNT: usize = 0x0c; + pub(super) const AVG_FILTER_PERIODS: usize = 0x10; + pub(super) const AVG_KI_DT: usize = 0x14; + pub(super) const AVG_KP: usize = 0x18; + pub(super) const AVG_MIN_DUTY_CYCLE: usize = 0x1c; + pub(super) const AVG_TARGET_FILTER_TC: usize = 0x20; + pub(super) const ZONES: usize = 0x24; + pub(super) const ZONE_STRIDE: usize = 12; + pub(super) const FAST_DIE_KP: usize = 0xb8; + pub(super) const FAST_DIE_KI_DT: usize = 0xbc; + pub(super) const MAX_POWER: usize = 0xd0; + pub(super) const PPM_KP: usize = 0xd4; + pub(super) const PPM_KI_DT: usize = 0xd8; + pub(super) const FLAG_244: usize = 0x244; +} + +/// The GPU node's power properties, each one u32 (gains are IEEE single bits), in the order of +/// [`PowerWords::values`]. +pub(crate) const PROPS: [&CStr; 11] = [ + c"apple,power-sample-period", + c"apple,idleoff-standby-timer", + c"apple,avg-power-filter-tc-ms", + c"apple,avg-power-ki-only", + c"apple,avg-power-kp", + c"apple,avg-power-min-duty-cycle", + c"apple,avg-power-target-filter-tc", + c"apple,fast-die0-proportional-gain", + c"apple,fast-die0-integral-gain", + c"apple,ppm-kp", + c"apple,ppm-ki", +]; +pub(crate) const PERIOD_MS: usize = 0; +pub(crate) const STANDBY_TIMER: usize = 1; +pub(crate) const AVG_POWER_FILTER_TC_MS: usize = 2; +pub(crate) const AVG_POWER_KI_ONLY: usize = 3; +pub(crate) const AVG_POWER_KP: usize = 4; +pub(crate) const AVG_POWER_MIN_DUTY_CYCLE: usize = 5; +pub(crate) const AVG_POWER_TARGET_FILTER_TC: usize = 6; +pub(crate) const FAST_DIE0_PROPORTIONAL_GAIN: usize = 7; +pub(crate) const FAST_DIE0_INTEGRAL_GAIN: usize = 8; +pub(crate) const PPM_KP: usize = 9; +pub(crate) const PPM_KI: usize = 10; + +/// The power zones, u32 triples {target, offset, filter time constant}; at most five. +pub(crate) const ZONES: &CStr = c"apple,power-zones"; +pub(crate) const MAX_ZONES: usize = 5; + +/// The GPU node's power words. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct PowerWords { + pub(crate) values: [u32; PROPS.len()], + pub(crate) zones: [[u32; 3]; MAX_ZONES], + pub(crate) zone_count: u32, +} + +impl PowerWords { + /// `values` in [`PROPS`] order and the `apple,power-zones` cells. + pub(crate) fn new(values: [u32; PROPS.len()], zones: &[u32]) -> Result { + if values[PERIOD_MS] == 0 { + return Err("the power sample period is 0"); + } + if zones.len() % 3 != 0 || zones.len() > 3 * MAX_ZONES { + return Err("the power zones are not up to five u32 triples"); + } + let mut out = Self { + values, + zones: [[0; 3]; MAX_ZONES], + zone_count: (zones.len() / 3) as u32, + }; + for (row, cells) in out.zones.iter_mut().zip(zones.chunks_exact(3)) { + row.copy_from_slice(cells); + } + Ok(out) + } + + /// The sample period in seconds, IEEE single bits. + fn period_s(&self) -> Result { + f32_div(f32_from_u32(self.values[PERIOD_MS]), f32_from_u32(1000)) + .ok_or("the power sample period has no single-precision quotient") + } + + /// A gain x the sample period, IEEE single bits. + fn per_period(&self, gain: usize) -> Result { + f32_mul(self.values[gain], self.period_s()?) + .ok_or("a gain is not a finite normal single, or its product with the period is not") + } +} + +fn get32(b: &[u8], offset: usize) -> Result { + let w = b.get(offset..offset + 4).ok_or("image too short")?; + Ok(u32::from_le_bytes([w[0], w[1], w[2], w[3]])) +} + +fn put32(b: &mut [u8], offset: usize, value: u32) -> Result<(), &'static str> { + b.get_mut(offset..offset + 4) + .ok_or("image too short")? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) +} + +/// Writes the J516C words into `image`, the expanded G15S Globals template. +pub(crate) fn globals( + image: &mut [u8], + max_power_mw: u32, + w: &PowerWords, +) -> Result<(), &'static str> { + if image.len() != GLOBALS_SIZE { + return Err("Globals image of another size"); + } + put32(image, globals::COMMAND_SUBMISSION, 1)?; + put32(image, globals::MAX_POWER, max_power_mw)?; + for at in globals::STATE_CAP { + put32(image, at, 100 * TOP_STATE)?; + } + put32(image, globals::STANDBY_TIMER, w.values[STANDBY_TIMER])?; + put32(image, globals::UNK_980, 100)?; + if get32(image, globals::SOFT_FAULT_G15S)? != globals::SOFT_FAULT { + return Err("Globals template without the soft-fault word at its G15S place"); + } + put32(image, globals::SOFT_FAULT_G15S, 0)?; + put32(image, globals::SOFT_FAULT_G15C, globals::SOFT_FAULT)?; + Ok(()) +} + +/// Writes the J516C words into `image`, the expanded G15S Power template. +pub(crate) fn power( + image: &mut [u8], + max_power_mw: u32, + w: &PowerWords, +) -> Result<(), &'static str> { + if image.len() != POWER_SIZE { + return Err("Power image of another size"); + } + let periods = w.values[AVG_POWER_FILTER_TC_MS] / w.values[PERIOD_MS]; + if periods == 0 { + return Err("the average-power filter time constant is shorter than the sample period"); + } + put32(image, power::ZONE_COUNT, w.zone_count)?; + put32(image, power::AVG_FILTER_PERIODS, periods)?; + put32(image, power::AVG_KI_DT, w.per_period(AVG_POWER_KI_ONLY)?)?; + put32(image, power::AVG_KP, w.values[AVG_POWER_KP])?; + put32( + image, + power::AVG_MIN_DUTY_CYCLE, + w.values[AVG_POWER_MIN_DUTY_CYCLE], + )?; + put32( + image, + power::AVG_TARGET_FILTER_TC, + w.values[AVG_POWER_TARGET_FILTER_TC], + )?; + for (i, zone) in w.zones.iter().enumerate() { + let at = power::ZONES + power::ZONE_STRIDE * i; + let (target, offset, filter) = (zone[0], zone[1], zone[2]); + let target_off = target + .checked_sub(offset) + .ok_or("a power zone's offset is larger than its target")?; + put32(image, at, target)?; + put32(image, at + 4, target_off)?; + put32(image, at + 8, filter)?; + } + put32( + image, + power::FAST_DIE_KP, + w.values[FAST_DIE0_PROPORTIONAL_GAIN], + )?; + put32( + image, + power::FAST_DIE_KI_DT, + w.per_period(FAST_DIE0_INTEGRAL_GAIN)?, + )?; + put32(image, power::MAX_POWER, max_power_mw)?; + put32(image, power::PPM_KP, w.values[PPM_KP])?; + put32(image, power::PPM_KI_DT, w.per_period(PPM_KI)?)?; + put32(image, power::FLAG_244, 1)?; + Ok(()) +} + +/// The first word (4-byte aligned offset) at which `a` and `b` differ, with both values; a +/// missing word reads 0. `None` when they are equal. +pub(crate) fn first_difference(a: &[u8], b: &[u8]) -> Option<(usize, u32, u32)> { + let word = |s: &[u8], at: usize| -> u32 { + let mut w = [0u8; 4]; + for (i, out) in w.iter_mut().enumerate() { + *out = s.get(at + i).copied().unwrap_or(0); + } + u32::from_le_bytes(w) + }; + let len = a.len().max(b.len()); + (0..len) + .step_by(4) + .map(|at| (at, word(a, at), word(b, at))) + .find(|&(_, x, y)| x != y) +} + +// --- IEEE binary32 arithmetic on bit patterns (the kernel has no floating point) --- +// +// Operands and results are finite normal numbers or zero; anything else is `None`. +// Rounding is to nearest, ties to even. + +/// Sign, unbiased exponent of the leading bit and mantissa with its implicit bit; zero is +/// `(sign, 0, 0)`. +fn unpack(bits: u32) -> Option<(bool, i32, u32)> { + let sign = bits >> 31 != 0; + let exponent = ((bits >> 23) & 0xff) as i32; + let mantissa = bits & 0x7f_ffff; + match exponent { + 0 if mantissa == 0 => Some((sign, 0, 0)), + 0 | 0xff => None, + _ => Some((sign, exponent - 127, mantissa | 0x80_0000)), + } +} + +/// The single nearest to `sign * m * 2^exp`, `m` with its leading bit anywhere; a sticky bit in +/// the lowest bit of `m` may stand for a nonzero remainder below it. +fn pack(sign: bool, exp: i32, m: u64) -> Option { + if m == 0 { + return Some((sign as u32) << 31); + } + let shift = m.leading_zeros(); + let m = m << shift; + let exp = exp - shift as i32 + 63; + let mut mantissa = (m >> 40) as u32; + let rest = m & ((1u64 << 40) - 1); + let half = 1u64 << 39; + if rest > half || (rest == half && mantissa & 1 == 1) { + mantissa += 1; + } + let mut exp = exp; + if mantissa == 1 << 24 { + mantissa >>= 1; + exp += 1; + } + let biased = exp + 127; + if !(1..=254).contains(&biased) { + return None; + } + Some((sign as u32) << 31 | (biased as u32) << 23 | (mantissa & 0x7f_ffff)) +} + +/// `n` as a single. +pub(crate) fn f32_from_u32(n: u32) -> u32 { + // Every u32 rounds to a normal single or zero. + pack(false, 0, n as u64).unwrap_or(0) +} + +/// `a * b`. +pub(crate) fn f32_mul(a: u32, b: u32) -> Option { + let (sa, ea, ma) = unpack(a)?; + let (sb, eb, mb) = unpack(b)?; + if ma == 0 || mb == 0 { + return Some(((sa ^ sb) as u32) << 31); + } + pack(sa ^ sb, ea + eb - 46, ma as u64 * mb as u64) +} + +/// `a / b`. +pub(crate) fn f32_div(a: u32, b: u32) -> Option { + let (sa, ea, ma) = unpack(a)?; + let (sb, eb, mb) = unpack(b)?; + if mb == 0 { + return None; + } + if ma == 0 { + return Some(((sa ^ sb) as u32) << 31); + } + // ma < 2^24, so ma << 40 fits; the quotient has at least 40 bits, 16 and more below the + // rounding position, so a sticky lowest bit stands in for a nonzero remainder. + let n = (ma as u64) << 40; + let mut q = n / mb as u64; + if n % mb as u64 != 0 { + q |= 1; + } + pack(sa ^ sb, ea - eb - 40, q) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn get32(b: &[u8], o: usize) -> u32 { + u32::from_le_bytes(b[o..o + 4].try_into().unwrap()) + } + + /// The J516C GPU node's power words, as the saved device tree of that machine carries them. + fn j516c_words() -> PowerWords { + PowerWords::new( + [ + 8, // apple,power-sample-period + 1500, // apple,idleoff-standby-timer + 32, // apple,avg-power-filter-tc-ms + 0x41480000, // apple,avg-power-ki-only (12.5) + 0x3e99999a, // apple,avg-power-kp (0.3) + 40, // apple,avg-power-min-duty-cycle + 1, // apple,avg-power-target-filter-tc + 0x42080000, // apple,fast-die0-proportional-gain (34.0) + 0x443b8000, // apple,fast-die0-integral-gain (750.0) + 0x3e99999a, // apple,ppm-kp (0.3) + 0x4198cccd, // apple,ppm-ki (19.1) + ], + &[], + ) + .unwrap() + } + + #[test] + fn single_arithmetic() { + // The products the power words need, against the host's IEEE singles. + let dt = f32_div(f32_from_u32(8), f32_from_u32(1000)).unwrap(); + assert_eq!(dt, (8.0f32 / 1000.0).to_bits()); + for ki in [0x443b8000u32, 0x41480000, 0x4198cccd] { + assert_eq!( + f32_mul(ki, dt).unwrap(), + (f32::from_bits(ki) * f32::from_bits(dt)).to_bits() + ); + } + // A sweep of normal operands, both signs. + let mut x = 0x2545f491u32; + for _ in 0..20000 { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + let a = f32::from_bits(x); + let b = f32::from_bits(x.rotate_left(11) ^ 0x5a5a5a5a); + if !a.is_normal() || !b.is_normal() { + continue; + } + let p = a * b; + if p.is_normal() { + assert_eq!( + f32_mul(a.to_bits(), b.to_bits()), + Some(p.to_bits()), + "{a} * {b}" + ); + } + let q = a / b; + if q.is_normal() { + assert_eq!( + f32_div(a.to_bits(), b.to_bits()), + Some(q.to_bits()), + "{a} / {b}" + ); + } + } + for n in [ + 0u32, + 1, + 8, + 1000, + 1500, + 0xffffff, + 0x1000000, + 0x1000001, + u32::MAX, + ] { + assert_eq!(f32_from_u32(n), (n as f32).to_bits()); + } + assert_eq!(f32_mul(0, 0x3f800000), Some(0)); + assert_eq!(f32_mul(0x3f800000, 0x80000000), Some(0x80000000)); + assert_eq!(f32_div(0x3f800000, 0), None); + assert_eq!(f32_div(0, 0x3f800000), Some(0)); + // Infinities, NaNs and subnormals are not operands. + assert_eq!(f32_mul(0x7f800000, 0x3f800000), None); + assert_eq!(f32_mul(0x00000001, 0x3f800000), None); + // Overflow and underflow of the result are refused, not rounded. + assert_eq!(f32_mul(0x7f000000, 0x7f000000), None); + assert_eq!(f32_mul(0x00800000, 0x00800000), None); + } + + #[test] + fn words_shape() { + let w = j516c_words(); + assert_eq!(w.zone_count, 0); + assert_eq!(PowerWords::new(w.values, &[1, 2, 3]).unwrap().zone_count, 1); + assert_eq!( + PowerWords::new(w.values, &[1, 2, 3]).unwrap().zones[0], + [1, 2, 3] + ); + assert!(PowerWords::new(w.values, &[1, 2]).is_err()); + assert!(PowerWords::new(w.values, &[0; 18]).is_err()); + let mut v = w.values; + v[PERIOD_MS] = 0; + assert!(PowerWords::new(v, &[]).is_err()); + assert_eq!(PROPS.len(), w.values.len()); + assert_eq!(PROPS[PERIOD_MS], c"apple,power-sample-period"); + assert_eq!(PROPS[STANDBY_TIMER], c"apple,idleoff-standby-timer"); + } + + #[test] + fn globals_words() { + let w = j516c_words(); + let mut img = vec![0u8; GLOBALS_SIZE]; + // A G15S-shaped image: the soft-fault word at its unaligned place. + img[0xdf9] = 3; + globals(&mut img, 54321, &w).unwrap(); + assert_eq!(get32(&img, 0x70), 1); + assert_eq!(get32(&img, 0x78), 54321); + assert_eq!(get32(&img, 0x7c), 800); + assert_eq!(get32(&img, 0x80), 800); + assert_eq!(get32(&img, 0x8c), 800); + assert_eq!(get32(&img, 0x7a8), 1500); + assert_eq!(get32(&img, 0x980), 100); + assert_eq!(get32(&img, 0xdf8), 0); + assert_eq!(get32(&img, 0x1200), 0x300); + // Everything else untouched. + let touched = [0x70, 0x78, 0x7c, 0x80, 0x8c, 0x7a8, 0x980, 0xdf8, 0x1200]; + for o in (0..GLOBALS_SIZE).step_by(4) { + if !touched.contains(&o) { + assert_eq!(get32(&img, o), 0, "+{o:#x}"); + } + } + // Without the soft-fault word where the G15S image has it, the image is not the one + // this rule is written for. + let mut bad = vec![0u8; GLOBALS_SIZE]; + assert!(globals(&mut bad, 1, &w).is_err()); + let mut short = vec![0u8; GLOBALS_SIZE - 1]; + assert!(globals(&mut short, 1, &w).is_err()); + } + + #[test] + fn power_words() { + let w = j516c_words(); + let mut img = vec![0u8; POWER_SIZE]; + power(&mut img, 54321, &w).unwrap(); + assert_eq!(get32(&img, 0x0c), 0); + assert_eq!(get32(&img, 0x10), 4); + assert_eq!(get32(&img, 0x14), 0x3dcccccd); + assert_eq!(get32(&img, 0x18), 0x3e99999a); + assert_eq!(get32(&img, 0x1c), 40); + assert_eq!(get32(&img, 0x20), 1); + assert_eq!(get32(&img, 0xb8), 0x42080000); + assert_eq!(get32(&img, 0xbc), 0x40c00001); + assert_eq!(get32(&img, 0xd0), 54321); + assert_eq!(get32(&img, 0xd4), 0x3e99999a); + assert_eq!(get32(&img, 0xd8), 0x3e1c779b); + assert_eq!(get32(&img, 0x244), 1); + let touched = [ + 0x10, 0x14, 0x18, 0x1c, 0x20, 0xb8, 0xbc, 0xd0, 0xd4, 0xd8, 0x244, + ]; + for o in (0..POWER_SIZE).step_by(4) { + if !touched.contains(&o) { + assert_eq!(get32(&img, o), 0, "+{o:#x}"); + } + } + // Power zones: count and {target, target - offset, filter} rows. + let z = PowerWords::new(w.values, &[5000, 200, 7, 9000, 1000, 3]).unwrap(); + let mut img = vec![0u8; POWER_SIZE]; + power(&mut img, 1, &z).unwrap(); + assert_eq!(get32(&img, 0x0c), 2); + assert_eq!(get32(&img, 0x24), 5000); + assert_eq!(get32(&img, 0x28), 4800); + assert_eq!(get32(&img, 0x2c), 7); + assert_eq!(get32(&img, 0x30), 9000); + assert_eq!(get32(&img, 0x34), 8000); + assert_eq!(get32(&img, 0x38), 3); + assert_eq!(get32(&img, 0x3c), 0); + // An offset larger than its target has no target - offset word. + let under = PowerWords::new(w.values, &[100, 101, 1]).unwrap(); + let mut img = vec![0u8; POWER_SIZE]; + assert!(power(&mut img, 1, &under).is_err()); + // A filter time constant shorter than the sample period has no period count. + let mut v = w.values; + v[AVG_POWER_FILTER_TC_MS] = 4; + let mut img = vec![0u8; POWER_SIZE]; + assert!(power(&mut img, 1, &PowerWords::new(v, &[]).unwrap()).is_err()); + let mut short = vec![0u8; POWER_SIZE - 1]; + assert!(power(&mut short, 1, &w).is_err()); + } + + #[test] + fn differences() { + let a = [1u8, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0]; + let mut b = a; + assert_eq!(first_difference(&a, &b), None); + b[9] = 1; + assert_eq!(first_difference(&a, &b), Some((8, 3, 0x103))); + assert_eq!(first_difference(&a, &b[..8]), Some((8, 3, 0))); + assert_eq!(first_difference(&a[..8], &b), Some((8, 0, 0x103))); + } + + // --- The fixture: a saved copy of a J516C's device tree and its private board file. --- + + fn be32(b: &[u8], o: usize) -> u32 { + u32::from_be_bytes(b[o..o + 4].try_into().unwrap()) + } + + /// Properties of the flattened device tree's node whose `compatible` names `compatible`. + fn fdt_node_props(dtb: &[u8], compatible: &[u8]) -> Option)>> { + assert_eq!(be32(dtb, 0), 0xd00dfeed); + let strings = be32(dtb, 12) as usize; + let mut at = be32(dtb, 8) as usize; + let mut stack: Vec)>> = Vec::new(); + loop { + let token = be32(dtb, at); + at += 4; + match token { + 1 => { + let end = dtb[at..].iter().position(|&b| b == 0).unwrap(); + at = (at + end + 1).next_multiple_of(4); + stack.push(Vec::new()); + } + 2 => { + let props = stack.pop().unwrap(); + let hit = props.iter().any(|(n, v)| { + n == "compatible" && v.split(|&b| b == 0).any(|s| s == compatible) + }); + if hit { + return Some(props); + } + } + 3 => { + let len = be32(dtb, at) as usize; + let name_off = strings + be32(dtb, at + 4) as usize; + let name_end = dtb[name_off..].iter().position(|&b| b == 0).unwrap(); + let name = + String::from_utf8(dtb[name_off..name_off + name_end].to_vec()).unwrap(); + let value = dtb[at + 8..at + 8 + len].to_vec(); + at = (at + 8 + len).next_multiple_of(4); + stack.last_mut().unwrap().push((name, value)); + } + 4 => {} + 9 => return None, + _ => panic!("fdt token {token}"), + } + } + } + + /// Builder on the saved device tree == the private file's Globals and Power, byte for byte. + /// Needs `G15_J516C_FIXTURES=` with `j516c.dtb` (`/sys/firmware/fdt` of the machine) + /// and `t6031-j516c-board.bin` (its board file); both stay off the tree, so the test is + /// ignored by default (`--ignored` runs it) and fails, rather than passes, without them. + #[test] + #[ignore = "needs the off-tree J516C fixtures (G15_J516C_FIXTURES)"] + fn j516c_fixture() { + let dir = std::env::var_os("G15_J516C_FIXTURES") + .expect("G15_J516C_FIXTURES names the directory with j516c.dtb and the board file"); + let dir = std::path::Path::new(&dir); + let dtb = std::fs::read(dir.join("j516c.dtb")).unwrap(); + let blob = std::fs::read(dir.join("t6031-j516c-board.bin")).unwrap(); + + // The board file: magic, version, count, 3 x (kind, size), images (board/blob.rs). + assert_eq!(&blob[..8], b"T6031BRD"); + assert_eq!(get32(&blob, 8), 1); + assert_eq!(get32(&blob, 12), 3); + let mut at = 16 + 3 * 8; + let mut images = Vec::new(); + for i in 0..3 { + assert_eq!(get32(&blob, 16 + i * 8) as usize, i); + let size = get32(&blob, 16 + i * 8 + 4) as usize; + images.push(blob[at..at + size].to_vec()); + at += size; + } + assert_eq!(at, blob.len()); + let (hardware, file_globals, file_power) = (&images[0], &images[1], &images[2]); + assert_eq!(hardware.len(), 0xb480); + assert_eq!(file_globals.len(), GLOBALS_SIZE); + assert_eq!(file_power.len(), POWER_SIZE); + + // The GPU node's words. + let props = fdt_node_props(&dtb, b"apple,agx-t6031").expect("a T6031 GPU node"); + let prop = |name: &core::ffi::CStr| -> Option<&[u8]> { + let name = name.to_str().unwrap(); + props + .iter() + .find(|(n, _)| n == name) + .map(|(_, v)| v.as_slice()) + }; + let mut values = [0u32; PROPS.len()]; + for (v, name) in values.iter_mut().zip(PROPS.iter()) { + let bytes = prop(name).unwrap_or_else(|| panic!("GPU node has no {name:?}")); + assert_eq!(bytes.len(), 4, "{name:?}"); + *v = be32(bytes, 0); + } + let zones: Vec = prop(ZONES) + .map(|b| b.chunks(4).map(|c| be32(c, 0)).collect()) + .unwrap_or_default(); + let words = PowerWords::new(values, &zones).unwrap(); + + // The images: the G15S templates with the J516C words, the hardware record's maximum + // power and the top state. + let max_power = get32(hardware, HWDATA_A + HWDATA_A_MAX_POWER); + let mut built_globals = vec![0u8; GLOBALS_SIZE]; + super::super::template::fill(&super::super::data::GLOBALS, &mut built_globals).unwrap(); + globals(&mut built_globals, max_power, &words).unwrap(); + let mut built_power = vec![0u8; POWER_SIZE]; + super::super::template::fill(&super::super::data::POWER, &mut built_power).unwrap(); + power(&mut built_power, max_power, &words).unwrap(); + + assert_eq!( + first_difference(&built_globals, file_globals), + None, + "Globals" + ); + assert_eq!(first_difference(&built_power, file_power), None, "Power"); + } +} diff --git a/drivers/gpu/drm/asahi/g15/boot.rs b/drivers/gpu/drm/asahi/g15/boot.rs new file mode 100644 index 0000000000000..e6278690d5f14 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/boot.rs @@ -0,0 +1,729 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Admission and firmware start to READY. +//! +//! Admission (no firmware-visible writes): machine check (the T6030 board check +//! `apple_pmp_t6030_board`, or the SoC's machine compatible), the PMP supplier on a SoC with +//! the PMP AGX vote (`apple_pmp_j514s_gpu_prepare`, may return EPROBE_DEFER), the GPU regions and +//! segments ([`super::adt`]: the ADT export, or the GPU device tree node), exact firmware TEXT +//! identity ([`super::identity`]), MMIO windows, runtime-PM hold of the GFX domain. +//! +//! Start: PMP AGX vote (`apple_pmp_j514s_gpu_get`) where the SoC takes one, hardware identity +//! and core masks (one word, or two above 32 core slots), `sgx_open` +//! 0x70001, cold-only check (ASC RUN clear, TTBAT all zero), UAT adoption and the PTE self-check +//! (nothing firmware-visible yet), RTKit init (the node's mailbox), wake, endpoints 0x20/0x21, +//! PPL/UAT bootstrap ([`super::uat`]), initdata graph ([`super::initdata`]), EP 0x20 +//! `0x0081 << 48 | ROOT`, DEVCTL 0x13 then 9, readiness (`Runtime+0x314 == 0xabcdabcd` and +//! `Flags+4 == 1`, 2 s, draining rings), DEVCTL 0x13 again. +//! +//! RTKit receive: EP 0x20 `0x0042 << 48` in interrupt context only kicks the poll work +//! ([`super::Shared::kick`]); a crash latches fail-stop. The callbacks are registered through +//! the C RTKit API directly: the only RTKit buffer accepted is the inherited one inside firmware +//! DATA, mapped as I/O memory, which the Rust RTKit abstraction (buffers in system memory only) +//! cannot express. +//! +//! Once the firmware has been touched nothing it can reach is ever freed: a failed start, or a +//! failed backend start, keeps the session as an [`Orphan`] whose rings are still drained. + +use core::ffi::c_void; +use core::sync::atomic::Ordering; + +use kernel::{ + bindings, device, + device::Core, + error::{from_err_ptr, to_result}, + platform, + prelude::*, + sync::{aref::ARef, Arc, ArcBorrow}, + time::{delay::fsleep, Delta, Instant, Monotonic}, +}; + +use super::adt::{self, GpuAdt}; +use super::channel::{self, Channel, Channels}; +use super::event::Events; +use super::fw::channels::{devctl, DevCtl}; +use super::fw::initdata::runtime; +use super::hw; +use super::initdata::{self, InitData}; +use super::object::Objects; +use super::{identity, regs, uat, Config, Shared}; +use crate::driver::AsahiDrmDevice as AsahiDevice; +use crate::mmu; + +/// The C shim (`g15/glue.c`): helpers without Rust bindings. +pub(crate) mod glue { + use core::sync::atomic::{AtomicBool, Ordering}; + use kernel::{device, prelude::*}; + + extern "C" { + fn asahi_g15_sha256(data: *const u8, len: usize, out: *mut u8); + fn asahi_g15_board() -> bool; + fn asahi_g15_machine(compatible: *const core::ffi::c_char) -> bool; + fn asahi_g15_hold_power(dev: *mut core::ffi::c_void); + fn asahi_g15_release_power(dev: *mut core::ffi::c_void); + fn asahi_g15_module_get(); + pub(crate) fn asahi_g15_reprobe(); + fn asahi_g15_adt_phram(start: *mut u64, size: *mut u64) -> core::ffi::c_int; + fn asahi_g15_ram_disjoint(start: u64, size: u64) -> bool; + fn asahi_g15_ram_within(start: u64, size: u64) -> bool; + fn asahi_g15_memory_mode(start: u64, size: u64) -> u32; + fn asahi_g15_ioremap_wc(start: u64, size: usize) -> *mut core::ffi::c_void; + fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; + pub(crate) fn asahi_g15_pmp_prepare(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_pmp_get(dev: *mut core::ffi::c_void) -> core::ffi::c_int; + pub(crate) fn asahi_g15_pmp_put(dev: *mut core::ffi::c_void); + fn asahi_g15_reserved_uat(index: u32, start: u64, size: u64) -> bool; + } + + /// SHA-256 (`sha256()`). + pub(crate) fn sha256(data: &[u8]) -> [u8; 32] { + let mut out = [0u8; 32]; + // SAFETY: `data` is valid for its length and `out` holds the 32-byte digest. + unsafe { asahi_g15_sha256(data.as_ptr(), data.len(), out.as_mut_ptr()) }; + out + } + + /// A J514S/J516S T6030 board (`apple_pmp_t6030_board()`). + pub(crate) fn board() -> bool { + // SAFETY: Reads the machine compatible strings only. + unsafe { asahi_g15_board() } + } + + /// Exact enabled UAT reservation from the maintained boot-loader handoff. + pub(crate) fn reserved_uat(index: u32, start: u64, size: u64) -> bool { + // SAFETY: Queries and releases OF nodes; no physical memory is accessed. + unsafe { asahi_g15_reserved_uat(index, start, size) } + } + + /// Whether the machine carries `compatible` (`of_machine_is_compatible()`). + pub(crate) fn machine(compatible: &CStr) -> bool { + // SAFETY: Reads the machine compatible strings only; the string is NUL-terminated. + unsafe { asahi_g15_machine(compatible.as_char_ptr()) } + } + + /// Holds the GFX power domain for the retained session. + pub(crate) fn hold_power(dev: &device::Device) { + // SAFETY: `dev` is a valid device; runtime-PM accounting on it has no other requirement. + unsafe { asahi_g15_hold_power(dev.as_raw().cast()) }; + } + + /// Gives the GFX power domain hold back after a start that touched nothing firmware-visible. + pub(crate) fn release_power(dev: &device::Device) { + // SAFETY: `dev` is the device `hold_power` was called with. + unsafe { asahi_g15_release_power(dev.as_raw().cast()) }; + } + + static RETAINED: AtomicBool = AtomicBool::new(false); + + /// Whether the module is pinned: the firmware has been touched. + pub(crate) fn retained() -> bool { + RETAINED.load(Ordering::Acquire) + } + + /// Pins this module until reboot, once (`__module_get(THIS_MODULE)`). + pub(crate) fn retain_module() { + if !RETAINED.swap(true, Ordering::AcqRel) { + // SAFETY: The module is live while its code runs; the reference is never put. + unsafe { asahi_g15_module_get() }; + } + } + + /// The phram "adt" export: start and size. + pub(crate) fn adt_phram() -> Result<(u64, u64)> { + let (mut start, mut size) = (0u64, 0u64); + // SAFETY: Both pointers are valid for a u64 write. + kernel::error::to_result(unsafe { asahi_g15_adt_phram(&mut start, &mut size) })?; + Ok((start, size)) + } + + /// The physical range is disjoint from System RAM. + pub(crate) fn ram_disjoint(start: u64, size: u64) -> bool { + // SAFETY: Only queries the resource tree. + unsafe { asahi_g15_ram_disjoint(start, size) } + } + + /// Every page is valid System RAM and remains reserved from the allocator. + pub(crate) fn ram_within(start: u64, size: u64) -> bool { + // SAFETY: Only queries the resource tree. + unsafe { asahi_g15_ram_within(start, size) } + } + + /// WB for complete reserved RAM, WC for aligned non-RAM carveouts; reject everything else. + pub(crate) fn memory_mode(start: u64, size: u64) -> Result { + // SAFETY: Only queries resources and page metadata; it does not create a mapping. + match unsafe { asahi_g15_memory_mode(start, size) } { + 0 => Err(EINVAL), + mode => Ok(mode), + } + } + + /// Whether the device's DMA mappings skip cache maintenance. + pub(crate) fn dma_coherent(dev: &device::Device) -> bool { + // SAFETY: Reads a flag of a valid device. + unsafe { asahi_g15_dma_coherent(dev.as_raw().cast()) } + } + + /// `ioremap_wc`, for the RTKit crash buffer. + pub(crate) fn ioremap_wc(start: u64, size: usize) -> *mut core::ffi::c_void { + // SAFETY: Creates a new mapping of a device range; no other effect. + unsafe { asahi_g15_ioremap_wc(start, size) } + } +} + +/// The PMP AGX vote of a start, released on drop until the session is retained ([`Vote::keep`]). +struct Vote(Option>); + +impl Vote { + /// The session is retained: the vote stays with it until reboot. + fn keep(mut self) { + self.0 = None; + } +} + +impl Drop for Vote { + fn drop(&mut self) { + // The vote is a level, not a count: never give it back under a retained session. + if let Some(d) = self.0.take().filter(|_| !glue::retained()) { + // SAFETY: `d` is the device `asahi_g15_pmp_get` voted for. + unsafe { glue::asahi_g15_pmp_put(d.as_raw().cast()) }; + } + } +} + +/// `EPROTO` (not among the Rust kernel's named error codes). +pub(crate) fn eproto() -> Error { + Error::from_errno(-(bindings::EPROTO as i32)) +} + +/// `Shared` from an RTKit cookie. +/// +/// # Safety +/// +/// `cookie` must be the pointer [`Rtkit::new`] registered. +unsafe fn shared<'a>(cookie: *mut c_void) -> ArcBorrow<'a, Shared> { + // SAFETY: The cookie is a leaked `Arc` (see `Rtkit::new`), live until reboot. + unsafe { ArcBorrow::from_raw(cookie as *const Shared) } +} + +/// RTKit crash callback: latches fail-stop. +unsafe extern "C" fn rtkit_crashed(cookie: *mut c_void, _log: *const c_void, _size: usize) { + // SAFETY: RTKit passes back the registered cookie. + let shared = unsafe { shared(cookie) }; + dev_err!( + shared.dev, + "GPU firmware crashed; retaining session until reboot\n" + ); + shared.fail_stop(EIO); +} + +/// RTKit message callback. +unsafe extern "C" fn rtkit_recv(cookie: *mut c_void, endpoint: u8, message: u64) { + // SAFETY: RTKit passes back the registered cookie. + let shared = unsafe { shared(cookie) }; + // EP 0x20 0x0042... asks for receive-ring draining; nothing else expected. + if endpoint == hw::ep::FIRMWARE && message == hw::ep::RING_SERVICE { + if shared.started.load(Ordering::Acquire) { + Arc::::from(shared).kick(); + } + return; + } + dev_info!( + shared.dev, + "firmware EP{:#x} message {:#x}\n", + endpoint, + message + ); +} + +/// EP 0x20 ring-service request, in the mailbox interrupt: wake only. The drain and the +/// backend's completion check run in the poll work; the shared state outlives the mailbox. +unsafe extern "C" fn rtkit_recv_early(cookie: *mut c_void, endpoint: u8, message: u64) -> bool { + // SAFETY: RTKit passes back the registered cookie. + let shared = unsafe { shared(cookie) }; + if !hw::tuning::EARLY_WAKE + || endpoint != hw::ep::FIRMWARE + || message != hw::ep::RING_SERVICE + || !shared.started.load(Ordering::Acquire) + { + return false; + } + Arc::::from(shared).kick(); + true +} + +/// Only the inherited crash buffer inside this session's firmware DATA, once. +unsafe extern "C" fn rtkit_shmem_setup( + cookie: *mut c_void, + bfr: *mut bindings::apple_rtkit_shmem, +) -> core::ffi::c_int { + // SAFETY: RTKit passes back the registered cookie. + let shared = unsafe { shared(cookie) }; + // SAFETY: RTKit passes a valid buffer descriptor it owns for the duration of the call. + let bfr = unsafe { &mut *bfr }; + let (iova, size) = (bfr.iova as u64, bfr.size as u64); + let data = &shared.data; + let data_size = data.end - data.start; + if shared.rtkit_buffer.load(Ordering::Acquire) != 0 + || iova == 0 + || size == 0 + || (iova | size) & 0xfff != 0 + || iova < data.start + || size > data_size + || iova - data.start > data_size - size + { + dev_err!( + shared.dev, + "refusing RTKit buffer {:#x}/{:#x}\n", + iova, + size + ); + shared.fail_stop(EINVAL); + return EINVAL.to_errno(); + } + let iomem = glue::ioremap_wc(iova, bfr.size); + if iomem.is_null() { + return ENOMEM.to_errno(); + } + bfr.iomem = iomem; + bfr.is_mapped = true; + shared.rtkit_buffer.store(iova, Ordering::Release); + dev_info!( + shared.dev, + "mapped firmware crash storage {:#x}/{:#x}\n", + iova, + size + ); + 0 +} + +/// Retained until reboot: no qualified stop path. +unsafe extern "C" fn rtkit_shmem_destroy( + _cookie: *mut c_void, + _bfr: *mut bindings::apple_rtkit_shmem, +) { +} + +/// The GPU's RTKit callbacks. +static RTKIT_OPS: bindings::apple_rtkit_ops = bindings::apple_rtkit_ops { + flags: 0, + crashed: Some(rtkit_crashed), + recv_message: Some(rtkit_recv), + recv_message_early: Some(rtkit_recv_early), + shmem_setup: Some(rtkit_shmem_setup), + shmem_destroy: Some(rtkit_shmem_destroy), +}; + +/// The GPU's RTKit client. Never freed: the firmware session lasts until reboot. +/// +/// # Invariants +/// +/// `rtk` is a live `struct apple_rtkit` whose cookie is a leaked `Arc`. +pub(crate) struct Rtkit { + rtk: *mut bindings::apple_rtkit, +} + +// SAFETY: The C RTKit API is callable from any thread; mutating calls take `&mut self`. +unsafe impl Send for Rtkit {} +// SAFETY: Shared calls (`has_endpoint`) only read RTKit state. +unsafe impl Sync for Rtkit {} + +impl Rtkit { + /// Initializes RTKit on the maintained GPU node's first, unnamed mailbox. + /// The session is not device-managed because it is retained until reboot. + fn new(dev: &device::Device, shared: &Arc) -> Result { + let cookie = Arc::into_raw(shared.clone()) as *mut c_void; + // SAFETY: `dev` is valid, the name is a NUL-terminated string or null and the ops are + // static. + let rtk = from_err_ptr(unsafe { + bindings::apple_rtkit_init(dev.as_raw(), cookie, core::ptr::null(), 0, &RTKIT_OPS) + }); + match rtk { + // INVARIANT: The cookie stays leaked from here on. + Ok(rtk) => Ok(Self { rtk }), + Err(e) => { + // SAFETY: Init failed, so RTKit holds no copy of the cookie from `into_raw`. + drop(unsafe { Arc::from_raw(cookie as *const Shared) }); + Err(e) + } + } + } + + fn wake(&mut self) -> Result { + // SAFETY: `rtk` is live by the type invariant. + to_result(unsafe { bindings::apple_rtkit_wake(self.rtk) }) + } + + fn has_endpoint(&self, endpoint: u8) -> bool { + // SAFETY: `rtk` is live by the type invariant. + unsafe { bindings::apple_rtkit_has_endpoint(self.rtk, endpoint) } + } + + fn start_endpoint(&mut self, endpoint: u8) -> Result { + // SAFETY: `rtk` is live by the type invariant. + to_result(unsafe { bindings::apple_rtkit_start_ep(self.rtk, endpoint) }) + } + + /// Sends a message without waiting. + pub(crate) fn send(&mut self, endpoint: u8, message: u64) -> Result { + // SAFETY: `rtk` is live by the type invariant. + to_result(unsafe { + bindings::apple_rtkit_send_message( + self.rtk, + endpoint, + message, + core::ptr::null_mut(), + false, + ) + }) + } +} + +/// The result of admission: everything probe-time checks established, nothing written. +pub(crate) struct Admitted { + pub(crate) regs: regs::Regs, + pub(crate) adt: GpuAdt, + pub(crate) board_images: initdata::BoardImages, +} + +/// The running firmware session: RTKit, MMIO and the adopted UAT. (The +/// host window lives in [`Objects`], which maps through it.) +pub(crate) struct Session { + pub(crate) dev: ARef, + pub(crate) shared: Arc, + pub(crate) rtkit: Rtkit, + pub(crate) regs: regs::Regs, + pub(crate) adt: GpuAdt, + uat: Arc, +} + +impl Session { + /// The adopted UAT (user VMs are created and bound through it). + pub(crate) fn uat(&self) -> Arc { + self.uat.clone() + } + + /// The latched error, if any. + pub(crate) fn check(&self) -> Result { + self.shared.check() + } + + /// Sends a mailbox message. + pub(crate) fn send(&mut self, endpoint: u8, message: u64) -> Result { + self.rtkit.send(endpoint, message) + } + + /// Rings a submit channel's doorbell: EP 0x21 `0x0083 << 48 | channel`. + pub(crate) fn ring(&mut self, channel: Channel) -> Result { + self.send(hw::ep::DOORBELL, hw::engine::DOORBELL | channel as u64) + } +} + +/// The firmware at READY. +pub(crate) struct Booted { + pub(crate) session: Session, + pub(crate) init: InitData, + pub(crate) objects: Objects, + pub(crate) channels: Channels, + pub(crate) events: Events, + pub(crate) id: regs::Identity, + pub(crate) max_frequency_khz: u32, +} + +/// A firmware session that did not complete: retained until reboot, its receive rings still +/// drained so the firmware never waits on the host. +pub(crate) struct Orphan { + pub(crate) session: Session, + /// Retained only: the firmware may still use them. + pub(crate) _objects: Objects, + pub(crate) channels: Option, + pub(crate) _init: Option, + pub(crate) events: Events, +} + +impl Orphan { + /// Drains the receive rings, if they exist. + pub(crate) fn drain(&mut self) -> Result { + if let Some(channels) = &mut self.channels { + channels.drain_rings(&self.session.dev, &mut self.events)?; + } + Ok(()) + } +} + +/// Probe-time admission. +pub(crate) fn admit( + pdev: &platform::Device>, + _dev: &AsahiDevice, + cfg: &'static Config, +) -> Result { + let dev = pdev.as_ref(); + let pmp = cfg.platform == hw::Platform::AdtWithPmp; + if pmp { + if !glue::board() { + return Err(ENODEV); + } + // Retain the PMP supplier (and its AGX bridge) before any vote. + // SAFETY: `dev` is the bound GPU device. + let ret = unsafe { glue::asahi_g15_pmp_prepare(dev.as_raw().cast()) }; + if ret != 0 { + let err = Error::from_errno(ret); + if err != EPROBE_DEFER { + dev_err!(dev, "PMP supplier not ready: {:?}\n", err); + } + return Err(err); + } + } else if !glue::machine(cfg.machine) { + dev_err!(dev, "G15: not a {} machine\n", cfg.name); + return Err(ENODEV); + } + let adt = adt::read(dev, cfg) + .inspect_err(|e| dev_err!(dev, "GPU regions not admitted: {:?}\n", e))?; + identity::firmware(dev, &adt, cfg.firmware) + .inspect_err(|e| dev_err!(dev, "loaded GFX TEXT identity mismatch: {:?}\n", e))?; + dev_info!(dev, "M3: firmware identity accepted\n"); + if cfg.board == hw::Board::None { + dev_err!( + dev, + "G15: no board data for the {}; the firmware is left as the boot loader left it\n", + cfg.name + ); + return Err(ENODEV); + } + // Firmware objects are cleaned to the point of coherency by their DMA mapping: that needs + // a device that is not DMA-coherent, as the synthesized node is. + if glue::dma_coherent(dev) { + dev_err!( + dev, + "G15: the GPU device is DMA-coherent; firmware objects would not be cleaned\n" + ); + return Err(EINVAL); + } + let board_images = initdata::prepare_board(dev, cfg, &adt)?; + let regs = regs::Regs::new(pdev)?; + // Keep the GFX domain powered for the whole (retained) session. + glue::hold_power(dev); + dev_info!(dev, "{} admitted, ASC={:#x}\n", cfg.name, regs.asc_control()?); + Ok(Admitted { regs, adt, board_images }) +} + +/// Posts one device control with only the opcode and waits for it. +fn devctl_op( + session: &mut Session, + channels: &mut Channels, + events: &mut Events, + init: &InitData, + opcode: u32, +) -> Result { + channel::devctl_submit(session, channels, events, init, &DevCtl::op(opcode), true) +} + +/// From the wake to READY. On failure the caller keeps everything as an [`Orphan`]. +fn run( + session: &mut Session, + cfg: &Config, + objects: &mut Objects, + events: &mut Events, + built: &mut Option<(InitData, Channels, u32)>, + board_images: initdata::BoardImages, +) -> Result { + let dev = session.dev.clone(); + session.rtkit.wake()?; + if !session.rtkit.has_endpoint(hw::ep::FIRMWARE) + || !session.rtkit.has_endpoint(hw::ep::DOORBELL) + { + return Err(ENODEV); + } + session.rtkit.start_endpoint(hw::ep::FIRMWARE)?; + session.rtkit.start_endpoint(hw::ep::DOORBELL)?; + let uat = session.uat.clone(); + objects.window.bootstrap(&dev, &uat)?; + let (init, channels, _) = built.insert(initdata::build(&dev, cfg, &session.adt, objects, board_images)?); + uat::tlbi_all(); + dev_info!( + dev, + "M3: publishing owned initdata root {:#x} ({} pages, {} L3)\n", + hw::va::ROOT, + objects.pages(), + objects.window.l3_count() + ); + session.send( + hw::ep::FIRMWARE, + hw::ep::INITDATA | (hw::va::ROOT & hw::ep::INITDATA_ADDR_MASK), + )?; + devctl_op(session, channels, events, init, devctl::IMPORT_GLOBALS)?; + devctl_op(session, channels, events, init, devctl::START)?; + let start = Instant::::now(); + loop { + let (magic, ready) = init.readiness()?; + if magic == runtime::READY_MAGIC && ready == 1 { + break; + } + channels.drain_rings(&dev, events)?; + session.check()?; + if start.elapsed() > Delta::from_millis(2000) { + let (magic, ready) = init.readiness()?; + dev_err!( + dev, + "readiness: runtime+0x314={:#x} flags+4={:#x}\n", + magic, + ready + ); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_micros(1000)); + } + let (magic, ready) = init.readiness()?; + dev_info!( + dev, + "M3 firmware readiness: magic={:#x} flags={}\n", + magic, + ready + ); + devctl_op(session, channels, events, init, devctl::IMPORT_GLOBALS)?; + channels.drain_rings(&dev, events)?; + dev_info!( + dev, + "M3: firmware accepted owned initdata and device controls\n" + ); + Ok(()) +} + +/// Brings the firmware to READY. +pub(crate) fn start( + admitted: Admitted, + dev: &AsahiDevice, + cfg: &'static Config, + shared: &Arc, +) -> Result { + let Admitted { regs, adt, board_images } = admitted; + let d: ARef = shared.dev.clone(); + // Released by an early failure below, until the session is retained. + let mut vote = Vote(None); + if cfg.platform == hw::Platform::AdtWithPmp { + // SAFETY: `d` is the bound GPU device. + let ret = unsafe { glue::asahi_g15_pmp_get(d.as_raw().cast()) }; + if ret != 0 { + dev_err!(d, "PMP AGX vote failed: {}\n", ret); + return Err(Error::from_errno(ret)); + } + vote = Vote(Some(d.clone())); + } + let id = regs.identity(cfg)?; + dev_info!( + d, + "{}: power acknowledged, version={:#x} counts={:#x} core-mask={:#x}\n", + cfg.name, + id.version, + id.counts, + id.cores + ); + if !cfg + .hw_ids + .iter() + .any(|h| h.version == id.version && h.counts == id.counts) + { + dev_err!( + d, + "unsupported {} hardware: version {:#x} counts {:#x}\n", + cfg.name, + id.version, + id.counts + ); + return Err(eproto()); + } + // Binning: any non-empty enabled set in each cluster, nothing above the core slots. + let clusters = cfg.num_clusters as usize; + let slots = cfg.core_slots(); + if clusters > regs::MAX_CLUSTERS + || id.core_masks[..clusters].iter().any(|m| *m == 0) + || (slots < 64 && id.cores >> slots != 0) + { + dev_err!(d, "implausible {} core mask {:#x}\n", cfg.name, id.cores); + return Err(eproto()); + } + let mut enabled = [0u32; regs::MAX_CLUSTERS]; + for (count, mask) in enabled.iter_mut().zip(id.core_masks.iter()) { + *count = mask.count_ones(); + } + dev_info!( + d, + "{}: cores enabled per cluster: {:?}\n", + cfg.name, + &enabled[..clusters] + ); + regs.open()?; + // Cold only: ASC RUN clear and an all-zero TTBAT before ownership. + let control = regs.asc_control()?; + if control & regs::ASC_RUN != 0 { + return Err(EBUSY); + } + let window = uat::Window::new(&adt)?; + if !window.ttbat_is_clear()? { + return Err(EBUSY); + } + // Adopt the UAT and prove the leaf words before anything firmware-visible is written: no + // context names the shadow window or the lower root yet. + let uat = uat::adopt(dev, cfg, &adt)?; + let mut objects = Objects::new(dev, uat.clone(), window); + if let Err(e) = objects.self_check() { + // Mapped with cached firmware classes: never unmapped. + core::mem::forget(objects); + return Err(e); + } + + // From here on the firmware is touched: nothing it can reach is freed, the vote included. + glue::retain_module(); + vote.keep(); + let rtkit = match Rtkit::new(&d, shared) { + Ok(rtkit) => rtkit, + Err(e) => { + dev_err!(d, "G15: RTKit mailbox initialization failed: {:?}\n", e); + core::mem::forget(objects); + return Err(e); + } + }; + shared.started.store(true, Ordering::Release); + let mut session = Session { + dev: d.clone(), + shared: shared.clone(), + rtkit, + regs, + adt, + uat, + }; + let mut events = Events::new(); + let mut built = None; + let result = events + .reserve() + .and_then(|_| session.regs.set_asc_control(control | regs::ASC_RUN)) + .and_then(|_| run(&mut session, cfg, &mut objects, &mut events, &mut built, board_images)); + if let Err(e) = result { + dev_err!( + d, + "G15 start failed: {:?}; session retained until reboot\n", + e + ); + let (init, channels) = match built { + Some((init, channels, _)) => (Some(init), Some(channels)), + None => (None, None), + }; + shared.orphan(Orphan { + session, + _objects: objects, + channels, + _init: init, + events, + }); + return Err(e); + } + let (init, channels, max_frequency_khz) = built.ok_or(EINVAL)?; + Ok(Booted { + session, + init, + objects, + channels, + events, + id, + max_frequency_khz, + }) +} diff --git a/drivers/gpu/drm/asahi/g15/channel.rs b/drivers/gpu/drm/asahi/g15/channel.rs new file mode 100644 index 0000000000000..60090f140081c --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/channel.rs @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Submit channels, the device-control ring, the receive rings and the doorbells. +//! +//! - Submit channels 0 (TA), 1 (FS), 2 (compute) of 12 allocated, 256 x 0x18 records each. +//! Posting writes the record, a write barrier, then the producer (+0x20 of the state block), +//! and another barrier. Doorbell: EP 0x21 `0x0083 << 48 | channel`. Records may be posted +//! behind unconsumed ones (submit-ahead, at most [`super::hw::sizes::AHEAD_RECORDS`]), never +//! over an unconsumed slot ([`Channels::post_ahead`]). +//! - Device-control ring: 256 x 0x38 records; doorbell EP 0x21 `0x0083000000000011`. +//! [`Firmware::devctl`] with `wait` posts and waits for consumption and completion (2 s); +//! without it, it does not wait (the kill request). Both return EBUSY while Control+0x4590 +//! says the firmware is halted, and wait for an earlier posted record's consumption first. +//! - Receive rings (event, 6 log sub-rings, trace, stats): consumed by setting read = producer; +//! the event ring is decoded by [`super::event`], the stats ring's tags and power-state +//! records are noted ([`super::power::pstate`]) before it is consumed. +//! - FwCtl ring: 256 x 0x18 = 0x1800 bytes (the 2419 firmware uses a 0x18 stride). Never used +//! by the host: no firmware cache flush path exists. + +use kernel::{ + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta, Instant, Monotonic}, +}; + +use super::boot::Session; +use super::event::Events; +use super::fw::channels::{ + DevCtl, SubmitRecord, RING_SLOTS, STATE_COMPLETION, STATE_READ, STATE_WRITE, +}; +use super::hw::{self, rings, va}; +use super::initdata::InitData; +use super::object::{Class, FwObject, Objects, Placement, Request}; +use super::power::pstate::{self, StatsSeen}; +use super::regs::{mb, rmb, wmb}; +use super::Firmware; + +/// A submit channel. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Channel { + Ta = 0, + Fs = 1, + Compute = 2, +} + +/// A ring and its state block. +pub(crate) struct Ring { + pub(crate) state: Arc, + pub(crate) ring: Arc, +} + +impl Ring { + /// The consumer's read index (state +0x00). + pub(crate) fn read(&self) -> Result { + self.state.r32(STATE_READ) + } + + /// The completion index (state +0x10). + pub(crate) fn completion(&self) -> Result { + self.state.r32(STATE_COMPLETION) + } + + /// The producer's write index (state +0x20). + pub(crate) fn write(&self) -> Result { + self.state.r32(STATE_WRITE) + } +} + +/// The channels and rings. +pub(crate) struct Channels { + /// The 12 submit channels (TA, FS and compute are used). + pub(crate) submit: KVec, + pub(crate) devctl: Ring, + pub(crate) event: Ring, + /// The six log sub-rings share one state object (6 x 0x30) and one ring object. + pub(crate) log: Ring, + pub(crate) trace: Ring, + pub(crate) stats: Ring, + /// What the stats ring carried, up to its consumer. + pub(crate) stats_seen: StatsSeen, + /// Allocated for the firmware; the host never uses it. + _fwctl: Ring, + /// Host producers of channels 0 TA, 1 FS, 2 compute. + producer: [u32; 3], + /// Host producer of the device-control ring. + devctl_producer: u32, +} + +fn fw_object(objects: &mut Objects, at: u64, size: usize) -> Result> { + objects.alloc(Request { + size, + class: Class::Fw, + placement: Placement::At(at), + offset: 0, + client: None, + lower: None, + cpu: true, + }) +} + +fn ring( + objects: &mut Objects, + state: u64, + state_size: usize, + ring: u64, + ring_size: usize, +) -> Result { + Ok(Ring { + state: fw_object(objects, state, state_size)?, + ring: fw_object(objects, ring, ring_size)?, + }) +} + +impl Channels { + /// Allocates every channel and ring of the initdata graph, firmware-only and uncached. + pub(crate) fn alloc(objects: &mut Objects) -> Result { + let mut submit = KVec::with_capacity(rings::CHANNELS as usize, GFP_KERNEL)?; + for i in 0..rings::CHANNELS { + submit.push( + ring( + objects, + va::ch_state(i), + rings::STATE_SIZE, + va::ch_ring(i), + rings::CHANNEL_RING_SIZE, + )?, + GFP_KERNEL, + )?; + } + Ok(Self { + submit, + devctl: ring( + objects, + va::DEVCTL_STATE, + rings::STATE_SIZE, + va::DEVCTL_RING, + rings::DEVCTL_RING_SIZE, + )?, + event: ring( + objects, + va::EVENT_STATE, + rings::STATE_SIZE, + va::EVENT_RING, + rings::EVENT_RING_SIZE, + )?, + log: ring( + objects, + va::LOG_STATE, + rings::LOG_STATE_SIZE, + va::LOG_RING, + rings::LOG_RING_SIZE, + )?, + trace: ring( + objects, + va::TRACE_STATE, + rings::STATE_SIZE, + va::TRACE_RING, + rings::TRACE_RING_SIZE, + )?, + stats: ring( + objects, + va::STATS_STATE, + rings::STATE_SIZE, + va::STATS_RING, + rings::STATS_RING_SIZE, + )?, + _fwctl: ring( + objects, + va::FWCTL_STATE, + rings::STATE_SIZE, + va::FWCTL_RING, + rings::FWCTL_RING_SIZE, + )?, + stats_seen: StatsSeen::default(), + producer: [0; 3], + devctl_producer: 0, + }) + } + + /// A submit channel's ring. + pub(crate) fn channel(&self, channel: Channel) -> &Ring { + &self.submit[channel as usize] + } + + /// The host producer of `channel`. + pub(crate) fn producer(&self, channel: Channel) -> u32 { + self.producer[channel as usize] + } + + /// The firmware's read index of `channel`. + pub(crate) fn read(&self, channel: Channel) -> Result { + self.channel(channel).read() + } + + /// The channel's write cell (state +0x20). + pub(crate) fn write_cell(&self, channel: Channel) -> Result { + self.channel(channel).write() + } + + /// Writes one submit record at the channel's producer and advances it, only if every earlier + /// record was consumed. Returns the new producer: the read index at which this record has + /// been consumed. + pub(crate) fn post(&mut self, channel: Channel, record: &SubmitRecord) -> Result { + self.post_inner(channel, record, false) + } + + /// Like [`Self::post`], but behind unconsumed records while fewer than + /// [`hw::sizes::AHEAD_RECORDS`] are outstanding. + pub(crate) fn post_ahead(&mut self, channel: Channel, record: &SubmitRecord) -> Result { + self.post_inner(channel, record, true) + } + + fn post_inner(&mut self, channel: Channel, record: &SubmitRecord, ahead: bool) -> Result { + let ch = channel as usize; + let producer = self.producer[ch]; + let ring = &self.submit[ch]; + let consumer = ring.read()?; + // Never over an unconsumed record: behind them only with room (ahead). + let busy = if ahead { + producer.wrapping_sub(consumer) % RING_SLOTS >= hw::sizes::AHEAD_RECORDS + } else { + consumer != producer + }; + if busy || ring.write()? != producer { + return Err(EBUSY); + } + ring.ring.write( + producer as usize * rings::CHANNEL_RECORD, + &record.to_bytes(), + )?; + wmb(); + let producer = (producer + 1) % RING_SLOTS; + ring.state.w32(STATE_WRITE, producer)?; + wmb(); + self.producer[ch] = producer; + Ok(producer) + } + + /// Consumes a receive ring to its firmware producer. + fn drain_one(ring: &Ring, offset: usize) -> Result { + let producer = ring.state.r32(offset + STATE_WRITE)?; + if ring.state.r32(offset + STATE_READ)? != producer { + ring.state.w32(offset + STATE_READ, producer)?; + wmb(); + } + Ok(()) + } + + /// Notes the stats records from the consumer to the producer into `seen`; returns the + /// producer, or `None` when an index is out of the ring. + fn note_stats(ring: &Ring, seen: &mut StatsSeen) -> Result> { + let producer = ring.state.r32(STATE_WRITE)?; + let read = ring.state.r32(STATE_READ)?; + let Some(slots) = pstate::pending(read, producer) else { + return Ok(None); + }; + rmb(); + for slot in slots { + let at = slot as usize * pstate::STATS_RECORD; + seen.note( + ring.ring.r32(at + pstate::STATS_TAG)?, + ring.ring.r32(at + pstate::POWER_STATE_INDEX)?, + ); + } + Ok(Some(producer)) + } + + /// The stats ring noted up to its producer, without consuming it. + pub(crate) fn peek_stats(&self) -> Result { + let mut seen = self.stats_seen; + Self::note_stats(&self.stats, &mut seen)?; + Ok(seen) + } + + /// Drains every receive ring: events (decoded into `events`), the six log sub-rings, trace + /// and stats (noted into [`Self::stats_seen`] first). + pub(crate) fn drain_rings( + &mut self, + dev: &kernel::device::Device, + events: &mut Events, + ) -> Result { + events.drain(dev, &self.event)?; + for i in 0..rings::LOG_RINGS { + Self::drain_one(&self.log, i * 0x30)?; + } + Self::drain_one(&self.trace, 0)?; + let mut seen = self.stats_seen; + if let Some(producer) = Self::note_stats(&self.stats, &mut seen)? { + self.stats_seen = seen; + if self.stats.read()? != producer { + // The record loads complete before the slots go back to the firmware. + mb(); + self.stats.state.w32(STATE_READ, producer)?; + wmb(); + } + return Ok(()); + } + Self::drain_one(&self.stats, 0) + } +} + +/// Posts one 0x38-byte device-control record (EP 0x21 `0x0083000000000011`) behind the earlier +/// ones (one posted without waiting may still be pending: wait for its consumption first) and, +/// with `wait`, waits for its consumption and completion. A firmware halted for recovery consumes +/// nothing until the host's resume: EBUSY then, so the caller can answer the halt instead of +/// timing out. Callers serialize (the device lock once the backend runs). +pub(crate) fn devctl_submit( + session: &mut Session, + channels: &mut Channels, + events: &mut Events, + init: &InitData, + record: &DevCtl, + wait: bool, +) -> Result { + let dev = session.dev.clone(); + let producer = channels.devctl_producer; + let next = (producer + 1) % RING_SLOTS; + let start = Instant::::now(); + let expired = || start.elapsed() > Delta::from_millis(2000); + let opcode = record.opcode(); + while channels.devctl.read()? != producer { + channels.drain_rings(&dev, events)?; + session.check()?; + if init.halted()? { + return Err(EBUSY); + } + if expired() { + dev_err!( + dev, + "control {:#x}: earlier record not consumed (read={} producer={})\n", + opcode, + channels.devctl.read()?, + producer + ); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_micros(1000)); + } + channels + .devctl + .ring + .write(producer as usize * rings::DEVCTL_RECORD, &record.to_bytes())?; + wmb(); + channels.devctl.state.w32(STATE_WRITE, next)?; + wmb(); + channels.devctl_producer = next; + session.send(hw::ep::DOORBELL, hw::engine::DEVCTL_DOORBELL)?; + if !wait { + return Ok(()); + } + loop { + channels.drain_rings(&dev, events)?; + session.check()?; + if channels.devctl.read()? == next && channels.devctl.completion()? == next { + return Ok(()); + } + if init.halted()? { + return Err(EBUSY); + } + if expired() { + dev_err!( + dev, + "control {:#x}: read={} completion={} producer={}\n", + opcode, + channels.devctl.read()?, + channels.devctl.completion()?, + next + ); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_micros(1000)); + } +} + +impl Firmware { + /// Posts a device-control record; with `wait` waits for its consumption and completion, + /// otherwise returns once posted. EBUSY while the firmware is halted; the transport error if + /// the session failed. + pub(crate) fn devctl(&mut self, record: &DevCtl, wait: bool) -> Result { + devctl_submit( + &mut self.session, + &mut self.channels, + &mut self.events, + &self.init, + record, + wait, + ) + } + + /// The latched transport error: an RTKit crash or a refused RTKit buffer. + pub(crate) fn transport_error(&self) -> Option { + self.session.check().err() + } + + /// Drains the receive rings and decodes events. Called first in every poll pass. + pub(crate) fn drain_rings(&mut self) -> Result { + let dev = self.session.dev.clone(); + self.channels.drain_rings(&dev, &mut self.events) + } +} diff --git a/drivers/gpu/drm/asahi/g15/command.rs b/drivers/gpu/drm/asahi/g15/command.rs new file mode 100644 index 0000000000000..45b65d1ddc026 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/command.rs @@ -0,0 +1,707 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! SUBMIT parsing and validation into engine commands. +//! +//! The rules, checked in this order: the whole stream's structure ([`check_stream`]), then the +//! timestamp handles and the historical priors ([`prepare`]), then each command's mapped spans +//! and gates ([`Prepared::validate`]): +//! +//! - Headers 8 bytes, payloads multiples of 8, at most 4 MiB; short payloads zero-filled, bytes +//! past the known struct must be zero; 1..=64 render+compute commands; attachments (types +//! 2..4) with NONE barriers, at most 16 x 24 bytes, pad/flags zero, inside the VM range. +//! - Header barriers: NONE or at most the number of earlier same-engine commands in this +//! SUBMIT; `prior_vdm`/`prior_cdm` at most the commands of that engine accepted on the queue +//! before this ioctl (from 1); NONE does not suppress a prior ([`check_order`], the semantics +//! of `g17::job::Order`). Validated only: the total order satisfies them. +//! - Render (256, 272 with EXACT_PRIOR, up to 384 bytes): flags PROCESS_EMPTY_TILES, +//! NO_VERTEX_CLUSTERING, RSRC_SPEC_HI, DBIAS_IS_INT (+ FRAGMENT_BARRIERS only with +//! `g15_early_tiling`); VERTEX_SCRATCH, FEW_PRIMITIVES, helpers, sampler heap/count, +//! `ray_query`, nonzero partial-EOT high word: EINVAL; RSRC_SPEC_HI needs >= 256 bytes; +//! without it every `*_spec_hi` is zero; utile 16 or 32; `sample_size_B` multiple of 8 within +//! the tile buffer; `ppp_ctrl & !0x203 == 0`; stencil compression, or depth compression with +//! layers or samples != 1: EOPNOTSUPP; VDM base 4-aligned inside the compact window; scissor +//! 8-aligned; every referenced span mapped with the right access (ENOENT/EACCES); geometry +//! too large: E2BIG. +//! - Compute (64, 136 bytes): flags, `scs_layout`, `cdm_auxiliary_allocation`, `reserved`, +//! `intersection_*`, helpers, samplers zero; stream base/end 4-aligned. +//! - Timestamps resolve BIND_OBJECT handles to firmware arena VAs ([`super::timestamp`]). +//! - Stream gates ([`super::gate`]) when `g15_stream_gate`. + +use core::mem::{offset_of, size_of}; +use core::ops::Range; +use kernel::{prelude::*, sync::Arc, xarray}; + +use super::wire as uapi; + +use super::{gate, hw}; +use crate::{file, mmu, util::AnyBitPattern, util::Reader}; + +/// Largest command buffer. +pub(crate) const MAX_CMDBUF: usize = 4 << 20; +/// Most input plus output syncs of one SUBMIT. +pub(crate) const MAX_SYNCS: usize = 4096; +/// Hardware commands per SUBMIT. +const MAX_COMMANDS: u32 = crate::file::MAX_COMMANDS_PER_SUBMISSION; + +const HEADER_SIZE: usize = size_of::(); +const ATTACHMENT_SIZE: usize = size_of::(); +/// A render payload with RSRC_SPEC_HI carries every high word. +const RSRC_SPEC_HI_MIN: usize = + offset_of!(uapi::drm_asahi_cmd_render, bg_partial_rsrc_spec_hi) + size_of::(); +/// Render payload bytes from the ray-query stages on (RENDER_RAY_QUERY is never advertised). +const RAY_QUERY: usize = offset_of!(uapi::drm_asahi_cmd_render, ray_query); +const NONE: u16 = uapi::DRM_ASAHI_BARRIER_NONE as u16; + +const VERTEX_SCRATCH: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH; +const PROCESS_EMPTY_TILES: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES; +const NO_VERTEX_CLUSTERING: u32 = + uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING; +const RSRC_SPEC_HI: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI; +const DBIAS_IS_INT: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT; +const FRAGMENT_BARRIERS: u32 = uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS; + +const CMD_RENDER: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER; +const CMD_COMPUTE: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE; +const SET_VERTEX_ATTACHMENTS: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS; +const SET_FRAGMENT_ATTACHMENTS: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_FRAGMENT_ATTACHMENTS; +const SET_COMPUTE_ATTACHMENTS: u32 = uapi::drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_ATTACHMENTS; + +const MAX_DIMENSION: u16 = 16384; +const MAX_LAYERS: u16 = 2048; +const MAX_TILE_BUFFER: u32 = 32768; +/// Size of a queue's USC window. +pub(crate) const USC_WINDOW: u64 = 1 << 32; + +/// Engine index of render commands (VDM) in the accepted counts. +pub(crate) const RENDER: usize = 0; +/// Engine index of compute commands (CDM). +pub(crate) const COMPUTE: usize = 1; + +/// Checks the ordering of one command (the semantics of `g17::job::Order`) after `preceding` +/// commands of this SUBMIT, on a queue that accepted `base` commands before it. A zero barrier +/// waits for every earlier SUBMIT; NONE waits for nothing in this SUBMIT; a prior names an +/// ordinal accepted before it. The ordinals themselves are not kept: the engine's total order +/// satisfies every valid one. +fn check_order(base: [u64; 2], preceding: [u16; 2], barriers: [u16; 2], prior: [u64; 2]) -> Result { + for index in 0..2 { + // Earlier commands in this ioctl are not historical commands. + if prior[index] > base[index] { + return Err(EINVAL); + } + base[index] + .checked_add(u64::from(preceding[index])) + .and_then(|value| value.checked_add(1)) + .ok_or(EOVERFLOW)?; + if barriers[index] != NONE { + if barriers[index] > preceding[index] { + return Err(EINVAL); + } + base[index] + .checked_add(u64::from(barriers[index])) + .ok_or(EOVERFLOW)?; + } + } + Ok(()) +} + +/// A validated render command. +pub(crate) struct RenderCommand { + /// The payload, zero-extended to the full UAPI structure. + pub(crate) cmd: uapi::drm_asahi_cmd_render, + /// Firmware arena VAs of the timestamp slots (vertex start/end, fragment start/end), 0 if + /// unused. + pub(crate) timestamps: [u64; 4], +} + +/// A validated compute command. +pub(crate) struct ComputeCommand { + pub(crate) cmd: uapi::drm_asahi_cmd_compute, + /// The compute attachments in effect (validated writable). + pub(crate) attachments: KVec, + /// Firmware arena VAs of the start and end timestamp slots, 0 if unused. + pub(crate) timestamps: [u64; 2], +} + +/// A validated command. +pub(crate) enum Command { + Render(KBox), + Compute(KBox), +} + +/// A validated SUBMIT. +pub(crate) struct Submission { + /// The commands, in submission order. + pub(crate) commands: KVec, + /// Render and compute commands, for the queue's EXACT_PRIOR ordinals. + pub(crate) counts: [u64; 2], + /// The timestamp buffers the commands name, retained with the job. + pub(crate) timestamps: KVec>, +} + +/// What the per-command checks need. +pub(crate) struct Context<'a> { + /// The submitting VM. + pub(crate) vm: &'a mmu::Vm, + /// The queue's USC window base. + pub(crate) usc_base: u64, +} + +/// A stream whose timestamps and ordinals are resolved ([`prepare`]), before its spans are +/// checked ([`Prepared::validate`]). +pub(crate) struct Prepared<'a> { + bytes: &'a [u8], + /// Arena VAs of each command's timestamp slots. + resolved: KVec<[u64; 4]>, + counts: [u64; 2], + timestamps: KVec>, +} + +/// One command of the stream. +struct Entry<'a> { + kind: u32, + barriers: [u16; 2], + payload: &'a [u8], +} + +/// Iterates the commands of a structurally valid stream. +struct Stream<'a> { + bytes: &'a [u8], +} + +impl<'a> Iterator for Stream<'a> { + type Item = Result>; + + fn next(&mut self) -> Option { + if self.bytes.is_empty() { + return None; + } + let entry = (|| { + if self.bytes.len() < HEADER_SIZE { + return Err(EINVAL); + } + let header: uapi::drm_asahi_cmd_header = Reader::new(self.bytes).read()?; + let size = header.size as usize; + let payload = self + .bytes + .get(HEADER_SIZE..HEADER_SIZE + size) + .ok_or(EINVAL)?; + if size % 8 != 0 { + return Err(EINVAL); + } + self.bytes = &self.bytes[HEADER_SIZE + size..]; + Ok(Entry { + kind: header.cmd_type.into(), + barriers: [header.vdm_barrier, header.cdm_barrier], + payload, + }) + })(); + if entry.is_err() { + self.bytes = &[]; + } + Some(entry) + } +} + +fn stream(bytes: &[u8]) -> Stream<'_> { + Stream { bytes } +} + +/// Decodes a payload: short ones are zero-filled, bytes past the known structure must be zero. +fn payload(bytes: &[u8]) -> Result { + if bytes + .get(size_of::()..) + .is_some_and(|tail| tail.iter().any(|&byte| byte != 0)) + { + return Err(EINVAL); + } + Reader::new(bytes).read_up_to(bytes.len()) +} + +/// Whether `[start, start + size)` is a nonempty range inside `range`. +fn inside(start: u64, size: u64, range: &Range) -> bool { + size != 0 && start >= range.start && start < range.end && size <= range.end - start +} + +fn helper_is_empty(helper: &uapi::drm_asahi_helper_program) -> bool { + helper.binary == 0 && helper.cfg == 0 && helper.data == 0 +} + +/// Structural validation of a whole command stream, before anything is looked up. +pub(crate) fn check_stream(bytes: &[u8], early_tiling: bool) -> Result { + if bytes.is_empty() || bytes.len() > MAX_CMDBUF || bytes.len() % 8 != 0 { + return Err(EINVAL); + } + let range = hw::USER_RANGE; + let mut counts = [0u32; 2]; + for entry in stream(bytes) { + let entry = entry?; + match entry.kind { + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS | SET_COMPUTE_ATTACHMENTS => { + if entry.barriers != [NONE; 2] + || entry.payload.len() % ATTACHMENT_SIZE != 0 + || entry.payload.len() / ATTACHMENT_SIZE > hw::sizes::MAX_ATTACHMENTS + { + return Err(EINVAL); + } + for bytes in entry.payload.chunks_exact(ATTACHMENT_SIZE) { + let attachment: uapi::drm_asahi_attachment = Reader::new(bytes).read()?; + if attachment.pad != 0 + || attachment.flags != 0 + || !inside(attachment.pointer, attachment.size, &range) + { + return Err(EINVAL); + } + } + continue; + } + _ => {} + } + if (entry.barriers[RENDER] != NONE && u32::from(entry.barriers[RENDER]) > counts[RENDER]) + || (entry.barriers[COMPUTE] != NONE + && u32::from(entry.barriers[COMPUTE]) > counts[COMPUTE]) + { + return Err(EINVAL); + } + match entry.kind { + CMD_RENDER => { + let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; + // G15S resource words are 64-bit: RSRC_SPEC_HI is always accepted. + let allowed = VERTEX_SCRATCH + | PROCESS_EMPTY_TILES + | NO_VERTEX_CLUSTERING + | DBIAS_IS_INT + | RSRC_SPEC_HI + | if early_tiling { FRAGMENT_BARRIERS } else { 0 }; + if cmd.flags & !allowed != 0 + || !(1..=MAX_DIMENSION).contains(&cmd.width_px) + || !(1..=MAX_DIMENSION).contains(&cmd.height_px) + || !(1..=MAX_LAYERS).contains(&cmd.layers) + || !matches!(cmd.samples, 1 | 2 | 4) + { + return Err(EINVAL); + } + // RENDER_RAY_QUERY is never advertised. + if entry + .payload + .get(RAY_QUERY..) + .is_some_and(|tail| tail.iter().any(|&byte| byte != 0)) + { + return Err(EINVAL); + } + // Never discard unflagged high words, nor accept the flag without them. + if cmd.flags & RSRC_SPEC_HI != 0 && entry.payload.len() < RSRC_SPEC_HI_MIN { + return Err(EINVAL); + } + if cmd.flags & RSRC_SPEC_HI == 0 + && (cmd.bg_eot_rsrc_spec_hi != 0 + || cmd.bg_eot_partial_rsrc_spec_hi != 0 + || cmd.bg_rsrc_spec_hi != 0 + || cmd.bg_partial_rsrc_spec_hi != 0) + { + return Err(EINVAL); + } + counts[RENDER] += 1; + } + CMD_COMPUTE => { + let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; + if cmd.flags != 0 + || cmd.cdm_ctrl_stream_end <= cmd.cdm_ctrl_stream_base + || !inside( + cmd.cdm_ctrl_stream_base, + cmd.cdm_ctrl_stream_end - cmd.cdm_ctrl_stream_base, + &range, + ) + { + return Err(EINVAL); + } + // The G17P scratch layout and intersection tables: never on G15. + if cmd.scs_layout != 0 + || cmd.cdm_auxiliary_allocation != 0 + || cmd.reserved != 0 + || cmd.intersection_pool != 0 + || cmd.intersection_pool_bytes != 0 + || cmd.intersection_first != 0 + || cmd.intersection_count != 0 + || cmd.intersection_default_program != 0 + || cmd.intersection_default_control != 0 + { + return Err(EINVAL); + } + counts[COMPUTE] += 1; + } + _ => return Err(EINVAL), + } + if counts[RENDER] + counts[COMPUTE] > MAX_COMMANDS { + return Err(EINVAL); + } + } + if counts == [0; 2] { + return Err(EINVAL); + } + Ok(()) +} + +/// The timestamp buffers named by a submission, resolved to firmware arena VAs. +struct Timestamps<'a> { + objects: Pin<&'a xarray::XArray>>, + held: KVec>, +} + +impl Timestamps<'_> { + /// The arena VA of one requested timestamp, 0 if none: unknown handle ENOENT; offset not + /// 8-aligned or not 8 bytes inside the bound range EINVAL. + fn resolve(&mut self, timestamp: &uapi::drm_asahi_timestamp) -> Result { + if timestamp.handle == 0 { + return Ok(0); + } + let object = self + .objects + .lock() + .get(timestamp.handle.try_into()?) + .ok_or(ENOENT)? + .clone(); + let file::Object::TimestampBuffer(mapping) = object; + let offset = timestamp.offset as usize; + if offset % 8 != 0 || offset >= mapping.size() || mapping.size() - offset < 8 { + return Err(EINVAL); + } + let va = mapping.iova() + offset as u64; + if !self.held.iter().any(|held| Arc::ptr_eq(held, &mapping)) { + self.held.push(mapping, GFP_KERNEL)?; + } + Ok(va) + } +} + +/// Checks that `[start, end)` (rounded out to pages) is user memory mapped with the requested +/// GPU access: a driver range ENOENT, outside the user range EINVAL, an unmapped page ENOENT, a +/// page without the access EACCES. +pub(crate) fn cover(vm: &mmu::Vm, start: u64, end: u64, read: bool, write: bool) -> Result { + let mask = mmu::UAT_PGMSK as u64; + if end <= start { + return Err(EINVAL); + } + let first = start & !mask; + let last = end.checked_add(mask).ok_or(EINVAL)? & !mask; + // Every G15 driver mapping of a user VM is a client alias inside the fixed reserved ranges + // (`vm::ClientAlias::check`), so testing those three ranges finds every driver range without + // walking the alias mappings under the VM's lifetime lock on each span. + if super::vm::reserved_ranges() + .iter() + .any(|reserved| first < reserved.end && reserved.start < last) + { + return Err(ENOENT); + } + let range = hw::USER_RANGE; + if first < range.start || last > range.end { + return Err(EINVAL); + } + if vm.covers_range(first, last - first, read, write) { + Ok(()) + } else if vm.covers_range(first, last - first, false, false) { + Err(EACCES) + } else { + Err(ENOENT) + } +} + +/// Attachment spans, 128-byte granular, actually mapped with `write` or read access. +fn check_attachments(vm: &mmu::Vm, list: &[uapi::drm_asahi_attachment], write: bool) -> Result { + for attachment in list { + let end = attachment + .pointer + .checked_add(attachment.size) + .ok_or(EINVAL)?; + if end > u64::MAX - 127 { + return Err(EINVAL); + } + let first = attachment.pointer & !127; + let last = (end + 127) & !127; + if (last - first) / 128 > u64::from(u32::MAX) { + return Err(EINVAL); + } + cover(vm, first, last, !write, write)?; + } + Ok(()) +} + +/// Parses the attachment list of a SET_*_ATTACHMENTS command. +fn attachment_list(bytes: &[u8]) -> Result> { + let mut list = KVec::with_capacity(bytes.len() / ATTACHMENT_SIZE, GFP_KERNEL)?; + for bytes in bytes.chunks_exact(ATTACHMENT_SIZE) { + list.push(Reader::new(bytes).read()?, GFP_KERNEL)?; + } + Ok(list) +} + +fn copy_list(lists: &[&[uapi::drm_asahi_attachment]]) -> Result> { + let mut copy = KVec::with_capacity(lists.iter().map(|list| list.len()).sum(), GFP_KERNEL)?; + for list in lists { + for attachment in list.iter() { + copy.push(*attachment, GFP_KERNEL)?; + } + } + Ok(copy) +} + +/// Whether a GVA can be named by a 32-bit compact pointer. +fn compact(gva: u64) -> bool { + hw::client::COMPACT_WINDOW.contains(&gva) +} + +/// TPC and tilemap bytes of a render pass (the encoder computes the full geometry with the same +/// formulas). +fn geometry_bytes(cmd: &uapi::drm_asahi_cmd_render) -> (u64, u64) { + let tiles_x = u64::from(cmd.width_px).div_ceil(32); + let tiles_y = u64::from(cmd.height_px).div_ceil(32); + let blocks_x = tiles_x.div_ceil(4).next_multiple_of(4); + let blocks_y = tiles_y.div_ceil(4).next_multiple_of(4); + let sx = blocks_x * if cmd.utile_width_px == 16 { 2 } else { 1 }; + let sy = blocks_y * if cmd.utile_height_px == 16 { 2 } else { 1 }; + let e = (4 * sx) * (4 * sy); + let layers = u64::from(cmd.layers); + let tilemap = (5 * e * layers).next_multiple_of(0x1000); + let tpc = 2 * e * 8 * layers; + (tpc, tilemap) +} + +/// Per-command render admission. +fn check_render(context: &Context<'_>, cmd: &uapi::drm_asahi_cmd_render) -> Result { + let vm = context.vm; + let allowed = PROCESS_EMPTY_TILES + | NO_VERTEX_CLUSTERING + | RSRC_SPEC_HI + | DBIAS_IS_INT + | if super::params::early_tiling() { + FRAGMENT_BARRIERS + } else { + 0 + }; + let tile_buffer = u32::from(cmd.sample_size_B) + * u32::from(cmd.utile_width_px) + * u32::from(cmd.utile_height_px) + * u32::from(cmd.samples); + if cmd.flags & !allowed != 0 + || !(1..=MAX_DIMENSION).contains(&cmd.width_px) + || !(1..=MAX_DIMENSION).contains(&cmd.height_px) + || !(1..=MAX_LAYERS).contains(&cmd.layers) + || !matches!(cmd.samples, 1 | 2 | 4) + || !matches!(cmd.utile_width_px, 16 | 32) + || !matches!(cmd.utile_height_px, 16 | 32) + || cmd.sample_size_B % 8 != 0 + || tile_buffer > MAX_TILE_BUFFER + || cmd.sampler_heap != 0 + || cmd.sampler_count != 0 + || !helper_is_empty(&cmd.vertex_helper) + || !helper_is_empty(&cmd.fragment_helper) + || cmd.ppp_ctrl & !0x203 != 0 + { + return Err(EINVAL); + } + if cmd.stencil.comp_base != 0 + || cmd.stencil.comp_stride != 0 + || (cmd.depth.comp_base != 0 && (cmd.layers != 1 || cmd.samples != 1)) + { + return Err(Error::from_errno(-(kernel::bindings::EOPNOTSUPP as i32))); + } + // The FS work's partial-EOT resource slots are 32 bits wide: a high word could only be + // dropped. + if cmd.flags & RSRC_SPEC_HI != 0 && cmd.bg_eot_partial_rsrc_spec_hi != 0 { + return Err(EINVAL); + } + // VDM: compact-addressable and readable; scissor: 8-aligned and readable. + let vdm = cmd.vdm_ctrl_stream_base; + let scissor = cmd.isp_scissor_base; + if vdm & 3 != 0 || !compact(vdm) || scissor == 0 || scissor & 7 != 0 { + return Err(EINVAL); + } + cover(vm, vdm, vdm + 4, true, false)?; + gate::vdm(vm, vdm)?; + cover(vm, scissor, scissor.checked_add(8).ok_or(EINVAL)?, true, false)?; + if cmd.isp_dbias_base != 0 { + let base = cmd.isp_dbias_base; + cover(vm, base, base.checked_add(8).ok_or(EINVAL)?, true, false)?; + } + if cmd.isp_oclqry_base != 0 { + let base = cmd.isp_oclqry_base & !15; + cover(vm, base, base.checked_add(8).ok_or(EINVAL)?, false, true)?; + } + for program in [&cmd.bg, &cmd.eot, &cmd.partial_bg, &cmd.partial_eot] { + if program.usc == 0 { + continue; + } + let address = context + .usc_base + .checked_add(u64::from(program.usc & !63)) + .ok_or(EINVAL)?; + cover( + vm, + address, + address.checked_add(64).ok_or(EINVAL)?, + true, + false, + )?; + } + let (tpc, tilemap) = geometry_bytes(cmd); + if tpc > hw::client::TPC_SIZE || tilemap > hw::client::TILEMAP_SIZE { + return Err(E2BIG); + } + Ok(()) +} + +/// Per-command compute admission. +fn check_compute( + context: &Context<'_>, + cmd: &uapi::drm_asahi_cmd_compute, + attachments: &[uapi::drm_asahi_attachment], +) -> Result { + let vm = context.vm; + let (base, end) = (cmd.cdm_ctrl_stream_base, cmd.cdm_ctrl_stream_end); + if cmd.flags != 0 + || cmd.sampler_count != 0 + || cmd.sampler_heap != 0 + || !helper_is_empty(&cmd.helper) + || base & 3 != 0 + || end & 3 != 0 + || end <= base + { + return Err(EINVAL); + } + cover(vm, base, end, true, false)?; + gate::cdm(vm, context.usc_base, base)?; + check_attachments(vm, attachments, true) +} + +/// Resolves the timestamps and EXACT_PRIOR ordinals of a structurally valid stream +/// ([`check_stream`]) for a queue that accepted `accepted` render and compute commands before +/// this ioctl. +pub(crate) fn prepare<'a>( + bytes: &'a [u8], + accepted: [u64; 2], + objects: Pin<&xarray::XArray>>, +) -> Result> { + // Timestamps of every command first. + let mut timestamps = Timestamps { + objects, + held: KVec::new(), + }; + let mut resolved: KVec<[u64; 4]> = KVec::new(); + for entry in stream(bytes) { + let entry = entry?; + let slots = match entry.kind { + CMD_RENDER => { + let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; + [ + timestamps.resolve(&cmd.ts_vtx.start)?, + timestamps.resolve(&cmd.ts_vtx.end)?, + timestamps.resolve(&cmd.ts_frag.start)?, + timestamps.resolve(&cmd.ts_frag.end)?, + ] + } + CMD_COMPUTE => { + let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; + [ + timestamps.resolve(&cmd.ts.start)?, + timestamps.resolve(&cmd.ts.end)?, + 0, + 0, + ] + } + _ => continue, + }; + resolved.push(slots, GFP_KERNEL)?; + } + + // Historical priors and header barriers. + let mut preceding = [0u16; 2]; + for entry in stream(bytes) { + let entry = entry?; + let (engine, prior) = match entry.kind { + CMD_RENDER => { + let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; + (RENDER, [cmd.prior_vdm, cmd.prior_cdm]) + } + CMD_COMPUTE => { + let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; + (COMPUTE, [cmd.prior_vdm, cmd.prior_cdm]) + } + _ => continue, + }; + check_order(accepted, preceding, entry.barriers, prior)?; + preceding[engine] += 1; + } + + Ok(Prepared { + bytes, + resolved, + counts: [u64::from(preceding[RENDER]), u64::from(preceding[COMPUTE])], + timestamps: timestamps.held, + }) +} + +impl Prepared<'_> { + /// Checks each command's spans, programs and stream gates, with a job reference on the VM + /// already held so that the checked mappings are retained until the job retires. + pub(crate) fn validate(self, context: &Context<'_>) -> Result { + let Prepared { + bytes, + resolved, + counts, + timestamps, + } = self; + let mut commands = KVec::with_capacity(resolved.len(), GFP_KERNEL)?; + let mut compute = KVec::new(); + let mut index = 0; + for entry in stream(bytes) { + let entry = entry?; + match entry.kind { + SET_VERTEX_ATTACHMENTS | SET_FRAGMENT_ATTACHMENTS => { + // Validated readable, never serialized on G15. + check_attachments(context.vm, &attachment_list(entry.payload)?, false)?; + } + SET_COMPUTE_ATTACHMENTS => compute = attachment_list(entry.payload)?, + CMD_RENDER => { + let cmd: uapi::drm_asahi_cmd_render = payload(entry.payload)?; + check_render(context, &cmd)?; + commands.push( + Command::Render(KBox::new( + RenderCommand { + cmd, + timestamps: *resolved.get(index).ok_or(EINVAL)?, + }, + GFP_KERNEL, + )?), + GFP_KERNEL, + )?; + index += 1; + } + CMD_COMPUTE => { + let cmd: uapi::drm_asahi_cmd_compute = payload(entry.payload)?; + check_compute(context, &cmd, &compute)?; + let slots = resolved.get(index).ok_or(EINVAL)?; + commands.push( + Command::Compute(KBox::new( + ComputeCommand { + cmd, + attachments: copy_list(&[&compute])?, + timestamps: [slots[0], slots[1]], + }, + GFP_KERNEL, + )?), + GFP_KERNEL, + )?; + index += 1; + } + _ => return Err(EINVAL), + } + } + if commands.is_empty() { + return Err(EINVAL); + } + Ok(Submission { + counts, + commands, + timestamps, + }) + } +} diff --git a/drivers/gpu/drm/asahi/g15/encode.rs b/drivers/gpu/drm/asahi/g15/encode.rs new file mode 100644 index 0000000000000..e46ecd05bdf78 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/encode.rs @@ -0,0 +1,1017 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Work-image and microsequence encoders. +//! +//! Stamps for ordinal n start from the seeds compute 0xc1000000, TA 0x7a000000 and FS +//! 0x3d000000. The compute image sets the compute fault mask at start +0x30 = 1 and interior +//! +0x804 = 1. [`Geometry`] holds the tiling geometry and register words derived from +//! `drm_asahi_cmd_render`; [`ClusterLayout`] the 64 segments and M1..M4 of clustered tiling. +//! Program addresses are `usc_base + (usc & !63)`; resource words take optional high 32 bits +//! (G15S resource words are 64-bit). The TA image sets register #33 PBREQ = initial PB size and +//! interior +0x880 = 1; the FS image carries the partial background/EOT programs for partial +//! renders (the partial-EOT resource word is 32-bit). The scene and InitBM records prefix a +//! render. +//! +//! Pure byte builders over [`super::fw::work`] layouts: no locks, no allocation, no +//! publication. Every address an image names comes in through [`ComputeRefs`], [`RenderRefs`] +//! and [`PassRefs`], resolved by the engine from its firmware objects; the firmware writes +//! timestamps in ns directly through the timestamp arena VAs. The words are written at literal +//! offsets in image order. This file only depends on `core`, the UAPI and the `hw`/`fw::work` +//! constants. + +use super::wire as uapi; + +use super::fw::work::{self, barrier, compute, fs, initbm, scene, ta, tail, ts_record, REG_RECORD}; +use super::hw; + +/// Stores `v` little-endian at `off`. Every offset is a literal inside a fixed-size image; an +/// out-of-range store is dropped rather than panicking. +#[inline(always)] +fn put(b: &mut [u8], off: usize, v: &[u8]) { + if let Some(dst) = b.get_mut(off..off + v.len()) { + dst.copy_from_slice(v); + } +} + +#[inline(always)] +fn put8(b: &mut [u8], off: usize, v: u8) { + put(b, off, &[v]); +} + +#[inline(always)] +fn put16(b: &mut [u8], off: usize, v: u16) { + put(b, off, &v.to_le_bytes()); +} + +#[inline(always)] +fn put32(b: &mut [u8], off: usize, v: u32) { + put(b, off, &v.to_le_bytes()); +} + +#[inline(always)] +fn put64(b: &mut [u8], off: usize, v: u64) { + put(b, off, &v.to_le_bytes()); +} + +/// `DIV_ROUND_UP`. +const fn div_round_up(n: u64, d: u64) -> u64 { + n.div_ceil(d) +} + +/// `round_up` / `ALIGN` to a power of two, wrapping on overflow (no panic). +const fn align(n: u64, a: u64) -> u64 { + n.wrapping_add(a - 1) & !(a - 1) +} + +/// Firmware stamp value of ordinal `n` on the queue with this seed. +/// Stamps step by 0x100 through 0xffffff values and never read 0. +pub(crate) const fn stamp_seed(seed: u32, n: u64) -> u32 { + let base = (seed / 0x100) as u64; + ((((base - 1 + n % 0xffffff) % 0xffffff) + 1) * 0x100) as u32 +} + +/// Compute stamp of ordinal `n`. +pub(crate) const fn stamp(n: u64) -> u32 { + stamp_seed(hw::engine::COMPUTE_SEED, n) +} + +/// 32-bit compact pointer of a GVA in the compact window. +pub(crate) const fn compact(gva: u64) -> u64 { + gva.wrapping_sub(0x1000000000) +} + +/// Addresses the compute encoder names: compute graph and compute slot buffers. +#[derive(Copy, Clone, Default)] +pub(crate) struct ComputeRefs { + /// HardwareData (firmware VA): the activity records the sequence names. + pub(crate) hwdata: u64, + /// Graph: notifier, queue info, user/firmware stamps, UMA manager (firmware VAs). + pub(crate) notifier: u64, + pub(crate) qinfo: u64, + pub(crate) user_stamp: u64, + pub(crate) fw_stamp: u64, + pub(crate) manager: u64, + /// Slot: work image (firmware VA and client GVA of the payload), sequence, preemption + /// buffer (client GVA), internal timestamp cells, tail and sequence scratch (firmware VAs). + pub(crate) work: u64, + pub(crate) work_gva: u64, + pub(crate) seq: u64, + pub(crate) preempt_gva: u64, + pub(crate) its: u64, + pub(crate) tail: u64, + pub(crate) scratch: [u64; 5], +} + +/// Encodes compute command `c` as ordinal `n`. The engine copies the result into the slot +/// after zeroing the slot's images. +#[allow(clippy::too_many_arguments)] +pub(crate) fn encode_compute( + r: &ComputeRefs, + regs_cfg: &hw::ComputeRegs, + c: &uapi::drm_asahi_cmd_compute, + att: &[uapi::drm_asahi_attachment], + ts_va: &[u64; 2], + ctx: u32, + n: u64, + fault_mask: u32, + w: &mut [u8; compute::WORK_SIZE], + q: &mut [u8; compute::SEQ_SIZE], +) { + let (wv, qv, h, pre) = (r.work, r.qinfo, r.hwdata, r.preempt_gva); + let t = stamp(n); + let att_count = att.len().min(hw::sizes::MAX_ATTACHMENTS); + // The saved-state pointers into the preemption buffer and the two 0x1c words; a SoC + // encoded "fresh" carries zero in all six, as a first command does. + let (saved, word) = if regs_cfg.fresh { + (0, 0) + } else { + (pre + regs_cfg.tail, 0x1c) + }; + let step = |i: u64| if saved == 0 { 0 } else { saved + 8 * i }; + let val: [u64; 22] = [ + pre, + c.cdm_ctrl_stream_base, + step(0), + step(1), + step(2), + step(3), + 0x154024201, + 0x10c08860, + word, + word, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + regs_cfg.reg_1a0e9, + 0xff0000, + regs_cfg.ptc, + regs_cfg.dupm, + ]; + + w.fill(0); + q.fill(0); + + // Work image. + put32(w, 0x0, work::work_type::COMPUTE); + put64(w, 0x4, (n as u32).wrapping_sub(1) as u64); + put32(w, 0x10, ctx); + put64(w, 0x14, r.notifier); + regs(&mut w[compute::REGS..], &compute::SEL, &val); + put64(w, 0x720, r.work_gva + compute::REGS as u64); + put16(w, 0x728, 22); + put16(w, 0x72a, (22 * REG_RECORD) as u16); + put64(w, 0x760, r.seq); + put32(w, 0x768, compute::SEQ_SIZE as u32); + put64(w, 0x794, pre); + put64(w, 0x79c, c.cdm_ctrl_stream_end.wrapping_sub(4)); + put64(w, 0x7c4, 0x154024201); + put32(w, 0x7e0, 0xffffffff); + put64(w, 0x7e8, r.user_stamp); + put64(w, 0x7f0, r.fw_stamp); + put32(w, 0x7f8, t); + put32(w, 0x7fc, hw::engine::EVENT_SLOT); + put32(w, 0x804, 1); + put32(w, 0x808, hw::engine::COMPUTE_UUID); + put32(w, 0x80c, (n as u32).wrapping_sub(1)); + put64(w, 0x818, r.its); + put64(w, 0x820, r.its + 8); + put64(w, 0x828, ts_va[0]); + put64(w, 0x830, ts_va[1]); + put64(w, 0x83e, r.manager); + put8(w, 0x846, 1); + put64(w, 0x847, 0xad0000); + put64(w, 0x84f, 0x1600000); + put64(w, 0x857, r.tail); + put8(w, 0x85f, 1); + + // Microsequence: start. + put32(q, 0x0, work::op::COMPUTE_START); + put64(q, 0x14, wv + 0x20); + put64(q, 0x1c, h + compute::HWDATA); + put64(q, 0x24, qv); + put32(q, 0x2c, ctx); + put32(q, 0x30, if fault_mask != 0 { 1 } else { 0 }); + put32(q, 0x34, 0); + put64(q, 0x44, wv + 0x76c); + put32(q, 0x50, hw::engine::COMPUTE_UUID); + for (a, at) in att.iter().take(att_count).enumerate() { + let base = at.pointer & !127; + let end = align(at.pointer.wrapping_add(at.size), 128); + let rec = compute::ATTACHMENTS + compute::ATTACHMENT_STRIDE * a; + + put64(q, rec, base); + put32(q, rec + 8, (end.wrapping_sub(base) / 128) as u32); + put16(q, rec + 0xc, 0x17); + put16(q, rec + 0xe, 1); + } + put32(q, 0x154, att_count as u32); + put64(q, 0x15c, r.manager); + put64(q, 0x174, r.scratch[0]); + put64(q, 0x184, r.scratch[1]); + put64(q, 0x18c, r.scratch[2]); + put64(q, 0x194, r.scratch[3]); + put64(q, 0x19c, r.scratch[4]); + put64(q, 0x1a4, wv + 0x880); + put64(q, 0x1b4, r.notifier + 0xa8); + // Start and end timestamp records: the 0x3c record over the cells at W + 0x810. Both + // records name the user pair base when any output is held. + let requested = ts_va[0] != 0 || ts_va[1] != 0; + for t_end in [false, true] { + let at = if t_end { + compute::TS_END + } else { + compute::TS_START + }; + ts_record( + &mut q[at..], + t_end, + wv, + compute::TS_CELLS, + qv, + requested, + hw::engine::COMPUTE_UUID, + ); + } + put32(q, 0x1f8, 1); + // Finalize and END. + let f = compute::FINALIZE; + put32(q, f, work::op::COMPUTE_FINALIZE); + put64(q, f + 0x4, h + compute::HWDATA); + put64(q, f + 0xc, qv); + put32(q, f + 0x14, ctx); + put64(q, f + 0x18, wv + 0x76c); + put32(q, f + 0x24, hw::engine::COMPUTE_UUID); + put64(q, f + 0x28, r.fw_stamp); + put32(q, f + 0x30, t); + put64(q, f + 0x58, wv + 0x83e); + put32(q, f + 0x60, 0u32.wrapping_sub(f as u32)); + put8(q, f + 0x64, if att_count != 0 { 1 } else { 0 }); + put64(q, f + 0x65, wv + 0x880); + put64(q, f + 0x6d, wv + 0x860); + put32(q, compute::END, work::op::END); +} + +/// Tiling geometry and register words of a render (accelerator count 2). +#[derive(Copy, Clone, Default, Debug, PartialEq, Eq)] +pub(crate) struct Geometry { + pub(crate) tx: u32, + pub(crate) ty: u32, + pub(crate) bx: u32, + pub(crate) by: u32, + pub(crate) sx: u32, + pub(crate) sy: u32, + pub(crate) pixels: u32, + pub(crate) screen: u32, + pub(crate) blocks: u32, + pub(crate) tiles: u32, + pub(crate) region_stride: u32, + pub(crate) tpc_stride: u32, + pub(crate) isp_blocks: u32, + pub(crate) utile_config: u32, + pub(crate) layer_mode: u32, + pub(crate) tilebuffer_control: u32, + pub(crate) tile_mode: u32, + pub(crate) isp_control: u32, + pub(crate) e: u64, + pub(crate) x_blocks: u64, + pub(crate) y_blocks: u64, + pub(crate) tilemap_bytes: u64, + pub(crate) tpc_bytes: u64, + pub(crate) layer_metadata: u64, + pub(crate) native_tiles: u64, +} + +impl Geometry { + /// The geometry of render `u` with layer metadata at `heap`. The render was validated + /// (width, height, layers >= 1, samples 1, 2 or 4, utiles 16 or 32). + pub(crate) fn new(u: &uapi::drm_asahi_cmd_render, heap: u64) -> Self { + let (w, h, l) = (u.width_px as u32, u.height_px as u32, u.layers as u32); + let empty = + u.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0; + let log2s = (u.samples as u32).max(1).ilog2(); + let mut g = Geometry::default(); + + g.tx = div_round_up(w as u64, 32) as u32; + g.ty = div_round_up(h as u64, 32) as u32; + g.bx = align(div_round_up(g.tx as u64, 4), 4) as u32; + g.by = align(div_round_up(g.ty as u64, 4), 4) as u32; + g.sx = g.bx * if u.utile_width_px == 16 { 2 } else { 1 }; + g.sy = g.by * if u.utile_height_px == 16 { 2 } else { 1 }; + g.e = (4 * g.sx as u64) * (4 * g.sy as u64); + g.pixels = w.wrapping_sub(1) | h.wrapping_sub(1) << 16; + g.screen = g.tx.wrapping_sub(1) | g.ty.wrapping_sub(1) << 12; + g.x_blocks = (g.bx as u64) << 10 | (g.bx as u64) << 18 | 3 * g.bx as u64; + g.y_blocks = (g.by as u64) << 18 | (g.by as u64) << 10 | 3 * g.by as u64; + g.blocks = g.bx * g.by; + g.tiles = g.tx * g.ty; + g.region_stride = (5 * g.e / 64) as u32; + g.tpc_stride = (g.e / 8) as u32; + g.isp_blocks = ((g.sx & 0x3fd) << 16) | (g.sy & 0x3fd); + let utile_w = if u.utile_width_px == 32 { + 0x2000 + } else { + 0x1000 + }; + let utile_h = if u.utile_height_px == 32 { + 0x8000 + } else { + 0x4000 + }; + g.utile_config = utile_w | utile_h | log2s; + g.tilemap_bytes = align(5 * g.e * l as u64, 0x1000); + g.tpc_bytes = 2 * g.e * 8 * l as u64; + g.layer_metadata = heap + if l > 1 { 0x200 } else { 0 }; + g.layer_mode = if l == 1 { + 0x8000 + } else { + 0xc000 | (if empty { 0x2000 } else { 0x1000 }) | l.wrapping_sub(1) + }; + g.tilebuffer_control = div_round_up( + u.sample_size_B as u64 + * u.utile_width_px as u64 + * u.utile_height_px as u64 + * u.samples as u64, + 2048, + ) as u32 + | log2s << 17; + g.tile_mode = 0x280 | (l > 1) as u32 | if empty { 1 << 16 } else { 0 }; + g.isp_control = 0xc000 | (u.flags & (1 << 18)); + g.native_tiles = 221201u64 + | ((l > 1) as u64) << 32 + | (g.tx.wrapping_sub(1) as u64) << 44 + | (g.ty.wrapping_sub(1) as u64) << 53 + | 137438953472u64 + | ((g.utile_config & 0xf000) as u64) << 28; + g + } +} + +/// Clustered tiling layout from `base`: 64 segments of 5E bytes, then M2 (0xc00), M3 (0xa00), +/// M4 ((Q + 1) x 512 for Q = pages of the owned PB / 0x1800, rounded up) and the 128-aligned +/// 0x80-byte M1. +#[derive(Copy, Clone, Default, Debug, PartialEq, Eq)] +pub(crate) struct ClusterLayout { + pub(crate) ct: u64, + pub(crate) m1: u64, + pub(crate) m2: u64, + pub(crate) m3: u64, + pub(crate) m4: u64, + pub(crate) bytes: u64, +} + +impl ClusterLayout { + pub(crate) fn new(geo: &Geometry, base: u64) -> Self { + let q = div_round_up(div_round_up(hw::client::PB_HEAP_SIZE, 0x8000), 0x1800); + let ct = base; + let m2 = base + align(64 * 5 * geo.e, 32); + let m3 = m2 + 0xc00; + let m4 = m3 + 0xa00; + let m1 = base + align(m4 - base + (q + 1) * 8 * 64, 128); + ClusterLayout { + ct, + m1, + m2, + m3, + m4, + bytes: m1 + 0x80 - base, + } + } +} + +/// Clustered tiling eligibility: one layer, one sample, 32x32 utiles, unless the submission +/// asked for single-cluster vertex work. The engine then also needs a clustering buffer that +/// holds [`ClusterLayout::bytes`]. +pub(crate) fn cluster_eligible(u: &uapi::drm_asahi_cmd_render, geo: &Geometry) -> bool { + hw::tuning::VERTEX_CLUSTERING + && u.layers == 1 + && geo.utile_config == 0xa000 + && u.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING == 0 +} + +/// Program address. +pub(crate) fn prog_addr(usc_base: u64, p: &uapi::drm_asahi_bg_eot) -> u64 { + usc_base.wrapping_add((p.usc & !63) as u64) +} + +/// Program resource word, 64-bit with RSRC_SPEC_HI. +pub(crate) fn prog_rsrc( + u: &uapi::drm_asahi_cmd_render, + p: &uapi::drm_asahi_bg_eot, + hi: u32, +) -> u64 { + if u.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI != 0 { + (hi as u64) << 32 | p.rsrc_spec as u64 + } else { + p.rsrc_spec as u64 + } +} + +/// Register records. +pub(crate) fn regs(dst: &mut [u8], sel: &[u32], val: &[u64]) { + for (i, (s, v)) in sel.iter().zip(val.iter()).enumerate() { + put32(dst, REG_RECORD * i, *s); + put64(dst, REG_RECORD * i + 4, *v); + } +} + +/// The 0x3c timestamp record. +pub(crate) fn ts_record( + rec: &mut [u8], + end: bool, + w: u64, + b: u64, + q: u64, + requested: bool, + uuid: u32, +) { + let rec = match rec.get_mut(..ts_record::SIZE) { + Some(rec) => rec, + None => return, + }; + put32( + rec, + 0x0, + if end { + work::op::TIMESTAMP_END + } else { + work::op::TIMESTAMP + }, + ); + put64(rec, 0x4, w + b); + put64(rec, 0xc, w + b + 8); + put64(rec, 0x14, w + b + if end { 0x10 } else { 8 }); + put64(rec, 0x1c, q); + put64(rec, 0x24, if requested { w + b + 0x18 } else { 0 }); + put64(rec, 0x2c, w + b + 0x58); + put32(rec, 0x34, uuid); +} + +/// Shared work tail (TA +0x8a8, FS +0xbf0); mirrors the compute tail. +pub(crate) fn work_tail( + t: &mut [u8], + its_start: u64, + its_end: u64, + user_start: u64, + user_end: u64, + uma_mgr: u64, + scratch: u64, +) { + put64(t, tail::ITS_START, its_start); + put64(t, tail::ITS_END, its_end); + put64(t, tail::USER_START, user_start); + put64(t, tail::USER_END, user_end); + put64(t, tail::UMA_MANAGER, uma_mgr); + put8(t, 0x36, 1); + put64(t, 0x37, 0xad0000); + put64(t, 0x3f, 0x1600000); + put64(t, tail::SCRATCH, scratch); + put8(t, 0x4f, 1); +} + +/// A barrier record waiting until the stamp at `stamp_va` reaches `wait`: +/// `event` names the producing engine (0 TA, 1 FS, 2 compute), `own` is the waiter's stamp. +pub(crate) fn barrier( + stamp_va: u64, + wait: u32, + event: u32, + own: u32, + uuid: u32, + ty: work::BarrierType, +) -> [u8; barrier::SIZE] { + let mut x = [0u8; barrier::SIZE]; + + put32(&mut x, 0x0, work::op::BARRIER); + put64(&mut x, barrier::STAMP_VA, stamp_va); + put64(&mut x, barrier::STAMP_VA2, stamp_va); + put32(&mut x, barrier::WAIT, wait); + put32(&mut x, barrier::EVENT, event); + put32(&mut x, barrier::OWN, own); + put32(&mut x, barrier::UUID, uuid); + put32(&mut x, barrier::TYPE, ty as u32); + x +} + +/// Addresses the render encoders name that belong to the render graph and its allocators +/// (the UMA/PB objects are in `Pools`). +#[derive(Copy, Clone, Default)] +pub(crate) struct RenderRefs { + /// HardwareData (firmware VA): the activity records the sequences name. + pub(crate) hwdata: u64, + pub(crate) notifier: u64, + pub(crate) legacy_empty: u64, + /// Queue info, user and firmware stamps of the TA (0) and FS (1) queues. + pub(crate) qinfo: [u64; 2], + pub(crate) user_stamp: [u64; 2], + pub(crate) fw_stamp: [u64; 2], + /// Allocators in use (after a rebuild possibly the spare set): UMA managers of TA and FS, + /// PB manager, scene list, misc record; and the initial PB blocks. + pub(crate) uma_mgr: [u64; 2], + pub(crate) pb_mgr: u64, + pub(crate) scene_list: u64, + pub(crate) mgr_misc: u64, + pub(crate) pb_init_blocks: u32, +} + +/// Addresses of one pass slot. +#[derive(Copy, Clone, Default)] +pub(crate) struct PassRefs { + /// Work images: firmware VA and client GVA of the payload. + pub(crate) ta_work: u64, + pub(crate) ta_work_gva: u64, + pub(crate) fs_work: u64, + pub(crate) fs_work_gva: u64, + pub(crate) ta_seq: u64, + pub(crate) fs_seq: u64, + pub(crate) scene: u64, + /// Client GVAs. + pub(crate) tpc_gva: u64, + pub(crate) tilemap_gva: u64, + pub(crate) heap_gva: u64, + pub(crate) scene_user_gva: u64, + pub(crate) pre_gva: [u64; 3], + pub(crate) fs_scratch_gva: u64, + pub(crate) aux_gva: u64, + /// Firmware VAs. + pub(crate) its: u64, + pub(crate) ta_scratch: u64, +} + +/// Encodes the TA work image and microsequence of a render. +#[allow(clippy::too_many_arguments)] +pub(crate) fn encode_ta( + r: &RenderRefs, + p: &PassRefs, + u: &uapi::drm_asahi_cmd_render, + ts_va: &[u64; 4], + ctx: u32, + tta: u32, + geo: &Geometry, + cl: Option<&ClusterLayout>, + w: &mut [u8; ta::WORK_SIZE], + q: &mut [u8; ta::SEQ_SIZE], +) { + let (wv, uv, qv, h) = (p.ta_work, p.ta_seq, r.qinfo[0], r.hwdata); + let (heap, su) = (p.heap_gva, p.scene_user_gva); + let ts = ts_va[0] != 0 || ts_va[1] != 0; + let mut val: [u64; 72] = [ + 1, + 0x200, + compact(p.tilemap_gva), + compact(p.tilemap_gva), + 0, + 0, + compact(p.tpc_gva), + compact(heap) | 1 << 63, + compact(heap) | 1 << 63, + 0x003a0012006b0003, + 1, + geo.utile_config as u64, + u.ppp_multisamplectl, + compact(p.pre_gva[0]), + compact(p.pre_gva[0]), + compact(p.pre_gva[1]), + compact(p.pre_gva[1]), + 1, + compact(p.pre_gva[2]) | 1 << 50, + 0, + compact(u.vdm_ctrl_stream_base), + 0, + 0, + 0, + 1, + 0, + compact(geo.layer_metadata), + compact(geo.layer_metadata), + 0, + 0, + 0, + 0, + 4 * r.pb_init_blocks as u64, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0x8860, + geo.region_stride as u64, + geo.region_stride as u64, + 0x88, + (2 | (u.ppp_ctrl & 0x201)) as u64, + geo.pixels as u64, + geo.screen as u64, + geo.x_blocks, + geo.y_blocks, + geo.blocks as u64, + geo.tpc_stride as u64, + 0x100, + geo.layer_mode as u64, + 0x1c, + 0x1c, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8, + 0xff0000, + 0x7f00400020, + 0x200000001, + 0, + compact(su + 0x40), + compact(su + 0x40), + 0xa000000000 | ((r.scene_list + 4) & 0xfffffffff) | 1, + ]; + + if let Some(cl) = cl { + // The ten clustered TA values, compact form. + val[4] = compact(cl.ct); + val[5] = compact(cl.ct); + val[22] = compact(cl.m2); + val[23] = compact(cl.m3); + val[24] = 0x200440; + val[25] = 4; + val[30] = compact(cl.m4); + val[31] = compact(cl.m4); + val[ta::REG_PBREQ] = hw::sizes::CLUSTER_PB_PAGES as u64; + val[33] = compact(cl.m1); + } + + w.fill(0); + q.fill(0); + + // TA work image. + put32(w, 0x0, work::work_type::TA); + put64(w, 0x4, 1); + put32(w, 0xc, ctx); + put64(w, 0x14, r.notifier); + put64(w, 0x24, r.pb_mgr); + put64(w, 0x2c, p.scene); + put64(w, 0x34, r.legacy_empty); + regs(&mut w[ta::REGS..], &ta::SEL, &val); + put64(w, 0x740, p.ta_work_gva + ta::REGS as u64); + put16(w, 0x748, 72); + put16(w, 0x74a, (72 * REG_RECORD) as u16); + put64(w, 0x760, p.tpc_gva); + put64(w, 0x768, geo.tpc_bytes); + put64(w, 0x770, uv); + put32(w, 0x778, ta::SEQ_SIZE as u32); + put32(w, 0x77c, 1); + put32(w, 0x780, tta); + put64(w, 0x7a8, compact(p.pre_gva[0])); + put32(w, 0x840, 0x67); + put32(w, 0x848, 0xffffffff); + put32(w, 0x854, 1); + put64(w, 0x864, r.user_stamp[0]); + put64(w, 0x86c, r.fw_stamp[0]); + put32(w, 0x874, tta); + put32(w, 0x880, 1); + put32(w, 0x884, hw::engine::TA_UUID); + work_tail( + &mut w[ta::TAIL..], + p.its, + p.its + 8, + ts_va[0], + ts_va[1], + r.uma_mgr[0], + p.ta_scratch, + ); + + // TA microsequence. + put32(q, 0x0, work::op::TA_START); + put64(q, 0x14, wv + 0x40); + put64(q, 0x1c, r.pb_mgr); + put64(q, 0x24, p.scene); + put64(q, 0x2c, h + ta::HWDATA); + put64(q, 0x34, qv); + put64(q, 0x3c, wv + 0x84c); + put32(q, 0x44, ctx); + put64(q, 0x64, wv + 0x784); + put64(q, 0x6c, wv + 0x890); + put64(q, 0x78, (hw::engine::TA_UUID as u64) << 32); + put64(q, 0x18c, r.uma_mgr[0]); + put64(q, 0x1a4, p.ta_scratch); + put64(q, 0x1b4, 1); + put64(q, 0x1bc, r.notifier + 0xa8); + ts_record( + &mut q[ta::TS_START..], + false, + wv, + ta::TAIL as u64, + qv, + ts, + hw::engine::TA_UUID, + ); + put32(q, 0x208, 1); + ts_record( + &mut q[ta::TS_END..], + true, + wv, + ta::TAIL as u64, + qv, + ts, + hw::engine::TA_UUID, + ); + let f = ta::FINALIZE; + put32(q, f, work::op::TA_FINALIZE); + put64(q, f + 0x4, p.scene); + put64(q, f + 0xc, r.pb_mgr); + put64(q, f + 0x14, h + ta::HWDATA); + put64(q, f + 0x1c, qv); + put64(q, f + 0x24, wv + 0x84c); + put32(q, f + 0x2c, ctx); + put64(q, f + 0x34, wv + 0x784); + put32(q, f + 0x40, hw::engine::TA_UUID); + put64(q, f + 0x44, r.fw_stamp[0]); + put32(q, f + 0x4c, tta); + put64(q, f + 0x74, uv + 0x18c); + put32(q, f + 0x7c, 0u32.wrapping_sub(f as u32)); + put64(q, f + 0x84, wv + 0x8f8); + put32(q, ta::END, work::op::END); +} + +/// Encodes the FS work image and microsequence of a render. +#[allow(clippy::too_many_arguments)] +pub(crate) fn encode_fs( + r: &RenderRefs, + p: &PassRefs, + u: &uapi::drm_asahi_cmd_render, + ts_va: &[u64; 4], + ctx: u32, + usc: u64, + n: u64, + tfs: u32, + geo: &Geometry, + w: &mut [u8; fs::WORK_SIZE], + q: &mut [u8; fs::SEQ_SIZE], +) { + let (wv, uv, qv, h) = (p.fs_work, p.fs_seq, r.qinfo[1], r.hwdata); + let (heap, su) = (p.heap_gva, p.scene_user_gva); + let ts = ts_va[2] != 0 || ts_va[3] != 0; + let (d, s) = (&u.depth, &u.stencil); + let val: [u64; 79] = [ + 1, + geo.utile_config as u64, + prog_rsrc(u, &u.eot, u.bg_eot_rsrc_spec_hi), + prog_addr(usc, &u.eot), + prog_rsrc(u, &u.bg, u.bg_rsrc_spec_hi), + prog_addr(usc, &u.bg), + u.isp_merge_upper_x as u64, + u.isp_merge_upper_y as u64, + 0, + 0, + 0, + 0, + u.ppp_multisamplectl, + geo.isp_blocks as u64, + geo.isp_blocks as u64, + geo.screen as u64, + 0xff0200, + heap, + u.isp_scissor_base, + u.isp_dbias_base, + geo.isp_control as u64, + (u.height_px as u64) << 32 | u.width_px as u64, + 0x100000, + geo.tilebuffer_control as u64, + u.isp_zls_pixels as u64, + u.isp_bgobjdepth as u64, + u.isp_bgobjvals as u64, + u.isp_oclqry_base & !15, + u.zls_ctrl, + 0x04040404, + 0, + d.base, + d.base, + s.base, + s.base, + 0, + 0, + 0, + 0, + d.stride as u64, + d.stride as u64, + s.stride as u64, + s.stride as u64, + d.comp_base, + d.comp_stride as u64, + d.comp_base, + d.comp_stride as u64, + s.comp_base, + s.comp_stride as u64, + s.comp_base, + s.comp_stride as u64, + p.tilemap_gva, + geo.layer_metadata, + (4 * geo.region_stride as u64) << 24, + geo.tile_mode as u64, + 0, + 0, + 0, + 0x8860, + p.aux_gva, + p.aux_gva, + 0x1c, + 0x1c, + 0, + geo.native_tiles, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 8, + 0, + compact(su + 0x40) & !15, + 0x6000400020, + 0x200000001, + ]; + let pbg_r = prog_rsrc(u, &u.partial_bg, u.bg_partial_rsrc_spec_hi); + let pbg_a = prog_addr(usc, &u.partial_bg); + let peot_a = prog_addr(usc, &u.partial_eot); + let pp = fs::PARAMS; + + w.fill(0); + q.fill(0); + + // FS work image. + put32(w, 0x0, work::work_type::FS); + put64(w, 0x4, 0); + put32(w, 0xc, ctx); + put64(w, 0x14, uv); + put32(w, 0x1c, fs::SEQ_SIZE as u32); + put64(w, 0x20, r.notifier); + put64(w, 0x28, r.pb_mgr); + put64(w, 0x30, p.scene); + put64(w, 0x38, r.legacy_empty); + put64(w, 0x40, p.tilemap_gva); + put64(w, 0x48, 0x88); + put32(w, 0x50, 1); + put16(w, 0x54, (geo.y_blocks >> 18) as u16); + put16(w, 0x56, (geo.x_blocks >> 18) as u16); + put32(w, 0x68, u.isp_merge_upper_x); + put32(w, 0x6c, u.isp_merge_upper_y); + put64(w, 0x78, geo.tiles as u64); + regs(&mut w[fs::REGS..], &fs::SEL, &val); + put64(w, 0x780, p.fs_work_gva + fs::REGS as u64); + put16(w, 0x788, 79); + put16(w, 0x78a, (79 * REG_RECORD) as u16); + // Parameters / firmware mirrors at +0x790. + put64(w, pp + 0x0, u.isp_dbias_base); + put64(w, pp + 0x10, u.isp_scissor_base); + put64(w, pp + 0x20, u.isp_oclqry_base); + put64(w, pp + 0x168, pbg_r); + put64(w, pp + 0x170, pbg_a); + put64(w, pp + 0x198, pbg_r); + put64(w, pp + 0x1a0, pbg_a); + put64(w, pp + 0x1a8, u.zls_ctrl); + put32(w, pp + 0x1b0, 0x04040404); + put64(w, pp + 0x1b8, d.base); + put64(w, pp + 0x1c0, d.stride as u64); + put64(w, pp + 0x1c8, d.comp_stride as u64); + put64(w, pp + 0x1d0, d.base); + put64(w, pp + 0x1d8, d.base); + put64(w, pp + 0x1e0, d.comp_base); + put64(w, pp + 0x1e8, s.base); + put64(w, pp + 0x1f0, s.stride as u64); + put64(w, pp + 0x1f8, s.comp_stride as u64); + put64(w, pp + 0x200, s.base); + put64(w, pp + 0x208, s.base); + put64(w, pp + 0x210, s.comp_base); + put32(w, pp + 0x228, geo.tilebuffer_control); + put32(w, pp + 0x230, geo.isp_control); + put32(w, pp + 0x238, u.width_px as u32); + put32(w, pp + 0x23c, u.height_px as u32); + put32(w, pp + 0x240, 0x100000); + put32(w, pp + 0x248, geo.tile_mode); + put32(w, pp + 0x264, u.partial_eot.rsrc_spec); + put64(w, pp + 0x26c, peot_a); + put32(w, pp + 0x284, u.partial_eot.rsrc_spec); + put64(w, pp + 0x28c, peot_a); + put32(w, pp + 0x298, u.isp_bgobjdepth); + put32(w, pp + 0x29c, u.isp_bgobjvals); + put32(w, pp + 0x2a0, u.sample_size_B as u32); + put64(w, pp + 0x2c0, u.isp_zls_pixels as u64); + put32(w, 0xb88, 0x67); + put32(w, 0xb90, 0xffffffff); + put32( + w, + 0xb94, + (u.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES != 0) as u32, + ); + put32(w, 0xb98, 1); + put64(w, 0xbac, r.user_stamp[1]); + put64(w, 0xbb4, r.fw_stamp[1]); + put32(w, 0xbbc, tfs); + put32(w, 0xbc0, 1); + put32(w, 0xbc8, 1); + put32(w, 0xbcc, hw::engine::FS_UUID); + put32(w, 0xbe8, 1); + work_tail( + &mut w[fs::TAIL..], + p.its + 0x10, + p.its + 0x18, + ts_va[2], + ts_va[3], + r.uma_mgr[1], + p.fs_scratch_gva, + ); + + // FS microsequence. + put32(q, 0x0, work::op::FS_START); + put64(q, 0x14, wv + 0x80); + put64(q, 0x1c, p.scene); + put64(q, 0x24, h + fs::HWDATA); + put64(q, 0x2c, wv + 0xb70); + put64(q, 0x34, wv + 0xb74); + put64(q, 0x3c, wv + 0xba0); + put64(q, 0x44, qv); + put64(q, 0x4c, wv); + put32(q, 0x54, ctx); + put32(q, 0x58, 1); + put64(q, 0x68, (n as u32).wrapping_sub(1) as u64); + put64(q, 0x74, wv + 0xa58); + put64(q, 0x7c, wv + 0xbd8); + put32(q, 0x98, hw::engine::FS_UUID); + put64(q, 0x1a4, r.uma_mgr[1]); + put64(q, 0x1bc, p.fs_scratch_gva); + put64(q, 0x1d4, r.notifier + 0xa8); + ts_record( + &mut q[fs::TS_START..], + false, + wv, + fs::TAIL as u64, + qv, + ts, + hw::engine::FS_UUID, + ); + put32(q, 0x218, 1); + ts_record( + &mut q[fs::TS_END..], + true, + wv, + fs::TAIL as u64, + qv, + ts, + hw::engine::FS_UUID, + ); + let f = fs::FINALIZE; + put32(q, f, work::op::FS_FINALIZE); + put32(q, f + 0x4, hw::engine::FS_UUID); + put64(q, f + 0xc, r.fw_stamp[1]); + put32(q, f + 0x14, tfs); + put64(q, f + 0x1c, p.scene); + put64(q, f + 0x24, r.pb_mgr); + put64(q, f + 0x2c, 1); + put64(q, f + 0x34, h + fs::HWDATA); + put64(q, f + 0x3c, wv + 0xba0); + put64(q, f + 0x44, wv + 0xb70); + put64(q, f + 0x4c, qv); + put64(q, f + 0x54, wv); + put64(q, f + 0x5c, ctx as u64); + put64(q, f + 0x64, wv + 0xa58); + put64(q, f + 0x9c, uv + 0x1a4); + put32(q, f + 0xa4, 0u32.wrapping_sub(f as u32)); + put32(q, f + 0xa8, 1); + put64(q, f + 0xac, wv + 0xc40); + put32(q, fs::END, work::op::END); +} + +/// The pass slot's scene record for draw ordinal `n`. +pub(crate) fn scene(r: &RenderRefs, p: &PassRefs, n: u64) -> [u8; scene::SIZE] { + let mut x = [0u8; scene::SIZE]; + let list = r.scene_list + 4 * (n % scene::LIST_SLOTS); + + put64(&mut x, 0x0, list); + put64(&mut x, 0x8, list); + put64(&mut x, 0x28, compact(p.scene_user_gva + 0x40)); + put32(&mut x, 0x30, compact(p.scene_user_gva + 0x900) as u32); + put64(&mut x, 0x40, r.mgr_misc); + x +} + +/// The InitBM record that (re)initializes the PB before a render. +pub(crate) fn initbm(ctx: u32, pb_init_blocks: u32, pb_mgr: u64, tta: u32) -> [u8; initbm::SIZE] { + let mut x = [0u8; initbm::SIZE]; + + put32(&mut x, 0x0, work::op::INIT_BM); + put32(&mut x, initbm::CTX, ctx); + put32(&mut x, initbm::BLOCKS, pb_init_blocks); + put64(&mut x, initbm::PB_MANAGER, pb_mgr); + put32(&mut x, initbm::STAMP, tta); + x +} diff --git a/drivers/gpu/drm/asahi/g15/engine.rs b/drivers/gpu/drm/asahi/g15/engine.rs new file mode 100644 index 0000000000000..e5c18a15e0c3b --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/engine.rs @@ -0,0 +1,1460 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The prefix/horizon/flight engine. +//! +//! One device-global compute queue and one TA+FS queue pair serve every logical queue and VM. +//! Jobs (one per SUBMIT) wait in `pending`; pick order: priority, then the least recently +//! served queue, then FIFO. A *prefix* is a run of consecutive commands of one job (up to 16 +//! computes and [`super::hw::tuning::PREFIX_RENDERS`] renders) published together: images, +//! then thresholds/counters once per prefix, then one queue WRITE per queue, then channel +//! records, then doorbells (compute, FS, TA). A compute-only job that ends its prefix pulls +//! further ready compute-only jobs of the same queue and VM into a *horizon* (up to 16, posted +//! ahead at the observed depth). At most [`super::hw::sizes::FLIGHTS`] prefixes are in flight, +//! the second only of the same VM, posted behind the first's unconsumed records +//! (submit-ahead). Ordering is a total order, which satisfies the header barriers and +//! EXACT_PRIOR ordinals validated at submission. Priming: after a recovered abort and for the +//! first render prefix after boot, an exclusive one-render prefix rebuilds the render +//! allocators and blocks publication behind it until it retires. +//! +//! Barrier types per edge (type 0 = firmware stamp only, 1 = also the predecessor job hold): +//! FS -> own TA 0; TA -> previous FS 0 only inside a prefix after a render with no +//! InitBM/resync, else 1; compute -> TA 1 on class 2; FS -> compute 1 (`CROSS_ENGINE_HOLD` +//! bit 0); compute -> compute 1 on class 2; early tiling (off) TA -> TA and FS -> FS 1. +//! +//! Completions: rather than signalling a fence under the device lock, the engine pushes the job +//! to [`Completions`] and the caller signals after dropping the lock; [`Job::complete`] runs +//! once per job, so a job may be pushed again when its last engine reference is released. + +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +use kernel::{prelude::*, sync::Arc, uapi}; + +use super::channel::Channel; +use super::command::{Command, ComputeCommand, RenderCommand}; +use super::encode::{self, ClusterLayout, ComputeRefs, Geometry, RenderRefs}; +use super::fw::channels::SubmitRecord; +use super::fw::graph::qstate; +use super::fw::work::{self, barrier::RENDER_WAIT_UUID, BarrierType}; +use super::hw::{engine as ids, sizes, tuning}; +use super::job::{Completions, Job}; +use super::slots::Claims; +use super::{params, Firmware, Shared}; + +/// `wmb()`: `dsb st` (a full fence on other architectures). +#[inline(always)] +pub(crate) fn wmb() { + #[cfg(target_arch = "aarch64")] + // SAFETY: A barrier instruction with no other effects. + unsafe { + core::arch::asm!("dsb st", options(nostack, preserves_flags)) + }; + #[cfg(not(target_arch = "aarch64"))] + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); +} + +/// `rmb()`: `dsb ld` (a full fence on other architectures). +#[inline(always)] +pub(crate) fn rmb() { + #[cfg(target_arch = "aarch64")] + // SAFETY: A barrier instruction with no other effects. + unsafe { + core::arch::asm!("dsb ld", options(nostack, preserves_flags)) + }; + #[cfg(not(target_arch = "aarch64"))] + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); +} + +/// `ktime_get_ns()`. +pub(crate) fn now_ns() -> u64 { + ::ktime_get() as u64 +} + +/// The engine's bookkeeping of one job, embedded in [`Job`]. Only the engine changes it, always +/// under the device lock (the atomics only make the shared `Arc` mutable). +pub(crate) struct Progress { + /// Commands still to run: the job's command count, cut to `sent` when a failure drops its + /// unpublished commands. + pub(crate) count: AtomicU32, + /// First command not yet published. + pub(crate) sent: AtomicU32, + /// First command not yet retired. + pub(crate) next: AtomicU32, + /// Waiting in `Engine::pending`. + pub(crate) pending: AtomicBool, + /// The fence was signalled: [`Job::complete`] runs once. + pub(crate) signaled: AtomicBool, +} + +impl Progress { + pub(crate) fn new(commands: usize) -> Self { + Self { + count: AtomicU32::new(commands as u32), + sent: AtomicU32::new(0), + next: AtomicU32::new(0), + pending: AtomicBool::new(false), + signaled: AtomicBool::new(false), + } + } + + pub(crate) fn count(&self) -> u32 { + self.count.load(Ordering::Relaxed) + } + + pub(crate) fn sent(&self) -> u32 { + self.sent.load(Ordering::Relaxed) + } + + pub(crate) fn next(&self) -> u32 { + self.next.load(Ordering::Relaxed) + } + + pub(crate) fn is_pending(&self) -> bool { + self.pending.load(Ordering::Relaxed) + } + + fn set_sent(&self, v: u32) { + self.sent.store(v, Ordering::Relaxed) + } + + fn set_pending(&self, v: bool) { + self.pending.store(v, Ordering::Relaxed) + } +} + +/// Engine of the previous member. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) enum Prev { + #[default] + None, + Compute, + Render, +} + +/// A published prefix: its job and commands, its own receipts (final +/// targets, exclusive ring ends and the ring positions before it, channel producers after its +/// records) and the storage it holds until its own retirement. +pub(crate) struct Flight { + /// The job whose commands the prefix holds, and horizon jobs folded into it (in order). + pub(crate) job: Arc, + pub(crate) group: KVec>, + /// Index of the first command of the job in this prefix, and the number of commands. + pub(crate) first: usize, + pub(crate) count: usize, + pub(crate) computes: u32, + pub(crate) renders: u32, + /// Compute: final FW stamp, ring end (exclusive) and ring position before the prefix. + pub(crate) target: u32, + pub(crate) head: u32, + pub(crate) c_start: u32, + /// Render: final TA/FS stamps, ring ends and ring positions before the prefix. + pub(crate) ta_target: u32, + pub(crate) fs_target: u32, + pub(crate) ta_head: u32, + pub(crate) fs_head: u32, + pub(crate) ta_start: u32, + pub(crate) fs_start: u32, + /// Channel producers (TA, FS, compute) after the prefix's records. + pub(crate) rec: [u32; 3], + /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none): + /// its predecessors in the prefix (for the compute-hang check). + pub(crate) cstamp: [u32; sizes::COMPUTE_SLOTS], + pub(crate) cwait: [u32; sizes::COMPUTE_SLOTS], + /// Compute slots, pass slots and clustering buffers it holds. + pub(crate) claims: Claims, + /// The prefix rebuilds the render allocators (priming); nothing is published behind it. + pub(crate) rebuild: bool, + /// A kill or recovery ended this prefix's VM: later prefixes of it fail with EIO. + pub(crate) doomed: bool, + /// Publication time, ns. + pub(crate) published_ns: u64, +} + +impl Flight { + fn new(job: Arc) -> Self { + Self { + job, + group: KVec::new(), + first: 0, + count: 0, + computes: 0, + renders: 0, + target: 0, + head: 0, + c_start: 0, + ta_target: 0, + fs_target: 0, + ta_head: 0, + fs_head: 0, + ta_start: 0, + fs_start: 0, + rec: [0; 3], + cstamp: [0; sizes::COMPUTE_SLOTS], + cwait: [0; sizes::COMPUTE_SLOTS], + claims: Claims::new(), + rebuild: false, + doomed: false, + published_ns: 0, + } + } +} + +/// A prefix under preparation: members in API order, each in a claimed +/// compute or pass slot. `prev` is the engine of the previous member, of this prefix or else +/// the last member of the newest published one; `last_ct`/`last_fs` its compute/FS target. +#[derive(Default)] +struct Prefix { + computes: u32, + renders: u32, + prev: Prev, + /// This prefix (re)initializes the PB with InitBM. + initbm: bool, + c_head: u32, + ta_head: u32, + fs_head: u32, + /// Last compute ordinal / last draw ordinal. + n: u64, + draw: u64, + last_ct: u32, + last_fs: u32, + /// The last compute / render member has API timestamps. + ct_ts: bool, + fs_ts: bool, +} + +/// Engine statistics, printed by `asahi-g15/stats`. +#[derive(Default)] +pub(crate) struct Stats { + pub(crate) pub_ns: u64, + pub(crate) idle_from_ns: u64, + pub(crate) starve_from_ns: u64, + pub(crate) prep_from_ns: u64, + /// Idle with nothing ready; prefix preparation. + pub(crate) starved_ns: u64, + pub(crate) prep_ns: u64, + /// Internal stage durations of retired prefixes (24 MHz ticks). + pub(crate) ta_ticks: u64, + pub(crate) fs_ticks: u64, + pub(crate) cdm_ticks: u64, + pub(crate) span_ticks: u64, + /// [0] compute, [1] render, [2] mixed. + pub(crate) count: [u64; 3], + pub(crate) prefixes: [u64; 3], + pub(crate) busy_ns: [u64; 3], + pub(crate) max_ns: [u64; 3], + pub(crate) first_ns: [u64; 3], + pub(crate) last_ns: [u64; 3], + pub(crate) gap_ns: u64, + pub(crate) gaps: u64, + pub(crate) yields: u64, + /// Prefixes published while another ran. + pub(crate) overlaps: u64, + pub(crate) jobs: u64, + pub(crate) job_cmds: u64, + pub(crate) job_max: u64, + pub(crate) early: u64, + /// Type-0 FS->compute / compute->TA waits. + pub(crate) fs2c0: u64, + pub(crate) c2ta0: u64, + pub(crate) clustered: u64, + /// Jobs published behind another in a compute horizon. + pub(crate) grouped: u64, + /// Compute-only prefixes by jobs; why grouping stopped (no further ready job, the cap, + /// compute slots, a job of the queue that does not qualify); behind a running prefix; of + /// those behind its unconsumed record. + pub(crate) hz: [u64; sizes::HORIZON_JOBS as usize], + pub(crate) hz_stop: [u64; 4], + pub(crate) hz_behind: u64, + pub(crate) hz_ahead: u64, + /// Why ready jobs stayed unpublished: both flights published, a channel record unconsumed, + /// no free slot. + pub(crate) hold: [u64; 3], +} + +/// Host-side phase timers of publication and polling, on with `debug_flags` bit 49 +/// (`Debug1`); printed as the `prof:` line of `asahi-g15/stats`. A measurement aid: nothing is +/// timed while the flag is off. +#[derive(Default)] +pub(crate) struct Prof { + /// Published prefixes timed. + pub(crate) prefixes: u64, + /// Member selection, channel checks, allocator and ring-head setup. + pub(crate) select_ns: u64, + /// Encoders (images into the scratch buffers). + pub(crate) encode_ns: u64, + /// Slot, pass and clustering-buffer claims, with synchronous scrubs and flush waits. + pub(crate) claim_ns: u64, + /// Object writes: images, scene, barriers, InitBM, ring entries. + pub(crate) write_ns: u64, + /// Thresholds, queue WRITEs, channel records and doorbells. + pub(crate) post_ns: u64, + /// Poll passes, and the time of their drain, retirement and whole pass. + pub(crate) polls: u64, + pub(crate) drain_ns: u64, + pub(crate) retire_ns: u64, + pub(crate) poll_ns: u64, +} + +/// A phase timer: reads the clock only while profiling is on. +pub(crate) struct Lap(Option); + +impl Lap { + pub(crate) fn start() -> Self { + Lap(crate::debug::debug_enabled(crate::debug::DebugFlags::Debug1).then(now_ns)) + } + + /// Adds the time since the start or the last lap to `acc`. + pub(crate) fn lap(&mut self, acc: &mut u64) { + if let Some(t) = self.0 { + let now = now_ns(); + *acc += now.wrapping_sub(t); + self.0 = Some(now); + } + } + + pub(crate) fn on(&self) -> bool { + self.0.is_some() + } +} + +/// Encoder output buffers, allocated once rather than per publication. +struct Scratch { + cw: KVec, + cq: KVec, + ta: KVec, + tas: KVec, + fs: KVec, + fss: KVec, +} + +impl Scratch { + fn new() -> Result { + let z = |n| KVec::from_elem(0u8, n, GFP_KERNEL); + Ok(Self { + cw: z(work::compute::WORK_SIZE)?, + cq: z(work::compute::SEQ_SIZE)?, + ta: z(work::ta::WORK_SIZE)?, + tas: z(work::ta::SEQ_SIZE)?, + fs: z(work::fs::WORK_SIZE)?, + fss: z(work::fs::SEQ_SIZE)?, + }) + } +} + +/// `&mut KVec` as a fixed-size image buffer. +fn image(v: &mut KVec) -> Result<&mut [u8; N]> { + <&mut [u8; N]>::try_from(&mut v[..]).map_err(|_| EINVAL) +} + +/// Engine state. +pub(crate) struct Engine { + /// Runnable jobs of every queue, in arrival order. + pub(crate) pending: KVec>, + /// Published prefixes, oldest first. + pub(crate) flights: KVec, + /// Jobs whose storage a failed prefix may still reference, retained until reboot. + pub(crate) retained: KVec>, + /// Publication frozen by a kill request (recovery may use this or `Recovery::armed`; both + /// freeze publication). + pub(crate) frozen: bool, + /// Service turn counter for the least-recently-served pick. + pub(crate) turn: u64, + /// The device's shared state, for the transport-error latch; set by the first poll or + /// submission. + pub(crate) shared: Option>, + /// Host compute ordinal and draw ordinal of the last publication. + pub(crate) ordinal: u64, + pub(crate) draw: u64, + /// Newest published receipts, the tail. + pub(crate) target: u32, + pub(crate) head: u32, + pub(crate) ta_target: u32, + pub(crate) fs_target: u32, + pub(crate) ta_head: u32, + pub(crate) fs_head: u32, + /// Event counts at the tail's publication. + pub(crate) events: u32, + pub(crate) stage_seen: u32, + /// Last member of the newest prefix. + pub(crate) tail_prev: Prev, + pub(crate) tail_ct: u32, + pub(crate) tail_fs: u32, + /// The compute queue / render queues were registered by a first record (`new_queue`). + pub(crate) queue_registered: bool, + pub(crate) render_registered: bool, + /// The rebuild (priming) prefix is published and not yet retired. + pub(crate) rebuild_live: bool, + /// A ready prefix waits only for a record's consumption: repoll sooner. + pub(crate) chan_wait: bool, + /// Progress tracking of the oldest prefix. + pub(crate) progress_flight: u64, + pub(crate) progress: [u32; 3], + pub(crate) stats: Stats, + /// Phase timers (debug_flags bit 49). + pub(crate) prof: Prof, + /// The `asahi-g15/stats` file. + pub(crate) debugfs: Option>>>>, + /// The `asahi-g15/pstate` file. + pub(crate) debugfs_pstate: Option>>>>, + scratch: Option>, +} + +impl Engine { + pub(crate) fn new() -> Self { + Self { + pending: KVec::new(), + flights: KVec::new(), + retained: KVec::new(), + frozen: false, + turn: 0, + shared: None, + ordinal: 0, + draw: 0, + target: 0, + head: 0, + ta_target: 0, + fs_target: 0, + ta_head: 0, + fs_head: 0, + events: 0, + stage_seen: 0, + tail_prev: Prev::None, + tail_ct: 0, + tail_fs: 0, + queue_registered: false, + render_registered: false, + rebuild_live: false, + chan_wait: false, + progress_flight: 0, + progress: [0; 3], + stats: Stats::default(), + prof: Prof::default(), + debugfs: None, + debugfs_pstate: None, + scratch: None, + } + } + + /// The oldest published prefix. + pub(crate) fn oldest(&self) -> Option<&Flight> { + self.flights.first() + } + + /// Jobs of the queue in published prefixes. + fn queue_published(&self, job: &Job) -> u32 { + self.flights + .iter() + .filter(|f| Arc::ptr_eq(&f.job.queue, &job.queue)) + .map(|f| 1 + f.group.len() as u32) + .sum() + } + + /// Index in `pending` of the next ready job: highest queue priority, then the queue served + /// least recently, then submission order. `QueueState::priority` is + /// `DRM_ASAHI_PRIORITY_REALTIME - priority` as `file.rs` passes it, so a smaller value + /// outranks. + fn pick(&self) -> Option { + let mut best: Option = None; + for (i, job) in self.pending.iter().enumerate() { + let better = match best { + None => true, + Some(b) => { + let (q, bq) = (&job.queue, &self.pending[b].queue); + let (t, bt) = ( + q.turn.load(Ordering::Relaxed), + bq.turn.load(Ordering::Relaxed), + ); + q.priority < bq.priority || (q.priority == bq.priority && t < bt) + } + }; + if better { + best = Some(i); + } + } + best + } + + /// Removes `job` from `pending`. + fn unpend(&mut self, job: &Arc) { + job.progress.set_pending(false); + self.pending.retain(|j| !Arc::ptr_eq(j, job)); + } +} + +/// The job's command `k`. +fn command(job: &Job, k: u32) -> Option<&Command> { + job.commands.get(k as usize) +} + +fn is_render(job: &Job, k: u32) -> bool { + matches!(command(job, k), Some(Command::Render(_))) +} + +/// Whole compute-only job. +fn compute_only(job: &Job) -> bool { + (0..job.progress.count()).all(|k| !is_render(job, k)) +} + +/// Pushes a job's completion; [`Job::complete`] only acts the first time. +pub(crate) fn complete(done: &mut Completions, job: &Arc, result: Result) { + if done.push((job.clone(), result), GFP_KERNEL).is_err() { + // Signalling must not be lost: complete in place, under the caller's device lock. + job.complete(result); + } +} + +impl Firmware { + /// Accepts a job from the scheduler; it is published by [`Self::publish`]. Records how many + /// jobs the queue owns at this arrival for the compute horizon. + pub(crate) fn enqueue(&mut self, job: Arc) -> Result { + if self.recovery.failed().is_some() { + return Err(EIO); + } + if job.progress.count() == 0 || job.commands.is_empty() { + return Err(EINVAL); + } + if self.engine.flights.capacity() < sizes::FLIGHTS { + self.engine.flights.reserve(sizes::FLIGHTS, GFP_KERNEL)?; + } + self.engine.pending.push(job.clone(), GFP_KERNEL)?; + job.progress.set_pending(true); + let q = &job.queue; + let owned = self.engine.queue_published(&job) + + self + .engine + .pending + .iter() + .filter(|j| Arc::ptr_eq(&j.queue, q)) + .count() as u32; + let i = q.owned_index.fetch_add(1, Ordering::Relaxed) as usize % q.owned.len(); + q.owned[i].store(owned.min(255), Ordering::Relaxed); + Ok(()) + } + + /// Publishes further prefixes: with nothing published any ready job; behind published + /// prefixes only the next job of the same VM, up to `max_inflight`, and not while recovery + /// state is pending: neither the prefix that rebuilds the render allocators after an abort + /// is published behind another one, nor anything behind it until it has retired. Jobs + /// whose first publication failed before any visible record complete with an error. + /// Returns Err only when this call latched the engine failed (a publication error after a + /// queue WRITE advanced, or a partly sent job): the caller latches the fail-stop. + pub(crate) fn publish(&mut self, done: &mut Completions) -> Result { + let max_inflight = tuning::MAX_INFLIGHT.clamp(1, sizes::FLIGHTS as u32) as usize; + let mut result = Ok(()); + + self.engine.chan_wait = false; + // Nothing is published while a hang request is armed. + while self.recovery.failed().is_none() + && !self.engine.frozen + && self.recovery.armed.is_none() + && !self.engine.pending.is_empty() + { + let Some(index) = self.engine.pick() else { + break; + }; + let job = self.engine.pending[index].clone(); + if let Some(tail) = self.engine.flights.last() { + if self.engine.flights.len() >= max_inflight + || !Arc::ptr_eq(&job.vm, &tail.job.vm) + || self.recovery.reinit + || self.recovery.fs_resync + || self.engine.rebuild_live + { + self.engine.stats.hold[0] += (self.engine.flights.len() >= max_inflight) as u64; + break; + } + if !Arc::ptr_eq(&tail.job, &job) && tail.job.progress.is_pending() { + self.engine.stats.yields += 1; // another queue's turn at a prefix boundary + } + } + self.engine.turn += 1; + job.queue.turn.store(self.engine.turn, Ordering::Relaxed); + match self.publish_prefix(&job, done) { + Ok(()) => { + if job.progress.sent() == job.progress.count() { + self.engine.unpend(&job); // fully published + } + continue; + } + Err(e) if e == EAGAIN => { + let i = if self.engine.chan_wait { 1 } else { 2 }; + self.engine.stats.hold[i] += 1; + break; // storage or channel still held: on a later retirement + } + Err(e) => { + self.engine.unpend(&job); + if e == EIO || job.progress.sent() != 0 { + // Uncertain, or earlier commands ran: retain, stop the engine. No + // recovery: every VM is lost, also those with no failed job. + dev_err!( + self.graph.dev.as_ref(), + "G15: publication of ctx {} failed after a queue WRITE ({:?})\n", + job.vm.slot, + e + ); + complete(done, &job, Err(EIO)); + self.retain(&job); + self.fail_stop(EIO, done); + result = Err(EIO); + break; + } + complete(done, &job, Err(e)); + } + } + } + result + } + + /// Whether some of `job`'s commands were published: such a job is retained after a + /// fail-stop. + pub(crate) fn job_partly_published(&self, job: &Job) -> bool { + job.progress.sent() != 0 + } + + /// Ready jobs the compute horizon of `job` could take now, besides `job` itself: the + /// grouping rules of [`Self::publish_prefix`], at most `max - 1`. + fn horizon_ready(&self, job: &Arc, mut computes: u32, free_c: u32, max: u32) -> u32 { + let mut n = 0; + for j2 in self.engine.pending.iter() { + if 1 + n >= max { + break; + } + if Arc::ptr_eq(j2, job) || !Arc::ptr_eq(&j2.queue, &job.queue) { + continue; + } + let count = j2.progress.count(); + if !compute_only(j2) + || !Arc::ptr_eq(&j2.vm, &job.vm) + || j2.progress.sent() != 0 + || count == 0 + || computes + count > free_c + { + break; + } + computes += count; + n += 1; + } + n + } + + /// The allocator and graph addresses the render encoders name. + fn render_refs(&self) -> RenderRefs { + let r = &self.graph.render; + let a = self.pools.allocator_refs(); + RenderRefs { + hwdata: self.cfg.layout.hwdata_va, + notifier: r.notifier.va, + legacy_empty: r.legacy_empty.va, + qinfo: [r.qinfo[0].va, r.qinfo[1].va], + user_stamp: [r.user_stamp[0].va, r.user_stamp[1].va], + fw_stamp: [r.fw_stamp[0].va, r.fw_stamp[1].va], + uma_mgr: a.uma_mgr, + pb_mgr: a.pb_mgr, + scene_list: a.scene_list, + mgr_misc: a.mgr_misc, + pb_init_blocks: a.init_blocks, + } + } + + /// Publishes the next prefix of `job`, from its first unpublished command. Members: + /// consecutive commands of one engine, or with `MIXED_PREFIX` of both engines in API order, + /// each in a free compute or pass slot. All images and ring entries precede one WRITE per + /// participating queue and the kicks (compute, FS, TA); host thresholds and counters name + /// the final ordinals. + /// + /// With prefixes already published (same VM only), the new one is appended behind them on + /// the running queues: it takes only slots they do not hold, its first member waits on + /// their last member like any member on its predecessor, and its SubmitQueue records + /// extend the queues' horizons (`new_queue` 0). Returns EAGAIN when it cannot be appended + /// yet (no free slot, or a channel record still unconsumed); EIO once a queue WRITE + /// advanced; any other error means nothing was published. + fn publish_prefix(&mut self, job: &Arc, done: &mut Completions) -> Result { + let mut f = Flight::new(job.clone()); + let result = self.publish_prefix_into(job, &mut f); + match result { + Err(e) if e != EIO => { + // Never published: its storage is free again, and horizon jobs taken into it + // fail with it. + self.slots.release(&mut f.claims); + for j2 in f.group.iter() { + complete(done, j2, Err(e)); + } + } + _ => {} + } + result + } + + fn publish_prefix_into(&mut self, job: &Arc, f: &mut Flight) -> Result { + // The PB backing is mapped into the job's VM before anything of the prefix is changed. + // Nothing is published when that fails, so the error must not be EIO (which reports a + // queue WRITE that advanced). + self.map_pb(&job.vm) + .map_err(|e| if e == EIO { EFAULT } else { e })?; + let overlap = !self.engine.flights.is_empty(); + let mut mixed = tuning::MIXED_PREFIX; + let mut pf = Prefix { + n: self.engine.ordinal, + draw: self.engine.draw, + ..Default::default() + }; + let i = job.progress.sent(); + let count = job.progress.count(); + let render0 = is_render(job, i); + let (mut computes, mut renders, mut end) = (0u32, 0u32, i); + let mut last_render = None; + let mut hz_stop: Option = None; + let mut rebuild = false; + let prefix_renders = tuning::PREFIX_RENDERS.clamp(1, sizes::RENDER_SLOTS as u32); + + self.engine.stats.prep_from_ns = now_ns(); + let mut lap = Lap::start(); + let free_c = self.slots.free_compute(); + // After an abort the prefix that rebuilds the render allocators holds renders only (see + // `publish`): a compute published with it, or behind it before it retired, crashed the + // 2419 firmware in its progress path (F+0x35568, FAR 0x58) after a recovered hang whose + // prefix had a compute waiting on the hung FS. + if self.recovery.reinit || self.recovery.fs_resync { + mixed = false; + } + while end < count { + let cmd = command(job, end).ok_or(EINVAL)?; + let render = matches!(cmd, Command::Render(_)); + if render != render0 && !mixed { + break; + } + match cmd { + Command::Compute(_) => { + if computes == free_c { + break; + } + computes += 1; + } + Command::Render(r) => { + if renders >= prefix_renders { + break; + } + // The prefix that primes freshly initialized render allocators (the first + // after boot, or after an abort's re-init) holds one render: their + // submitted cell is 1 and their completed count 0, so the first completed + // TA (FS) catches its allocator up and clears its registration. A second + // render of the prefix then holds a position of that allocator with no + // registered manager until it completes, and the firmware's + // progress/preemption path dereferences the missing manager: RTKit crash + // at F+0x35568, FAR 0x58. + if renders != 0 && (self.recovery.reinit || self.engine.draw == 0) { + break; + } + let geo = Geometry::new(&r.cmd, 0); + let Some(k) = self.slots.pick_pass(&geo, f.claims.pass_mask) else { + break; + }; + f.claims.pass_mask |= 1 << k; + f.claims.pass[renders as usize] = k as u32; + renders += 1; + } + } + end += 1; + } + if end == i { + // Idle: every slot is free. + return Err(if overlap { EAGAIN } else { E2BIG }); + } + // The picked pass slots are not claimed yet: an early return below must not release + // them. + let picked = f.claims.pass_mask; + f.claims.pass_mask = 0; + // Participating channels: every earlier record consumed, or (a prefix with renders + // behind a running one, SUBMIT_AHEAD) room behind the unconsumed ones. Behind a running + // prefix the firmware takes its record within microseconds: retry soon rather than at + // that prefix's retirement. + let mut ahead = overlap && renders != 0 && tuning::SUBMIT_AHEAD; + if overlap && renders == 0 && end == count && tuning::SUBMIT_AHEAD { + let max = tuning::COMPUTE_HORIZON.clamp(1, sizes::HORIZON_JOBS); + let ready = 1 + self.horizon_ready(job, computes, free_c, max); + let depth = job + .queue + .owned + .iter() + .map(|o| o.load(Ordering::Relaxed)) + .max() + .unwrap_or(0); + ahead = ready >= max || ready + self.engine.queue_published(job) >= depth; + } + for ch in [Channel::Ta, Channel::Fs, Channel::Compute] { + let used = if ch == Channel::Compute { + computes != 0 + } else { + renders != 0 + }; + if !used { + continue; + } + let consumer = self.channels.read(ch)?; + let producer = self.channels.producer(ch); + let busy = if ahead { + producer.wrapping_sub(consumer) % 256 >= sizes::AHEAD_RECORDS + } else { + consumer != producer + }; + if busy || self.channels.write_cell(ch)? != producer { + self.engine.chan_wait = overlap; + return Err(if overlap { EAGAIN } else { EBUSY }); + } + } + f.claims.pass_mask = picked; + self.slots.pass_busy |= f.claims.pass_mask; + if renders != 0 { + let cursor = match f.claims.pass[renders as usize - 1] { + 0 => self.slots.pass_cursor, + k => k, + }; + self.slots.pass_cursor = cursor % (sizes::RENDER_SLOTS as u32 - 1).max(1); + // After a recovered render abort the allocator state is not trusted. The priming + // prefix (also the first after boot) retires before anything is published behind + // it. + rebuild = self.recovery.reinit || self.engine.draw == 0; + if self.recovery.reinit { + self.init_render_pools(true)?; + dev_info!( + self.graph.dev.as_ref(), + "G15: render allocators re-initialized; InitBM at draw {}\n", + self.engine.draw + 1 + ); + } + } + if self.engine.scratch.is_none() { + self.engine.scratch = Some(KBox::new(Scratch::new()?, GFP_KERNEL)?); + } + if overlap { + pf.prev = self.engine.tail_prev; + pf.last_ct = self.engine.tail_ct; + pf.last_fs = self.engine.tail_fs; + } + let cap = sizes::QUEUE_CAP; + pf.c_head = self.graph.compute.qstate.r32(qstate::WRITE)? % cap; + pf.ta_head = self.graph.render.qstate[0].r32(qstate::WRITE)? % cap; + pf.fs_head = self.graph.render.qstate[1].r32(qstate::WRITE)? % cap; + f.c_start = pf.c_head; + f.ta_start = pf.ta_head; + f.fs_start = pf.fs_head; + for k in i..end { + if is_render(job, k) { + last_render = Some(k); + } + } + let refs = self.render_refs(); + lap.lap(&mut self.engine.prof.select_ns); + for k in i..end { + match command(job, k).ok_or(EINVAL)? { + Command::Render(r) => self.prefix_render(job, r, &refs, &mut pf, f)?, + Command::Compute(c) => { + let later_render = last_render.is_some_and(|l| k < l); + self.prefix_compute(job, c, &mut pf, f, later_render)? + } + } + } + // Compute horizon: whole compute-only jobs of the same queue and VM that already wait + // ready (their inputs resolved before the scheduler ran them), in queue order, into + // slots still free. A job of that queue that does not qualify ends the group: no later + // job overtakes it. + if renders == 0 && end == count { + let max = tuning::COMPUTE_HORIZON.clamp(1, sizes::HORIZON_JOBS); + let mut idx = 0; + hz_stop = Some(0); + while idx < self.engine.pending.len() { + if 1 + f.group.len() as u32 >= max { + hz_stop = Some(1); + break; + } + let j2 = self.engine.pending[idx].clone(); + if Arc::ptr_eq(&j2, job) || !Arc::ptr_eq(&j2.queue, &job.queue) { + idx += 1; + continue; + } + let n2 = j2.progress.count(); + if !compute_only(&j2) + || !Arc::ptr_eq(&j2.vm, &job.vm) + || j2.progress.sent() != 0 + || n2 == 0 + { + hz_stop = Some(3); + break; + } + if computes + n2 > free_c { + hz_stop = Some(2); + break; + } + // Without room for another group entry the horizon ends here: the base job + // is published already, and nothing of `j2` has been touched or sent, so a + // transient allocation failure must not fail the prefix (which, after a queue + // WRITE, would be taken for an uncertain post-WRITE state and stop the engine). + if f.group.reserve(1, GFP_KERNEL).is_err() { + hz_stop = Some(1); + break; + } + for k in 0..n2 { + match command(&j2, k).ok_or(EINVAL)? { + Command::Compute(c) => self.prefix_compute(&j2, c, &mut pf, f, false)?, + Command::Render(_) => return Err(EINVAL), + } + } + computes += n2; + j2.progress.set_sent(n2); + self.engine.turn += 1; + j2.queue.turn.store(self.engine.turn, Ordering::Relaxed); + j2.progress.set_pending(false); + let _ = self.engine.pending.remove(idx); + // Reserved above. + let _ = f.group.push(j2, GFP_KERNEL); + self.engine.stats.grouped += 1; + } + } + // Host thresholds/counters after the whole prefix. The notifiers' context (+0x24) is + // not ours to write: every completing work stores its own context there before the + // service reads it. + let mut lap = Lap::start(); + if computes != 0 { + self.graph.compute.threshold.w32(0, pf.n as u32)?; + self.graph.compute.counter.w32(0, pf.n as u32)?; + } + if renders != 0 { + self.graph + .render + .threshold + .w32(0, 2u32.wrapping_mul(pf.draw as u32))?; + // pb_counter = draw, then C1 once per (re)initialization. + self.publish_pb(pf.draw as u32)?; + } + wmb(); + // The queue WRITEs: from here on a failure is uncertain (EIO). + let writes = (|| -> Result { + if computes != 0 { + self.graph.compute.qstate.w32(qstate::WRITE, pf.c_head)?; + } + if renders != 0 { + self.graph.render.qstate[0].w32(qstate::WRITE, pf.ta_head)?; + self.graph.render.qstate[1].w32(qstate::WRITE, pf.fs_head)?; + } + Ok(()) + })(); + wmb(); + + f.first = i as usize; + f.count = (end - i) as usize; + f.computes = computes; + f.renders = renders; + if computes != 0 { + self.engine.ordinal = pf.n; + f.target = pf.last_ct; + self.engine.target = pf.last_ct; + f.head = pf.c_head; + self.engine.head = pf.c_head; + } + if renders != 0 { + f.ta_target = encode::stamp_seed(ids::TA_SEED, pf.draw); + self.engine.ta_target = f.ta_target; + f.fs_target = pf.last_fs; + self.engine.fs_target = pf.last_fs; + f.ta_head = pf.ta_head; + self.engine.ta_head = pf.ta_head; + f.fs_head = pf.fs_head; + self.engine.fs_head = pf.fs_head; + self.engine.draw = pf.draw; + self.recovery.fs_resync = false; + self.recovery.reinit = false; + } + self.engine.tail_prev = pf.prev; + self.engine.tail_ct = pf.last_ct; + self.engine.tail_fs = pf.last_fs; + self.engine.events = self.events.compute_events; + self.engine.stage_seen = self.events.stage_events; + self.stat_publish(f); + if overlap { + self.engine.stats.overlaps += 1; + } else { + self.renew_deadlines(); + } + job.progress.set_sent(end); + if let Some(stop) = hz_stop { + let s = &mut self.engine.stats; + s.hz[f.group.len()] += 1; + s.hz_stop[stop] += 1; + s.hz_behind += overlap as u64; + s.hz_ahead += ahead as u64; + } + if rebuild { + f.rebuild = true; + self.engine.rebuild_live = true; + } + // Compute on channel 2, then FS on channel 1, then TA on channel 0. + let mut posted = writes; + if posted.is_ok() && computes != 0 { + let rec = SubmitRecord { + qinfo: self.graph.compute.qinfo.va, + kind: 2, + head: pf.c_head as u16, + event: ids::EVENT_SLOT as u8, + new_queue: !self.engine.queue_registered as u8, + ..Default::default() + }; + posted = self.post(Channel::Compute, &rec, ahead); + if posted.is_ok() { + self.engine.queue_registered = true; + posted = self.session.ring(Channel::Compute); + } + } + if posted.is_ok() && renders != 0 { + let new_queue = !self.engine.render_registered as u8; + let fs_rec = SubmitRecord { + qinfo: self.graph.render.qinfo[1].va, + kind: 1, + head: pf.fs_head as u16, + event: 1, + new_queue, + ..Default::default() + }; + let ta_rec = SubmitRecord { + qinfo: self.graph.render.qinfo[0].va, + kind: 0, + head: pf.ta_head as u16, + event: 0, + new_queue, + ..Default::default() + }; + posted = self.post(Channel::Fs, &fs_rec, ahead); + if posted.is_ok() { + posted = self.session.ring(Channel::Fs); + } + if posted.is_ok() { + posted = self.post(Channel::Ta, &ta_rec, ahead); + } + if posted.is_ok() { + posted = self.session.ring(Channel::Ta); + } + self.engine.render_registered = true; + } + f.rec = [ + self.channels.producer(Channel::Ta), + self.channels.producer(Channel::Fs), + self.channels.producer(Channel::Compute), + ]; + if lap.on() { + lap.lap(&mut self.engine.prof.post_ns); + self.engine.prof.prefixes += 1; + } + // The flight is published: move it into the engine (capacity reserved in `enqueue`). + let mut flight = Flight::new(job.clone()); + core::mem::swap(&mut flight, f); + let _ = self.engine.flights.push(flight, GFP_KERNEL); + crate::cls_dev_dbg!( + Queue, + self.graph.dev, + "G15: prefix ctx={} computes {} (n={} head {}) renders {} (draw {} ta/fs head {}/{}) behind {}: {:?}\n", + job.vm.slot, + computes, + pf.n, + pf.c_head, + renders, + pf.draw, + pf.ta_head, + pf.fs_head, + self.engine.flights.len() - 1, + posted + ); + // A queue WRITE already advanced: any later failure is uncertain. + posted.map_err(|_| EIO) + } + + /// Posts a submit record, behind unconsumed ones with `ahead`. + fn post(&mut self, channel: Channel, record: &SubmitRecord, ahead: bool) -> Result { + if ahead { + self.channels.post_ahead(channel, record)?; + } else { + self.channels.post(channel, record)?; + } + Ok(()) + } + + /// Compute member: encode into its slot and append to the CDM ring. After a render of the + /// same prefix it first waits on that render's FS: a distinct barrier entry ahead of the + /// work. + fn prefix_compute( + &mut self, + job: &Arc, + cmd: &ComputeCommand, + pf: &mut Prefix, + f: &mut Flight, + later_render: bool, + ) -> Result { + let n = pf.n + 1; + let t = encode::stamp(n); + let ctx = job.vm.slot; + let ts = &cmd.timestamps; + let mut lap = Lap::start(); + // Nothing published yet on failure. + let index = self + .slots + .claim_compute(&mut f.claims, pf.computes)? + .ok_or(EBUSY)?; + lap.lap(&mut self.engine.prof.claim_ns); + let scrub = self.slots.scrub.clone(); + let s = &scrub.compute[index]; + let g = &self.graph.compute; + let refs = ComputeRefs { + hwdata: self.cfg.layout.hwdata_va, + notifier: g.notifier.va, + qinfo: g.qinfo.va, + user_stamp: g.user_stamp.va, + fw_stamp: g.fw_stamp.va, + manager: g.manager.va, + work: s.work.va, + work_gva: s.work.gva.unwrap_or(0), + seq: s.seq.va, + preempt_gva: s.preempt.gva.unwrap_or(0), + its: s.its.va, + tail: s.tail.va, + scratch: [ + s.scratch[0].va, + s.scratch[1].va, + s.scratch[2].va, + s.scratch[3].va, + s.scratch[4].va, + ], + }; + let scratch = self.engine.scratch.as_deref_mut().ok_or(ENOMEM)?; + let (w, q) = (&mut scratch.cw, &mut scratch.cq); + encode::encode_compute( + &refs, + &self.cfg.compute, + &cmd.cmd, + &cmd.attachments, + ts, + ctx, + n, + tuning::COMPUTE_FAULT_MASK, + image(w)?, + image(q)?, + ); + lap.lap(&mut self.engine.prof.encode_ns); + // Rewrite the slot's images (its runtime storage is already scrubbed). + s.work.fill(0, 0, work::compute::WORK_SIZE)?; + s.seq.fill(0, 0, work::compute::SEQ_OBJECT)?; + s.work.write(0, &w[..])?; + s.seq.write(0, &q[..])?; + + let g = &self.graph.compute; + if pf.prev == Prev::Render { + // Type 0 when the FS is of this prefix and a later render of the prefix follows + // it: its X names a later scene, so this FS takes no PB catch-up at this point. + let type0 = pf.renders != 0 + && later_render + && !pf.initbm + && !self.recovery.fs_resync + && !pf.fs_ts + && ts[0] == 0 + && ts[1] == 0 + && tuning::CROSS_ENGINE_HOLD & 1 == 0; + let b = encode::barrier( + self.graph.render.fw_stamp[1].va, + pf.last_fs, + 1, + t, + ids::COMPUTE_UUID, + BarrierType::from_stamp_only(type0), + ); + s.barrier.write(0, &b)?; + self.engine.stats.fs2c0 += type0 as u64; + g.ring.w64(8 * pf.c_head as usize, s.barrier.va)?; + pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP; + } else if pf.prev == Prev::Compute && tuning::COMPUTE_QUEUE_CLASS != 0 { + // Class 2 runs computes of the queue concurrently: keep the ring order with a wait + // on the previous compute's FW stamp. Type 1: a successor admitted with only the + // stamp wait makes the firmware decline the compute request that ends a hang. + let b = encode::barrier( + g.fw_stamp.va, + pf.last_ct, + ids::EVENT_SLOT, + t, + ids::COMPUTE_UUID, + BarrierType::Hold, + ); + s.barrier.write(0, &b)?; + g.ring.w64(8 * pf.c_head as usize, s.barrier.va)?; + pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP; + } + let j = pf.computes as usize; + f.cstamp[j] = t; + f.cwait[j] = if pf.prev == Prev::Render { + pf.last_fs + } else { + 0 + }; + g.ring.w64(8 * pf.c_head as usize, s.work.va)?; + pf.c_head = (pf.c_head + 1) % sizes::QUEUE_CAP; + lap.lap(&mut self.engine.prof.write_ns); + if n <= 4 { + crate::cls_dev_dbg!( + Compute, + self.graph.dev, + "G15: compute n={} slot {} ctx={} cdm {:#x}..{:#x} att={} ts={:#x}/{:#x}\n", + n, + index, + ctx, + cmd.cmd.cdm_ctrl_stream_base, + cmd.cmd.cdm_ctrl_stream_end, + cmd.attachments.len(), + ts[0], + ts[1] + ); + } + pf.computes += 1; + pf.n = n; + pf.last_ct = t; + pf.ct_ts = ts[0] != 0 || ts[1] != 0; + pf.prev = Prev::Compute; + Ok(()) + } + + /// Render member: TA and FS work, sequences, scene and barriers in its pass slot, appended + /// to the TA and FS rings. Its TA waits on the previous FS (full order), on the previous + /// TA (early tiling: a flagged later render after a render, its FS then also waiting on the + /// previous FS), or on the previous compute of the prefix; its FS always waits on its own + /// TA. + fn prefix_render( + &mut self, + job: &Arc, + cmd: &RenderCommand, + refs: &RenderRefs, + pf: &mut Prefix, + f: &mut Flight, + ) -> Result { + let u = &cmd.cmd; + let ts = &cmd.timestamps; + let k = f.claims.pass[pf.renders as usize] as usize; + let n = pf.draw + 1; + let usc = job.queue.usc_base; + let ctx = job.vm.slot; + let tta = encode::stamp_seed(ids::TA_SEED, n); + let tfs = encode::stamp_seed(ids::FS_SEED, n); + // Early tiling: a later render member after a render of the same prefix whose render + // dependencies are fragment-only (FRAGMENT_BARRIERS) and that has no start timestamps + // lets its TA wait for the previous TA; its FS still waits for the previous FS, then + // its own TA. Compute dependencies are already ahead of the previous render's TA. + let early = pf.computes + pf.renders != 0 + && pf.prev == Prev::Render + && params::early_tiling() + && u.flags & uapi::drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS != 0 + && ts[0] == 0 + && ts[2] == 0; + let scrub = self.slots.scrub.clone(); + let p = &scrub.passes[k]; + let pass = p.refs(); + let mut lap = Lap::start(); + let geo = Geometry::new(u, pass.heap_gva); + // Clustered tiling: one layer, one sample, 32x32 utiles, unless the submission asked + // for single-cluster vertex work, and a zeroed buffer holds this geometry. + let mut cl = None; + if encode::cluster_eligible(u, &geo) { + let bytes = ClusterLayout::new(&geo, 0).bytes; + if let Some(c) = self.slots.claim_ct(&mut f.claims, bytes)? { + let base = scrub.cts[c].gva.unwrap_or(0); + cl = Some(ClusterLayout::new(&geo, base)); + } + } + lap.lap(&mut self.engine.prof.claim_ns); + let scratch = self.engine.scratch.as_deref_mut().ok_or(ENOMEM)?; + encode::encode_ta( + refs, + &pass, + u, + ts, + ctx, + tta, + &geo, + cl.as_ref(), + image(&mut scratch.ta)?, + image(&mut scratch.tas)?, + ); + encode::encode_fs( + refs, + &pass, + u, + ts, + ctx, + usc, + n, + tfs, + &geo, + image(&mut scratch.fs)?, + image(&mut scratch.fss)?, + ); + + lap.lap(&mut self.engine.prof.encode_ns); + // The slot is unpublished: its previous prefix fully retired. Its storage was reset in + // the background since then (flight release), or is reset here. + self.slots.claim_pass(k)?; + self.slots + .set_pass_dirty(k, geo.tpc_bytes, geo.tilemap_bytes); + lap.lap(&mut self.engine.prof.claim_ns); + if cl.is_some() { + self.engine.stats.clustered += 1; // its buffer is zero: cleared since its last use + } + let scratch = self.engine.scratch.as_deref().ok_or(ENOMEM)?; + p.ta_work.write(0, &scratch.ta[..])?; + p.fs_work.write(0, &scratch.fs[..])?; + p.ta_seq.write(0, &scratch.tas[..])?; + p.fs_seq.write(0, &scratch.fss[..])?; + p.scene.write(0, &encode::scene(refs, &pass, n))?; + + let r = &self.graph.render; + if pf.prev == Prev::Compute { + // Compute -> render: the TA waits on the previous compute; type 0 when that compute + // is of this prefix and this render starts no InitBM, on the class-0 compute queue + // only (on class 2 a TA admitted with only the stamp wait makes the firmware + // decline the request that ends a hung compute). + let type0 = pf.computes != 0 + && !pf.initbm + && !self.recovery.fs_resync + && n != 1 + && !(pf.renders == 0 && self.recovery.reinit) + && !pf.ct_ts + && ts.iter().all(|t| *t == 0) + && tuning::CROSS_ENGINE_HOLD & 2 == 0 + && tuning::COMPUTE_QUEUE_CLASS == 0; + let b = encode::barrier( + self.graph.compute.fw_stamp.va, + pf.last_ct, + ids::EVENT_SLOT, + tta, + RENDER_WAIT_UUID, + BarrierType::from_stamp_only(type0), + ); + p.prev_barrier.write(0, &b)?; + self.engine.stats.c2ta0 += type0 as u64; + } else if early { + let b = encode::barrier( + r.fw_stamp[0].va, + encode::stamp_seed(ids::TA_SEED, n - 1), + 0, + tta, + RENDER_WAIT_UUID, + BarrierType::Hold, + ); + p.prev_barrier.write(0, &b)?; + let b = encode::barrier( + r.fw_stamp[1].va, + pf.last_fs, + 1, + tfs, + RENDER_WAIT_UUID, + BarrierType::Hold, + ); + p.fs_dep.write(0, &b)?; + self.engine.stats.early += 1; + } else { + let mut prev_fs = if pf.prev != Prev::None { + pf.last_fs + } else if n == 1 { + ids::FS_SEED + } else { + encode::stamp_seed(ids::FS_SEED, n - 1) + }; + // Restricted type 0: predecessor render in this prefix, no InitBM/resync. + let internal = pf.prev == Prev::Render + && pf.renders != 0 + && !pf.initbm + && !self.recovery.fs_resync + && tuning::RENDER_INTERNAL_HOLD == 0; + // After a recovered failure the previous FS may never have written. + if pf.prev == Prev::None && self.recovery.fs_resync { + prev_fs = r.fw_stamp[1].r32(0)?; + } + let b = encode::barrier( + r.fw_stamp[1].va, + prev_fs, + 1, + tta, + RENDER_WAIT_UUID, + BarrierType::from_stamp_only(internal), + ); + p.prev_barrier.write(0, &b)?; + } + let b = encode::barrier( + r.fw_stamp[0].va, + tta, + 0, + tfs, + ids::FS_UUID, + BarrierType::Stamp, + ); + p.own_barrier.write(0, &b)?; + let cap = sizes::QUEUE_CAP; + if n == 1 || (pf.renders == 0 && self.recovery.reinit) { + pf.initbm = true; + let bm = encode::initbm(ctx, refs.pb_init_blocks, refs.pb_mgr, tta); + r.initbm.write(0, &bm)?; + r.ring[0].w64(8 * pf.ta_head as usize, r.initbm.va)?; + pf.ta_head = (pf.ta_head + 1) % cap; + } + // Rings: TA [InitBM,] predecessor wait, TA; FS [prior-FS wait if early,] own-TA wait, + // FS. + r.ring[0].w64(8 * pf.ta_head as usize, p.prev_barrier.va)?; + pf.ta_head = (pf.ta_head + 1) % cap; + r.ring[0].w64(8 * pf.ta_head as usize, p.ta_work.va)?; + pf.ta_head = (pf.ta_head + 1) % cap; + if early { + r.ring[1].w64(8 * pf.fs_head as usize, p.fs_dep.va)?; + pf.fs_head = (pf.fs_head + 1) % cap; + } + r.ring[1].w64(8 * pf.fs_head as usize, p.own_barrier.va)?; + pf.fs_head = (pf.fs_head + 1) % cap; + r.ring[1].w64(8 * pf.fs_head as usize, p.fs_work.va)?; + pf.fs_head = (pf.fs_head + 1) % cap; + lap.lap(&mut self.engine.prof.write_ns); + if n <= 4 { + crate::cls_dev_dbg!( + Render, + self.graph.dev, + "G15: render n={} slot {} ctx={} {}x{} L{} s{} utile {}x{} ss{} flags={:#x} vdm {:#x} ta/fs head {}/{} tpc {:#x} tilemap {:#x}\n", + n, + k, + ctx, + u.width_px, + u.height_px, + u.layers, + u.samples, + u.utile_width_px, + u.utile_height_px, + u.sample_size_B, + u.flags, + u.vdm_ctrl_stream_base, + pf.ta_head, + pf.fs_head, + geo.tpc_bytes, + geo.tilemap_bytes + ); + } + pf.renders += 1; + pf.draw = n; + pf.last_fs = tfs; + pf.fs_ts = ts.iter().any(|t| *t != 0); + pf.prev = Prev::Render; + Ok(()) + } + + /// Records the publication time of `f` and the preparation, gap and starvation times. + fn stat_publish(&mut self, f: &mut Flight) { + let s = &mut self.engine.stats; + s.pub_ns = now_ns(); + f.published_ns = s.pub_ns; + s.prep_ns += s.pub_ns.wrapping_sub(s.prep_from_ns); + if s.idle_from_ns != 0 { + s.gap_ns += s.pub_ns.wrapping_sub(s.idle_from_ns); + s.gaps += 1; + s.idle_from_ns = 0; + } else if s.starve_from_ns != 0 { + s.starved_ns += s.pub_ns.wrapping_sub(s.starve_from_ns); + } + s.starve_from_ns = 0; + } +} diff --git a/drivers/gpu/drm/asahi/g15/event.rs b/drivers/gpu/drm/asahi/g15/event.rs new file mode 100644 index 0000000000000..d8883476e3c25 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/event.rs @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Decoded firmware events. +//! +//! Event ring records are 0x38 bytes: type 1 stage flags (u64 at +4: bit 0 TA, bit 1 FS, bit 2 +//! compute), type 0 fault, type 4 timeout (i32 culprit slot at +0xc), type 7 PB growth request +//! (copied whole into an 8-entry queue; never answered by the drain itself), type 8 channel +//! error. The drain runs under the device lock from the poll work (and inline in the boot and +//! device-control waits); the EP 0x20 interrupt only kicks the poll. Notifications only: +//! retirement has its own receipts. + +use kernel::{bindings, device, prelude::*}; + +use super::channel::Ring; +use super::fw::channels::{event, RING_SLOTS, STATE_READ, STATE_WRITE}; +use super::hw::rings; +use super::regs::{mb, rmb, wmb}; + +/// Number of queued PB growth requests. +pub(crate) const GROW_QUEUE: usize = 8; + +/// A copied firmware error event (type 0 fault or type 4 timeout). +#[derive(Copy, Clone, Default)] +pub(crate) struct ErrorEvent { + pub(crate) kind: u32, + pub(crate) raw: [u32; 14], + /// Type 4: the culprit event slot (possibly -1); -1 for type 0. + pub(crate) slot: i32, +} + +/// Event state consumed by the engine, recovery and growth. +pub(crate) struct Events { + /// Compute completions and stage events seen. + pub(crate) compute_events: u32, + pub(crate) stage_events: u32, + /// Error events seen, and the last one. + pub(crate) errors: u32, + pub(crate) last_error: Option, + /// Channel errors seen. + pub(crate) channel_errors: u32, + /// Last firing time per stage bit (TA, FS, compute), ns. + pub(crate) event_ns: [u64; 3], + /// PB growth requests, oldest first; at most [`GROW_QUEUE`]. + pub(crate) grow: KVec<[u32; 14]>, + /// Requests dropped because the queue was full. + pub(crate) grow_lost: u32, +} + +impl Events { + pub(crate) fn new() -> Self { + Self { + compute_events: 0, + stage_events: 0, + errors: 0, + last_error: None, + channel_errors: 0, + event_ns: [0; 3], + grow: KVec::new(), + grow_lost: 0, + } + } + + /// Reserves the growth queue, so that the drain never allocates. + pub(crate) fn reserve(&mut self) -> Result { + self.grow.reserve(GROW_QUEUE, GFP_KERNEL)?; + Ok(()) + } + + fn words(ring: &Ring, index: u32) -> Result<[u32; 14]> { + let base = index as usize * rings::EVENT_RECORD; + let mut w = [0u32; 14]; + for (j, word) in w.iter_mut().enumerate() { + *word = ring.ring.r32(base + 4 * j)?; + } + Ok(w) + } + + /// Decodes and consumes the event ring. + pub(crate) fn drain(&mut self, dev: &device::Device, ring: &Ring) -> Result { + let producer = ring.state.r32(STATE_WRITE)?; + let mut read = ring.state.r32(STATE_READ)?; + // SAFETY: ktime_get() has no preconditions. + let now = unsafe { bindings::ktime_get() } as u64; + if producer >= RING_SLOTS || read >= RING_SLOTS { + return Ok(()); + } + rmb(); + while read != producer { + let w = Self::words(ring, read)?; + let kind = w[0]; + let flags = w[1] as u64 | (w[2] as u64) << 32; + if kind == event::STAGE { + for b in 0..3 { + if flags & (1 << b) != 0 { + self.event_ns[b] = now; + } + } + if flags & (1 << 2) != 0 { + self.compute_events = self.compute_events.wrapping_add(1); + } + self.stage_events = self.stage_events.wrapping_add((flags & 7).count_ones()); + } + if kind == event::FAULT || kind == event::TIMEOUT { + self.last_error = Some(ErrorEvent { + kind, + raw: w, + slot: if kind == event::TIMEOUT { + w[3] as i32 + } else { + -1 + }, + }); + self.errors = self.errors.wrapping_add(1); + dev_err!( + dev, + "firmware event {}: {:08x} {:08x} {:08x} {:08x} {:08x} {:08x}\n", + kind, + w[1], + w[2], + w[3], + w[4], + w[5], + w[6] + ); + } else if kind == event::PB_GROW { + if self.grow.len() < GROW_QUEUE { + self.grow.push(w, GFP_KERNEL)?; + } else { + self.grow_lost = self.grow_lost.wrapping_add(1); + } + dev_dbg!( + dev, + "firmware event 7 (PB growth): ctx {} slot {} counter {} position {} epoch {:#x}\n", + w[1], + w[2], + w[3], + w[4], + w[5] as u64 | (w[6] as u64) << 32 + ); + } else if kind == event::CHANNEL_ERROR { + // Channel error (DMKill): needs a DEVCTL 8 acknowledgement. + dev_warn!( + dev, + "firmware event 8: {:08x} {:08x} {:08x} {:08x} {:08x}\n", + w[1], + w[2], + w[3], + w[4], + w[5] + ); + self.channel_errors = self.channel_errors.wrapping_add(1); + } else { + dev_dbg!(dev, "firmware event {} flags {:#x}\n", kind, flags); + } + read = (read + 1) % RING_SLOTS; + } + mb(); + ring.state.w32(STATE_READ, producer)?; + wmb(); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g15/fence.rs b/drivers/gpu/drm/asahi/g15/fence.rs new file mode 100644 index 0000000000000..a959ff00d4bb1 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/fence.rs @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Job fences. +//! +//! One fence per job (= per SUBMIT, the aggregate of its commands) on a per-queue context +//! ([`Timeline`]), signalled strictly in queue order. The VM status is latched (and mirrored) +//! before a failed job's fence signals ([`super::job::Job`]). Signalling happens after the device +//! lock is dropped. +//! +//! The engine retires a queue's jobs in order, but a later job may still be completed first +//! (a prefix failed before an earlier job's last prefix retired, or a never-run job cancelled at +//! queue teardown). Fences of one context must signal in order, so a [`Timeline`] holds a +//! completed fence back until every earlier fence of the queue has completed. + +use kernel::{ + c_str, + dma_fence::{Fence, FenceContexts, FenceObject, FenceOps, RawDmaFence, UserFence}, + new_mutex, + prelude::*, + str::CStr, + sync::{Arc, LockClassKey, Mutex}, +}; + +use crate::file::SyncItem; + +static TIMELINE_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); +static ERROR_KEY: Pin<&LockClassKey> = kernel::static_lock_class!(); + +/// Fence operations of G15 job fences. +pub(crate) struct JobFenceOps; + +#[vtable] +impl FenceOps for JobFenceOps { + fn get_driver_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("asahi") + } + + fn get_timeline_name<'a>(self: &'a FenceObject) -> &'a CStr { + c_str!("g15") + } +} + +/// An accepted fence of a timeline that has not signalled yet. +struct Pending { + fence: UserFence, + /// The result, once the job completed. + result: Option, +} + +/// The fence timeline of one logical queue. +#[pin_data] +pub(crate) struct Timeline { + /// One context for job fences, signalled in creation order. + jobs: FenceContexts, + /// One context for the error fences of refused submissions, each signalled at creation. + errors: FenceContexts, + /// Accepted fences that have not signalled, oldest first. + #[pin] + pending: Mutex>, + /// Pre-signalled ENOMEM fence for a refused submission when no error fence can be allocated. + allocation_failure: Fence, +} + +impl Timeline { + pub(crate) fn new() -> Result> { + let errors = FenceContexts::new(1, c_str!("asahi_g15_refused"), ERROR_KEY)?; + let allocation_failure = error_fence(&errors, ENOMEM)?; + Arc::pin_init( + try_pin_init!(Self { + jobs: FenceContexts::new(1, c_str!("asahi_g15_queue"), TIMELINE_KEY)?, + errors, + pending <- new_mutex!(KVec::new(), "g15::Timeline::pending"), + allocation_failure, + }), + GFP_KERNEL, + ) + } + + /// Creates the fence of the next accepted job. Fences are created in submission order: the + /// caller holds the queue (SUBMIT is serialized per queue). + pub(crate) fn next(self: &Arc) -> Result { + let mut pending = self.pending.lock(); + pending.reserve(1, GFP_KERNEL)?; + let fence: UserFence = self.jobs.new_fence(0, JobFenceOps)?.into(); + pending + .push_within_capacity(Pending { + fence: fence.clone(), + result: None, + }) + .map_err(|_| ENOMEM)?; + Ok(JobFence { + fence, + timeline: self.clone(), + }) + } + + /// Records the result of `fence` and signals every completed fence at the head, in order. + fn complete(&self, fence: &UserFence, result: Result) { + let mut pending = self.pending.lock(); + let Some(entry) = pending + .iter_mut() + .find(|entry| entry.fence.raw() == fence.raw()) + else { + // Already signalled. + return; + }; + if entry.result.is_some() { + return; + } + entry.result = Some(result); + let mut signalled = 0; + for entry in pending.iter() { + let Some(result) = entry.result else { + break; + }; + if let Err(error) = result { + entry.fence.set_error(error); + } + entry.fence.signal(); + signalled += 1; + } + drop(pending.drain(..signalled)); + } + + /// A signalled fence carrying `error`, for the output syncs of a refused submission. + pub(crate) fn refused(&self, error: Error) -> Fence { + error_fence(&self.errors, error).unwrap_or_else(|_| self.allocation_failure.clone()) + } +} + +/// A fence on `contexts` signalled with `error`. Every fence of an error context signals at +/// creation, so the context's fences signal in order. +fn error_fence(contexts: &FenceContexts, error: Error) -> Result { + let fence = contexts.new_fence(0, JobFenceOps)?; + fence.set_error(error); + fence.signal(); + Ok(Fence::from_fence(&fence)) +} + +/// The hardware fence of one job. +pub(crate) struct JobFence { + fence: UserFence, + timeline: Arc, +} + +impl JobFence { + /// Signals the fence, with an error on failure, after every earlier fence of the queue. Only + /// the first call has an effect. + pub(crate) fn signal(&self, result: Result) { + self.timeline.complete(&self.fence, result); + } + + /// A reference to the fence. + pub(crate) fn fence(&self) -> Fence { + Fence::from_fence(&self.fence) + } +} + +impl Drop for JobFence { + fn drop(&mut self) { + // Every fence of a timeline must complete, or no later fence of the queue would signal. + // A no-op for a completed fence (a job that was never accepted, or a bug, is cancelled). + self.timeline.complete(&self.fence, Err(ECANCELED)); + } +} + +/// Installs a fence in the output syncs of a submission. +fn install(outputs: impl Iterator, fence: &Fence) { + for mut sync in outputs { + if let Some(chain) = sync.chain_fence.take() { + sync.syncobj.add_point(chain, fence, sync.timeline_value); + } else { + sync.syncobj.replace_fence(Some(fence)); + } + } +} + +/// The syncs of one SUBMIT. Every output sync gets a fence: the job's fence once the job is +/// accepted, a signalled error fence when the submission is refused (the shared `file.rs` +/// contract of `tracked_submission`). +pub(crate) struct Syncs { + syncs: KVec, + input_count: usize, +} + +impl Syncs { + pub(crate) fn new(syncs: KVec, input_count: usize) -> Self { + Self { syncs, input_count } + } + + /// The input syncs (all resolved to fences or already signalled). + pub(crate) fn inputs(&self) -> &[SyncItem] { + &self.syncs[..self.input_count.min(self.syncs.len())] + } + + /// Publishes the accepted job's fence in every output sync. + pub(crate) fn accept(mut self, fence: &Fence) { + let first = self.input_count.min(self.syncs.len()); + install(self.syncs.drain(first..), fence); + } + + /// Publishes a signalled error fence in every output sync of a refused submission. + pub(crate) fn refuse(mut self, timeline: &Timeline, error: Error) { + let first = self.input_count.min(self.syncs.len()); + if first < self.syncs.len() { + install(self.syncs.drain(first..), &timeline.refused(error)); + } + } +} diff --git a/drivers/gpu/drm/asahi/g15/fw/buffer.rs b/drivers/gpu/drm/asahi/g15/fw/buffer.rs new file mode 100644 index 0000000000000..37d2ffb09ef8a --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/fw/buffer.rs @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Parameter-buffer and UMA allocator records of the 2419 firmware. +//! +//! The PB is one 512 MiB heap of [`MAX_PAGES`] pages of 0x8000 bytes in [`MAX_BLOCKS`] blocks of +//! four pages. The firmware sees it through the page list (u32 records `G >> 15` of compact +//! addresses `G = gva - 0x1000000000`, one per page), the block list (8-byte records, the low +//! word `G >> 15` of the block's first page) and the PB manager P: P+0x28 = 4 * blocks pages, +//! P+0x2c K = 4096 (truthful: 8 * K bytes is the block-list span the firmware synchronizes on +//! PB reconstruction, also during recovery), P+0x30 = blocks, P+0x48 = 4 * blocks - 1, +//! P+0x70/+0x74 = 4 * the firmware's growth maximum in pages (its asynchronous growth stops +//! there; a partial render may still ask for more). block_ctl +0 = blocks, +4 = the host +//! producer of block-list records, modulo K, advanced by every growth answer. +//! +//! Nothing here allocates, locks or publishes: the functions build the exact record bytes (a +//! zero fill followed by the field stores) and are host-tested. + +/// Bytes of one PB page. +pub(crate) const PAGE_BYTES: u64 = 0x8000; +/// Pages per block, and bytes of one block. +pub(crate) const PAGES_PER_BLOCK: u32 = 4; +pub(crate) const BLOCK_BYTES: u64 = 0x20000; +/// Blocks and pages of the mapped 512 MiB heap. +pub(crate) const MAX_BLOCKS: u32 = 4096; +pub(crate) const MAX_PAGES: u32 = 16384; +/// Records of the 0x8000-byte block list: the truthful K. +pub(crate) const BLOCK_LIST_RECORDS: u32 = 4096; +/// Sizes of the page list and the block list. +pub(crate) const PAGE_LIST_SIZE: usize = 0x28000; +pub(crate) const BLOCK_LIST_SIZE: usize = 0x8000; +/// Bytes of one block-list record. +pub(crate) const BLOCK_RECORD_BYTES: usize = 8; +/// Blocks per growth request (the native growImmediately). +pub(crate) const GROW_STEP: u32 = 10; +/// Smallest initial PB with growth. +pub(crate) const MIN_BLOCKS: u32 = 64; +/// Base of the 32-bit compact pointer window. +pub(crate) const COMPACT_BASE: u64 = 0x1000000000; + +/// Compact pointer of a GPU VA. +pub(crate) const fn compact(gva: u64) -> u64 { + gva.wrapping_sub(COMPACT_BASE) +} + +/// Page-list record of PB page `page` of the heap at `heap_gva`. +pub(crate) const fn page_record(heap_gva: u64, page: u32) -> u32 { + (compact(heap_gva + page as u64 * PAGE_BYTES) / PAGE_BYTES) as u32 +} + +/// Block-list record of PB block `block`: `(G - 0x1000000000) >> 15` of its first page. +pub(crate) const fn block_record(heap_gva: u64, block: u32) -> u32 { + (compact(heap_gva + block as u64 * BLOCK_BYTES) / PAGE_BYTES) as u32 +} + +/// Byte offset of the block-list record that block `block` goes to: `8 * (block % K)`. +pub(crate) const fn block_record_offset(block: u32) -> usize { + BLOCK_RECORD_BYTES * (block % BLOCK_LIST_RECORDS) as usize +} + +/// PB manager offsets (0xac bytes, firmware-only). +pub(crate) mod pb_manager { + pub(crate) const SIZE: usize = 0xac; + /// u64 page-list GVA, and its size in bytes. + pub(crate) const PAGE_LIST: usize = 0x1c; + pub(crate) const PAGE_LIST_BYTES: usize = 0x24; + pub(crate) const PAGES: usize = 0x28; + pub(crate) const K: usize = 0x2c; + pub(crate) const K_VALUE: u32 = super::BLOCK_LIST_RECORDS; + pub(crate) const BLOCKS: usize = 0x30; + /// u64 block-list GVA. + pub(crate) const BLOCK_LIST: usize = 0x38; + /// u64 block_ctl firmware VA. + pub(crate) const BLOCK_CTL: usize = 0x40; + pub(crate) const LAST_PAGE: usize = 0x48; + pub(crate) const BLOCK_BYTES: usize = 0x4c; + /// u64 PB counter firmware VA. + pub(crate) const COUNTER: usize = 0x58; + pub(crate) const MAX_PAGES_A: usize = 0x70; + pub(crate) const MAX_PAGES_B: usize = 0x74; +} + +/// UMA allocator manager offsets (0x84 bytes; render managers firmware+GPU class). +pub(crate) mod uma_manager { + pub(crate) const SIZE: usize = 0x84; + pub(crate) const ID: usize = 0x00; + pub(crate) const INDEX: usize = 0x08; + pub(crate) const ONE: usize = 0x0c; + /// u64 freelist GVA. + pub(crate) const FREELIST: usize = 0x14; + pub(crate) const UNK_1C: usize = 0x1c; + pub(crate) const UNK_1C_VALUE: u32 = 0x80000; + /// Freelist seeds available and total. + pub(crate) const SEEDS_A: usize = 0x24; + pub(crate) const SEEDS_B: usize = 0x2c; + pub(crate) const SEEDS: u32 = 8192; + /// u64 pool-table GVA. + pub(crate) const POOL_TABLE: usize = 0x34; + pub(crate) const UNK_44: usize = 0x44; + pub(crate) const UNK_44_VALUE: u32 = 0x40; + /// u64 counter firmware VA. + pub(crate) const COUNTER: usize = 0x48; +} + +/// UMA pools of one render engine: eight 4 MiB pools seeded 0x1000 at a time into the freelist +/// (8 * 1024 = [`uma_manager::SEEDS`] u64 records), pool-table rows of 0x40 bytes. +pub(crate) mod uma_pools { + pub(crate) const POOLS: usize = 8; + pub(crate) const POOL_SIZE: u64 = 0x400000; + pub(crate) const SEED_BYTES: u64 = 0x1000; + pub(crate) const FREELIST_SIZE: usize = 0x400000; + pub(crate) const POOL_TABLE_SIZE: usize = 0x1000; + pub(crate) const POOL_TABLE_OFFSET: usize = 0x3000; + pub(crate) const POOL_TABLE_STRIDE: usize = 0x40; + /// Pool-table row: the pool's GVA with bit 62. + pub(crate) const fn row(pool_gva: u64) -> u64 { + pool_gva | (1 << 62) + } +} + +/// Block control (0x40 bytes): +0 blocks, +4 the live producer of block-list records. +pub(crate) mod block_ctl { + pub(crate) const SIZE: usize = 0x40; + pub(crate) const BLOCKS: usize = 0; + pub(crate) const PRODUCER: usize = 4; +} + +/// Misc record (0x40 bytes): +0x20 = 1. +pub(crate) mod misc { + pub(crate) const SIZE: usize = 0x40; + pub(crate) const ENABLE: usize = 0x20; +} + +/// UMA and PB counters (0x40 bytes): +0 = 1 at initialization. +pub(crate) mod counter { + pub(crate) const SIZE: usize = 0x40; +} + +/// Scene list (0xc0 bytes), zeroed at initialization. +pub(crate) const SCENE_LIST_SIZE: usize = 0xc0; + +/// PB growth reply (DEVCTL 7) words. +pub(crate) mod grow_reply { + pub(crate) const OPCODE: usize = 0; + /// 1: grown, 0: refused (no new pages). + pub(crate) const GREW: usize = 1; + pub(crate) const SLOT: usize = 2; + pub(crate) const CONTEXT: usize = 3; + pub(crate) const COUNTER: usize = 4; + pub(crate) const POSITION: usize = 5; + /// u64 epoch (the Control halt count) at words 6..7. + pub(crate) const EPOCH: usize = 6; +} + +/// PB growth request (event type 7) words, copied whole by the event drain. +pub(crate) mod grow_request { + pub(crate) const TYPE: usize = 0; + pub(crate) const CONTEXT: usize = 1; + pub(crate) const SLOT: usize = 2; + /// The firmware's request counter (P+0x60 before its increment). + pub(crate) const COUNTER: usize = 3; + /// The TA position. + pub(crate) const POSITION: usize = 4; + /// u64 halt epoch at words 5..6. + pub(crate) const EPOCH: usize = 5; +} + +/// The C1 record written once per (re)initialization before the first render prefix that +/// follows it: {0x07400000, 4 * initial blocks}. +pub(crate) const fn c1_record(initial_blocks: u32) -> [u8; 16] { + let a = 0x07400000u32.to_le_bytes(); + let b = (PAGES_PER_BLOCK * initial_blocks).to_le_bytes(); + [ + a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3], 0, 0, 0, 0, 0, 0, 0, 0, + ] +} + +fn put32(image: &mut [u8], offset: usize, value: u32) { + image[offset..offset + 4].copy_from_slice(&value.to_le_bytes()); +} + +fn put64(image: &mut [u8], offset: usize, value: u64) { + image[offset..offset + 8].copy_from_slice(&value.to_le_bytes()); +} + +/// The PB manager image. +pub(crate) fn pb_manager_image( + initial: u32, + max: u32, + page_list_gva: u64, + block_list_gva: u64, + block_ctl_va: u64, + counter_va: u64, +) -> [u8; pb_manager::SIZE] { + let mut p = [0u8; pb_manager::SIZE]; + put64(&mut p, pb_manager::PAGE_LIST, page_list_gva); + put32(&mut p, pb_manager::PAGE_LIST_BYTES, PAGE_LIST_SIZE as u32); + put32(&mut p, pb_manager::PAGES, PAGES_PER_BLOCK * initial); + put32(&mut p, pb_manager::K, pb_manager::K_VALUE); + put32(&mut p, pb_manager::BLOCKS, initial); + put64(&mut p, pb_manager::BLOCK_LIST, block_list_gva); + put64(&mut p, pb_manager::BLOCK_CTL, block_ctl_va); + put32(&mut p, pb_manager::LAST_PAGE, PAGES_PER_BLOCK * initial - 1); + put32(&mut p, pb_manager::BLOCK_BYTES, BLOCK_BYTES as u32); + put64(&mut p, pb_manager::COUNTER, counter_va); + put32(&mut p, pb_manager::MAX_PAGES_A, PAGES_PER_BLOCK * max); + put32(&mut p, pb_manager::MAX_PAGES_B, PAGES_PER_BLOCK * max); + p +} + +/// A UMA manager image. The render managers have `id` = engine + 1, `index` = engine; the +/// compute graph's manager is id 3, index 2. +pub(crate) fn uma_manager_image( + id: u32, + index: u32, + freelist_gva: u64, + pool_table_gva: u64, + counter_va: u64, +) -> [u8; uma_manager::SIZE] { + let mut m = [0u8; uma_manager::SIZE]; + put32(&mut m, uma_manager::ID, id); + put32(&mut m, uma_manager::INDEX, index); + put32(&mut m, uma_manager::ONE, 1); + put64(&mut m, uma_manager::FREELIST, freelist_gva); + put32(&mut m, uma_manager::UNK_1C, uma_manager::UNK_1C_VALUE); + put32(&mut m, uma_manager::SEEDS_A, uma_manager::SEEDS); + put32(&mut m, uma_manager::SEEDS_B, uma_manager::SEEDS); + put64(&mut m, uma_manager::POOL_TABLE, pool_table_gva); + put32(&mut m, uma_manager::UNK_44, uma_manager::UNK_44_VALUE); + put64(&mut m, uma_manager::COUNTER, counter_va); + m +} + +/// The block-control image. +pub(crate) fn block_ctl_image(initial: u32) -> [u8; block_ctl::SIZE] { + let mut b = [0u8; block_ctl::SIZE]; + put32(&mut b, block_ctl::BLOCKS, initial); + put32(&mut b, block_ctl::PRODUCER, initial); + b +} + +/// The DEVCTL 7 reply to growth request `request`. +pub(crate) fn grow_reply_words(request: &[u32; 14], grew: bool) -> [u32; 14] { + let mut w = [0u32; 14]; + w[grow_reply::OPCODE] = super::channels::devctl::GROW_REPLY; + w[grow_reply::GREW] = u32::from(grew); + w[grow_reply::SLOT] = request[grow_request::SLOT]; + w[grow_reply::CONTEXT] = request[grow_request::CONTEXT]; + w[grow_reply::COUNTER] = request[grow_request::COUNTER]; + w[grow_reply::POSITION] = request[grow_request::POSITION]; + w[grow_reply::EPOCH] = request[grow_request::EPOCH]; + w[grow_reply::EPOCH + 1] = request[grow_request::EPOCH + 1]; + w +} + +#[cfg(test)] +mod tests { + use super::*; + + fn get32(b: &[u8], o: usize) -> u32 { + u32::from_le_bytes(b[o..o + 4].try_into().unwrap()) + } + + fn get64(b: &[u8], o: usize) -> u64 { + u64::from_le_bytes(b[o..o + 8].try_into().unwrap()) + } + + const HEAP: u64 = 0x70c0000000; + + #[test] + fn records() { + // (0x70c0000000 - 0x1000000000) >> 15 = 0xc18000. + assert_eq!(page_record(HEAP, 0), 0xc18000); + assert_eq!(page_record(HEAP, 1), 0xc18001); + assert_eq!(block_record(HEAP, 0), 0xc18000); + assert_eq!(block_record(HEAP, 1), 0xc18004); + assert_eq!(block_record(HEAP, 2047), 0xc18000 + 4 * 2047); + // The last page of the 512 MiB heap. + assert_eq!(page_record(HEAP, MAX_PAGES - 1), 0xc18000 + 16383); + assert_eq!(block_record_offset(0), 0); + assert_eq!(block_record_offset(1023), 8 * 1023); + assert_eq!(block_record_offset(4096), 0); + assert_eq!(block_record_offset(4097), 8); + // The block list holds K records exactly. + assert_eq!( + BLOCK_LIST_RECORDS as usize * BLOCK_RECORD_BYTES, + BLOCK_LIST_SIZE + ); + // The page list holds every page of the heap. + assert!(MAX_PAGES as usize * 4 <= PAGE_LIST_SIZE); + assert_eq!(MAX_BLOCKS as u64 * BLOCK_BYTES, 0x20000000); + } + + #[test] + fn pb_manager() { + let p = pb_manager_image( + 1024, + 2048, + 0x1074000000, + 0x70e0000000, + 0xfffffc2d00010000, + 0xfffffc2d00020000, + ); + assert_eq!(get64(&p, 0x1c), 0x1074000000); + assert_eq!(get32(&p, 0x24), 0x28000); + assert_eq!(get32(&p, 0x28), 4096); + assert_eq!(get32(&p, 0x2c), 4096); + assert_eq!(get32(&p, 0x30), 1024); + assert_eq!(get64(&p, 0x38), 0x70e0000000); + assert_eq!(get64(&p, 0x40), 0xfffffc2d00010000); + assert_eq!(get32(&p, 0x48), 4095); + assert_eq!(get32(&p, 0x4c), 0x20000); + assert_eq!(get64(&p, 0x58), 0xfffffc2d00020000); + assert_eq!(get32(&p, 0x70), 8192); + assert_eq!(get32(&p, 0x74), 8192); + // Every other byte is zero (the zero fill). + let written = [ + (0x1c, 8), + (0x24, 4), + (0x28, 4), + (0x2c, 4), + (0x30, 4), + (0x38, 8), + (0x40, 8), + (0x48, 4), + (0x4c, 4), + (0x58, 8), + (0x70, 4), + (0x74, 4), + ]; + for (i, byte) in p.iter().enumerate() { + if !written.iter().any(|&(o, n)| i >= o && i < o + n) { + assert_eq!(*byte, 0, "byte {i:#x}"); + } + } + } + + #[test] + fn uma_manager() { + let m = uma_manager_image(2, 1, 0x7014490000, 0x701489b000, 0xfffffc2d00030000); + assert_eq!(get32(&m, 0), 2); + assert_eq!(get32(&m, 8), 1); + assert_eq!(get32(&m, 0xc), 1); + assert_eq!(get64(&m, 0x14), 0x7014490000); + assert_eq!(get32(&m, 0x1c), 0x80000); + assert_eq!(get32(&m, 0x24), 8192); + assert_eq!(get32(&m, 0x2c), 8192); + assert_eq!(get64(&m, 0x34), 0x701489b000); + assert_eq!(get32(&m, 0x44), 0x40); + assert_eq!(get64(&m, 0x48), 0xfffffc2d00030000); + assert_eq!(&m[0x50..], &[0u8; 0x84 - 0x50][..]); + // Freelist seeds = 8 pools of 4 MiB in 0x1000 units. + assert_eq!( + uma_pools::POOLS as u64 * uma_pools::POOL_SIZE / uma_pools::SEED_BYTES, + uma_manager::SEEDS as u64 + ); + // The seeded records fit the freelist. + assert!(uma_manager::SEEDS as usize * 8 <= uma_pools::FREELIST_SIZE); + } + + #[test] + fn small_records() { + let b = block_ctl_image(1024); + assert_eq!(get32(&b, 0), 1024); + assert_eq!(get32(&b, 4), 1024); + assert!(b[8..].iter().all(|&x| x == 0)); + assert_eq!(uma_pools::row(0x7010000000), 0x4000007010000000); + let c1 = c1_record(1024); + assert_eq!(get32(&c1, 0), 0x07400000); + assert_eq!(get32(&c1, 4), 4096); + assert!(c1[8..].iter().all(|&x| x == 0)); + } + + #[test] + fn reply() { + let mut q = [0u32; 14]; + q[0] = 7; + q[1] = 5; // context + q[2] = 0; // slot + q[3] = 12; // counter + q[4] = 0x345; // position + q[5] = 3; // epoch low + q[6] = 1; // epoch high + let w = grow_reply_words(&q, true); + assert_eq!(&w[..8], &[7, 1, 0, 5, 12, 0x345, 3, 1]); + assert!(w[8..].iter().all(|&x| x == 0)); + let w = grow_reply_words(&q, false); + assert_eq!(w[1], 0); + } +} diff --git a/drivers/gpu/drm/asahi/g15/fw/channels.rs b/drivers/gpu/drm/asahi/g15/fw/channels.rs new file mode 100644 index 0000000000000..7a002d0836c47 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/fw/channels.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Channel, device-control and event records. + +use kernel::prelude::*; + +/// Records per ring. +pub(crate) const RING_SLOTS: u32 = 256; + +/// Ring state cells (offsets in a 0x30-byte state block): the consumer's read index, the +/// completion index and the producer's write index. +pub(crate) const STATE_READ: usize = 0x00; +pub(crate) const STATE_COMPLETION: usize = 0x10; +pub(crate) const STATE_WRITE: usize = 0x20; + +/// A submit channel record. +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub(crate) struct SubmitRecord { + pub(crate) unk_0: u64, + /// Queue-info VA of the queue to run. + pub(crate) qinfo: u64, + pub(crate) kind: u32, + /// Work-ring head the queue runs to. + pub(crate) head: u16, + pub(crate) event: u8, + pub(crate) new_queue: u8, +} + +static_assert!(core::mem::size_of::() == 0x18); +static_assert!(core::mem::offset_of!(SubmitRecord, qinfo) == 0x8); +static_assert!(core::mem::offset_of!(SubmitRecord, kind) == 0x10); +static_assert!(core::mem::offset_of!(SubmitRecord, head) == 0x14); +static_assert!(core::mem::offset_of!(SubmitRecord, event) == 0x16); +static_assert!(core::mem::offset_of!(SubmitRecord, new_queue) == 0x17); + +impl SubmitRecord { + /// The little-endian record bytes. + pub(crate) fn to_bytes(&self) -> [u8; 0x18] { + let mut b = [0u8; 0x18]; + b[0..8].copy_from_slice(&self.unk_0.to_le_bytes()); + b[8..16].copy_from_slice(&self.qinfo.to_le_bytes()); + b[16..20].copy_from_slice(&self.kind.to_le_bytes()); + b[20..22].copy_from_slice(&self.head.to_le_bytes()); + b[22] = self.event; + b[23] = self.new_queue; + b + } +} + +/// Device-control opcodes used with 2419. +pub(crate) mod devctl { + /// Re-import Globals. + pub(crate) const IMPORT_GLOBALS: u32 = 0x13; + /// Start. + pub(crate) const START: u32 = 9; + /// PB growth reply. + pub(crate) const GROW_REPLY: u32 = 7; +} + +/// A device-control record (0x38 bytes, opcode first). +#[repr(C)] +#[derive(Copy, Clone)] +pub(crate) struct DevCtl { + pub(crate) words: [u32; 14], +} + +static_assert!(core::mem::size_of::() == 0x38); + +impl DevCtl { + pub(crate) const fn op(opcode: u32) -> Self { + let mut words = [0; 14]; + words[0] = opcode; + Self { words } + } + + /// The opcode. + pub(crate) const fn opcode(&self) -> u32 { + self.words[0] + } + + /// The little-endian record bytes. + pub(crate) fn to_bytes(&self) -> [u8; 0x38] { + let mut b = [0u8; 0x38]; + for (i, w) in self.words.iter().enumerate() { + b[4 * i..4 * i + 4].copy_from_slice(&w.to_le_bytes()); + } + b + } +} + +/// Event types. +pub(crate) mod event { + pub(crate) const FAULT: u32 = 0; + pub(crate) const STAGE: u32 = 1; + pub(crate) const TIMEOUT: u32 = 4; + pub(crate) const PB_GROW: u32 = 7; + pub(crate) const CHANNEL_ERROR: u32 = 8; +} + +/// An event record. +#[repr(C)] +#[derive(Copy, Clone)] +pub(crate) struct EventRecord { + pub(crate) words: [u32; 14], +} + +static_assert!(core::mem::size_of::() == 0x38); diff --git a/drivers/gpu/drm/asahi/g15/fw/graph.rs b/drivers/gpu/drm/asahi/g15/fw/graph.rs new file mode 100644 index 0000000000000..d88b0fc0f6c15 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/fw/graph.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Queue-info rows, queue state, scheduling contexts, notifiers and the compute UMA manager. +//! +//! Queue-info scheduling row 2 (class 2, used for render and compute): +0x30 = 2, +0x34 = 2, +//! +0x38 = 0xffff000000000000, +0x40 = 0, +0x44 = 0, +0x48 = 2. Notifiers are initialized +//! once before first exposure and never written by the host again; the threshold is a +//! separate host-written uncached cell. + +/// Queue info (0xb4 bytes, firmware-only, ASC-cached). +pub(crate) mod qinfo { + pub(crate) const SIZE: usize = 0xb4; + /// u64 VAs of the queue state, the work ring, the job list and the queue scratch. + pub(crate) const STATE: usize = 0x00; + pub(crate) const RING: usize = 0x08; + pub(crate) const JOB_LIST: usize = 0x10; + pub(crate) const SCRATCH: usize = 0x18; + pub(crate) const UNK_2C: usize = 0x2c; + pub(crate) const UNK_4C: usize = 0x4c; + pub(crate) const UUID: usize = 0x50; + /// u64 scheduling context VA, then u64 4. + pub(crate) const SCHED_CTX: usize = 0xa4; + pub(crate) const UNK_AC: usize = 0xac; +} + +/// Queue-info scheduling rows (the firmware's recognized policy rows). +pub(crate) mod qinfo_row2 { + pub(crate) const CLASS: usize = 0x30; + pub(crate) const CLASS_B: usize = 0x34; + pub(crate) const MASK: usize = 0x38; + pub(crate) const MASK_VALUE: u64 = 0xffff000000000000; + pub(crate) const UNK_40: usize = 0x40; + pub(crate) const UNK_44: usize = 0x44; + pub(crate) const UNK_48: usize = 0x48; + /// Row 0 (class 0): +0x38 mask, +0x40 = 1, +0x48 = 1. + pub(crate) const ROW0_MASK_VALUE: u64 = 0xffffffffffff0000; +} + +/// Queue state (0x60 bytes, firmware-only, never cached). +pub(crate) mod qstate { + pub(crate) const SIZE: usize = 0x60; + /// Firmware DONE cursor. + pub(crate) const DONE: usize = 0x00; + /// Firmware READ cursor. + pub(crate) const READ: usize = 0x30; + /// Host WRITE cursor. + pub(crate) const WRITE: usize = 0x40; + /// Ring capacity in entries. + pub(crate) const CAP: usize = 0x50; +} + +/// Scheduling context (0x40 bytes): 0xff at +0..+1, 1 at +5, 0xff at +0x1e, 2 at +0x23. +pub(crate) mod sched_ctx { + pub(crate) const SIZE: usize = 0x40; +} + +/// Notifier (0x108 bytes): u64 threshold VA at +0, 0x50 at +0x10, 8 x 0xff at +0xa8; the +/// completing works store their context at +0x24 (firmware-owned once exposed). +pub(crate) mod notifier { + pub(crate) const SIZE: usize = 0x108; + pub(crate) const THRESHOLD: usize = 0x00; + pub(crate) const UNK_10: usize = 0x10; + pub(crate) const UNK_10_VALUE: u32 = 0x50; + /// The 8-byte mask the microsequences name (notifier + 0xa8). + pub(crate) const MASK: usize = 0xa8; +} + +/// Job list (0x18 bytes): u64 self VA at +8. +pub(crate) mod job_list { + pub(crate) const SIZE: usize = 0x18; + pub(crate) const SELF: usize = 0x08; +} + +/// The compute UMA allocator manager (0x84 bytes). The render managers have the same layout +/// (`fw::buffer`). +pub(crate) mod manager { + pub(crate) const SIZE: usize = 0x84; + pub(crate) const ID: usize = 0x00; + pub(crate) const INDEX: usize = 0x08; + pub(crate) const ONE: usize = 0x0c; + pub(crate) const FREELIST: usize = 0x14; + pub(crate) const UNK_1C: usize = 0x1c; + pub(crate) const UNK_1C_VALUE: u32 = 0x80000; + pub(crate) const SEEDS_A: usize = 0x24; + pub(crate) const SEEDS_B: usize = 0x2c; + pub(crate) const POOL_TABLE: usize = 0x34; + pub(crate) const UNK_44: usize = 0x44; + pub(crate) const UNK_44_VALUE: u32 = 0x40; + pub(crate) const COUNTER: usize = 0x48; + /// Compute manager identity: id 3, index 2. + pub(crate) const COMPUTE_ID: u32 = 3; + pub(crate) const COMPUTE_INDEX: u32 = 2; +} + +/// Compute pool table rows (0x40 bytes each, `gva | 1 << 62`) and freelist seeds (u64 VAs of +/// 0x1000-byte units of every pool). Seeding only the first eight pools (8192 pages, 32 MiB) let +/// ten compute workgroups of more than ~576 threads run at once and hung the eleventh (ticket +/// malik-na/linux-aurora#69): such groups take their memory from this pool. +pub(crate) mod pools { + pub(crate) const TABLE_STRIDE: usize = 0x40; + pub(crate) const TABLE_VALID: u64 = 1 << 62; + pub(crate) const SEED_BYTES: u64 = 0x1000; + pub(crate) const SEEDED_POOLS: usize = crate::g15::hw::sizes::POOLS; + /// Seeds in the freelist. + pub(crate) const SEEDS: u32 = + (SEEDED_POOLS as u64 * crate::g15::hw::sizes::POOL_SIZE / SEED_BYTES) as u32; +} diff --git a/drivers/gpu/drm/asahi/g15/fw/initdata.rs b/drivers/gpu/drm/asahi/g15/fw/initdata.rs new file mode 100644 index 0000000000000..d5a1f0188b368 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/fw/initdata.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Initdata layout of the 2419 build. +//! +//! The driver writes sparse words into large objects and board images, so this module holds the +//! offsets that the initdata build and recovery use, not full structure definitions. Several +//! canonical objects (Runtime, HardwareData, Globals, Power, Control) start at unaligned VAs, so +//! their words are written as bytes. + +/// Root object (0xc0 bytes). +pub(crate) mod root { + pub(crate) const SIZE: usize = 0xc0; + pub(crate) const MAGIC: u64 = 0x0c08e21e83800490; + pub(crate) const REGION_A: usize = 0x08; + pub(crate) const RUNTIME: usize = 0x18; + pub(crate) const GLOBALS: usize = 0x20; + /// u32 1. + pub(crate) const ENABLE: usize = 0x2c; + /// u16 page size, u8 page shift, u8 levels. + pub(crate) const GEOMETRY: usize = 0x30; + /// Three 0x20-byte level descriptors. + pub(crate) const LEVELS: usize = 0x34; + pub(crate) const LEVEL_STRIDE: usize = 0x20; + pub(crate) const FLAGS: usize = 0xa8; + pub(crate) const CONTROL: usize = 0xb0; + pub(crate) const POWER: usize = 0xb8; +} + +/// Region A (one page). +pub(crate) mod region_a { + pub(crate) const SIZE: usize = 0x4000; +} + +/// Runtime object (0x4b3 bytes). +pub(crate) mod runtime { + pub(crate) const SIZE: usize = 0x4b3; + pub(crate) const HWDATA: usize = 0x00; + pub(crate) const PAIR: usize = 0x08; + pub(crate) const PAIR2: usize = 0x10; + /// 13 x {state, completion cell, write cell, ring}: the 12 submit channels, then DEVCTL. + pub(crate) const CHANNELS: usize = 0x18; + pub(crate) const CHANNEL_STRIDE: usize = 0x20; + /// 4 x {state, ring}: event, log, trace, stats. + pub(crate) const RX: usize = 0x1b8; + pub(crate) const RX_STRIDE: usize = 0x10; + pub(crate) const SMALL: usize = 0x1f8; + pub(crate) const TA_ACTIVITY: usize = 0x234; + pub(crate) const FS_ACTIVITY: usize = 0x23c; + pub(crate) const COMPUTE_ACTIVITY: usize = 0x244; + pub(crate) const FAULT_INFO: usize = 0x24c; + pub(crate) const C0: usize = 0x2a8; + pub(crate) const C1: usize = 0x2b0; + pub(crate) const C1_2: usize = 0x2b8; + pub(crate) const C3: usize = 0x2c0; + pub(crate) const C3_2: usize = 0x2c8; + /// u32 4. + pub(crate) const UNK_2D0: usize = 0x2d0; + /// Readiness magic written by the firmware. + pub(crate) const READY: usize = 0x314; + pub(crate) const READY_MAGIC: u32 = 0xabcdabcd; + /// u32 0xff. + pub(crate) const UNK_3B0: usize = 0x3b0; + pub(crate) const HWDATA_A: usize = 0x441; +} + +/// Pair (0x100 bytes) and Small (0x10 bytes). +pub(crate) mod pair { + pub(crate) const SIZE: usize = 0x100; +} +pub(crate) mod small { + pub(crate) const SIZE: usize = 0x10; +} + +/// C0 (0x1000 bytes), C1 and C3 (one page each): firmware + GPU shared. +pub(crate) mod cbuf { + pub(crate) const C0_SIZE: usize = 0x1000; + pub(crate) const C1_SIZE: usize = 0x4000; + pub(crate) const C3_SIZE: usize = 0x4000; +} + +/// Control object (0xc3d0 bytes). +pub(crate) mod control { + pub(crate) const SIZE: usize = 0xc3d0; + /// u32 1 at initdata. + pub(crate) const ENABLE: usize = 0x4; + pub(crate) const PENDING: usize = 0x40a0; + pub(crate) const FWCTL_STATE: usize = 0x4568; + pub(crate) const FWCTL_RING: usize = 0x4570; + pub(crate) const HALT_COUNT: usize = 0x4580; + /// u32: firmware halted for recovery (it sets 1, the host clears). + pub(crate) const HALTED: usize = 0x4590; + pub(crate) const RESUME: usize = 0x45a0; + pub(crate) const FLAG: usize = 0x45b0; + /// u32 1 at initdata. + pub(crate) const ENABLE2: usize = 0x45c4; +} + +/// Globals (15,868 bytes) words used after READY. +pub(crate) mod globals { + pub(crate) const SIZE: usize = 15868; + pub(crate) const COMPUTE_KILL: usize = 0x1c; + pub(crate) const FRAGMENT_KILL: usize = 0x24; + pub(crate) const IDLE_ENABLE: usize = 0x2c; + pub(crate) const IDLE_PREDICTIVE: usize = 0x30; + pub(crate) const IDLE_STANDBY_US: usize = 0x7a8; + pub(crate) const IDLE_MIN_CONF: usize = 0x7c4; + /// Template 10, written 100. + pub(crate) const UNK_980: usize = 0x980; + pub(crate) const IDLE_DELAY: usize = 0x9a0; + pub(crate) const COMPUTE_MONITOR: usize = 0x9b0; +} + +/// Power (2,292 bytes). +pub(crate) mod power { + pub(crate) const SIZE: usize = 2292; +} + +/// HardwareData (35,332 bytes). +pub(crate) mod hwdata { + pub(crate) const SIZE: usize = 35332; + /// Bytes of the first pages mapped ASC-cached (HardwareDataB and the activity records): + /// up to the end of the page after the one holding byte 0. + pub(crate) const CACHED: usize = 0x4a04; + pub(crate) const TIMESTAMPS: usize = 0x28; + /// IOMapping records: 32 bytes per slot, the firmware VA at +8. + pub(crate) const IOMAPS: usize = 0x640; + pub(crate) const IOMAP_STRIDE: usize = 32; + pub(crate) const IOMAP_VA: usize = 8; + /// Template checks: u32 1, and the number of DVFS states minus one. + pub(crate) const CHECK_A38: usize = 0xa38; + pub(crate) const DVFS_COUNT: usize = 0xb58; + pub(crate) const TTBAT: usize = 0xb44; + pub(crate) const TA_ACTIVITY: usize = 0x1880; + pub(crate) const FS_ACTIVITY: usize = 0x24c0; + pub(crate) const COMPUTE_ACTIVITY: usize = 0x3740; + /// HardwareDataA: the maintained S8 request words. + pub(crate) const HWDATA_A: usize = 0x4580; + /// HardwareDataA word holding the machine's maximum GPU power in mW (Globals +0x78 and + /// Power +0xd0 repeat it). + pub(crate) const MAX_POWER_A: usize = 0x6a4; + pub(crate) const FAULT_INFO: usize = 0x8900; +} + +/// Flags (0x40 bytes). +pub(crate) mod flags { + pub(crate) const SIZE: usize = 0x40; + pub(crate) const READY: usize = 4; +} diff --git a/drivers/gpu/drm/asahi/g15/fw/mod.rs b/drivers/gpu/drm/asahi/g15/fw/mod.rs new file mode 100644 index 0000000000000..234805b081923 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/fw/mod.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G15 firmware 2419 structures. +//! +//! Layouts of the records, work images and objects the host shares with the 2419 firmware, +//! in the style of `g17/fw`: `#[repr(C)]` structures with `static_assert!` offsets where the +//! layout is a fixed record, and offset constants where the driver writes sparse words into a +//! larger image. Nothing here allocates, locks or publishes. + +pub(crate) mod buffer; +pub(crate) mod channels; +pub(crate) mod graph; +pub(crate) mod initdata; +pub(crate) mod work; diff --git a/drivers/gpu/drm/asahi/g15/fw/work.rs b/drivers/gpu/drm/asahi/g15/fw/work.rs new file mode 100644 index 0000000000000..929101852333f --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/fw/work.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Compute, TA and FS work images and microsequence records of the 2419 firmware. +//! +//! Compute work: 0x893 bytes at client +0x3760, sequence 0x2b8. TA work: 0x92b bytes, 72 +//! register records, sequence 0x2d8. FS work: 0xc73 bytes, 79 register records and the +//! parameter block, sequence 0x310. Register records are 12-byte `(u32 selector, u64 value)` +//! pairs; timestamp records are 0x3c bytes (0x80000003/3). Barrier records are 0x3e bytes: +4 +//! wait stamp VA, +0x14 value, +0x18 producing engine, +0x1c own stamp, +0x20 UUID, +0x2a type +//! (0 = firmware-stamp wait only, 1 = also the predecessor software-job hold). +//! +//! Microsequence opcodes: compute start 0x0b / finalize 0x0c, TA 5 / 6, FS 7 / 8, barrier 4, +//! InitBM 6, END 0x40000002. Work type word: compute 3, TA 0, FS 1. HardwareData references: +//! compute +0x3740, TA +0x1884, FS +0x24c8. +//! +//! The encoders ([`super::super::encode`]) write the sparse words of these images at literal +//! offsets (kept inline there); this module holds the sizes, the fixed records and the register +//! selector tables. + +/// Microsequence opcodes. +pub(crate) mod op { + pub(crate) const COMPUTE_START: u32 = 0x0b; + pub(crate) const COMPUTE_FINALIZE: u32 = 0x0c; + pub(crate) const TA_START: u32 = 5; + pub(crate) const TA_FINALIZE: u32 = 6; + pub(crate) const FS_START: u32 = 7; + pub(crate) const FS_FINALIZE: u32 = 8; + pub(crate) const BARRIER: u32 = 4; + pub(crate) const INIT_BM: u32 = 6; + pub(crate) const END: u32 = 0x40000002; + /// Start timestamp record; the end record is `TIMESTAMP & 0xff` (3). + pub(crate) const TIMESTAMP: u32 = 0x80000003; + pub(crate) const TIMESTAMP_END: u32 = 3; +} + +/// Barrier wait types (+0x2a of a barrier record). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum BarrierType { + /// Wait for the producer's firmware stamp only. + Stamp = 0, + /// Also hold for the predecessor software job (0x537/0x538). + Hold = 1, +} + +impl BarrierType { + /// Type 0 if `stamp_only`, else type 1. + pub(crate) const fn from_stamp_only(stamp_only: bool) -> Self { + if stamp_only { + BarrierType::Stamp + } else { + BarrierType::Hold + } + } +} + +/// Work type words. +pub(crate) mod work_type { + pub(crate) const TA: u32 = 0; + pub(crate) const FS: u32 = 1; + pub(crate) const COMPUTE: u32 = 3; +} + +/// One `(u32 selector, u64 value)` register record. +pub(crate) const REG_RECORD: usize = 12; + +/// Compute work image and microsequence. +pub(crate) mod compute { + /// Work image bytes. + pub(crate) const WORK_SIZE: usize = 0x893; + /// Microsequence bytes; the sequence object is 0x1000 bytes. + pub(crate) const SEQ_SIZE: usize = 0x2b8; + pub(crate) const SEQ_OBJECT: usize = 0x1000; + /// Register records at +0x20. + pub(crate) const REGS: usize = 0x20; + /// HardwareData compute activity record. + pub(crate) const HWDATA: u64 = 0x3740; + /// Attachment records in the start record (+0x54, 16 bytes each). + pub(crate) const ATTACHMENTS: usize = 0x54; + pub(crate) const ATTACHMENT_STRIDE: usize = 16; + /// Start/end timestamp records and the base of their cells in the work image. + pub(crate) const TS_START: usize = 0x1bc; + pub(crate) const TS_END: usize = 0x1fc; + pub(crate) const TS_CELLS: u64 = 0x810; + /// Finalize record. + pub(crate) const FINALIZE: usize = 0x238; + pub(crate) const END: usize = 0x2b4; + + /// Register selectors of the compute work image. + pub(crate) const SEL: [u32; 22] = [ + 0x1a510, 0x1a420, 0x1a4d0, 0x1a4d8, 0x1a4e0, 0x1a4e8, 0x1a440, 0x1a458, 0x12091, 0x101d9, + 0x1a089, 0x1a091, 0x1a059, 0x1a061, 0x1a0b9, 0x1a0c1, 0x101d1, 0xd479, 0x1a0e9, 0x107a1, + 0xa599, 0xd411, + ]; +} + +/// TA work image and microsequence. +pub(crate) mod ta { + pub(crate) const WORK_SIZE: usize = 0x92b; + pub(crate) const SEQ_SIZE: usize = 0x2d8; + pub(crate) const SEQ_OBJECT: usize = 0x1000; + /// Register records at +0x40. + pub(crate) const REGS: usize = 0x40; + pub(crate) const HWDATA: u64 = 0x1884; + /// Work tail (shared layout, [`super::tail`]). + pub(crate) const TAIL: usize = 0x8a8; + pub(crate) const TS_START: usize = 0x1cc; + pub(crate) const TS_END: usize = 0x20c; + pub(crate) const FINALIZE: usize = 0x248; + pub(crate) const END: usize = 0x2d4; + + /// Register selectors of the TA work image. + pub(crate) const SEL: [u32; 72] = [ + 0x1748, 0x10141, 0x1c039, 0x1c9c8, 0x1c041, 0x1c9d0, 0x1c0a1, 0x1c031, 0x1c9c0, 0x1c051, + 0x1c061, 0x10149, 0x10139, 0x10111, 0x1c9b0, 0x10119, 0x1c9b8, 0x1c958, 0x1c950, 0x1c930, + 0x1c880, 0x1c898, 0x1c948, 0x1c888, 0x1c890, 0x1c918, 0x1c079, 0x1c9d8, 0x1c089, 0x1c9e0, + 0x16c41, 0x1ca40, 0x1c9a8, 0x1c920, 0x10151, 0x1c199, 0x1c1a1, 0x1c1a9, 0x1c1b1, 0x1c1b9, + 0x1c8f8, 0x1c0b1, 0x1c850, 0x10131, 0x10121, 0x10129, 0x101b9, 0x1c069, 0x1c071, 0x1c081, + 0x1c0a9, 0x10171, 0x10169, 0x12099, 0x101e1, 0x1c9e8, 0x1a099, 0x1a0a1, 0x1a069, 0x1a071, + 0x1a0c9, 0x1a0d1, 0x101c9, 0xd471, 0x1a0f1, 0x10799, 0xa5a1, 0xd419, 0x1c830, 0x1ca30, + 0x16c39, 0x1c910, + ]; + + /// Index of register #33 (PBREQ: the initial PB size in pages, 0x3c when clustered). + pub(crate) const REG_PBREQ: usize = 32; +} + +/// FS work image and microsequence. +pub(crate) mod fs { + pub(crate) const WORK_SIZE: usize = 0xc73; + pub(crate) const SEQ_SIZE: usize = 0x310; + pub(crate) const SEQ_OBJECT: usize = 0x1000; + /// Register records at +0x80. + pub(crate) const REGS: usize = 0x80; + /// Parameter block / firmware mirrors. + pub(crate) const PARAMS: usize = 0x790; + pub(crate) const HWDATA: u64 = 0x24c8; + pub(crate) const TAIL: usize = 0xbf0; + pub(crate) const TS_START: usize = 0x1dc; + pub(crate) const TS_END: usize = 0x21c; + pub(crate) const FINALIZE: usize = 0x258; + pub(crate) const END: usize = 0x30c; + + /// Register selectors of the FS work image. + pub(crate) const SEL: [u32; 79] = [ + 0x1739, 0x10009, 0x15379, 0x15381, 0x15369, 0x15371, 0x15131, 0x15139, 0x100a1, 0x15069, + 0x15071, 0x16058, 0x10019, 0x100b1, 0x16030, 0x100d9, 0x10791, 0x16098, 0x15109, 0x15101, + 0x15021, 0x15211, 0x15049, 0x10051, 0x15321, 0x15301, 0x15309, 0x15311, 0x15319, 0x15349, + 0x15351, 0x15329, 0x15331, 0x15339, 0x15341, 0x15231, 0x15221, 0x15239, 0x15229, 0x15401, + 0x15421, 0x15409, 0x15429, 0x153c1, 0x15411, 0x153c9, 0x15431, 0x153d1, 0x15419, 0x153d9, + 0x15439, 0x16429, 0x16060, 0x16431, 0x10039, 0x16020, 0x16451, 0x15359, 0x100b8, 0x16461, + 0x16090, 0x120a1, 0x101e9, 0x160a8, 0x16068, 0x160b8, 0x1a0a9, 0x1a0b1, 0x1a079, 0x1a081, + 0x1a0d9, 0x1a0e1, 0x101c1, 0xd469, 0x1a0f9, 0x1c838, 0x1ca28, 0xa5a9, 0xd429, + ]; +} + +/// The 0x3c timestamp record: `+0` opcode, `+4/+0xc/+0x14` cells of the work image at `B`, +/// `+0x1c` queue info, `+0x24` user pair (0 unless requested), `+0x2c` `W + B + 0x58`, `+0x34` +/// UUID. +pub(crate) mod ts_record { + pub(crate) const SIZE: usize = 0x3c; +} + +/// The work tail shared by TA (+0x8a8) and FS (+0xbf0); the compute image has the same layout +/// at +0x810. +pub(crate) mod tail { + pub(crate) const ITS_START: usize = 0x8; + pub(crate) const ITS_END: usize = 0x10; + pub(crate) const USER_START: usize = 0x18; + pub(crate) const USER_END: usize = 0x20; + pub(crate) const UMA_MANAGER: usize = 0x2e; + pub(crate) const SCRATCH: usize = 0x47; +} + +/// Barrier record. +pub(crate) mod barrier { + pub(crate) const SIZE: usize = 0x3e; + pub(crate) const STAMP_VA: usize = 0x4; + pub(crate) const STAMP_VA2: usize = 0xc; + pub(crate) const WAIT: usize = 0x14; + pub(crate) const EVENT: usize = 0x18; + pub(crate) const OWN: usize = 0x1c; + pub(crate) const UUID: usize = 0x20; + pub(crate) const TYPE: usize = 0x2a; + /// UUID of render predecessor waits. + pub(crate) const RENDER_WAIT_UUID: u32 = 0x3d420; +} + +/// InitBM record (0x20 bytes). +pub(crate) mod initbm { + pub(crate) const SIZE: usize = 0x20; + pub(crate) const CTX: usize = 0x4; + pub(crate) const BLOCKS: usize = 0x10; + pub(crate) const PB_MANAGER: usize = 0x14; + pub(crate) const STAMP: usize = 0x1c; +} + +/// Per-pass scene record (0x80 bytes). +pub(crate) mod scene { + pub(crate) const SIZE: usize = 0x80; + /// Scene-list slots cycled by draw ordinal (`scene_list + 4 * (n % 48)`). + pub(crate) const LIST_SLOTS: u64 = 48; +} diff --git a/drivers/gpu/drm/asahi/g15/gate.rs b/drivers/gpu/drm/asahi/g15/gate.rs new file mode 100644 index 0000000000000..f41b2e3f75f57 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/gate.rs @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Stream admission gates (safety nets for userspace encoders). +//! +//! VDM: the root stream's first word is 0x21000960. CDM walk from `cdm_ctrl_stream_base` through +//! links until TERMINATE (0x40000000), at most 8192 records and 64 links (a longer or cyclic +//! stream is rejected): barriers 0x60000160, 0x60001560, 0x600215fe, 0x600201fe; +//! link 0x2000XXXX; direct launch (40 B, the G15 layout), indirect launches 36 B +//! (`w0 & 0xff7fffff == 0x08080000`) and 24 B (0x10080000); every program VA inside the queue's +//! 4 GiB USC window, 64-aligned, first 64 bytes readable. Client memory is read through the VM's +//! page tables with a point-of-coherency clean+invalidate of the lines ([`mmu::Vm::read_user`]: +//! the GPU may have written it). `g15_stream_gate=0` disables both gates (encoder bring-up only). + +use core::sync::atomic::{AtomicU32, Ordering}; +use kernel::prelude::*; + +use super::{command::cover, command::USC_WINDOW, params}; +use crate::mmu; + +const VDM_ROOT: u32 = 0x21000960; +const CDM_TERMINATE: u32 = 0x40000000; +const CDM_BARRIERS: [u32; 4] = [0x60000160, 0x600215fe, 0x60001560, 0x600201fe]; +const CDM_LINK: u32 = 0x20000000; +const CDM_DIRECT: u32 = 0x00080000; +const CDM_INDIRECT_SIZES: u32 = 0x08080000; +const CDM_INDIRECT: u32 = 0x10080000; +const CDM_MAX_RECORDS: u32 = 8192; +const CDM_MAX_LINKS: u32 = 64; +/// Rejections logged per boot. +const MAX_REPORTS: u32 = 32; + +static REPORTS: AtomicU32 = AtomicU32::new(0); + +/// Logs a refused record and refuses it. +fn reject(what: &str, va: u64, w: &[u32]) -> Error { + if REPORTS.fetch_add(1, Ordering::Relaxed) < MAX_REPORTS { + let word = |i: usize| w.get(i).copied().unwrap_or(0); + pr_warn!( + "G15: rejecting {} record at {:#x}: {:08x} {:08x} {:08x} {:08x}\n", + what, + va, + word(0), + word(1), + word(2), + word(3) + ); + } + EINVAL +} + +/// Up to 256 stream bytes read at once. +struct Window { + va: u64, + len: usize, + bytes: [u8; 256], +} + +impl Window { + /// The `count` words at `va`, reading whole spans of up to 256 bytes (not past the page + /// unless the record crosses it). Unreadable memory: EFAULT. + fn words(&mut self, vm: &mmu::Vm, va: u64, count: usize) -> Result<[u32; 10]> { + let n = count * 4; + let cached = self.len != 0 + && va >= self.va + && va + .checked_add(n as u64) + .is_some_and(|end| end <= self.va + self.len as u64); + if !cached { + let in_page = mmu::UAT_PGSZ - (va as usize & mmu::UAT_PGMSK); + let len = n.max(self.bytes.len().min(in_page)); + self.len = 0; + vm.read_user(va, &mut self.bytes[..len]) + .map_err(|_| EFAULT)?; + self.va = va; + self.len = len; + } + let start = (va - self.va) as usize; + let mut words = [0u32; 10]; + for (index, word) in words.iter_mut().take(count).enumerate() { + let at = start + index * 4; + let mut le = [0u8; 4]; + le.copy_from_slice(&self.bytes[at..at + 4]); + *word = u32::from_le_bytes(le); + } + Ok(words) + } +} + +/// A launch's program: inside the queue's 4 GiB USC window, 64-aligned, first block readable. +/// `checked` caches the last proven program page. +fn program(vm: &mmu::Vm, usc: u64, address: u64, checked: &mut Option) -> Result { + if address < usc || address - usc >= USC_WINDOW || address & 63 != 0 { + return Err(EINVAL); + } + // 64-aligned: the block never crosses its page. + let page = address & !(mmu::UAT_PGMSK as u64); + if *checked == Some(page) { + return Ok(()); + } + cover(vm, address, address + 64, true, false)?; + *checked = Some(page); + Ok(()) +} + +/// VDM admission: a normal root stream opens with exactly the G15 barrier. +pub(crate) fn vdm(vm: &mmu::Vm, va: u64) -> Result { + if !params::stream_gate() { + return Ok(()); + } + let mut bytes = [0u8; 4]; + vm.read_user(va, &mut bytes).map_err(|_| EFAULT)?; + let word = u32::from_le_bytes(bytes); + if word != VDM_ROOT { + return Err(reject("VDM root", va, &[word])); + } + Ok(()) +} + +/// CDM admission: walks the stream from its base through no-return links until TERMINATE, +/// admitting only the known G15S records: direct launch (40), indirect launch (36/24), barriers +/// (4), link (8). +pub(crate) fn cdm(vm: &mmu::Vm, usc: u64, base: u64) -> Result { + if !params::stream_gate() { + return Ok(()); + } + let mut window = Window { + va: 0, + len: 0, + bytes: [0; 256], + }; + let mut checked = None; + let mut va = base; + let mut links = 0; + let mut records = 0; + while records < CDM_MAX_RECORDS { + records += 1; + let w = window.words(vm, va, 1)?; + if CDM_BARRIERS.contains(&w[0]) { + va += 4; + continue; + } + if w[0] == CDM_TERMINATE { + return Ok(()); + } + if w[0] & 0xffff0000 == CDM_LINK { + let w = window.words(vm, va, 2)?; + links += 1; + if links > CDM_MAX_LINKS { + return Err(reject("CDM link budget", va, &w)); + } + va = (u64::from(w[0] & 0xffff) << 32) | u64::from(w[1]); + if va & 3 != 0 { + return Err(reject("CDM link", va, &w)); + } + continue; + } + if w[0] == CDM_DIRECT { + let w = window.words(vm, va, 10)?; + let address = (u64::from(w[1] >> 16) << 32) | (u64::from(w[2]) << 6); + if w[1] & 0xffff != 0 || w[3] != 0x40 || (w[2] >> 26) != ((w[1] >> 16) & 0x3f) { + return Err(reject("CDM direct launch", va, &w)); + } + match program(vm, usc, address, &mut checked) { + Err(error) if error == EINVAL => return Err(reject("CDM program", va, &w)), + result => result?, + } + va += 40; + continue; + } + let kind = w[0] & 0xff7fffff; + if kind == CDM_INDIRECT_SIZES || kind == CDM_INDIRECT { + let sizes = kind == CDM_INDIRECT_SIZES; + let w = window.words(vm, va, if sizes { 9 } else { 6 })?; + let address = (u64::from(w[1] >> 22) << 38) | (u64::from(w[2]) << 6); + let indirect = (u64::from(w[4] & 0xffff) << 32) | u64::from(w[5]); + let bad_size = |size: u32| size == 0 || size > 0x7ff; + if w[3] != (0x40000000 | ((address >> 40) & 0xff) as u32) + || w[4] >> 16 != 0 + || indirect & 3 != 0 + || (sizes && (bad_size(w[6]) || bad_size(w[7]) || bad_size(w[8]))) + { + return Err(reject("CDM indirect launch", va, &w)); + } + let result = program(vm, usc, address, &mut checked) + .and_then(|()| cover(vm, indirect, indirect + 12, true, false)); + match result { + Err(error) if error == EINVAL => return Err(reject("CDM indirect", va, &w)), + result => result?, + } + va += if sizes { 36 } else { 24 }; + continue; + } + return Err(reject("CDM", va, &w)); + } + Err(reject("CDM record budget", va, &[])) +} diff --git a/drivers/gpu/drm/asahi/g15/glue.c b/drivers/gpu/drm/asahi/g15/glue.c new file mode 100644 index 0000000000000..662c9c8258e21 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/glue.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * G15 (T6030/T6031, M3 Pro/Max) helpers that have no Rust bindings: static inlines, + * macros and THIS_MODULE. Built only with CONFIG_DRM_ASAHI_G15. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void asahi_g15_sha256(const u8 *data, size_t len, u8 *out); +bool asahi_g15_board(void); +bool asahi_g15_machine(const char *compatible); +void asahi_g15_hold_power(struct device *dev); +void asahi_g15_release_power(struct device *dev); +void asahi_g15_module_get(void); +void asahi_g15_reprobe(void); +int asahi_g15_adt_phram(u64 *start, u64 *size); +bool asahi_g15_ram_disjoint(u64 start, u64 size); +bool asahi_g15_ram_within(u64 start, u64 size); +unsigned int asahi_g15_memory_mode(u64 start, u64 size); +void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size); +bool asahi_g15_dma_coherent(struct device *dev); +int asahi_g15_pmp_prepare(struct device *dev); +int asahi_g15_pmp_get(struct device *dev); +void asahi_g15_pmp_put(struct device *dev); +bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size); + +/* + * The maintained m1n1 handoff keeps these four UAT regions in the linear map. + * Require the exact enabled reserved-memory owner, in addition to the ADT + * bounds/size/disjointness and per-page PageReserved checks in admission. + */ +bool asahi_g15_reserved_uat(u32 index, u64 start, u64 size) +{ + static const char *const paths[] = { + "/reserved-memory/uat-ttbs", + "/reserved-memory/uat-pagetables", + "/reserved-memory/uat-handoff", + "/reserved-memory/uat-pagetables-l2", + }; + struct device_node *np; + struct resource res; + bool match; + + if (index >= ARRAY_SIZE(paths)) + return false; + np = of_find_node_by_path(paths[index]); + match = np && of_device_is_available(np) && + !of_property_read_bool(np, "reusable") && + !of_get_address(np, 1, NULL, NULL) && + !of_address_to_resource(np, 0, &res) && + res.start == start && resource_size(&res) == size; + of_node_put(np); + return match; +} + +/* + * Keep a single PMP RTKit owner. The maintained bridge serializes requests, + * waits for supplier readiness and acknowledgement, and latches lost replies. + * The GPU retains both its module and supplier dependency until reboot. + */ +int asahi_g15_pmp_prepare(struct device *dev) +{ + int ret; + + if (!asahi_g15_board() || !dev->of_node || + !of_device_is_compatible(dev->of_node, "apple,agx-t6030")) + return -ENODEV; + /* DISP ACK alone does not prove the native DRM component set is up. */ + /* The direct import waits for the display module to become LIVE; a + * symbol_get during its probe could defer without any post-LIVE retry. + */ + ret = apple_drm_t6030_link_device(dev); + if (ret) + return ret; + return apple_pmp_link_device(dev); +} + +int asahi_g15_pmp_get(struct device *dev) +{ + if (!asahi_g15_board() || !dev->of_node || + !of_device_is_compatible(dev->of_node, "apple,agx-t6030")) + return -ENODEV; + return apple_pmp_set_device_power(0x0f, 5, 1); + +} + +/* + * Give back the vote of asahi_g15_pmp_get() after a start that failed before + * the firmware was touched. The full M3 stack's J514S bridge has no release: + * its vote stays, as before. + */ +void asahi_g15_pmp_put(struct device *dev) +{ + int ret = apple_pmp_set_device_power(0x0f, 5, 0); + + if (ret) + dev_warn(dev, "G15: PMP AGX vote release failed: %d\n", ret); + +} + +/* SHA-256 of the firmware TEXT and the board templates. */ +void asahi_g15_sha256(const u8 *data, size_t len, u8 *out) +{ + sha256(data, len, out); +} + +/* Whether this is a J514S/J516S T6030 board. */ +bool asahi_g15_board(void) +{ + return of_machine_is_compatible("apple,t6030") && + (of_machine_is_compatible("apple,j514s") || + of_machine_is_compatible("apple,j516s")); +} + +/* Whether the machine carries this compatible (of_machine_is_compatible()). */ +bool asahi_g15_machine(const char *compatible) +{ + return of_machine_is_compatible(compatible) > 0; +} + +/* + * Keep the GFX domain powered for the whole (retained) session, like the + * firmware session itself; released only by a start that failed before the + * firmware was touched (asahi_g15_release_power()). + */ +void asahi_g15_hold_power(struct device *dev) +{ + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); +} + +/* + * Give the hold of asahi_g15_hold_power() back: the start failed before + * anything firmware-visible was written, so the session is not retained. + * Only where the AGX vote is given back as well (asahi_g15_pmp_put()): the + * full M3 stack's J514S bridge keeps its vote, and with it the hold stays. + */ +void asahi_g15_release_power(struct device *dev) +{ + pm_runtime_put_noidle(dev); + pm_runtime_disable(dev); + pm_runtime_set_suspended(dev); + +} + +/* Pin this module until reboot. */ +void asahi_g15_module_get(void) +{ + __module_get(THIS_MODULE); +} + +/* + * The boot loader's phram "adt" export: the first /reserved-memory child + * compatible with "phram" and labelled "adt". + */ +int asahi_g15_adt_phram(u64 *start, u64 *size) +{ + struct device_node *rmem, *np; + struct resource res; + int ret = -ENODEV; + + rmem = of_find_node_by_path("/reserved-memory"); + for_each_child_of_node(rmem, np) { + const char *label; + + if (of_device_is_compatible(np, "phram") && + !of_property_read_string(np, "label", &label) && !strcmp(label, "adt") && + !of_address_to_resource(np, 0, &res)) { + of_node_put(np); + ret = 0; + break; + } + } + of_node_put(rmem); + if (ret) + return ret; + *start = res.start; + *size = resource_size(&res); + return 0; +} + +/* The range is disjoint from System RAM (ADT region admission). */ +bool asahi_g15_ram_disjoint(u64 start, u64 size) +{ + return region_intersects(start, size, IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) == + REGION_DISJOINT; +} + +/* + * Whether DMA mappings of the device skip cache maintenance. The G15 firmware + * objects rely on the DMA mapping cleaning their pages to the point of + * coherency, so the device must not be DMA-coherent. + */ +bool asahi_g15_dma_coherent(struct device *dev) +{ + return dev_is_dma_coherent(dev); +} + +/* + * Every page must be valid, System RAM and still reserved from the allocator. + * REGION_INTERSECTS alone also accepts RAM mixed with undefined memory holes. + */ +bool asahi_g15_ram_within(u64 start, u64 size) +{ + u64 end, pa; + + if (!size || !IS_ALIGNED(start, PAGE_SIZE) || + !IS_ALIGNED(size, PAGE_SIZE) || check_add_overflow(start, size, &end)) + return false; + for (pa = start; pa < end; pa += PAGE_SIZE) { + unsigned long pfn = PHYS_PFN(pa); + + if (!pfn_valid(pfn) || !page_is_ram(pfn) || + !PageReserved(pfn_to_page(pfn))) + return false; + } + return true; +} + +/* One classification for every adopted-UAT and firmware-region CPU mapping. */ +unsigned int asahi_g15_memory_mode(u64 start, u64 size) +{ + u64 end; + + if (!size || !IS_ALIGNED(start, PAGE_SIZE) || + !IS_ALIGNED(size, PAGE_SIZE) || check_add_overflow(start, size, &end)) + return 0; + if (asahi_g15_ram_within(start, size)) + return MEMREMAP_WB; + if (asahi_g15_ram_disjoint(start, size)) + return MEMREMAP_WC; + return 0; +} + +/* The inherited RTKit buffer inside firmware DATA. */ +void __iomem *asahi_g15_ioremap_wc(u64 start, size_t size) +{ + return ioremap_wc(start, size); +} + +/* + * Called only by the final infallible DriverModule initializer, after both + * platform registrations have completed. driver_find() returns a borrowed + * pointer, not an owned reference. The pinned registration is still owned by + * DriverModule throughout this call; module exit cannot race its initialization. + * No put_driver() is appropriate. A failed attach must not fail module init: + * another device in this pass may already have started retained firmware. + */ +void asahi_g15_reprobe(void) +{ + const struct device_driver *drv; + int ret; + + drv = driver_find("asahi-g15", &platform_bus_type); + if (!drv) { + pr_err("asahi-g15: registered adapter not found for activation\n"); + return; + } + ret = driver_attach(drv); + if (ret) + pr_err("asahi-g15: activation attach failed: %d\n", ret); +} diff --git a/drivers/gpu/drm/asahi/g15/graph.rs b/drivers/gpu/drm/asahi/g15/graph.rs new file mode 100644 index 0000000000000..9a62f25f19a63 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/graph.rs @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The persistent compute and render graphs. +//! +//! The compute graph holds 22 x 4 MiB compute pools, the freelist (every pool's pages), the pool +//! table, the manager, scheduling context, notifier, threshold, job list, queue info/state, ring, +//! queue scratch and the user and firmware stamps. The render graph holds the render queues, +//! scheduling context, notifier, threshold, job list, the empty legacy record and InitBM. The +//! compute slots and the render pass slots are [`super::slots`]; the render UMA/PB allocators +//! and their spare set are [`super::pools`], initialized from [`Firmware::init_graphs`]. +//! Classes per [`super::hw::tuning`]. Graph buffers with a client GVA are aliased into every +//! user VM ([`Request::client`]). +//! +//! The compute queue's Globals +0x9b0 = 0 (unchanged-work monitor off) is written with a waited +//! DEVCTL 0x13 before the queue is first exposed. + +use kernel::{ + device, + prelude::*, + sync::{aref::ARef, Arc}, +}; + +use super::channel::Channel; +use super::fw::graph::{job_list, manager, notifier, pools, qinfo, qinfo_row2, qstate, sched_ctx}; +use super::hw::{client, engine, pte, sizes, tuning}; +use super::object::{Class, FwObject, Objects, Placement, Request}; +use super::Firmware; +use crate::driver::AsahiDrmDevice as AsahiDevice; + +/// Firmware class of firmware-maintained command objects. +pub(crate) const fn cmd_fw() -> Class { + if tuning::FW_CACHED { + Class::FwCached + } else { + Class::Fw + } +} + +/// The firmware + GPU twin of [`cmd_fw`]. +pub(crate) const fn cmd_shared() -> Class { + if tuning::FW_CACHED { + Class::SharedCached + } else { + Class::Shared + } +} + +/// Notifiers: firmware-only, cached as selected by `tuning::NOTIFIER_CACHED` (bit 0 compute, +/// bit 1 render). +pub(crate) const fn notifier_class(bit: u32) -> Class { + if tuning::NOTIFIER_CACHED & bit != 0 { + Class::FwCached + } else { + Class::Fw + } +} + +/// Allocator managers: their index-2 class or its AttrIndex 0 twin. +pub(crate) const fn manager_class(class: Class) -> Class { + if !tuning::MANAGER_CACHED { + return class; + } + match class { + Class::Shared => Class::SharedCached, + _ => Class::FwCached, + } +} + +/// Queue scratch: firmware-private cached, or the shared index-2 class. +pub(crate) const fn queue_scratch() -> Class { + if tuning::FW_CACHED { + Class::FwCached + } else { + Class::Shared + } +} + +/// A client alias: the GVA of the page holding the payload and the leaf word. +#[derive(Copy, Clone)] +pub(crate) struct Client { + pub(crate) page: u64, + pub(crate) leaf: u64, +} + +impl Client { + /// GPU-only NC RW. + pub(crate) const fn nc(page: u64) -> Option { + Some(Self { + page, + leaf: pte::LEAF_NC, + }) + } + + /// GPU-cached. + pub(crate) const fn cached(page: u64) -> Option { + Some(Self { + page, + leaf: pte::LEAF_CACHED, + }) + } +} + +/// Allocates a zeroed firmware object of `size` payload bytes at `offset` in its first page, +/// mapped in the firmware window with `class`, optionally aliased in every user VM and, with +/// `lower`, in the context-0 lower tree at the client page. +pub(crate) fn alloc( + objects: &mut Objects, + size: usize, + class: Class, + client: Option, + offset: usize, + lower: bool, +) -> Result> { + objects.alloc(Request { + size, + class, + placement: Placement::Anywhere, + offset, + client: client.map(|c| (c.page, c.leaf)), + lower: if lower { client.map(|c| c.page) } else { None }, + cpu: true, + }) +} + +/// The compute graph, without its slots. +pub(crate) struct ComputeGraph { + /// The empty legacy record: allocated for the firmware's layout, never written. + _legacy: Arc, + pub(crate) pools: KVec>, + pub(crate) freelist: Arc, + pub(crate) pool_table: Arc, + pub(crate) manager: Arc, + pub(crate) counter: Arc, + pub(crate) sched_ctx: Arc, + pub(crate) notifier: Arc, + pub(crate) threshold: Arc, + pub(crate) job_list: Arc, + pub(crate) qinfo: Arc, + pub(crate) qstate: Arc, + pub(crate) ring: Arc, + pub(crate) qscratch: Arc, + pub(crate) user_stamp: Arc, + pub(crate) fw_stamp: Arc, +} + +impl ComputeGraph { + /// Allocates the compute graph objects. + fn new(objects: &mut Objects) -> Result { + let (sh, fw) = (Class::Shared, Class::Fw); + let o = objects; + + let legacy = alloc(o, 0x4000, sh, None, 0, false)?; + let mut pools = KVec::with_capacity(sizes::POOLS, GFP_KERNEL)?; + for i in 0..sizes::POOLS as u64 { + let pool = alloc( + o, + sizes::POOL_SIZE as usize, + sh, + Client::nc(client::pool(i)), + 0, + false, + )?; + pools.push(pool, GFP_KERNEL)?; + } + Ok(Self { + _legacy: legacy, + pools, + freelist: alloc(o, 0x400000, sh, Client::nc(client::FREELIST), 0, false)?, + pool_table: alloc( + o, + 0x1000, + sh, + Client::nc(client::POOL_TABLE_PAGE), + client::POOL_TABLE_OFF as usize, + false, + )?, + manager: alloc(o, manager::SIZE, manager_class(sh), None, 0, false)?, + counter: alloc(o, 0x40, sh, None, 0, false)?, + sched_ctx: alloc(o, sched_ctx::SIZE, cmd_fw(), None, 0, false)?, + notifier: alloc(o, notifier::SIZE, notifier_class(1), None, 0, false)?, + threshold: alloc(o, 4, fw, None, 0, false)?, + job_list: alloc(o, job_list::SIZE, cmd_fw(), None, 0, false)?, + qinfo: alloc(o, qinfo::SIZE, cmd_fw(), None, 0, false)?, + qstate: alloc(o, qstate::SIZE, fw, None, 0, false)?, + ring: alloc(o, sizes::QUEUE_CAP as usize * 8, fw, None, 0, false)?, + qscratch: alloc(o, 0x40000, queue_scratch(), None, 0, false)?, + user_stamp: alloc(o, 4, sh, None, 0, false)?, + fw_stamp: alloc(o, 4, sh, None, 0, false)?, + }) + } + + /// Initial contents; the firmware owns them afterwards. The slots are scrubbed by + /// [`super::slots`]. + fn init(&self) -> Result { + let mut entry = 0u64; + + // Freelist: every 4 MiB pool in 0x1000 units. + for pool in self.pools.iter().take(pools::SEEDED_POOLS) { + let gva = pool.gva.ok_or(EINVAL)?; + let mut off = 0; + while off < sizes::POOL_SIZE { + self.freelist.w64(8 * entry as usize, gva + off)?; + entry += 1; + off += pools::SEED_BYTES; + } + } + for (k, pool) in self.pools.iter().enumerate() { + let gva = pool.gva.ok_or(EINVAL)?; + self.pool_table + .w64(pools::TABLE_STRIDE * k, gva | pools::TABLE_VALID)?; + } + let m = &self.manager; + m.w32(manager::ID, manager::COMPUTE_ID)?; + m.w32(manager::INDEX, manager::COMPUTE_INDEX)?; + m.w32(manager::ONE, 1)?; + m.w64(manager::FREELIST, self.freelist.gva.ok_or(EINVAL)?)?; + m.w32(manager::UNK_1C, manager::UNK_1C_VALUE)?; + m.w32(manager::SEEDS_A, pools::SEEDS)?; + m.w32(manager::SEEDS_B, pools::SEEDS)?; + m.w64(manager::POOL_TABLE, self.pool_table.gva.ok_or(EINVAL)?)?; + m.w32(manager::UNK_44, manager::UNK_44_VALUE)?; + m.w64(manager::COUNTER, self.counter.va)?; + self.counter.w32(0, 1)?; + + init_sched_ctx(&self.sched_ctx)?; + init_notifier(&self.notifier, &self.threshold)?; + self.job_list.w64(job_list::SELF, self.job_list.va)?; + self.threshold.w32(0, 1)?; + + let q = &self.qinfo; + q.w64(qinfo::STATE, self.qstate.va)?; + q.w64(qinfo::RING, self.ring.va)?; + q.w64(qinfo::JOB_LIST, self.job_list.va)?; + q.w64(qinfo::SCRATCH, self.qscratch.va)?; + q.w32(qinfo::UNK_2C, 0xffffffff)?; + q.w64(qinfo_row2::MASK, qinfo_row2::ROW0_MASK_VALUE)?; + q.w32(qinfo_row2::UNK_40, 1)?; + q.w32(qinfo_row2::UNK_48, 1)?; + if tuning::COMPUTE_QUEUE_CLASS == 2 { + // Row 2: 2, 2, ffff000000000000, 0, 0, 2. + q.w32(qinfo_row2::CLASS, 2)?; + q.w32(qinfo_row2::CLASS_B, 2)?; + q.w64(qinfo_row2::MASK, qinfo_row2::MASK_VALUE)?; + q.w32(qinfo_row2::UNK_40, 0)?; + q.w32(qinfo_row2::UNK_44, 0)?; + q.w32(qinfo_row2::UNK_48, 2)?; + } + q.w32(qinfo::UNK_4C, 0xffffffff)?; + q.w32(qinfo::UUID, engine::COMPUTE_UUID)?; + q.w64(qinfo::SCHED_CTX, self.sched_ctx.va)?; + q.w64(qinfo::UNK_AC, 4)?; + self.qstate.w32(qstate::CAP, sizes::QUEUE_CAP)?; + + self.user_stamp.w32(0, engine::COMPUTE_SEED)?; + self.fw_stamp.w32(0, 0) + } +} + +/// The render graph, without the pass slots, the clustering buffers and the render +/// allocators. +pub(crate) struct RenderGraph { + /// TA (0) and FS (1) queues. + pub(crate) qinfo: [Arc; 2], + pub(crate) qstate: [Arc; 2], + pub(crate) ring: [Arc; 2], + pub(crate) qscratch: [Arc; 2], + pub(crate) user_stamp: [Arc; 2], + pub(crate) fw_stamp: [Arc; 2], + pub(crate) sched_ctx: Arc, + pub(crate) notifier: Arc, + pub(crate) threshold: Arc, + pub(crate) job_list: Arc, + pub(crate) legacy_empty: Arc, + pub(crate) initbm: Arc, +} + +/// One render engine's queue objects, allocated together. +struct RenderQueue { + qinfo: Arc, + qstate: Arc, + ring: Arc, + qscratch: Arc, + user_stamp: Arc, + fw_stamp: Arc, +} + +impl RenderQueue { + fn new(o: &mut Objects) -> Result { + Ok(Self { + qinfo: alloc(o, qinfo::SIZE, cmd_fw(), None, 0, false)?, + qstate: alloc(o, qstate::SIZE, Class::Fw, None, 0, false)?, + ring: alloc(o, sizes::QUEUE_CAP as usize * 8, Class::Fw, None, 0, false)?, + qscratch: alloc(o, 0x40000, queue_scratch(), None, 0, false)?, + user_stamp: alloc(o, 4, Class::Shared, None, 0, false)?, + fw_stamp: alloc(o, 4, Class::Shared, None, 0, false)?, + }) + } +} + +impl RenderGraph { + /// Allocates the render graph objects. + fn new(o: &mut Objects) -> Result { + let [ta, fs] = [RenderQueue::new(o)?, RenderQueue::new(o)?]; + let sched_ctx = alloc(o, sched_ctx::SIZE, cmd_fw(), None, 0, false)?; + let notifier = alloc(o, notifier::SIZE, notifier_class(2), None, 0, false)?; + let threshold = alloc(o, 4, Class::Fw, None, 0, false)?; + let job_list = alloc(o, job_list::SIZE, cmd_fw(), None, 0, false)?; + let legacy_empty = alloc(o, 0x40, Class::Shared, None, 0, false)?; + let initbm = alloc(o, super::fw::work::initbm::SIZE, cmd_fw(), None, 0, false)?; + Ok(Self { + qinfo: [ta.qinfo, fs.qinfo], + qstate: [ta.qstate, fs.qstate], + ring: [ta.ring, fs.ring], + qscratch: [ta.qscratch, fs.qscratch], + user_stamp: [ta.user_stamp, fs.user_stamp], + fw_stamp: [ta.fw_stamp, fs.fw_stamp], + sched_ctx, + notifier, + threshold, + job_list, + legacy_empty, + initbm, + }) + } + + /// Initial contents; the render allocators are initialized separately. + fn init(&self) -> Result { + let uuid = [engine::TA_UUID, engine::FS_UUID]; + let seed = [engine::TA_SEED, engine::FS_SEED]; + + for e in 0..2 { + let q = &self.qinfo[e]; + q.w64(qinfo::STATE, self.qstate[e].va)?; + q.w64(qinfo::RING, self.ring[e].va)?; + q.w64(qinfo::JOB_LIST, self.job_list.va)?; + q.w64(qinfo::SCRATCH, self.qscratch[e].va)?; + q.w32(qinfo::UNK_2C, 0xffffffff)?; + if tuning::RENDER_QUEUE_CLASS == 2 { + // Recognized row 2: class/policy 2, mask, +0x40/+0x44 0. + q.w32(qinfo_row2::CLASS, 2)?; + q.w32(qinfo_row2::CLASS_B, 2)?; + q.w64(qinfo_row2::MASK, qinfo_row2::MASK_VALUE)?; + q.w32(qinfo_row2::UNK_48, 2)?; + } else { + q.w64(qinfo_row2::MASK, qinfo_row2::ROW0_MASK_VALUE)?; + q.w32(qinfo_row2::UNK_40, 1)?; + q.w32(qinfo_row2::UNK_48, 1)?; + } + q.w32(qinfo::UNK_4C, 0xffffffff)?; + q.w32(qinfo::UUID, uuid[e])?; + q.w64(qinfo::SCHED_CTX, self.sched_ctx.va)?; + q.w64(qinfo::UNK_AC, 4)?; + self.qstate[e].w32(qstate::CAP, sizes::QUEUE_CAP)?; + self.ring[e].fill(0, 0xcc, sizes::QUEUE_CAP as usize * 8)?; + self.user_stamp[e].w32(0, seed[e])?; + self.fw_stamp[e].w32(0, seed[e])?; + } + init_sched_ctx(&self.sched_ctx)?; + init_notifier(&self.notifier, &self.threshold)?; + self.job_list.w64(job_list::SELF, self.job_list.va)?; + self.threshold.w32(0, 2) + } +} + +/// Scheduling context initial contents, shared by both graphs. +fn init_sched_ctx(s: &FwObject) -> Result { + s.fill(0x00, 0xff, 2)?; + s.fill(0x05, 0x01, 1)?; + s.fill(0x1e, 0xff, 1)?; + s.fill(0x23, 0x02, 1) +} + +/// Notifier initial contents: the only host stores the notifier ever gets. +fn init_notifier(n: &FwObject, threshold: &FwObject) -> Result { + n.w64(notifier::THRESHOLD, threshold.va)?; + n.w32(notifier::UNK_10, notifier::UNK_10_VALUE)?; + n.fill(notifier::MASK, 0xff, 8) +} + +/// The compute and render graph objects. +pub(crate) struct Graphs { + /// The GPU device, for the engine's log messages. + pub(crate) dev: ARef, + pub(crate) compute: ComputeGraph, + pub(crate) render: RenderGraph, +} + +impl Graphs { + /// Allocates both graphs. + pub(crate) fn new(dev: &AsahiDevice, objects: &mut Objects) -> Result { + Ok(Self { + dev: dev.as_ref().as_ref().into(), + compute: ComputeGraph::new(objects)?, + render: RenderGraph::new(objects)?, + }) + } +} + +impl Firmware { + /// Initializes both graphs, the slots and the render allocators, then turns the compute + /// monitor off before the queues are first exposed. + pub(crate) fn init_graphs(&mut self) -> Result { + // TA/FS/compute channels must start empty. + for ch in [Channel::Ta, Channel::Fs, Channel::Compute] { + if self.channels.read(ch)? != 0 || self.channels.write_cell(ch)? != 0 { + return Err(Error::from_errno(-(kernel::bindings::EPROTO as i32))); + } + } + self.graph.compute.init()?; + self.slots.init()?; + super::engine::wmb(); + self.graph.render.init()?; + self.init_render_pools(false)?; + // New firmware and context-0 translations before any reference. + super::uat::tlbi_all(); + dev_info!( + self.graph.dev.as_ref(), + "G15: compute graph: work {:#x}/{:#x} pools {:#x} freelist {:#x} table {:#x}\n", + self.slots.scrub.compute[0].work.va, + self.slots.scrub.compute[0].work.gva.unwrap_or(0), + self.graph.compute.pools[0].gva.unwrap_or(0), + self.graph.compute.freelist.gva.unwrap_or(0), + self.graph.compute.pool_table.gva.unwrap_or(0) + ); + if tuning::COMPUTE_QUEUE_CLASS == 2 { + // Class-2 compute profile: the unchanged-work compute monitor (Globals +0x9b0) + // off before the queue's first exposure. + self.compute_monitor_off()?; + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g15/hw.rs b/drivers/gpu/drm/asahi/g15/hw.rs new file mode 100644 index 0000000000000..d566a54953e07 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/hw.rs @@ -0,0 +1,590 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G15 hardware and firmware-ABI constants (layout values are shared by several modules; change +//! one only together with its users). The qualified T6030 values are the module-level +//! constants; what differs per SoC is a [`super::Config`] row ([`t6030`], [`t6031`]). The tuned +//! policy values are in [`tuning`]. + +use core::ops::Range; + +pub(crate) mod t6031; + +/// Identification register values of a qualified G15S. +pub(crate) struct HwId { + /// SGX identity word (`sgx + 0xd04000`). + pub(crate) version: u32, + /// SGX layout word (`sgx + 0xd04010`). + pub(crate) counts: u32, +} + +/// A qualified GPU firmware build, identified by its loaded TEXT. +pub(crate) struct FirmwareBuild { + pub(crate) name: &'static str, + /// Optional board restriction in addition to the SoC admission. + pub(crate) machine: Option<&'static kernel::str::CStr>, + pub(crate) text_size: u64, + pub(crate) data_size: u64, + /// Offset of the GKTS marker in TEXT; the 8 entropy bytes after it are zeroed before hashing. + pub(crate) marker: u32, + pub(crate) text_sha256: [u8; 32], +} + +/// Where a SoC's GPU regions, firmware segments and performance states come from, and how its +/// GPU is powered. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum Platform { + /// T6030: the boot loader's ADT export ([`super::adt`]), a GPU node synthesized by the PMP + /// handoff, and the PMP AGX vote through the J514S supplier. + AdtWithPmp, + /// The GPU node of the kernel device tree, filled by the boot loader (regions, firmware + /// segments and operating points); the GPU is powered by its power domain alone. + DeviceTree, +} + +/// The source of the board images (HardwareData, Globals, Power) of a SoC. +#[derive(Copy, Clone, PartialEq, Eq)] +pub(crate) enum Board { + /// The hash-checked T6030 templates ([`super::board`]). + Templates, + /// The three images built from the device tree ([`super::board::record`], + /// [`super::board::device_tree`]) and cross-checked against a firmware file holding the + /// machine's own ([`super::board::blob`], pinned by its SHA-256), which also supplies the + /// record's words the device tree does not carry ([`super::board::record::FROM_FILE`]); a + /// difference refuses admission, before any firmware is touched. + Blob { + name: &'static kernel::str::CStr, + sha256: [u8; 32], + }, + /// No board data: admission refuses the SoC before anything firmware-visible. + None, +} + +/// Per-SoC placement and size of the initdata objects that differ between G15 firmware +/// builds; everything else keeps the canonical [`va`] values. +pub(crate) struct Layout { + /// Root object version word. + pub(crate) root_magic: u64, + /// HardwareData VA, size, ASC-cached prefix (`None`: the whole object), and the offset of + /// the fourth HardwareDataB view (fault information). + pub(crate) hwdata_va: u64, + pub(crate) hwdata_size: usize, + pub(crate) hwdata_cached: Option, + pub(crate) hwdata_view: usize, + /// HardwareDataB IOMapping records. + pub(crate) iomaps: &'static [IoMap], + /// For a record built from the device tree ([`super::board::record`]): per IOMapping + /// record, in `iomaps` order, the mapped bytes, the element bytes and whether the firmware + /// may write the window; and the unmapped slots whose flag word is nevertheless 1. + pub(crate) record_slots: &'static [(u32, u32, bool)], + pub(crate) record_empty_writable: &'static [u8], +} + +/// Compute command register values that scale with the cluster count. +pub(crate) struct ComputeRegs { + /// Offset of the four saved-state pointers in the preemption buffer. + pub(crate) tail: u64, + /// UMA PTC (0xa599) and DUPM (0xd411). + pub(crate) ptc: u64, + pub(crate) dupm: u64, + /// The 0x1a0e9 word. + pub(crate) reg_1a0e9: u64, + /// Encode every command the way the first command of a fresh queue is encoded: zero + /// saved-state pointers and zero 0x12091/0x101d9. + pub(crate) fresh: bool, +} + +/// The T6030 configuration. +pub(crate) mod t6030 { + use super::super::Config; + use super::{ComputeRegs, FirmwareBuild, HwId, Layout}; + + /// The qualified graph: the canonical VAs and sizes of [`super::va`] and + /// [`super::super::fw::initdata`]. + pub(crate) const LAYOUT: Layout = Layout { + root_magic: super::super::fw::initdata::root::MAGIC, + hwdata_va: super::va::HWDATA, + hwdata_size: super::super::fw::initdata::hwdata::SIZE, + hwdata_cached: Some(super::super::fw::initdata::hwdata::CACHED), + hwdata_view: super::super::fw::initdata::hwdata::FAULT_INFO, + iomaps: &super::IOMAPS, + record_slots: &[], + record_empty_writable: &[], + }; + + /// The two-cluster compute register values. + pub(crate) const COMPUTE: ComputeRegs = ComputeRegs { + tail: 0x1480, + ptc: 0x13200400020, + dupm: 0x200000001, + reg_1a0e9: 8, + fresh: false, + }; + + pub(crate) const CONFIG: Config = Config { + name: "T6030 G15S", + chip_id: 0x6030, + gpu_variant: crate::hw::GpuVariant::S, + gpu_revision: 0x11, + uat_ias: 42, + uat_oas: 42, + num_clusters: 2, + num_cores_per_cluster: 10, + hw_ids: &[HwId { + version: 0x07031100, + counts: 0x00110209, + }], + firmware: &[ + FirmwareBuild { + name: "macOS 14.7-14.8.x stub firmware (iBoot-10151.140.19.700.2)", + machine: None, + text_size: 0x5c000, + data_size: 0x114000, + marker: 0x58e88, + text_sha256: [ + 0x11, 0xe4, 0x9f, 0x75, 0x0b, 0x67, 0x1a, 0x2b, 0x13, 0xd0, 0x92, 0xdb, 0x1c, 0xbc, + 0xa3, 0xa8, 0x77, 0x55, 0x86, 0xd6, 0xb1, 0xe3, 0xaa, 0xe8, 0xa6, 0xa4, 0x64, 0xa7, + 0x80, 0x02, 0x93, 0xd5, + ], + }, + // The maintained J516S runtime already identifies this B0 image by + // UUID dbf37c40-ead5-3760-9441-995018782955 and uses the same + // 0x0c08e21e83800490 InitData ABI. Pin its complete measured TEXT + // digest here, rather than enabling the bring-up prefix override. + FirmwareBuild { + name: "J516S B0 RTKit-2419.140.12 (maintained 14.x handoff)", + machine: Some(kernel::c_str!("apple,j516s")), + text_size: 0x5c000, + data_size: 0x114000, + marker: 0x58e88, + text_sha256: [ + 0xda, 0x26, 0xec, 0xca, 0xb1, 0xf8, 0x59, 0x8d, + 0xa8, 0x0c, 0x85, 0x5e, 0xca, 0x91, 0x65, 0x20, + 0x40, 0x6d, 0x1d, 0x06, 0x70, 0xf1, 0xd4, 0x70, + 0xe7, 0x9b, 0xb9, 0xb0, 0x22, 0x09, 0xdb, 0x53, + ], + }, + ], + machine: kernel::c_str!("apple,t6030"), + platform: super::Platform::AdtWithPmp, + layout: &LAYOUT, + board: super::Board::Templates, + compute: COMPUTE, + }; +} + +/// User VA range reported as `vm_start..vm_end`. Not the full 42-bit range: the userspace VA +/// layout on G15S was qualified with this end. +pub(crate) const USER_RANGE: Range = 0x4000..0x2ffffff8000; + +/// Bits per cluster in the enabled-core mask (`sgx + 0xe01500`). +pub(crate) const CLUSTER_BITS: u32 = 10; + +/// UAT page size. +pub(crate) const PAGE: u64 = 0x4000; +/// Physical address bits of UAT leaves. +pub(crate) const PA_MASK: u64 = 0x3ffffffc000; + +/// Firmware upper half: root[0..1] are firmware-owned, root[2] is the host window. +pub(crate) const UPPER: u64 = 0xfffffc0000000000; +pub(crate) const HOST: u64 = 0xfffffc2000000000; +pub(crate) const HOST_SIZE: u64 = 0x1000000000; + +/// Firmware UAT leaf words. Shared-layer `Prot` values for them are built with +/// `mmu::Prot::from_pte` and must round-trip to these exact words. +pub(crate) mod pte { + /// Firmware-only shared RW data, AttrIndex 2 (inner non-cacheable for the ASC). + pub(crate) const FW: u64 = 0x00c000000000044b; + /// Firmware + GPU shared, AttrIndex 2. + pub(crate) const SHARED: u64 = 0x00e000000000040b; + /// The same two classes with AttrIndex 0 (inner+outer write-back for the ASC). + pub(crate) const FW_CACHED: u64 = 0x00c0000000000443; + pub(crate) const SHARED_CACHED: u64 = 0x00e0000000000403; + /// Firmware device RW. + pub(crate) const MMIO: u64 = 0x00c0000000000447; + /// Table descriptor bits. + pub(crate) const TABLE: u64 = 3; + /// Context-0 lower work alias: GPU RW NC, non-global. + pub(crate) const LOWER: u64 = 0x00c0000000000c8b; + /// Client (user VM) driver leaves: NC, cached, PB list. + pub(crate) const LEAF_NC: u64 = 0x00c0000000000c8b; + pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c83; + pub(crate) const LEAF_PB_LIST: u64 = 0x00e0000000000c03; +} + +/// PPL handoff magic. +pub(crate) const PPL_MAGIC: u64 = 0x4b1d000000000002; + +/// UAT handoff region sizes of this firmware ABI (TTBAT page, TTBAT tables (the firmware kernel +/// root), PPL handoff page, shared L2 page); the addresses come from the ADT. +pub(crate) const TTBS_SIZE: u64 = 0x4000; +pub(crate) const TABLES_SIZE: u64 = 0x40000; +pub(crate) const HANDOFF_SIZE: u64 = 0x4000; +pub(crate) const SHARED_L2_SIZE: u64 = 0x4000; + +/// Canonical initdata object VAs of the qualified build. +pub(crate) mod va { + pub(crate) const ROOT: u64 = 0xfffffc204062ff40; + pub(crate) const REGION_A: u64 = 0xfffffc2040670000; + pub(crate) const RUNTIME: u64 = 0xfffffc20406b7b4d; + pub(crate) const HWDATA: u64 = 0xfffffc20406fb5fc; + pub(crate) const PAIR: u64 = 0xfffffc2040747f00; + pub(crate) const SMALL: u64 = 0xfffffc204078bff0; + pub(crate) const C0: u64 = 0xfffffc2070003000; + pub(crate) const C1: u64 = 0xfffffc2070008000; + pub(crate) const C3: u64 = 0xfffffc2070010000; + pub(crate) const GLOBALS: u64 = 0xfffffc20407cc204; + pub(crate) const POWER: u64 = 0xfffffc204081370c; + pub(crate) const CONTROL: u64 = 0xfffffc20003bbc30; + pub(crate) const FLAGS: u64 = 0xfffffc2040857fc0; + pub(crate) const DEVCTL_STATE: u64 = 0xfffffc2040333fd0; + pub(crate) const DEVCTL_RING: u64 = 0xfffffc2000330800; + pub(crate) const EVENT_STATE: u64 = 0xfffffc2040377fd0; + pub(crate) const EVENT_RING: u64 = 0xfffffc20403b8800; + pub(crate) const LOG_STATE: u64 = 0xfffffc20403ffee0; + pub(crate) const LOG_RING: u64 = 0xfffffc2040443000; + pub(crate) const TRACE_STATE: u64 = 0xfffffc20404d7fd0; + pub(crate) const TRACE_RING: u64 = 0xfffffc2040519000; + pub(crate) const STATS_STATE: u64 = 0xfffffc2040563fd0; + pub(crate) const STATS_RING: u64 = 0xfffffc20405a4000; + pub(crate) const FWCTL_STATE: u64 = 0xfffffc20405ebfd0; + pub(crate) const FWCTL_RING: u64 = 0xfffffc2000376c00; + /// Firmware timestamp arena (64 MiB). + pub(crate) const TIMESTAMPS: u64 = 0xfffffc2ffc000000; + pub(crate) const TIMESTAMPS_SIZE: u64 = 0x4000000; + /// Firmware VA allocator window for backend objects. + pub(crate) const FW_ALLOC: core::ops::Range = 0xfffffc2d00000000..0xfffffc2e00000000; + + /// State block of submit channel `i` (12 channels). + pub(crate) const fn ch_state(i: u64) -> u64 { + 0xfffffc2040003fd0 + i * 0x44000 + } + /// Ring of submit channel `i`. + pub(crate) const fn ch_ring(i: u64) -> u64 { + 0xfffffc2000002800 + i * 0x44000 + } + + /// Scratch page of the boot-time PTE class self-check (host window; never published). + pub(crate) const SELF_CHECK: u64 = 0xfffffc2ff8000000; + /// Scratch page of the PTE self-check in the context-0 lower half (never published). + pub(crate) const SELF_CHECK_LOWER: u64 = 0x7ff0000000; +} + +/// Firmware channels and rings of the initdata graph. +pub(crate) mod rings { + /// Submit channels allocated (TA, FS and compute are used). + pub(crate) const CHANNELS: u64 = 12; + /// Bytes of a channel state block. + pub(crate) const STATE_SIZE: usize = 0x30; + /// Submit channel ring: 256 x 0x18. + pub(crate) const CHANNEL_RING_SIZE: usize = 0x1800; + /// Device-control and event rings: 256 x 0x38. + pub(crate) const DEVCTL_RING_SIZE: usize = 0x3800; + pub(crate) const EVENT_RING_SIZE: usize = 0x3800; + /// Six log sub-rings: their state blocks and their rings. + pub(crate) const LOG_RINGS: usize = 6; + pub(crate) const LOG_STATE_SIZE: usize = 0x120; + pub(crate) const LOG_RING_SIZE: usize = 0x51000; + pub(crate) const TRACE_RING_SIZE: usize = 0x7000; + pub(crate) const STATS_RING_SIZE: usize = 0x4000; + /// FwCtl ring: 256 x 0x18 (the record stride of this firmware). Never used by the host: no + /// firmware cache flush path exists. + pub(crate) const FWCTL_RING_SIZE: usize = 0x1800; + /// Device-control and event record sizes. + pub(crate) const DEVCTL_RECORD: usize = 0x38; + pub(crate) const EVENT_RECORD: usize = 0x38; + /// Submit channel record size. + pub(crate) const CHANNEL_RECORD: usize = 0x18; +} + +/// One HardwareDataB IOMapping record: firmware device alias of an MMIO range. +#[derive(Copy, Clone)] +pub(crate) struct IoMap { + pub(crate) slot: u8, + pub(crate) pa: u64, + pub(crate) map_bytes: u64, + pub(crate) va: u64, + /// Added to `va` in the HardwareData record. + pub(crate) delta: u64, +} + +/// HardwareDataB IOMapping records of the qualified build. +pub(crate) const IOMAPS: [IoMap; 15] = [ + IoMap { + slot: 0, + pa: 0x290d00000, + map_bytes: 0x144000, + va: 0xfffffc2068000000, + delta: 0, + }, + IoMap { + slot: 1, + pa: 0x20e100000, + map_bytes: 0x4000, + va: 0xfffffc2068148000, + delta: 0x1000, + }, + IoMap { + slot: 2, + pa: 0x351014000, + map_bytes: 0x4000, + va: 0xfffffc2068150000, + delta: 0, + }, + IoMap { + slot: 3, + pa: 0x290000000, + map_bytes: 0x20000, + va: 0xfffffc2068158000, + delta: 0, + }, + IoMap { + slot: 7, + pa: 0x3502bc000, + map_bytes: 0x4000, + va: 0xfffffc206817c000, + delta: 0, + }, + IoMap { + slot: 9, + pa: 0x290e08000, + map_bytes: 0x8000, + va: 0xfffffc2068184000, + delta: 0, + }, + IoMap { + slot: 11, + pa: 0x220000000, + map_bytes: 0x12c000, + va: 0xfffffc2068190000, + delta: 0, + }, + IoMap { + slot: 12, + pa: 0x35104c000, + map_bytes: 0x4000, + va: 0xfffffc20682c0000, + delta: 0, + }, + IoMap { + slot: 18, + pa: 0x3503d0000, + map_bytes: 0x4000, + va: 0xfffffc20682c8000, + delta: 0, + }, + IoMap { + slot: 19, + pa: 0x3503c0000, + map_bytes: 0x4000, + va: 0xfffffc20682d0000, + delta: 0, + }, + IoMap { + slot: 20, + pa: 0x3503d8000, + map_bytes: 0x4000, + va: 0xfffffc20682d8000, + delta: 0, + }, + IoMap { + slot: 23, + pa: 0x293000000, + map_bytes: 0x400000, + va: 0xfffffc20682e0000, + delta: 0, + }, + IoMap { + slot: 25, + pa: 0x30945c000, + map_bytes: 0x4000, + va: 0xfffffc20686e4000, + delta: 0, + }, + IoMap { + slot: 26, + pa: 0x350280000, + map_bytes: 0x8000, + va: 0xfffffc20686ec000, + delta: 0, + }, + IoMap { + slot: 29, + pa: 0x290e5c000, + map_bytes: 0x4000, + va: 0xfffffc20686f8000, + delta: 0, + }, +]; + +/// RTKit endpoints and messages. +pub(crate) mod ep { + /// Firmware endpoint: the initdata doorbell and ring-service requests. + pub(crate) const FIRMWARE: u8 = 0x20; + /// Doorbell endpoint: channel and device-control notifications. + pub(crate) const DOORBELL: u8 = 0x21; + /// Host message handing the firmware its initdata root. + pub(crate) const INITDATA: u64 = 0x0081 << 48; + /// Address bits of the initdata root carried by [`INITDATA`]. + pub(crate) const INITDATA_ADDR_MASK: u64 = 0xfffffffffff; + /// Firmware message requesting that the receive rings be drained. + pub(crate) const RING_SERVICE: u64 = 0x0042 << 48; +} + +/// Fixed per-VM client placement, identical in every VM, inside [`CLIENT_APERTURE`], plus the +/// pass slots and PB page list in the compact window. User binds into any of these fail. +pub(crate) mod client { + use core::ops::Range; + + /// Per-VM kernel objects (the hole the userspace VA layout leaves for the runtime graph). + pub(crate) const CLIENT_APERTURE: Range = 0x7000000000..0x7100000000; + /// Compact-pointer window that VDM stream bases must lie in. + pub(crate) const COMPACT_WINDOW: Range = 0x1000000000..0x9000000000; + + pub(crate) const fn pool(k: u64) -> u64 { + 0x7000000000 + k * 0x404000 + } + pub(crate) const FREELIST: u64 = 0x7005858000; + pub(crate) const POOL_TABLE_PAGE: u64 = 0x7005c5c000; + pub(crate) const POOL_TABLE_OFF: u64 = 0x3000; + pub(crate) const PREEMPT: u64 = 0x7005c64000; + pub(crate) const PREEMPT_STRIDE: u64 = 0x10000; + pub(crate) const WORK_PAGE: u64 = 0x7021000000; + pub(crate) const WORK_OFF: u64 = 0x3760; + pub(crate) const CWORK_STRIDE: u64 = 0x4000; + pub(crate) const FS_WORK_PAGE: u64 = 0x7020000000; + pub(crate) const FS_WORK_OFF: u64 = 0x3380; + pub(crate) const TA_WORK_PAGE: u64 = 0x7020010000; + pub(crate) const TA_WORK_OFF: u64 = 0x36c0; + pub(crate) const WORK_STRIDE: u64 = 0x20000; + pub(crate) const fn uma(engine: u64, k: u64) -> u64 { + 0x7010000000 + engine * 0x2450000 + k * 0x408000 + } + pub(crate) const fn uma_freelist(engine: u64) -> u64 { + 0x7012040000 + engine * 0x2450000 + } + pub(crate) const fn uma_table_page(engine: u64) -> u64 { + 0x7012448000 + engine * 0x2450000 + } + pub(crate) const AUX: u64 = 0x70148a0000; + pub(crate) const AUX_STRIDE: u64 = 0x98000; + pub(crate) const TPC: u64 = 0x7040000000; + pub(crate) const TPC_SIZE: u64 = 0x1000000; + pub(crate) const TILEMAP: u64 = 0x7042000000; + pub(crate) const TILEMAP_SIZE: u64 = 0x800000; + pub(crate) const TPC_SLOT_SIZE: u64 = 0x200000; + pub(crate) const TILEMAP_SLOT_SIZE: u64 = 0x100000; + pub(crate) const TPC_SLOT_STRIDE: u64 = 0x4000000; + pub(crate) const CT: u64 = 0x7080000000; + pub(crate) const CT_STRIDE: u64 = 0x1000000; + pub(crate) const CT_SIZE: u64 = 0x400000; + pub(crate) const PB_HEAP: u64 = 0x70c0000000; + pub(crate) const PB_HEAP_SIZE: u64 = 0x20000000; + pub(crate) const PB_BLOCK_LIST: u64 = 0x70f0000000; + /// Compact-window objects (32-bit compact pointers: `gva - 0x1000000000`). + pub(crate) const PRE0_PAGE: u64 = 0x1006080000; + pub(crate) const PRE1_PAGE: u64 = 0x1006098000; + pub(crate) const PRE2_PAGE: u64 = 0x10060b0000; + pub(crate) const HEAP_META_PAGE: u64 = 0x1006140000; + pub(crate) const SCENE_USER: u64 = 0x1006158000; + pub(crate) const FS_SCRATCH: u64 = 0x1006178000; + pub(crate) const PASS_STRIDE: u64 = 0x118000; + pub(crate) const PB_PAGE_LIST: u64 = 0x1074000000; + pub(crate) const PB_PAGE_LIST_SIZE: u64 = 0x28000; +} + +/// Sizes and counts of the engine's objects. +pub(crate) mod sizes { + pub(crate) const POOLS: usize = 22; + pub(crate) const POOL_SIZE: u64 = 0x400000; + pub(crate) const FREELIST_SEED: u32 = 8192; + pub(crate) const QUEUE_CAP: u32 = 0x500; + pub(crate) const COMPUTE_SLOTS: usize = 16; + pub(crate) const RENDER_SLOTS: usize = 16; + pub(crate) const CT_BUFS: usize = 2 * RENDER_SLOTS; + pub(crate) const CLUSTER_PB_PAGES: u32 = 0x3c; + pub(crate) const MAX_ATTACHMENTS: usize = 16; + pub(crate) const MAX_CLIENT: usize = 320; + pub(crate) const PB_PAGES: u32 = 16384; + pub(crate) const PB_BLOCKS: u32 = 4096; + pub(crate) const PB_LIST_RECORDS: u32 = 4096; + pub(crate) const PB_GROW_STEP: u32 = 10; + /// Flights (published prefixes) in flight at most; the second must be of the same VM. + pub(crate) const FLIGHTS: usize = 2; + /// Jobs per compute horizon, and scheduler credits per queue. + pub(crate) const HORIZON_JOBS: u32 = 16; + /// Unconsumed records per 256-entry channel ring a prefix may be posted behind. + pub(crate) const AHEAD_RECORDS: u32 = 64; + /// Policy bound on logical queues reported in QUEUE_LIMITS (not a firmware limit). + pub(crate) const MAX_QUEUES: u32 = 58; + /// TTBAT user slots: user VMs hold one each for their lifetime (ASIDs 1..63). + pub(crate) const MAX_VMS: u32 = 63; +} + +/// Firmware channels, doorbells, seeds and identifiers. +pub(crate) mod engine { + pub(crate) const DOORBELL: u64 = 0x0083000000000000; + pub(crate) const DEVCTL_DOORBELL: u64 = 0x0083000000000011; + pub(crate) const EVENT_SLOT: u32 = 2; + pub(crate) const COMPUTE_SEED: u32 = 0xc1000000; + pub(crate) const TA_SEED: u32 = 0x7a000000; + pub(crate) const FS_SEED: u32 = 0x3d000000; + pub(crate) const COMPUTE_UUID: u32 = 0xc10000; + pub(crate) const TA_UUID: u32 = 0x7a0000; + pub(crate) const FS_UUID: u32 = 0x3d0000; +} + +/// Tuned policy. Every value was chosen by measurement on hardware; only the ones in +/// `params.rs` are tunable. +pub(crate) mod tuning { + /// Firmware idle standby time, us, of a machine whose device tree does not provide one (the + /// T6030): with the template's 700 us the GPU powers down between works ~100 us apart. + pub(crate) const IDLE_STANDBY_US: u32 = 2500; + /// Command objects (work images, microsequences, barriers, queue info, scheduling contexts, + /// job lists, FW-only queue scratch) map ASC-cached (AttrIndex 0). + pub(crate) const FW_CACHED: bool = true; + /// The first HardwareData pages map ASC-cached. + pub(crate) const HWDATA_CACHED: bool = true; + /// Render notifier ASC-cached (bit 1), compute notifier not (bit 0). + pub(crate) const NOTIFIER_CACHED: u32 = 2; + /// UMA/PB managers, block_ctl and misc ASC-cached, with an uncached spare set. + pub(crate) const MANAGER_CACHED: bool = true; + /// Scheduling class of the render and compute queue-info rows (2: killable compute). + pub(crate) const RENDER_QUEUE_CLASS: u32 = 2; + pub(crate) const COMPUTE_QUEUE_CLASS: u32 = 2; + /// Intra-prefix FS->TA barrier type. + pub(crate) const RENDER_INTERNAL_HOLD: u32 = 0; + /// Bit 0: FS->compute type 1; bit 1: compute->TA type 1. Never both type 0. + pub(crate) const CROSS_ENGINE_HOLD: u32 = 1; + /// Compute-only jobs folded into one prefix. + pub(crate) const COMPUTE_HORIZON: u32 = 16; + /// Renders per prefix (half of the pass slots), with submit-ahead. + pub(crate) const PREFIX_RENDERS: u32 = 8; + pub(crate) const SUBMIT_AHEAD: bool = true; + /// Computes and renders in one prefix. + pub(crate) const MIXED_PREFIX: bool = true; + /// Two-cluster vertex clustering. + pub(crate) const VERTEX_CLUSTERING: bool = true; + /// Background zeroing of retired storage; also zero aux, heap and scene-user. + pub(crate) const CT_ASYNC: bool = true; + pub(crate) const PASS_FULL_RESET: bool = true; + /// Rotating compute-slot cursor. + pub(crate) const SLOT_ROTATE: bool = true; + /// Flights per VM. + pub(crate) const MAX_INFLIGHT: u32 = 2; + /// PB growth through event 7 / DEVCTL 7, initial and maximum blocks. + pub(crate) const PB_GROW: bool = true; + /// Initial PB blocks, and the most that may be posted: one below the mapped heap's block + /// count, so the block-list producer (modulo K = 4096) never returns to 0. + pub(crate) const PB_BLOCKS: u32 = 1024; + pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1; + /// At the host deadline, arm the firmware kill. + pub(crate) const HANG_PREEMPT: bool = true; + /// Compute start-sequence MMU fault mask (unmapped stores absorbed). + pub(crate) const COMPUTE_FAULT_MASK: u32 = 1; + /// Firmware idle decision: finite delay, not predictive. + pub(crate) const IDLE_PREDICTIVE: bool = false; + /// Handle EP 0x20 drain requests in the mailbox interrupt. + pub(crate) const EARLY_WAKE: bool = true; + /// Scheduler timeout per job, ms; the engine's deadlines own failure (always "no hang"). + pub(crate) const SCHED_TIMEOUT_MS: u32 = 3000; +} diff --git a/drivers/gpu/drm/asahi/g15/hw/t6031.rs b/drivers/gpu/drm/asahi/g15/hw/t6031.rs new file mode 100644 index 0000000000000..4af32f6068cd7 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/hw/t6031.rs @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! T6031 (M3 Max, G15C) configuration, admitted on J516C only. +//! +//! One die, four clusters of ten core slots. The firmware is the same RTKit 2419 generation as +//! the T6030's and the initdata graph keeps the T6030 canonical VAs; what differs is the SoC +//! identity, the 40-bit enabled-core mask (two words), the source of the GPU regions (the +//! device tree node the boot loader fills, not an ADT export), the GPU power (its power domain; +//! there is no PMP vote), the initdata root version, a longer HardwareData record (0xb480 +//! bytes, its fourth view at +0xb400) placed aligned and ASC-cached, the IOMapping table (17 +//! slots at the T6031 register blocks) and the board images, built from the device tree and +//! checked against this machine's firmware file, which supplies the record's non-device-tree +//! words. Every +//! value was measured on a J516C: the SGX identification +//! registers, the loaded firmware segment, the device tree the boot loader hands over and the +//! initdata of that machine. + +use super::super::Config; +use super::{FirmwareBuild, HwId, IoMap, Layout}; + +/// SGX register window (the `sgx` resource), physical. +const SGX: u64 = 0x4_0800_0000; +/// The die 0 interrupt controller. +const AIC: u64 = 0x2_9240_0000; + +/// Base of the IOMapping firmware VAs and the gap left between consecutive mappings. +const IOMAP_BASE: u64 = 0xfffffc2068000000; +const IOMAP_GAP: u64 = 0x4000; +const PAGE: u64 = 0x4000; + +/// Mappings in slot order: slot, physical address, mapped bytes (rounded up to pages below), +/// element bytes, writable. Slots 0 (Fender, 0x144000), 1 (AIC timer page), 2 (AIC software +/// interrupts), 3 (RGX), 7, 9 (metrology sensors), 11 (memory cache, eight 0xd8000 +/// instances), 12 (AIC banked registers), 18-20 (telemetry dashboard), 21, 24, 25 (ANE), 26 +/// (PMS metrology), 28 and 29 (GPU clock generator at `sgx + 0xe1c000`). The record carries +/// each slot's physical address, sizes and access flag. +const SLOTS: [(u8, u64, u64, u64, bool); 17] = [ + (0, SGX + 0xd0_0000, 0x14_4000, 0x14_4000, true), + (1, 0x2_0e10_0000, 0x4000, 0x4000, false), + (2, AIC + 0x1_4000, 0x4000, 0x4000, true), + (3, SGX, 0x2_0000, 0x2_0000, true), + (7, 0x2_9044_4000, 0x4000, 0x4000, false), + (9, SGX + 0xe0_8000, 0x8000, 0x8000, true), + (11, 0x2_2000_0000, 8 * 0xd_8000, 0xd_8000, true), + (12, AIC + 0x4_c000, 0x4000, 0x4000, false), + (18, 0x2_903d_0000, 0x4000, 0x4000, true), + (19, 0x2_903c_0000, 0x4000, 0x4000, false), + (20, 0x2_903d_8000, 0x4000, 0x4000, true), + (21, SGX + 0xeb_4000, 0x4000, 0x4000, true), + (24, SGX + 0x100_0000, 0x8000, 0x8000, true), + (25, 0x3_c945_c000, 0x4000, 0x4000, true), + (26, 0x2_9040_0000, 0x8000, 0x8000, false), + (28, 0x2_a4dc_0000, 0x2_4000, 0x2_4000, false), + (29, SGX + 0xe1_c000, 0x4000, 0x4000, false), +]; + +/// The record's per-slot sizes and access flags, in [`SLOTS`] order. +const fn record_slots() -> [(u32, u32, bool); 17] { + let mut out = [(0, 0, false); 17]; + let mut i = 0; + while i < SLOTS.len() { + let (_, _, total, element, writable) = SLOTS[i]; + out[i] = (total as u32, element as u32, writable); + i += 1; + } + out +} +const RECORD_SLOTS: [(u32, u32, bool); 17] = record_slots(); + +/// Unmapped slots whose IOMapping flag word the J516C record holds as 1. +const RECORD_EMPTY_WRITABLE: [u8; 9] = [4, 6, 8, 10, 13, 14, 15, 16, 17]; + +/// Packs the slots' firmware VAs: consecutive from [`IOMAP_BASE`], each mapping followed by one +/// unmapped page. +const fn iomaps() -> [IoMap; 17] { + let mut out = [IoMap { + slot: 0, + pa: 0, + map_bytes: 0, + va: 0, + delta: 0, + }; 17]; + let mut va = IOMAP_BASE; + let mut i = 0; + while i < SLOTS.len() { + let (slot, pa, bytes, _, _) = SLOTS[i]; + let delta = pa & (PAGE - 1); + let map_bytes = (delta + bytes + PAGE - 1) & !(PAGE - 1); + out[i] = IoMap { + slot, + pa: pa & !(PAGE - 1), + map_bytes, + va, + delta, + }; + va += map_bytes + IOMAP_GAP; + i += 1; + } + out +} + +/// HardwareDataB IOMapping records of the J516C. +pub(crate) const IOMAPS: [IoMap; 17] = iomaps(); + +/// The J516C initdata layout: the T6030 graph with the root version this firmware's host +/// driver sends and the longer HardwareData record, placed 16-byte aligned (the firmware's MTR +/// alarm handler does a 64-bit atomic in HardwareDataA; at the T6030 placement the record is +/// 4-byte aligned, which faulted on T8122) and mapped ASC-cached as a whole. +pub(crate) const LAYOUT: Layout = Layout { + root_magic: 0x0c08e21e84820490, + hwdata_va: 0xfffffc20406fb580, + hwdata_size: 0xb480, + hwdata_cached: None, + hwdata_view: 0xb400, + iomaps: &IOMAPS, + record_slots: &RECORD_SLOTS, + record_empty_writable: &RECORD_EMPTY_WRITABLE, +}; + +/// The J516C board file (`asahi/t6031-j516c-board.bin`, built into the kernel): the hardware +/// record, Globals and the power block that this machine's firmware accepted, as the earlier +/// J516C driver uploaded them (its device-tree-built records with the per-board words of this +/// machine applied). Kept out of the tree; pinned by its SHA-256. The three images are built +/// from the device tree and must match the file's; the file supplies the record's words the +/// device tree does not carry (`board::record::FROM_FILE`). +const BOARD_SHA256: [u8; 32] = [ + 0x4e, 0x03, 0xf2, 0x9f, 0xa4, 0x47, 0xdc, 0xeb, 0xfd, 0x4d, 0x1c, 0x22, 0x5a, 0x64, 0x7b, + 0xfa, 0x31, 0xb3, 0x63, 0x45, 0x8f, 0x28, 0x33, 0x82, 0x3a, 0x5c, 0xa8, 0xa1, 0x0e, 0xdf, + 0xf3, 0xfc, +]; + +pub(crate) const CONFIG: Config = Config { + name: "T6031 G15C", + chip_id: 0x6031, + gpu_variant: crate::hw::GpuVariant::C, + gpu_revision: 0x12, + uat_ias: 42, + uat_oas: 42, + num_clusters: 4, + num_cores_per_cluster: 10, + hw_ids: &[HwId { + version: 0x07041100, + counts: 0x0021040a, + }], + // The stubs the boot chains of this J516C leave loaded (RTKit-2419.140.12): the macOS 14.7 + // chain's and the macOS 14.8 boot volume's. Same segment sizes and marker; the layout, board + // images and initdata below do not depend on which one is loaded. A further build can be + // admitted for one boot with `g15_accept_text_prefix`, which logs its digest. + firmware: &[ + FirmwareBuild { + name: "macOS 14.7 stub firmware (iBoot-10151.140.19.700.2), J516C", + machine: None, + text_size: 0x64000, + data_size: 0x140000, + marker: 0x5cf28, + text_sha256: [ + 0xc2, 0x6e, 0xc8, 0x9c, 0x11, 0x50, 0xeb, 0x65, 0x49, 0xcf, 0x60, 0x2e, 0x01, + 0x2d, 0x0c, 0x5f, 0x49, 0xeb, 0x75, 0x6e, 0x89, 0x6d, 0x6b, 0xb8, 0x34, 0x2c, + 0xb2, 0xc5, 0xd9, 0x8b, 0x33, 0x2c, + ], + }, + FirmwareBuild { + name: "macOS 14.8 stub firmware, J516C", + machine: None, + text_size: 0x64000, + data_size: 0x140000, + marker: 0x5cf28, + text_sha256: [ + 0x1b, 0xf2, 0xfd, 0x34, 0x76, 0xbe, 0xfe, 0x73, 0x6b, 0xc2, 0xc0, 0x70, 0xba, + 0xd4, 0x5f, 0x8b, 0xe3, 0x6b, 0x68, 0xcd, 0x11, 0xbc, 0xff, 0x16, 0xc9, 0x02, + 0x9d, 0x6b, 0xcd, 0xa0, 0x26, 0xcf, + ], + }, + ], + // The pinned board images and measured layout belong to the J516C, not + // every T6031 board (in particular, not the 14-inch J514C). + machine: kernel::c_str!("apple,j516c"), + platform: super::Platform::DeviceTree, + layout: &LAYOUT, + board: super::Board::Blob { + name: kernel::c_str!("asahi/t6031-j516c-board.bin"), + sha256: BOARD_SHA256, + }, + // Four clusters: the saved-state pointers at +0x2980 of the preemption buffer (0xd80 plus + // 0x700 per cluster), UMA PTC 0x6000400020, DUPM 0x3c000000140 (32 N | 96 N << 32 for + // N = 10 cores per cluster) and 0x1a0e9 = 0, with the saved-state pointers and the + // 0x12091/0x101d9 words zero in every command, as this machine's driver encodes a queue's + // first command. Read from 38 compute commands on a J516C; the two-cluster values fault + // the first job on this GPU. + compute: super::ComputeRegs { + tail: 0x2980, + ptc: 0x6000400020, + dupm: 0x3c000000140, + reg_1a0e9: 0, + fresh: true, + }, +}; diff --git a/drivers/gpu/drm/asahi/g15/identity.rs b/drivers/gpu/drm/asahi/g15/identity.rs new file mode 100644 index 0000000000000..2bf95c8ad98db --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/identity.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact firmware and hardware identity. +//! +//! The firmware build is the SHA-256 of TEXT with the 8 entropy bytes after the single +//! `GKTS\x08\0\0\0` marker zeroed, plus the segment sizes, looked up in the SoC +//! configuration's `firmware` list; the hardware revision is checked against its `hw_ids` at +//! start ([`super::boot`]). An unknown build or revision is refused with its measured identity; +//! an unknown build's refusal also says what to report so that it can be pinned. The pure +//! lookup is in [`lookup`] (host-tested). +//! The canonical object VAs, the initdata graph, the board images and the work layouts all +//! belong to one build, so nothing is written before the build is identified. +//! +//! Bring-up escape hatch: `g15_accept_text_prefix=` admits an unpinned build whose digest +//! starts with those 8 bytes (the full digest and marker offset are logged, so that the build +//! can be pinned by the next kernel). + +use kernel::{bindings, device, fmt, prelude::*}; + +use super::boot::glue; +use super::hw::FirmwareBuild; + +mod lookup; +mod matching; + +use lookup::{Measured, Verdict}; + +/// Lowercase hex of a byte string, as `%*phN` prints it. +pub(crate) struct Hex<'a>(pub(crate) &'a [u8]); + +impl fmt::Display for Hex<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for b in self.0 { + write!(f, "{:02x}", b)?; + } + Ok(()) + } +} + +/// Copies `size` bytes at physical `phys`, a range outside the kernel's System RAM, into a +/// private copy through a temporary write-back mapping. +pub(crate) fn copy_physical(phys: u64, size: usize) -> Result> { + // SAFETY: memremap() of a boot-loader range creates a temporary mapping; it has no other + // side effects. + let map = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WB as _) }; + if map.is_null() { + return Err(ENOMEM); + } + let copy = KVVec::from_elem(0u8, size, GFP_KERNEL).map(|mut copy| { + // SAFETY: `map` is a live mapping of `size` bytes and `copy` holds `size` bytes; the two do + // not overlap. + unsafe { core::ptr::copy_nonoverlapping(map as *const u8, copy.as_mut_ptr(), size) }; + copy + }); + // SAFETY: `map` came from memremap() above and is not used afterwards. + unsafe { bindings::memunmap(map) }; + Ok(copy?) +} + +impl From<&FirmwareBuild> for Measured { + fn from(build: &FirmwareBuild) -> Self { + Measured { + text_size: build.text_size, + data_size: build.data_size, + marker: build.marker, + sha256: build.text_sha256, + } + } +} + +/// Identifies the loaded firmware build: the pinned build, or `None` for a build admitted by +/// its digest prefix (`g15_accept_text_prefix`). +pub(crate) fn firmware( + dev: &device::Device, + adt: &super::adt::GpuAdt, + builds: &'static [FirmwareBuild], +) -> Result> { + let size = usize::try_from(adt.text.end - adt.text.start)?; + let data_size = adt.data.end - adt.data.start; + let mut copy = copy_physical(adt.text.start, size)?; + let at = lookup::marker(©).map_err(|found| { + dev_err!( + dev, + "GPU firmware TEXT has {} GKTS markers, need one\n", + found + ); + EINVAL + })?; + copy[at + 8..at + 16].fill(0); + let m = Measured { + text_size: size as u64, + data_size, + marker: u32::try_from(at)?, + sha256: glue::sha256(©), + }; + // A qualified build pins only on the machine it names. + let qualified = builds + .iter() + .map(|b| (Measured::from(b), b.machine.map_or(true, glue::machine))); + match lookup::verdict(&m, qualified, super::params::accept_text_prefix()) { + Verdict::Pinned(i) => { + let build = builds.get(i).ok_or(EINVAL)?; + dev_info!(dev, "GPU firmware build: {}\n", build.name); + return Ok(Some(build)); + } + Verdict::Admitted => { + dev_warn!( + dev, + "GPU firmware build admitted by g15_accept_text_prefix: TEXT {:#x} DATA {:#x} GKTS {:#x} sha256 {}\n", + size, + data_size, + at, + Hex(&m.sha256) + ); + return Ok(None); + } + Verdict::Refused => {} + } + dev_err!( + dev, + "unsupported GPU firmware build: TEXT {:#x} DATA {:#x} GKTS {:#x} sha256 {}; {}\n", + size, + data_size, + at, + Hex(&m.sha256), + lookup::REPORT_HINT + ); + for build in builds { + dev_err!(dev, "supported: {}\n", build.name); + } + Err(EINVAL) +} diff --git a/drivers/gpu/drm/asahi/g15/identity/lookup.rs b/drivers/gpu/drm/asahi/g15/identity/lookup.rs new file mode 100644 index 0000000000000..efb8bbcd08090 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/identity/lookup.rs @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pure firmware-build identification: the GKTS marker search, the lookup of a measured build in +//! the qualified list, the `g15_accept_text_prefix` escape and the refusal hint. +//! +//! A qualified build pins only through [`super::matching::exact`]: its sizes, marker and digest, +//! and the machine it names. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. + +use super::matching; + +/// The GKTS marker: the 8 bytes after it are per-boot entropy. +pub(crate) const MARKER: [u8; 8] = [0x47, 0x4b, 0x54, 0x53, 0x08, 0, 0, 0]; + +/// Appended to the refusal of an unknown build: what a user reports so that it can be pinned. +pub(crate) const REPORT_HINT: &str = + "to have it supported, report this line and the macOS version of the boot chain that loaded it"; + +/// The identity of a loaded firmware build: segment sizes, marker offset and the TEXT digest with +/// the marker's entropy zeroed. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Measured { + pub(crate) text_size: u64, + pub(crate) data_size: u64, + pub(crate) marker: u32, + pub(crate) sha256: [u8; 32], +} + +/// The outcome of identifying a measured build. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Verdict { + /// The qualified build at this index. + Pinned(usize), + /// Not qualified, admitted for this boot because its digest prefix was named. + Admitted, + /// Not qualified and not admitted. + Refused, +} + +/// Offsets of the 8-byte-aligned GKTS markers in `text` that have their 8 entropy bytes in +/// range: `Ok(offset)` for exactly one, `Err(count)` otherwise. +pub(crate) fn marker(text: &[u8]) -> Result { + let mut at = 0usize; + let mut found = 0usize; + let mut off = 0usize; + while off + 16 <= text.len() { + if text[off..off + 8] == MARKER { + at = off; + found += 1; + } + off += 8; + } + if found == 1 { + Ok(at) + } else { + Err(found) + } +} + +/// The first 8 digest bytes, big-endian, as `g15_accept_text_prefix` names them. +pub(crate) fn prefix(sha256: &[u8; 32]) -> u64 { + let mut first = [0u8; 8]; + first.copy_from_slice(&sha256[..8]); + u64::from_be_bytes(first) +} + +/// Looks `m` up in `qualified`, each build with whether this machine is the one it names; an +/// unqualified build is admitted only when `accept_prefix` is nonzero and equals its digest +/// prefix. +pub(crate) fn verdict( + m: &Measured, + qualified: impl IntoIterator, + accept_prefix: u64, +) -> Verdict { + if let Some(i) = qualified.into_iter().position(|(q, machine)| { + matching::exact( + (q.text_size, q.data_size), + (m.text_size, m.data_size), + q.marker, + m.marker as usize, + &q.sha256, + &m.sha256, + machine, + ) + }) { + return Verdict::Pinned(i); + } + if accept_prefix != 0 && accept_prefix == prefix(&m.sha256) { + Verdict::Admitted + } else { + Verdict::Refused + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build(n: u8) -> Measured { + let mut sha256 = [0u8; 32]; + sha256[0] = 0x1b; + sha256[7] = n; + sha256[31] = n; + Measured { + text_size: 0x64000, + data_size: 0x140000, + marker: 0x5cf28, + sha256, + } + } + + #[test] + fn finds_one_marker() { + let mut text = [0u8; 64]; + text[16..24].copy_from_slice(&MARKER); + assert_eq!(marker(&text), Ok(16)); + } + + #[test] + fn refuses_none_two_or_misaligned_markers() { + let mut text = [0u8; 64]; + assert_eq!(marker(&text), Err(0)); + text[12..20].copy_from_slice(&MARKER); + assert_eq!(marker(&text), Err(0)); + text = [0u8; 64]; + text[8..16].copy_from_slice(&MARKER); + text[32..40].copy_from_slice(&MARKER); + assert_eq!(marker(&text), Err(2)); + } + + #[test] + fn needs_the_entropy_bytes_in_range() { + let mut text = [0u8; 64]; + text[56..64].copy_from_slice(&MARKER); + assert_eq!(marker(&text), Err(0)); + } + + #[test] + fn prefix_is_the_first_eight_bytes_big_endian() { + let mut sha256 = [0u8; 32]; + sha256[..8].copy_from_slice(&[0x1b, 0xf2, 0xfd, 0x34, 0x76, 0xbe, 0xfe, 0x73]); + sha256[8] = 0xff; + assert_eq!(prefix(&sha256), 0x1bf2fd3476befe73); + } + + #[test] + fn pins_the_matching_build() { + assert_eq!( + verdict(&build(2), [(build(1), true), (build(2), true)], 0), + Verdict::Pinned(1) + ); + } + + #[test] + fn pinned_wins_over_the_prefix() { + let m = build(1); + assert_eq!(verdict(&m, [(m, true)], prefix(&m.sha256)), Verdict::Pinned(0)); + } + + #[test] + fn every_field_must_match() { + let q = build(1); + let mut m = q; + m.text_size += 0x1000; + assert_eq!(verdict(&m, [(q, true)], 0), Verdict::Refused); + m = q; + m.data_size += 0x1000; + assert_eq!(verdict(&m, [(q, true)], 0), Verdict::Refused); + m = q; + m.marker += 8; + assert_eq!(verdict(&m, [(q, true)], 0), Verdict::Refused); + m = q; + m.sha256[31] ^= 1; + assert_eq!(verdict(&m, [(q, true)], 0), Verdict::Refused); + } + + #[test] + fn another_machines_build_is_not_pinned() { + let m = build(1); + assert_eq!(verdict(&m, [(m, false)], 0), Verdict::Refused); + assert_eq!( + verdict(&m, [(m, false), (m, true)], 0), + Verdict::Pinned(1) + ); + // The digest-prefix escape still applies to it, as to any unqualified build. + assert_eq!( + verdict(&m, [(m, false)], prefix(&m.sha256)), + Verdict::Admitted + ); + } + + #[test] + fn admits_only_the_named_prefix() { + let m = build(3); + assert_eq!( + verdict(&m, [(build(1), true)], prefix(&m.sha256)), + Verdict::Admitted + ); + assert_eq!( + verdict(&m, [(build(1), true)], prefix(&build(4).sha256)), + Verdict::Refused + ); + } + + #[test] + fn zero_prefix_admits_nothing() { + let m = Measured { + sha256: [0u8; 32], + ..build(0) + }; + assert_eq!(verdict(&m, [(build(1), true)], 0), Verdict::Refused); + } + + #[test] + fn hint_says_what_to_report() { + assert!(REPORT_HINT.contains("report this line")); + assert!(REPORT_HINT.contains("macOS version")); + } +} diff --git a/drivers/gpu/drm/asahi/g15/identity/matching.rs b/drivers/gpu/drm/asahi/g15/identity/matching.rs new file mode 100644 index 0000000000000..23b24afd41159 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/identity/matching.rs @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Exact firmware metadata admission, shared with the host regression checks. + +pub(crate) fn exact( + expected_sizes: (u64, u64), actual_sizes: (u64, u64), + expected_marker: u32, actual_marker: usize, + expected_digest: &[u8; 32], actual_digest: &[u8; 32], + machine_matches: bool, +) -> bool { + machine_matches && expected_sizes == actual_sizes && + expected_marker as usize == actual_marker && expected_digest == actual_digest +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn requires_board_full_digest_sizes_and_marker() { + let sizes = (0x5c000, 0x114000); + let hash = [0x5a; 32]; + assert!(exact(sizes, sizes, 0x58e88, 0x58e88, &hash, &hash, true)); + assert!(!exact(sizes, sizes, 0x58e88, 0x58e88, &hash, &hash, false)); + assert!(!exact(sizes, (sizes.0 + 1, sizes.1), 0x58e88, 0x58e88, &hash, &hash, true)); + assert!(!exact(sizes, (sizes.0, sizes.1 + 1), 0x58e88, 0x58e88, &hash, &hash, true)); + assert!(!exact(sizes, sizes, 0x58e88, 0x58e80, &hash, &hash, true)); + let mut same_prefix = hash; + same_prefix[31] ^= 1; + assert!(!exact(sizes, sizes, 0x58e88, 0x58e88, &hash, &same_prefix, true)); + } +} diff --git a/drivers/gpu/drm/asahi/g15/initdata.rs b/drivers/gpu/drm/asahi/g15/initdata.rs new file mode 100644 index 0000000000000..f3f5232e92a28 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/initdata.rs @@ -0,0 +1,487 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The initdata graph (49 objects at canonical VAs) and its live words. +//! +//! The build allocates the objects, channels and IOMappings, writes the board images and the +//! root, runtime and control pointers, and patches HardwareData and Globals (TTBAT PA at +//! HardwareData +0xb44, IOMapping VAs, the timestamp arena at +0x28, the tuned idle policy, +//! Globals +0x980 = 100, HardwareDataA +0x4580 words). The object classes follow +//! [`super::hw::tuning`]: HardwareData pages 0-1 ASC-cached, the fault-information page and +//! everything else uncached; channel rings NEVER cached (caching them hangs the firmware). +//! +//! Live words used after READY: Globals (+0x1c/+0x24 kill divisors, +0x2c/+0x30/+0x7a8/ +//! +0x7c4/+0x9a0 idle policy, +0x9b0 compute monitor), Control (+0x40a0 pending, +0x4580 halt +//! count / PB epoch, +0x4590 halted, +0x45a0 resume), HardwareData +0x8900 fault information. +//! Globals changes are re-imported with DEVCTL 0x13. All objects here are firmware-only +//! (AttrIndex 2) unless noted; offsets are relative to each object's canonical VA. + +use kernel::{device, prelude::*, sync::Arc}; + +use super::adt::GpuAdt; +use super::board::{self, data}; +use super::boot::glue; +use super::channel::Channels; +use super::fw::initdata::{ + cbuf, control, flags, globals, hwdata, pair, power, region_a, root, runtime, small, +}; +use super::hw::{self, va}; +use super::identity::Hex; +use super::object::{Class, FwObject, Objects, Placement, Request}; +use super::Config; + +/// The built initdata graph. +pub(crate) struct InitData { + /// Globals, Control and HardwareData: the live words of recovery, power and faults. + pub(crate) globals: Option>, + pub(crate) control: Option>, + pub(crate) hwdata: Option>, + /// Runtime (readiness) and Flags. + pub(crate) runtime: Option>, + pub(crate) flags: Option>, + /// C1 (firmware + GPU shared), written by the PB (re)initialization. + pub(crate) c1: Option>, + /// The rest of the graph, retained (root, region A, pair, small, C0, C1, C3, power). + others: KVec>, +} + +impl InitData { + pub(crate) fn new() -> Self { + Self { + globals: None, + control: None, + hwdata: None, + runtime: None, + flags: None, + c1: None, + others: KVec::new(), + } + } + + /// Control +0x4590: the firmware halted for recovery. + pub(crate) fn halted(&self) -> Result { + Ok(self.control.as_ref().ok_or(ENODEV)?.r32(control::HALTED)? == 1) + } + + /// The readiness words: Runtime +0x314 and Flags +4. + pub(crate) fn readiness(&self) -> Result<(u32, u32)> { + Ok(( + self.runtime.as_ref().ok_or(ENODEV)?.r32(runtime::READY)?, + self.flags.as_ref().ok_or(ENODEV)?.r32(flags::READY)?, + )) + } +} + +impl super::Firmware { + /// Writes the 16-byte C1 record into the initdata C1 object. + pub(crate) fn write_c1(&mut self, record: &[u8; 16]) -> Result { + self.init.c1.as_ref().ok_or(ENODEV)?.write(0, record) + } +} + +/// A canonical firmware-only object at `va`. +fn canonical(objects: &mut Objects, at: u64, size: usize, class: Class) -> Result> { + objects.alloc(Request { + size, + class, + placement: Placement::At(at), + offset: 0, + client: None, + lower: None, + cpu: true, + }) +} + +fn get32(b: &[u8], offset: usize) -> Result { + Ok(u32::from_le_bytes( + b.get(offset..offset + 4) + .ok_or(EINVAL)? + .try_into() + .map_err(|_| EINVAL)?, + )) +} + +fn put32(b: &mut [u8], offset: usize, value: u32) -> Result { + b.get_mut(offset..offset + 4) + .ok_or(EINVAL)? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) +} + +fn put64(b: &mut [u8], offset: usize, value: u64) -> Result { + b.get_mut(offset..offset + 8) + .ok_or(EINVAL)? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) +} + + +/// The T6030 board images: the verified templates with DVFS from the ADT. Returns the images +/// and the top DVFS frequency in kHz. +fn templates( + dev: &device::Device, + adt: &GpuAdt, +) -> Result<(KVVec, KVVec, KVVec, u32)> { + let mut hw_img = KVVec::from_elem(0u8, hwdata::SIZE, GFP_KERNEL)?; + let mut glob = KVVec::from_elem(0u8, globals::SIZE, GFP_KERNEL)?; + let mut pwr = KVVec::from_elem(0u8, power::SIZE, GFP_KERNEL)?; + if board::expand(&data::HARDWARE, &mut hw_img).is_err() + || board::expand(&data::GLOBALS, &mut glob).is_err() + || board::expand(&data::POWER, &mut pwr).is_err() + { + dev_err!(dev, "board data template hash mismatch\n"); + return Err(EINVAL); + } + if get32(&hw_img, hwdata::CHECK_A38)? != 1 + || get32(&hw_img, hwdata::DVFS_COUNT)? != board::DVFS_STATES as u32 - 1 + || get32(&glob, globals::UNK_980)? != 10 + || get32(&glob, globals::IDLE_STANDBY_US)? != 700 + || get32(&glob, globals::IDLE_MIN_CONF)? != 0x3f333333 + { + return Err(EINVAL); + } + let max_khz = board::from_adt(dev, adt, &mut hw_img)?; + let reference = glue::sha256(&hw_img) == data::HARDWARE.sha256; + dev_info!( + dev, + "board data: ADT performance states, {} MHz top{}\n", + max_khz / 1000, + if reference { + " (reference machine)" + } else { + " (differs from the reference machine)" + } + ); + if !reference { + dev_info!(dev, "board data sha256 {}\n", Hex(&glue::sha256(&hw_img))); + } + Ok((hw_img, glob, pwr, max_khz)) +} + +/// HardwareDataA word holding the system counter at initdata creation. +const HWDATA_A_INIT_TIMESTAMP: usize = 0x424c; + +/// Board images of a SoC with a firmware file. The hardware record ([`board::hardware_record`]), +/// Globals and Power ([`board::device_tree`]) are built from the device tree; the file, this +/// machine's own, supplies the record's words the device tree does not carry and is otherwise +/// a cross-check: admission ([`prepare_board`], before any firmware is touched) is refused at +/// the first differing word. Returns the images and the top operating-point frequency in kHz. +fn blob( + dev: &device::Device, + cfg: &Config, + adt: &GpuAdt, + name: &CStr, + sha256: &[u8; 32], +) -> Result<(KVVec, KVVec, KVVec, u32)> { + let layout = cfg.layout; + let images = board::blob::load( + dev, + name, + sha256, + [layout.hwdata_size, globals::SIZE, power::SIZE], + )?; + let hw_img = board::hardware_record(dev, cfg, adt, &images.hardware)?; + let (glob, pwr, max_power_mw) = board::device_tree(dev, &hw_img)?; + for (what, built, file) in [ + ("hardware record", &hw_img, &images.hardware), + ("Globals", &glob, &images.globals), + ("Power", &pwr, &images.power), + ] { + let diff = if what == "hardware record" { + board::record::cross_check(built, file) + } else { + board::words::first_difference(built, file) + }; + if let Some((at, b, f)) = diff { + dev_err!( + dev, + "board data: device-tree {} differs from {} at +{:#x}: built {:#x}, file {:#x}\n", + what, + name, + at, + b, + f + ); + return Err(EINVAL); + } + } + let max_mhz = adt.perf_mhz.iter().copied().max().unwrap_or(0); + dev_info!( + dev, + "board data: {} images from the device tree match {} ({} record words from the file, {} file words left zero), {} mW maximum power, {} MHz top operating point\n", + cfg.name, + name, + board::record::FROM_FILE.len(), + board::record::UNNEEDED.len(), + max_power_mw, + max_mhz + ); + let max_khz = max_mhz.checked_mul(1000).ok_or_else(|| { + dev_err!(dev, "board data: top operating point {} MHz does not fit in kHz\n", max_mhz); + EINVAL + })?; + Ok((hw_img, glob, pwr, max_khz)) +} + +/// Verified board bytes, prepared before any GPU firmware is started. +pub(crate) type BoardImages = (KVVec, KVVec, KVVec, u32); + +/// Check availability, digest, layout and machine-derived tables before firmware start. +/// Keep the verified bytes so a later firmware request cannot change the admitted data. +pub(crate) fn prepare_board( + dev: &device::Device, + cfg: &Config, + adt: &GpuAdt, +) -> Result { + match cfg.board { + hw::Board::Templates => templates(dev, adt), + hw::Board::Blob { name, sha256 } => blob(dev, cfg, adt, name, &sha256), + hw::Board::None => Err(ENODEV), + } +} + +/// Builds the initdata graph. Returns it with its channels and the top DVFS frequency in kHz. +/// Nothing is published here. +pub(crate) fn build( + dev: &device::Device, + cfg: &Config, + adt: &GpuAdt, + objects: &mut Objects, + board_images: BoardImages, +) -> Result<(InitData, Channels, u32)> { + let mut init = InitData::new(); + let fw = Class::Fw; + let layout = cfg.layout; + let hwdata_va = layout.hwdata_va; + + // Owned objects and channels. + let root_obj = canonical(objects, va::ROOT, root::SIZE, fw)?; + let region_a_obj = canonical(objects, va::REGION_A, region_a::SIZE, fw)?; + let runtime_obj = canonical(objects, va::RUNTIME, runtime::SIZE, fw)?; + // Activity records cached; fault info and HardwareDataA timestamps not (T6030), or the + // whole record cached (a layout without a cached prefix). + let cached = if hw::tuning::HWDATA_CACHED { + Class::FwCached + } else { + fw + }; + let hwdata_obj = match layout.hwdata_cached { + Some(prefix) => objects.alloc_split( + Request { + size: layout.hwdata_size, + class: fw, + placement: Placement::At(hwdata_va), + offset: 0, + client: None, + lower: None, + cpu: true, + }, + Some(((hwdata_va as usize & (hw::PAGE as usize - 1)) + prefix, cached)), + )?, + None => canonical(objects, hwdata_va, layout.hwdata_size, cached)?, + }; + let pair_obj = canonical(objects, va::PAIR, pair::SIZE, fw)?; + let small_obj = canonical(objects, va::SMALL, small::SIZE, fw)?; + let c0 = canonical(objects, va::C0, cbuf::C0_SIZE, Class::Shared)?; + let c1 = canonical(objects, va::C1, cbuf::C1_SIZE, Class::Shared)?; + let c3 = canonical(objects, va::C3, cbuf::C3_SIZE, Class::Shared)?; + let globals_obj = canonical(objects, va::GLOBALS, globals::SIZE, fw)?; + let power_obj = canonical(objects, va::POWER, power::SIZE, fw)?; + let control_obj = canonical(objects, va::CONTROL, control::SIZE, fw)?; + let flags_obj = canonical(objects, va::FLAGS, flags::SIZE, fw)?; + let channels = Channels::alloc(objects)?; + // Firmware Device RW aliases for the IOMapping table. + for map in layout.iomaps.iter() { + objects.map_mmio(map.va, map.pa, map.map_bytes)?; + } + + // Board images, verified, then the owned replacements. + let (mut hw_img, mut glob, pwr, max_khz) = board_images; + if matches!(cfg.board, hw::Board::Blob { .. }) { + // This timestamp belongs to initdata creation, not board-data preflight. + let now: u64; + // SAFETY: Reading the architectural counter has no side effects. + unsafe { + core::arch::asm!("mrs {}, cntpct_el0", out(reg) now, options(nomem, nostack, preserves_flags)) + }; + put64(&mut hw_img, hwdata::HWDATA_A + HWDATA_A_INIT_TIMESTAMP, now)?; + } + if hw_img.len() != layout.hwdata_size { + return Err(EINVAL); + } + put64(&mut hw_img, hwdata::TTBAT, adt.ttbs.start)?; + for map in layout.iomaps.iter() { + put64( + &mut hw_img, + hwdata::IOMAPS + hwdata::IOMAP_STRIDE * map.slot as usize + hwdata::IOMAP_VA, + map.va + map.delta, + )?; + } + put64(&mut hw_img, hwdata::TIMESTAMPS, va::TIMESTAMPS)?; + put32(&mut glob, globals::UNK_980, 100)?; + // The device tree's standby time (a machine with a firmware file; the templates' machine + // has none). + let dt_standby = match cfg.board { + hw::Board::Blob { .. } => { + let prop = board::words::PROPS[board::words::STANDBY_TIMER]; + match dev + .of_node() + .ok_or(ENODEV) + .and_then(|n| n.get_property::(prop)) + { + Ok(v) => Some(v), + Err(_) => { + dev_warn!( + dev, + "GPU node has no u32 {}, using the tuned idle standby\n", + prop + ); + None + } + } + } + _ => None, + }; + let (standby_us, standby_src, dt_rejected) = super::power::standby( + super::params::idle_standby_override_us(), + dt_standby, + hw::tuning::IDLE_STANDBY_US, + ); + if dt_rejected { + dev_warn!( + dev, + "apple,idleoff-standby-timer is outside 100..100000 us, ignored\n" + ); + } + dev_info!( + dev, + "idle standby {} us from {}\n", + standby_us, + standby_src.name() + ); + put32(&mut glob, globals::IDLE_STANDBY_US, standby_us)?; + put32( + &mut glob, + globals::IDLE_MIN_CONF, + super::power::permille_f32(super::params::idle_min_conf()), + )?; + put32( + &mut glob, + globals::IDLE_PREDICTIVE, + hw::tuning::IDLE_PREDICTIVE as u32, + )?; + // HardwareDataA at +0x4580: maintained S8 request (the templates' machine; a board image + // from a file carries its own machine's words). + if cfg.board == hw::Board::Templates { + for (offset, value) in [ + (0x2c, 8), + (0x30, 8), + (0x44, 800), + (0x7d8, 800), + (0x7e0, 800), + (0x81c, 800), + ] { + put32(&mut hw_img, hwdata::HWDATA_A + offset, value)?; + } + } + hwdata_obj.write(0, &hw_img)?; + globals_obj.write(0, &glob)?; + power_obj.write(0, &pwr)?; + + // Root. + let r = &root_obj; + r.w64(0x00, layout.root_magic)?; + r.w64(root::REGION_A, va::REGION_A)?; + r.w64(root::RUNTIME, va::RUNTIME)?; + r.w64(root::GLOBALS, va::GLOBALS)?; + r.w32(root::ENABLE, 1)?; + r.write(root::GEOMETRY, &[0x00, 0x40, 14, 3])?; + const SHIFTS: [u8; 3] = [36, 25, 14]; + const ENTRIES: [u16; 3] = [64, 2048, 2048]; + for i in 0..3 { + let at = root::LEVELS + root::LEVEL_STRIDE * i; + let e = ENTRIES[i].to_le_bytes(); + r.write(at, &[8, 14, 14, SHIFTS[i], e[0], e[1], 0x00, 0x40])?; + r.w64(at + 8, 1)?; + r.w64(at + 0x10, 0x3ffffffc000)?; + r.w64(at + 0x18, ((ENTRIES[i] - 1) as u64) << SHIFTS[i])?; + } + r.w64(root::FLAGS, va::FLAGS)?; + r.w64(root::CONTROL, va::CONTROL)?; + r.w64(root::POWER, va::POWER)?; + + // Runtime pointers. + let rt = &runtime_obj; + rt.w64(runtime::HWDATA, hwdata_va)?; + rt.w64(runtime::PAIR, va::PAIR)?; + rt.w64(runtime::PAIR2, va::PAIR)?; + for i in 0..13u64 { + let (state, ring) = if i < 12 { + (va::ch_state(i), va::ch_ring(i)) + } else { + (va::DEVCTL_STATE, va::DEVCTL_RING) + }; + let at = runtime::CHANNELS + runtime::CHANNEL_STRIDE * i as usize; + rt.w64(at, state)?; + rt.w64(at + 8, state + 0x10)?; + rt.w64(at + 0x10, state + 0x20)?; + rt.w64(at + 0x18, ring)?; + } + let rx = [ + (va::EVENT_STATE, va::EVENT_RING), + (va::LOG_STATE, va::LOG_RING), + (va::TRACE_STATE, va::TRACE_RING), + (va::STATS_STATE, va::STATS_RING), + ]; + for (i, (state, ring)) in rx.iter().enumerate() { + rt.w64(runtime::RX + runtime::RX_STRIDE * i, *state)?; + rt.w64(runtime::RX + 8 + runtime::RX_STRIDE * i, *ring)?; + } + rt.w64(runtime::SMALL, va::SMALL)?; + rt.w64( + runtime::TA_ACTIVITY, + hwdata_va + hwdata::TA_ACTIVITY as u64, + )?; + rt.w64( + runtime::FS_ACTIVITY, + hwdata_va + hwdata::FS_ACTIVITY as u64, + )?; + rt.w64( + runtime::COMPUTE_ACTIVITY, + hwdata_va + hwdata::COMPUTE_ACTIVITY as u64, + )?; + rt.w64(runtime::FAULT_INFO, hwdata_va + layout.hwdata_view as u64)?; + rt.w64(runtime::C0, va::C0)?; + rt.w64(runtime::C1, va::C1)?; + rt.w64(runtime::C1_2, va::C1)?; + rt.w64(runtime::C3, va::C3)?; + rt.w64(runtime::C3_2, va::C3)?; + rt.w32(runtime::UNK_2D0, 4)?; + rt.w32(runtime::UNK_3B0, 0xff)?; + rt.w64(runtime::HWDATA_A, hwdata_va + hwdata::HWDATA_A as u64)?; + + // Control and flags. + let c = &control_obj; + c.w32(control::ENABLE, 1)?; + c.w64(control::FWCTL_STATE, va::FWCTL_STATE)?; + c.w64(control::FWCTL_RING, va::FWCTL_RING)?; + c.w32(control::ENABLE2, 1)?; + + init.c1 = Some(c1); + for o in [ + root_obj, + region_a_obj, + pair_obj, + small_obj, + c0, + c3, + power_obj, + ] { + init.others.push(o, GFP_KERNEL)?; + } + init.globals = Some(globals_obj); + init.control = Some(control_obj); + init.hwdata = Some(hwdata_obj); + init.runtime = Some(runtime_obj); + init.flags = Some(flags_obj); + Ok((init, channels, max_khz)) +} diff --git a/drivers/gpu/drm/asahi/g15/job.rs b/drivers/gpu/drm/asahi/g15/job.rs new file mode 100644 index 0000000000000..a360b58987c81 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/job.rs @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Jobs: one per SUBMIT, scheduled by a per-queue DRM scheduler and executed by the engine. +//! +//! - One `drm_sched` scheduler and entity per logical queue ([`Scheduler`], credit limit +//! [`super::hw::sizes::HORIZON_JOBS`] = 16, one credit per job); every input sync is a +//! dependency of the whole job (PER_COMMAND_INPUT_SYNCS holds a fortiori). The host priority +//! lives in [`super::queue::QueueState::priority`] for the engine's pick: with one entity per +//! scheduler, the entity priority orders nothing. +//! - Timeout [`super::hw::tuning::SCHED_TIMEOUT_MS`], always "no hang": the engine's +//! progress-based deadlines own failure. +//! - A failed input refuses unpublished work and reports the error before its fence signals. +//! A foreign device-loss error does not become this VM's sticky hardware-admission failure. +//! - `run` hands the job to the engine ([`super::Firmware::enqueue`], then +//! [`super::Firmware::publish`]) under the device lock and returns the job's fence; FINISHED +//! fences stay in queue order ([`super::fence::Timeline`]). +//! - A failed job latches its VM error before its fence signals ([`Job::complete`]). +//! +//! # For the engine +//! +//! The engine owns an `Arc` from [`super::Firmware::enqueue`] on. It completes the job +//! exactly once with [`Job::complete`] (later calls are ignored), through [`Completions`] +//! collected under the device lock and [`complete_all`] after it is dropped. A job whose storage +//! the firmware may still reference stays in `Engine::retained` (failed jobs until reboot): its +//! [`mmu::VmJobGuard`] keeps the VM's mappings and its VM state keeps the VM's TTBAT slot until +//! the last `Arc` drops. + +use core::sync::atomic::{AtomicBool, Ordering}; +use kernel::{ + bindings, c_str, device, + dma_fence::{Fence, RawDmaFence}, + drm::sched, + prelude::*, + sync::Arc, +}; + +use super::{command::Command, fence::JobFence, hw, queue::QueueState, vm::VmState, Shared}; +use crate::mmu; + +/// A job as the engine sees it. +pub(crate) struct Job { + /// Submission ID (debugging). + pub(crate) id: u64, + /// The submitting queue (priority, service turn, horizon history). + pub(crate) queue: Arc, + /// The job's VM (TTBAT slot, status). + pub(crate) vm: Arc, + /// Validated commands, in submission order (at most 64). + pub(crate) commands: KVec, + /// The job's hardware fence. + pub(crate) fence: JobFence, + /// The engine's bookkeeping; `progress.signaled` makes [`Job::complete`] act once. + pub(crate) progress: super::engine::Progress, + /// The timestamp buffers the commands name (their arena VAs stay mapped while the job lives). + _timestamps: KVec>, + /// Retains the VM's mappings (deferred unmaps, deferred VM close) while the firmware may + /// access them. + _guard: mmu::VmJobGuard, + /// Pushed to the scheduler: from now on the job counts as submitted work of its VM. + accepted: AtomicBool, + /// Handed to the engine (or refused at `run`): the scheduler no longer cancels it. + started: AtomicBool, +} + +impl Job { + pub(crate) fn new( + id: u64, + queue: Arc, + vm: Arc, + commands: KVec, + fence: JobFence, + timestamps: KVec>, + guard: mmu::VmJobGuard, + ) -> Result> { + let progress = super::engine::Progress::new(commands.len()); + Ok(Arc::new( + Self { + id, + queue, + vm, + commands, + fence, + progress, + _timestamps: timestamps, + _guard: guard, + accepted: AtomicBool::new(false), + started: AtomicBool::new(false), + }, + GFP_KERNEL, + )?) + } + + /// Completes the job: latches the VM status on failure, then signals the fence (after every + /// earlier job of the queue). Called by the engine, without the device lock. Only the first + /// call has an effect. + /// + /// Any failure makes the VM's error sticky: later SUBMITs, QUEUE_CREATEs and mapping changes + /// of the VM are refused (EIO), and mapping changes stay blocked while the job lives (the + /// engine retains a job the firmware may still reference). + #[track_caller] + pub(crate) fn complete(&self, result: Result) { + self.finish(result, true); + } + + /// Completes the job; `engine`: the engine had it, so a failure quarantines its mappings. + /// ECANCELED (never-run work at queue teardown) is only reported, not sticky. + #[track_caller] + fn finish(&self, result: Result, engine: bool) { + if self.progress.signaled.swap(true, Ordering::AcqRel) { + return; + } + if let Err(error) = result { + if error == ECANCELED { + self.vm.status.report_failure(error); + } else { + if engine { + // Before the error fence becomes visible (G17's rule). + self._guard.quarantine(); + } + self.vm.status.record(error); + } + } + self.fence.signal(result); + } + + /// Reports an input failure without quarantining or poisoning a healthy consumer VM. + fn finish_dependency(&self, error: Error) { + if self.progress.signaled.swap(true, Ordering::AcqRel) { + return; + } + self.vm.status.report_failure(error); + self.fence.signal(Err(error)); + } + + /// Hands the job to the engine, or completes it before publication on GPU/input failure. + fn start(self: &Arc, shared: &Arc, inputs: &[Fence]) { + self.started.store(true, Ordering::Release); + if shared.crashed.load(Ordering::Acquire) { + self.finish(Err(EIO), false); + return; + } + // SAFETY: Each retained fence owns its pointer for this read. drm_sched has + // waited for every dependency; inspect errors even while this VM is healthy. + let failed = inputs + .iter() + .map(|fence| unsafe { bindings::dma_fence_get_status(fence.raw()) }) + .find(|&status| status < 0); + if let Some(status) = failed { + let error = Error::from_errno(status); + if self.vm.status.get() != 0 { + // Preserve the existing own-failed-VM completion semantics. + self.finish(Err(error), false); + } else { + self.finish_dependency(crate::g17::inherited_error(error)); + } + return; + } + // `Shared::submit`: enqueue, publish and rearm under the device lock, + // completions after it. + if let Err(error) = shared.submit(self.clone()) { + // An engine that refuses work is a device failure for the job's VM. + self.finish(Err(if error == ENODEV { EIO } else { error }), false); + } + } + + /// Cancels a job the scheduler never ran (queue teardown, or a failed submission). + fn cancel_unstarted(&self) { + if self.started.swap(true, Ordering::AcqRel) { + return; + } + if self.accepted.load(Ordering::Acquire) { + self.finish(Err(ECANCELED), false); + } else if !self.progress.signaled.swap(true, Ordering::AcqRel) { + // A refused submission's job: its fence orders nothing and was never published. + self.fence.signal(Err(ECANCELED)); + } + } +} + +/// Jobs retired or failed under the device lock, completed after it is dropped. +pub(crate) type Completions = KVec<(Arc, Result)>; + +/// Completes jobs in order. +#[track_caller] +pub(crate) fn complete_all(done: Completions) { + for (job, result) in done { + job.complete(result); + } +} + +/// The scheduler's view of a job. +struct SchedJob { + job: Arc, + shared: Arc, + /// The input fences, kept for the dependency-error check at `run`. + inputs: KVec, +} + +impl sched::JobImpl for SchedJob { + type TimeoutData = (); + + const MODULE: Option<&'static kernel::ThisModule> = Some(&crate::THIS_MODULE); + + fn timeout_data(&self) {} + + fn run(job: &mut Self) -> Result> { + let fence = job.job.fence.fence(); + job.job.start(&job.shared, &job.inputs); + Ok(Some(fence)) + } + + /// The engine's own deadlines own failure; late delivery is not a hang. + fn timed_out(_data: &mut (), _finished: bool) -> sched::Status { + sched::Status::NoHang + } + + fn cancel(job: &mut Self) { + job.job.cancel_unstarted(); + } +} + +impl Drop for SchedJob { + fn drop(&mut self) { + // Entity teardown frees jobs it never ran without calling `cancel`. + self.job.cancel_unstarted(); + } +} + +/// The scheduler and entity of one logical queue. +pub(crate) struct Scheduler { + // Dropped first: the entity flush waits until every queued job has run (QUEUE_DESTROY + // drains), and only then the scheduler is finalized. + entity: sched::Entity, + _scheduler: sched::Scheduler, +} + +impl Scheduler { + pub(crate) fn new(dev: &device::Device) -> Result { + let scheduler = sched::Scheduler::new( + dev, + 1, + hw::sizes::HORIZON_JOBS, + 0, + hw::tuning::SCHED_TIMEOUT_MS, + c_str!("asahi_g15"), + )?; + // With one run queue, only the kernel priority is valid. + let entity = sched::Entity::new(&scheduler, sched::Priority::Kernel)?; + Ok(Self { + entity, + _scheduler: scheduler, + }) + } + + /// Pushes a job that waits for `inputs`. On failure the job is cancelled (not counted as + /// submitted work of its VM) and the error is returned. + pub(crate) fn push( + &mut self, + job: Arc, + shared: Arc, + inputs: KVec, + ) -> Result { + let mut dependencies = KVec::with_capacity(inputs.len(), GFP_KERNEL)?; + for fence in inputs.iter() { + dependencies.push(fence.clone(), GFP_KERNEL)?; + } + let handle = job.clone(); + let mut pending = self.entity.new_job( + 1, + SchedJob { + job, + shared, + inputs, + }, + )?; + for fence in dependencies { + pending.add_dependency(fence)?; + } + let armed = pending.arm(); + handle.accepted.store(true, Ordering::Release); + armed.push(); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g15/mod.rs b/drivers/gpu/drm/asahi/g15/mod.rs new file mode 100644 index 0000000000000..75d3ecc30ffe5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/mod.rs @@ -0,0 +1,662 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G15 GPU support (T6030, M3 Pro: J514S, J516S; firmware RTKit 2419, the macOS 14.7-14.8 stub). +//! +//! G15 firmware is *classic*: one ring per engine (TA, FS, compute) and ordering expressed by +//! in-ring barrier records that wait on another queue's stamp, unlike the G17 kick-ring/QID +//! model. The boot loader leaves the firmware loaded; this driver identifies the exact build, +//! adopts the running PPL handoff, builds the 49-object initdata graph from this machine's ADT +//! and brings the firmware to READY. All logical queues and VMs share one persistent compute +//! graph and one render graph; the commands of a submission are encoded into compute and pass +//! slots and published as *prefixes* on the firmware's queues, then retired on their stamps. +//! Everything firmware-visible that a failed command may still reference is retained until +//! reboot. +//! +//! # Plugging into the shared driver +//! +//! - [`Driver`] is a separate platform driver. T6030 has a module alias; its probe must wait +//! for the native display and PMP supplier before booting firmware. T6031 retains explicit +//! module loading. It calls [`crate::driver::AsahiPlatformDriver::probe_g15`]. +//! - [`Gpu`] implements [`crate::gpu::Gpu`]: GET_PARAMS identity, VM creation (`new_vm`), +//! queues (`new_queue`, see [`queue`]) and timestamp buffers. +//! - User VMs are shared-layer [`mmu::Vm`]s on a [`mmu::Uat::new_adopted`] UAT with tracked +//! status ([`mmu::Vm::with_tracked_status`]); `file.rs` handles VM_BIND, GEM, syncobjs, +//! VM_STATUS(_MIRROR) and MADVISE. +//! +//! # Locking +//! +//! [`Shared::state`] is the device lock. It serializes all firmware-visible publication, the +//! device-control ring and every firmware-shared object, including the bounded waits for the +//! firmware that publication, recovery and the device controls need (up to 2 s). Fences are +//! signalled after dropping it ([`job::Completions`]). Lock order: `Shared::state` -> UAT shared +//! state -> VM exec lock -> GEM reservation. The RTKit receive callbacks run in interrupt +//! context and only set atomics and queue the poll work. +//! +//! # Module structure +//! +//! - Platform, firmware boot and UAT handoff: `boot`, `hw`, `adt`, `identity`, `regs`, `uat`, +//! `object`, `board`, `initdata`, `channel`, `event`, `fw/{mod,initdata,channels}`. +//! - Submission engine: `engine`, `encode`, `slots`, `graph`, `poll`, `fw/{work,graph}`. +//! - Recovery, parameter buffer and power: `recovery`, `pools`, `power`, `fw/buffer`. +//! - UAPI glue and VM integration: `queue`, `command`, `gate`, `job`, `fence`, `vm`, +//! `timestamp`, `params`, and the hooks in the shared driver (`file.rs`, `gem.rs`, `gpu.rs`, +//! `mmu.rs`, `pgtable.rs`). + +mod adt; +mod board; +mod boot; + +// One admission rule for the generation mappings and shared adopted UAT. +pub(crate) use boot::glue::memory_mode; +mod channel; +mod command; +mod encode; +mod engine; +mod event; +mod fence; +pub(crate) mod fw; +mod gate; +mod graph; +pub(crate) mod hw; +mod identity; +mod initdata; +mod job; +mod object; +mod params; +mod poll; +mod pools; +mod power; +mod queue; +mod recovery; +mod regs; +mod slots; +mod timestamp; +mod uat; +mod vm; +mod wire; + +use core::any::Any; +use core::ops::Range; +use core::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, AtomicU64, Ordering}; + +use kernel::{ + c_str, + device::{self, Core}, + impl_has_delayed_work, new_delayed_work, new_mutex, of, platform, + prelude::*, + sync::{aref::ARef, Arc, Mutex}, + time::hrtimer::{ArcHrTimerHandle, HrTimer}, + uapi, + workqueue::{DelayedWork, WorkItem}, +}; + +use crate::driver::AsahiDrmDevice as AsahiDevice; +use crate::{alloc, gem, gpu, mmu}; + +/// Name of the G15 platform driver. + +/// GPU generation reported to userspace. +pub(crate) const GPU_GENERATION: u32 = 15; + +/// Static configuration of a SoC with a G15 GPU. +pub(crate) struct Config { + /// SoC and GPU name, for the log. + pub(crate) name: &'static str, + /// SoC identifier reported to userspace. + pub(crate) chip_id: u32, + /// GPU variant reported to userspace. + pub(crate) gpu_variant: crate::hw::GpuVariant, + /// GPU revision reported to userspace. + pub(crate) gpu_revision: u32, + /// Input address bits translated by each UAT root. + pub(crate) uat_ias: u32, + /// Output (physical) address bits of the UAT. + pub(crate) uat_oas: u32, + /// Clusters and cores per cluster. + pub(crate) num_clusters: u32, + pub(crate) num_cores_per_cluster: u32, + /// Qualified identification register values. + pub(crate) hw_ids: &'static [hw::HwId], + /// Qualified firmware builds. + pub(crate) firmware: &'static [hw::FirmwareBuild], + /// Machine compatible this SoC's boards carry. + pub(crate) machine: &'static CStr, + /// Where regions, segments and performance states come from, and how the GPU is powered. + pub(crate) platform: hw::Platform, + /// Initdata placement and sizes that differ between firmware builds. + pub(crate) layout: &'static hw::Layout, + /// Board images. + pub(crate) board: hw::Board, + /// Compute register values that scale with the cluster count. + pub(crate) compute: hw::ComputeRegs, +} + +/// Core slots of a SoC: the width of its enabled-core mask. +impl Config { + pub(crate) const fn core_slots(&self) -> u32 { + self.num_clusters * self.num_cores_per_cluster + } +} + +/// OF match table of the G15 platform driver. +const OF_TABLE: kernel::device_id::IdArray = + kernel::device_id::IdArray::new([ + ( + of::DeviceId::new(c_str!("apple,agx-t6030")), + &hw::t6030::CONFIG, + ), + ( + of::DeviceId::new(c_str!("apple,agx-t6031")), + &hw::t6031::CONFIG, + ), + ]); + +// T6030 probe admission requires the native display and PMP supplier before +// firmware starts. T6031 retains explicit loading until it has an equivalent +// display-order admission gate. +kernel::of_device_table!( + AUTOLOAD_OF_TABLE, + MODULE_OF_TABLE_G15, + (), + [(of::DeviceId::new(c_str!("apple,agx-t6030")), ())] +); + +/// The G15 platform adapter: the shared DRM registration probed through +/// [`crate::driver::AsahiPlatformDriver::probe_g15`]. +/// +/// `None` after a probe that failed once the firmware had been touched: the device stays bound +/// (unbinding would detach and power off the GFX domain under the running firmware), the +/// firmware session is retained until reboot and no DRM device exists. +#[pin_data] +pub(crate) struct Driver<'bound> { + _driver: Option>, +} + +impl platform::Driver for Driver<'_> { + type IdInfo = &'static Config; + type Data<'bound> = Driver<'bound>; + const OF_ID_TABLE: Option> = Some(&OF_TABLE); + + /// The firmware session cannot be stopped once started. + fn suppress_bind_attrs() -> bool { + true + } + + fn probe<'bound>( + pdev: &'bound platform::Device>, + info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound { + // `Device::set_drvdata` allocates its KBox before running this initializer. + // Keep every firmware-visible action inside it: otherwise an allocation failure + // after probe_g15 started firmware would return an unbound device. + try_pin_init!(Driver { + _driver: { + if !MODULE_READY.load(Ordering::Acquire) { + return Err(EPROBE_DEFER); + } + let cfg = *info.ok_or(ENODEV)?; + dev_info!(pdev.as_ref(), "G15: probing ({})\n", cfg.name); + match crate::driver::AsahiPlatformDriver::probe_g15(pdev, cfg) { + Ok(driver) => Some(driver), + Err(error) if boot::glue::retained() => { + dev_err!( + pdev.as_ref(), + "G15: probe failed after the firmware start ({:?}); device kept bound, firmware retained until reboot\n", + error + ); + None + } + Err(error) => return Err(error), + } + }, + }) + } + +} + +/// Firmware starts only after every fallible module/driver registration has succeeded. +/// A deferred probe racing the final activation sees the Release/Acquire publication. +static MODULE_READY: AtomicBool = AtomicBool::new(false); + +/// Last, infallible module initializer: enable native probes and retry this adapter. +/// No fallible module initialization may follow this call. +pub(crate) fn activate() { + MODULE_READY.store(true, Ordering::Release); + // SAFETY: Both pinned platform registrations are fully initialized and remain owned + // by DriverModule. The C helper borrows their live driver pointer only for this call. + unsafe { boot::glue::asahi_g15_reprobe() }; +} + +/// Work item ID of the poll work. +const POLL_WORK: u64 = 0; + +/// Device state shared with the RTKit callbacks and the poll work. +#[pin_data] +pub(crate) struct Shared { + /// Reserved host logical queues, shared across every file and VM on this GPU. + pub(crate) logical_queues: AtomicU32, + pub(crate) dev: ARef, + /// Latched fail-stop: every VM fails with EIO until reboot. + pub(crate) crashed: AtomicBool, + /// Set by the firmware's EP 0x20 event message; consumed by the poll work. + pub(crate) evented: AtomicBool, + // --- RTKit callback state and a retained failed session. --- + /// The first latched error, as a negative errno. + pub(crate) error: AtomicI32, + /// The RTKit receive callbacks may queue the poll work. + pub(crate) started: AtomicBool, + /// Firmware DATA, where the only RTKit buffer accepted lies. + pub(crate) data: Range, + /// Physical address of the accepted RTKit buffer, 0 before. + pub(crate) rtkit_buffer: AtomicU64, + /// A firmware session whose start or backend start failed: retained, rings still drained. + pub(crate) orphaned: AtomicBool, + #[pin] + pub(crate) orphan: Mutex>>, + #[pin] + pub(crate) poll: DelayedWork, + // --- The completion/repoll hrtimer and the handle of its current expiry (dropping it + // cancels the expiry; see `Shared::rearm`). --- + #[pin] + pub(crate) resnap: HrTimer, + #[pin] + pub(crate) resnap_handle: Mutex>>, + /// The device lock. `None` before boot completes. + #[pin] + pub(crate) state: Mutex>>, +} + +impl_has_delayed_work! { + impl HasDelayedWork for Shared { self.poll } +} + +impl WorkItem for Shared { + type Pointer = Arc; + fn run(this: Arc) { + if this.orphaned.load(Ordering::Acquire) { + this.drain_orphan(); + return; + } + this.run_poll(); + } +} + +impl Shared { + /// Queues the poll work immediately. Callable from any context. + pub(crate) fn kick(self: &Arc) { + self.evented.store(true, Ordering::Release); + // `mod_delayed_work(.., 0)` so that a kick also runs a poll whose 1-jiffy backup is + // pending (a plain enqueue is refused then). + self.poll_now(); + } + + /// Latches the fail-stop state. + pub(crate) fn fail_stop(&self, error: Error) { + let _ = + self.error + .compare_exchange(0, error.to_errno(), Ordering::AcqRel, Ordering::Acquire); + if !self.crashed.swap(true, Ordering::AcqRel) { + dev_err!( + self.dev.as_ref(), + "G15: fail-stop ({:?}); GPU unusable until reboot\n", + error + ); + } + } + + /// The latched error, if any. + pub(crate) fn check(&self) -> Result { + if !self.crashed.load(Ordering::Acquire) { + return Ok(()); + } + match self.error.load(Ordering::Acquire) { + 0 => Err(EIO), + e => Err(Error::from_errno(e)), + } + } + + /// Keeps a firmware session that did not complete until reboot; its receive rings stay + /// drained by the poll work. + pub(crate) fn orphan(&self, orphan: boot::Orphan) { + let mut slot = self.orphan.lock(); + match KBox::::new_uninit(GFP_KERNEL) { + Ok(storage) => { + *slot = Some(KBox::write(storage, orphan)); + self.orphaned.store(true, Ordering::Release); + } + Err(_) => { + // Allocation must not consume the live session: KBox::new would drop it + // on failure. Keep its mappings and backing even without ring draining. + core::mem::forget(orphan); + dev_err!( + self.dev.as_ref(), + "G15: cannot allocate failed-session storage; session leaked without ring draining\n" + ); + } + } + } + + /// Drains the receive rings of an orphaned session. + fn drain_orphan(&self) { + if let Some(orphan) = self.orphan.lock().as_deref_mut() { + let _ = orphan.drain(); + } + } +} + +/// Everything that exists while the firmware runs, under the device lock. +/// +/// Nothing in here is ever freed while the firmware can reach it: a G15 firmware session cannot +/// be stopped, so dropping this structure leaks the firmware-visible parts on purpose +/// ([`Gpu::retain`]). +pub(crate) struct Firmware { + /// The SoC configuration. + pub(crate) cfg: &'static Config, + /// RTKit, MMIO, regions, the adopted UAT and the host window. + pub(crate) session: boot::Session, + /// The initdata graph and its live words (Globals, Control, HardwareData, ...). + pub(crate) init: initdata::InitData, + /// Firmware objects: the firmware VA allocator and the retained objects. + pub(crate) objects: object::Objects, + /// Submit channels, the device-control ring and the receive rings. + pub(crate) channels: channel::Channels, + /// Decoded firmware events (stage counters, last error, growth requests). + pub(crate) events: event::Events, + /// The compute and render graphs. + pub(crate) graph: graph::Graphs, + /// Compute slots, pass slots and clustering buffers. + pub(crate) slots: slots::Slots, + /// Pending jobs, prefixes, horizons and flights. + pub(crate) engine: engine::Engine, + /// Halt/resume, kills, deadlines and fail-stop. + pub(crate) recovery: recovery::Recovery, + /// Render UMA/PB allocators, their spare set and parameter-buffer growth. + pub(crate) pools: pools::Pools, + /// Live VMs and their exclusive TTBAT slots, and the client aliases of every user VM. + pub(crate) vms: Arc, +} + +/// A G15 GPU. +pub(crate) struct Gpu { + cfg: &'static Config, + /// Identification read at boot. + pub(crate) id: regs::Identity, + /// Highest firmware performance state, kHz. + pub(crate) max_frequency_khz: u32, + ids: gpu::SequenceIDs, + pub(crate) shared: Arc, + /// The adopted UAT, for creating and binding user VMs without the device lock. + pub(crate) uat: Arc, +} + +impl Gpu { + /// Admits and boots a G15 GPU: identity checks, PMP AGX vote, RTKit, UAT handoff, initdata, + /// READY, then the compute and render graphs. On success the module is pinned: the + /// firmware session lasts until reboot. + pub(crate) fn new( + pdev: &platform::Device>, + dev: &AsahiDevice, + cfg: &'static Config, + ) -> Result> { + // Admission (probe-time checks), then the firmware start to READY. + let admitted = boot::admit(pdev, dev, cfg)?; + // Admission holds the GFX domain. A failure before the firmware is touched gives it + // back (a start that touched the firmware keeps it, retained until reboot). + let unretained = |error: Error| -> Error { + if !boot::glue::retained() { + boot::glue::release_power(dev.as_ref().as_ref()); + } + error + }; + let data = admitted.adt.data.clone(); + let shared = Arc::pin_init( + pin_init!(Shared { + logical_queues: AtomicU32::new(0), + dev: dev.as_ref().as_ref().into(), + crashed: AtomicBool::new(false), + evented: AtomicBool::new(false), + error: AtomicI32::new(0), + started: AtomicBool::new(false), + data, + rtkit_buffer: AtomicU64::new(0), + orphaned: AtomicBool::new(false), + orphan <- new_mutex!(None, "g15::Shared::orphan"), + poll <- new_delayed_work!("g15::Shared::poll"), + resnap <- HrTimer::new(), + resnap_handle <- new_mutex!(None, "g15::Shared::resnap_handle"), + state <- new_mutex!(None, "g15::Shared::state"), + }), + GFP_KERNEL, + ) + .map_err(unretained)?; + let booted = boot::start(admitted, dev, cfg, &shared).map_err(unretained)?; + let uat = booted.session.uat(); + let (id, max_frequency_khz) = (booted.id, booted.max_frequency_khz); + // The backend: graphs, slots, pools, engine, recovery and VM registry. + let firmware = Firmware::start_backend(dev, cfg, booted, &shared)?; + *shared.state.lock() = Some(firmware); + // The engine's statistics file (a debugfs failure does not fail the probe). + if let Err(error) = shared.start_engine() { + dev_warn!(dev.as_ref(), "G15: no asahi-g15 debugfs ({:?})\n", error); + } + let gpu = Arc::new( + Gpu { + cfg, + id, + max_frequency_khz, + ids: Default::default(), + shared, + uat, + }, + GFP_KERNEL, + )?; + gpu.retain(); + Ok(gpu) + } + + /// Keeps the firmware session and this module alive until reboot. + /// + /// Takes the module reference once. Nothing firmware-reachable is freed on any path: the + /// RTKit callbacks hold [`Shared`], which holds the [`Firmware`] (or the [`boot::Orphan`] of a + /// failed start) until reboot. + pub(crate) fn retain(&self) { + boot::glue::retain_module(); + } +} + +impl Firmware { + /// Builds the graphs and the engine after READY. On failure the firmware stays at READY, + /// retained as an orphan of `shared`. + fn start_backend( + dev: &AsahiDevice, + cfg: &'static Config, + booted: boot::Booted, + shared: &Arc, + ) -> Result> { + let boot::Booted { + session, + init, + mut objects, + channels, + events, + .. + } = booted; + dev_info!( + dev.as_ref(), + "G15: firmware at READY, starting the backend\n" + ); + let orphan = |session: boot::Session, + objects: object::Objects, + channels: channel::Channels, + init: initdata::InitData, + events: event::Events, + error: Error| + -> Error { + dev_err!( + dev.as_ref(), + "G15: backend start failed ({:?}); firmware retained at READY until reboot\n", + error + ); + shared.orphan(boot::Orphan { + session, + _objects: objects, + channels: Some(channels), + _init: Some(init), + events, + }); + error + }; + let backend = (|| -> Result<_> { + let graph = graph::Graphs::new(dev, &mut objects)?; + let slots = slots::Slots::new(&mut objects)?; + let pools = pools::Pools::new(dev, &mut objects)?; + let vms = vm::Registry::new()?; + let slot = KBox::::new_uninit(GFP_KERNEL)?; + Ok((graph, slots, pools, vms, slot)) + })(); + let (graph, slots, pools, vms, slot) = match backend { + Ok(backend) => backend, + Err(e) => return Err(orphan(session, objects, channels, init, events, e)), + }; + let mut firmware = KBox::write( + slot, + Firmware { + cfg, + session, + init, + objects, + channels, + events, + graph, + slots, + engine: engine::Engine::new(), + recovery: recovery::Recovery::new(), + pools, + vms, + }, + ); + if let Err(e) = firmware.init_graphs() { + let Firmware { + session, + init, + objects, + channels, + events, + .. + } = KBox::into_inner(firmware); + return Err(orphan(session, objects, channels, init, events, e)); + } + Ok(firmware) + } +} + +impl gpu::Gpu for Gpu { + fn stats_snapshot(&self) -> Option> { + None + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn arc_as_any(self: Arc) -> Arc { + self as Arc + } + + fn manager(&self) -> Option<&dyn gpu::GpuManager> { + None + } + + fn init(&self) -> Result { + Ok(()) + } + + fn ids(&self) -> &gpu::SequenceIDs { + &self.ids + } + + fn is_crashed(&self) -> bool { + self.shared.crashed.load(Ordering::Acquire) + } + + fn get_params(&self, params: &mut uapi::drm_asahi_params_global) -> Result { + params::fill(self, params) + } + + fn feature_flags(&self) -> u64 { + params::feature_flags() + } + + fn supports_vm_status(&self) -> bool { + true + } + + fn queue_limits(&self) -> Option { + Some(params::queue_limits()) + } + + fn base_clock_hz(&self) -> u32 { + let frequency: u64; + // SAFETY: CNTFRQ_EL0 is a read-only register containing the architectural counter rate. + unsafe { core::arch::asm!("mrs {frequency}, CNTFRQ_EL0", frequency = out(reg) frequency) }; + frequency as u32 + } + + fn uat_geometry(&self) -> mmu::UatGeometry { + mmu::UatGeometry { + ias: self.cfg.uat_ias, + oas: self.cfg.uat_oas, + } + } + + fn user_range(&self) -> Range { + hw::USER_RANGE + } + + fn tracked_submission(&self) -> bool { + true + } + + fn soft_faults(&self) -> bool { + false + } + + fn prepare_new_objects(&self) -> bool { + true + } + + // VM_BIND operations per call at most. + fn bind_batch_limit(&self) -> Option { + Some(params::MAX_BINDS) + } + + fn vm_closed(&self) { + vm::closed(self) + } + + fn new_vm(&self, kernel_range: Range) -> Result { + vm::new_vm(self, kernel_range) + } + + fn new_queue( + &self, + vm: mmu::Vm, + _ualloc: Arc>, + _ualloc_priv: Arc>, + priority: u32, + usc_exec_base: u64, + ) -> Result> { + Ok(KBox::new( + queue::Queue::new(self, vm, priority, usc_exec_base)?, + GFP_KERNEL, + )?) + } + + fn map_timestamp_buffer( + &self, + bo: gem::ObjectRef, + range: Range, + ) -> Result { + timestamp::map(self, bo, range) + } +} diff --git a/drivers/gpu/drm/asahi/g15/object.rs b/drivers/gpu/drm/asahi/g15/object.rs new file mode 100644 index 0000000000000..227d8a8ac8e3b --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/object.rs @@ -0,0 +1,529 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Firmware objects and their cache classes. +//! +//! Each object is a GEM kernel object (write-combined CPU mapping) mapped into the host window +//! at a fixed VA or in [`super::hw::va::FW_ALLOC`] (a bump allocator with an unmapped guard page +//! after each object), with these optional aliases of the same pages: +//! +//! - a context-0 lower alias (`Uat::kernel_lower_vm`, [`super::hw::pte::LOWER`]); +//! - a client alias at a fixed GVA in every user VM: recorded here ([`Objects::clients`]) and +//! installed by `vm.rs` as driver mappings when the VM is created. +//! +//! The backing is populated and DMA-mapped before any CPU view exists: on this non-DMA-coherent +//! device the DMA mapping cleans the freshly zeroed pages' cacheable alias to the point of +//! coherency, so no dirty line can later overwrite what the host writes through the +//! write-combined view. The firmware-side leaf word is chosen by [`Class`] and must be exact +//! ([`Objects::self_check`] proves the shared page-table code writes it). Host accesses +//! after publication use single, aligned volatile accesses (`r32`/`w32`/`w64`). Objects are +//! retained until reboot: nothing the firmware can reach is ever unmapped or freed. + +mod geometry; + +use core::ptr::NonNull; + +use kernel::io::Io; +use kernel::{drm::gem::BaseObject, prelude::*, sync::aref::ARef, sync::Arc}; + +use super::hw::{self, pte}; +use super::regs::wmb; +use super::uat; +use crate::driver::AsahiDrmDevice as AsahiDevice; +use crate::{gem, mmu}; + +/// Firmware-side cache class of an object (the leaf words of [`super::hw::pte`]). +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Class { + /// Firmware-only, AttrIndex 2. + Fw, + /// Firmware-only, ASC-cached (AttrIndex 0). Never for rings or anything the host writes + /// after the firmware first touched it. + FwCached, + /// Firmware and GPU, AttrIndex 2. + Shared, + /// Firmware and GPU, ASC-cached. + SharedCached, + /// Firmware device mapping. + Mmio, +} + +impl Class { + /// The exact firmware leaf word of this class. + pub(crate) const fn pte(self) -> u64 { + match self { + Class::Fw => pte::FW, + Class::FwCached => pte::FW_CACHED, + Class::Shared => pte::SHARED, + Class::SharedCached => pte::SHARED_CACHED, + Class::Mmio => pte::MMIO, + } + } +} + +/// Where an object is placed in the host window. +pub(crate) enum Placement { + /// At this firmware VA (canonical initdata objects). The payload starts at this VA; the + /// object starts at its page. + At(u64), + /// Anywhere in the firmware allocator window, followed by an unmapped guard page. + Anywhere, +} + +/// A firmware object. +/// +/// # Invariants +/// +/// `cpu`, if set, points to a live write-combined kernel mapping of the whole backing (`bytes` +/// bytes), kept alive by `backing`. +pub(crate) struct FwObject { + /// Firmware (host-window) VA of byte 0 of the payload. + pub(crate) va: u64, + /// Client GVA of byte 0, if aliased into every user VM. + pub(crate) gva: Option, + /// Payload size. + pub(crate) size: usize, + /// Payload offset inside the first page. + offset: usize, + /// Mapped bytes: the payload rounded up to whole pages. + bytes: usize, + /// Client alias leaf word. + client_leaf: u64, + cpu: Option>, + /// Host-window and lower mappings; never dropped while the firmware runs. + _mappings: KVec, + /// Holds the pages for the device. + _sgt: HeldPages, + backing: gem::ObjectRef, +} + +// SAFETY: The object only refers to kernel memory shared with the firmware; every host access +// goes through raw pointers with volatile or byte-copy semantics, serialized by the device lock +// where the protocol needs it. +unsafe impl Send for FwObject {} +// SAFETY: See `Send`. +unsafe impl Sync for FwObject {} + +impl FwObject { + /// Pointer to payload byte `offset` for `len` bytes, bounded by the mapped pages. + fn ptr(&self, offset: usize, len: usize) -> Result<*mut u8> { + let cpu = self.cpu.ok_or(EINVAL)?; + let at = self.offset.checked_add(offset).ok_or(EINVAL)?; + if at.checked_add(len).ok_or(EINVAL)? > self.bytes { + return Err(EINVAL); + } + // SAFETY: By the type invariant `cpu` maps `bytes` bytes; `at + len` was checked. + Ok(unsafe { cpu.as_ptr().add(at) }) + } + + /// Copies bytes into the object before it is published, or into host-owned words. + pub(crate) fn write(&self, offset: usize, bytes: &[u8]) -> Result { + let p = self.ptr(offset, bytes.len())?; + // SAFETY: `p` is valid for `bytes.len()` bytes of the object's mapping, which no Rust + // reference aliases. + unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), p, bytes.len()) }; + Ok(()) + } + + /// Fills bytes. + pub(crate) fn fill(&self, offset: usize, value: u8, len: usize) -> Result { + let p = self.ptr(offset, len)?; + // SAFETY: As for `write`. + unsafe { core::ptr::write_bytes(p, value, len) }; + Ok(()) + } + + /// 32-bit store: a single aligned access, or bytes at an unaligned offset. + pub(crate) fn w32(&self, offset: usize, value: u32) -> Result { + let p = self.ptr(offset, 4)?; + if p as usize % 4 == 0 { + // SAFETY: `p` is valid and aligned for a u32. + unsafe { (p as *mut u32).write_volatile(value.to_le()) }; + Ok(()) + } else { + self.write(offset, &value.to_le_bytes()) + } + } + + /// 64-bit store: a single aligned access, or bytes at an unaligned offset. + pub(crate) fn w64(&self, offset: usize, value: u64) -> Result { + let p = self.ptr(offset, 8)?; + if p as usize % 8 == 0 { + // SAFETY: `p` is valid and aligned for a u64. + unsafe { (p as *mut u64).write_volatile(value.to_le()) }; + Ok(()) + } else { + self.write(offset, &value.to_le_bytes()) + } + } + + /// 32-bit load of a firmware- or GPU-written word: a single aligned access, or an unaligned + /// load (words of the unaligned canonical objects). + pub(crate) fn r32(&self, offset: usize) -> Result { + let p = self.ptr(offset, 4)?; + // SAFETY: `p` is valid for 4 bytes; Normal non-cacheable memory permits an unaligned + // load. + Ok(u32::from_le(unsafe { + if p as usize % 4 == 0 { + (p as *const u32).read_volatile() + } else { + (p as *const u32).read_unaligned() + } + })) + } + + /// Mapped bytes (the payload offset plus size, rounded up to pages). + pub(crate) fn bytes(&self) -> usize { + self.bytes + } + + /// The client alias: GVA of the first page and its leaf word. `vm.rs` maps [`Self::gem`] + /// there in every user VM. + pub(crate) fn client(&self) -> Option<(u64, u64)> { + self.gva + .map(|gva| (gva - self.offset as u64, self.client_leaf)) + } + + /// The backing GEM object, for aliases in other address spaces. + pub(crate) fn gem(&self) -> ARef { + self.backing.gem.clone() + } +} + +/// The pinned pages of a firmware-visible object, as [`gpu_backing`] returns them. +pub(crate) type HeldPages = ARef; + +// The host tests of the geometry use the hardware's page size and page-frame mask. +static_assert!(geometry::PAGE == hw::PAGE && geometry::PA_MASK == hw::PA_MASK); + +/// A zeroed GEM kernel object of exactly `bytes` (a nonzero number of whole pages, EINVAL +/// otherwise), populated and DMA-mapped. Its DMA segments must back exactly `bytes` with pages +/// the UAT can name ([`geometry::backing_ok`], EPERM otherwise). The DMA mapping cleans the +/// freshly zeroed pages' cacheable alias to the point of coherency, before any mapping of them +/// exists. +pub(crate) fn gpu_backing(dev: &AsahiDevice, bytes: usize) -> Result<(gem::ObjectRef, HeldPages)> { + if bytes == 0 || bytes as u64 & (hw::PAGE - 1) != 0 { + return Err(EINVAL); + } + let backing = gem::new_kernel_object_wc(dev, bytes)?; + if backing.gem.size() != bytes { + return Err(EINVAL); + } + // SAFETY: Backing is populated only while probing the bound G15 platform device, + // or through its retained session. Manual unbind is suppressed for this adapter. + let sgt = gem::held_sg_table(&backing.gem, unsafe { dev.as_ref().as_ref().as_bound() })?; + let segments = sgt + .iter() + .map(|range| (range.dma_address() as u64, range.dma_len() as u64)); + if !geometry::backing_ok(bytes as u64, hw::PAGE, hw::PA_MASK, segments) { + return Err(EPERM); + } + let held = backing.gem.clone(); + Ok((backing, held)) +} + +/// Allocation request. +pub(crate) struct Request { + pub(crate) size: usize, + pub(crate) class: Class, + pub(crate) placement: Placement, + /// Payload offset inside the first page (e.g. work images at +0x3760). Ignored + /// for [`Placement::At`], whose VA gives it. + pub(crate) offset: usize, + /// Client alias GVA of the page holding the payload, and its leaf word + /// ([`pte::LEAF_NC`], [`pte::LEAF_CACHED`] or [`pte::LEAF_PB_LIST`]). + pub(crate) client: Option<(u64, u64)>, + /// Context-0 lower alias GVA of the first page (the client alias page). + pub(crate) lower: Option, + /// GPU-only objects need no CPU mapping (zeroed at allocation). + pub(crate) cpu: bool, +} + +/// The firmware object allocator and the set of retained objects. +pub(crate) struct Objects { + dev: ARef, + uat: Arc, + /// The host window: shared L2 mirror and PPL handoff. + pub(crate) window: uat::Window, + /// Next free VA of the firmware allocator window. + fw_next: u64, + /// Every object, retained until reboot. + retained: KVec>, + /// Objects aliased into every user VM. + clients: KVec>, + /// Device aliases and self-check mappings, retained until reboot. + mappings: KVec, + /// Host-window pages backed by objects. + pages: usize, +} + +impl Objects { + pub(crate) fn new(dev: &AsahiDevice, uat: Arc, window: uat::Window) -> Self { + Self { + dev: dev.into(), + uat, + window, + fw_next: hw::va::FW_ALLOC.start, + retained: KVec::new(), + clients: KVec::new(), + mappings: KVec::new(), + pages: 0, + } + } + + /// Objects aliased into every user VM, in allocation order. + pub(crate) fn clients(&self) -> &[Arc] { + &self.clients + } + + /// Host-window pages backed by firmware objects. + pub(crate) fn pages(&self) -> usize { + self.pages + } + + /// Mirrors host-window mappings made outside this allocator (timestamp buffers) into the + /// firmware's view before their addresses are used. + pub(crate) fn sync_window(&mut self, range: core::ops::Range) -> Result { + self.window.sync_range(&self.uat, range) + } + + /// Allocates and maps a zeroed firmware object. + pub(crate) fn alloc(&mut self, req: Request) -> Result> { + self.alloc_split(req, None) + } + + /// Like [`Self::alloc`], with the first `cached` mapped bytes (a whole number of pages) in + /// `class` and the rest in the request's class (HardwareData). + pub(super) fn alloc_split( + &mut self, + req: Request, + prefix: Option<(usize, Class)>, + ) -> Result> { + let page = hw::PAGE as usize; + let (page_va, offset) = match req.placement { + Placement::At(va) => (va & !(hw::PAGE - 1), (va & (hw::PAGE - 1)) as usize), + Placement::Anywhere => (self.fw_next, req.offset), + }; + if req.size == 0 || offset >= page { + return Err(EINVAL); + } + let bytes = offset + .checked_add(req.size) + .ok_or(EINVAL)? + .next_multiple_of(page); + if page_va < hw::HOST || page_va + bytes as u64 > hw::HOST + hw::HOST_SIZE { + return Err(EINVAL); + } + if let Placement::Anywhere = req.placement { + if page_va + bytes as u64 + hw::PAGE > hw::va::FW_ALLOC.end { + return Err(ENOSPC); + } + } + if let Some((gpage, _)) = req.client { + if gpage & (hw::PAGE - 1) != 0 || gpage + bytes as u64 > hw::USER_RANGE.end { + return Err(EINVAL); + } + if self.clients.len() == hw::sizes::MAX_CLIENT { + return Err(ENOSPC); + } + } + if let Some((cached, _)) = prefix { + if cached == 0 || cached % page != 0 || cached >= bytes { + return Err(EINVAL); + } + } + + // Populate and DMA-map first: this cleans the zeroed pages' cacheable alias to the + // point of coherency before any write-combined access. + let (mut backing, sgt) = gpu_backing(&self.dev, bytes)?; + let cpu = if req.cpu { + let vmap = backing.vmap()?; + for i in 0..bytes { + vmap.try_write8(0, i)?; + } + let ptr = NonNull::new(vmap.addr.cast::()).ok_or(ENOMEM)?; + Some(ptr) + } else { + None + }; + + let mut mappings = KVec::new(); + let kernel_vm = self.uat.kernel_vm(); + let guard = matches!(req.placement, Placement::Anywhere); + match prefix { + None => { + mappings.push( + kernel_vm.map_at( + page_va, + bytes, + &backing.gem, + mmu::Prot::from_pte(req.class.pte()), + guard, + )?, + GFP_KERNEL, + )?; + } + Some((cached, class)) => { + mappings.push( + kernel_vm.map_in_range( + &backing.gem, + 0..cached, + hw::PAGE, + page_va..page_va + cached as u64, + mmu::Prot::from_pte(class.pte()), + false, + )?, + GFP_KERNEL, + )?; + let rest = page_va + cached as u64; + mappings.push( + kernel_vm.map_in_range( + &backing.gem, + cached..bytes, + hw::PAGE, + rest..page_va + bytes as u64, + mmu::Prot::from_pte(req.class.pte()), + guard, + )?, + GFP_KERNEL, + )?; + if mappings[0].iova() != page_va || mappings[1].iova() != rest { + return Err(EINVAL); + } + } + } + self.window + .sync_range(&self.uat, page_va..page_va + bytes as u64)?; + if let Placement::Anywhere = req.placement { + // Skip an unmapped guard page after the object. + self.fw_next += bytes as u64 + hw::PAGE; + } + if let Some(lower) = req.lower { + if lower & (hw::PAGE - 1) != 0 { + return Err(EINVAL); + } + mappings.push( + self.uat.kernel_lower_vm().map_at( + lower, + bytes, + &backing.gem, + mmu::Prot::from_pte(pte::LOWER), + false, + )?, + GFP_KERNEL, + )?; + uat::clean_lower(&self.uat)?; + } + wmb(); + + // INVARIANT: `cpu` came from the owned vmap that `backing` keeps for its lifetime. + let object = Arc::new( + FwObject { + va: page_va + offset as u64, + gva: req.client.map(|(gpage, _)| gpage + offset as u64), + size: req.size, + offset, + bytes, + client_leaf: req.client.map_or(pte::LEAF_NC, |(_, leaf)| leaf), + cpu, + _mappings: mappings, + _sgt: sgt, + backing, + }, + GFP_KERNEL, + )?; + self.pages += bytes / page; + self.retained.push(object.clone(), GFP_KERNEL)?; + if req.client.is_some() { + self.clients.push(object.clone(), GFP_KERNEL)?; + } + Ok(object) + } + + /// Maps a device range (an IOMapping record) for the firmware as [`Class::Mmio`]. + pub(crate) fn map_mmio(&mut self, va: u64, pa: u64, size: u64) -> Result { + let mapping = self.uat.kernel_vm().map_io( + self.dev.as_ref().as_ref(), + va, + usize::try_from(pa)?, + usize::try_from(size)?, + mmu::Prot::from_pte(Class::Mmio.pte()), + )?; + self.mappings.push(mapping, GFP_KERNEL)?; + self.window.sync_range(&self.uat, va..va + size) + } + + /// Boot-time check that the shared page-table code writes every firmware leaf word exactly: + /// each class word is mapped on a scratch page with `Prot::from_pte(word)`, the leaf is read + /// back from the tables and must equal `pa | word`. Host-window classes are checked in the + /// (global) host window, the lower and client classes in the (non-global) context-0 lower + /// tree, which user VMs share the leaf format with. Runs before anything firmware-visible is + /// written; the scratch mappings are retained (they are never published). A mismatch refuses + /// the boot. + pub(crate) fn self_check(&mut self) -> Result { + let window = [ + pte::FW, + pte::SHARED, + pte::FW_CACHED, + pte::SHARED_CACHED, + pte::MMIO, + ]; + let lower = [ + pte::LOWER, + pte::LEAF_NC, + pte::LEAF_CACHED, + pte::LEAF_PB_LIST, + ]; + let scratch = gem::new_kernel_object_wc(&self.dev, hw::PAGE as usize)?; + // SAFETY: The self-check runs during the bound platform probe. Its retained + // mappings and GEM owner keep the backing pinned after the check. + let sgt = gem::held_sg_table(&scratch.gem, unsafe { self.dev.as_ref().as_ref().as_bound() })?; + let pa = sgt.iter().next().ok_or(EINVAL)?.dma_address() as u64; + let checks = Iterator::chain( + window + .iter() + .enumerate() + .map(|(i, &w)| (true, hw::va::SELF_CHECK + i as u64 * hw::PAGE, w)), + lower + .iter() + .enumerate() + .map(|(i, &w)| (false, hw::va::SELF_CHECK_LOWER + i as u64 * hw::PAGE, w)), + ); + let mut count = 0; + for (host, va, word) in checks { + let (vm, root) = if host { + (self.uat.kernel_vm(), self.uat.kernel_window_root()) + } else { + (self.uat.kernel_lower_vm(), self.uat.kernel_lower_root()) + }; + let mapping = vm.map_at( + va, + hw::PAGE as usize, + &scratch.gem, + mmu::Prot::from_pte(word), + false, + )?; + self.mappings.push(mapping, GFP_KERNEL)?; + let leaf = uat::read_leaf(root, va)?; + if leaf != Some(pa | word) { + dev_err!( + self.dev.as_ref(), + "G15: UAT leaf self-check failed at {:#x}: want {:#x}, tables hold {:#x}\n", + va, + pa | word, + leaf.unwrap_or(0) + ); + return Err(super::boot::eproto()); + } + count += 1; + } + // The scratch page stays mapped: retain its backing with the mappings. + // SG backing is permanently pinned by held_sg_table; scratch remains retained. + core::mem::forget(scratch); + dev_info!( + self.dev.as_ref(), + "G15: UAT leaf self-check passed ({} classes)\n", + count + ); + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g15/object/geometry.rs b/drivers/gpu/drm/asahi/g15/object/geometry.rs new file mode 100644 index 0000000000000..3dc47ca02f534 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/object/geometry.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The DMA geometry a firmware- or GPU-visible backing must have. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. + +/// The page size and page-frame mask [`backing_ok`] is host-tested with (`hw::PAGE`, +/// `hw::PA_MASK`; `object.rs` asserts they are the same). +pub(crate) const PAGE: u64 = 0x4000; +pub(crate) const PA_MASK: u64 = 0x3ffffffc000; + +/// Whether the DMA segments `(address, length)` back exactly `bytes` with whole pages of `page` +/// bytes that the UAT can name: +/// +/// - `page` is a power of two and `bytes` a nonzero multiple of it; +/// - every segment is nonzero, starts and ends on a page boundary, does not start at 0 and its +/// last byte lies within `pa_mask` (the page-frame bits), with no overflow; +/// - the segments' lengths add up to `bytes`, no more and no less. +pub(crate) fn backing_ok( + bytes: u64, + page: u64, + pa_mask: u64, + segments: impl IntoIterator, +) -> bool { + if !page.is_power_of_two() || bytes == 0 || bytes & (page - 1) != 0 { + return false; + } + let mut covered = 0u64; + for (pa, len) in segments { + if pa == 0 || len == 0 || (pa | len) & (page - 1) != 0 { + return false; + } + let Some(last) = pa.checked_add(len - 1) else { + return false; + }; + if last & !(pa_mask | (page - 1)) != 0 { + return false; + } + match covered.checked_add(len) { + Some(sum) if sum <= bytes => covered = sum, + _ => return false, + } + } + covered == bytes +} + +#[cfg(test)] +mod tests { + use super::*; + + fn ok(bytes: u64, segments: &[(u64, u64)]) -> bool { + backing_ok(bytes, PAGE, PA_MASK, segments.iter().copied()) + } + + #[test] + fn exact_coverage() { + assert!(ok(PAGE, &[(0x8_0000_0000, PAGE)])); + assert!(ok( + 3 * PAGE, + &[(0x8_0000_0000, PAGE), (0x9_0000_0000, 2 * PAGE)] + )); + // A short or an oversized scatterlist. + assert!(!ok(3 * PAGE, &[(0x8_0000_0000, 2 * PAGE)])); + assert!(!ok( + 2 * PAGE, + &[(0x8_0000_0000, 2 * PAGE), (0x9_0000_0000, PAGE)] + )); + assert!(!ok(PAGE, &[])); + } + + #[test] + fn sizes() { + assert!(!ok(0, &[])); + assert!(!ok(PAGE + 1, &[(0x8_0000_0000, PAGE)])); + assert!(!backing_ok(PAGE, 0, PA_MASK, [(0x8_0000_0000, PAGE)])); + assert!(!backing_ok( + 0x6000, + 0x3000, + PA_MASK, + [(0x8_0000_0000, 0x6000)] + )); + } + + #[test] + fn segments() { + // Address 0, a zero-length segment, unaligned starts and lengths. + assert!(!ok(PAGE, &[(0, PAGE)])); + assert!(!ok(PAGE, &[(0x8_0000_0000, 0), (0x9_0000_0000, PAGE)])); + assert!(!ok(PAGE, &[(0x8_0000_1000, PAGE)])); + assert!(!ok( + 2 * PAGE, + &[ + (0x8_0000_0000, PAGE + 0x1000), + (0x9_0000_0000, PAGE - 0x1000) + ] + )); + } + + #[test] + fn physical_bounds() { + // The last page the UAT names, and the one beyond it. + let top = PA_MASK & !(PAGE - 1); + assert!(ok(PAGE, &[(top, PAGE)])); + assert!(!ok(PAGE, &[(top + PAGE, PAGE)])); + assert!(!ok(2 * PAGE, &[(top, 2 * PAGE)])); + // The end overflows instead of wrapping. + assert!(!ok(2 * PAGE, &[(u64::MAX & !(PAGE - 1), 2 * PAGE)])); + // With every bit nameable: a segment ending at the last byte is fine, one page further + // overflows. + let len = u64::MAX & !(PAGE - 1); + assert!(backing_ok(len, PAGE, u64::MAX, [(PAGE, len)])); + assert!(!backing_ok(len, PAGE, u64::MAX, [(2 * PAGE, len)])); + } + + #[test] + fn coverage_does_not_overflow() { + // Two halves wrap the total to 0, and a third segment then makes it look exact. + let bytes = u64::MAX & !(PAGE - 1); + let half = 1u64 << 63; + assert!(!backing_ok( + bytes, + PAGE, + u64::MAX, + [(PAGE, half), (PAGE, half), (PAGE, bytes)] + )); + assert!(!backing_ok( + PAGE, + PAGE, + u64::MAX, + [(PAGE, half), (PAGE, half)] + )); + } +} diff --git a/drivers/gpu/drm/asahi/g15/params.rs b/drivers/gpu/drm/asahi/g15/params.rs new file mode 100644 index 0000000000000..6eabb033a0edf --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/params.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Module parameters and the GET_PARAMS identity. +//! +//! With the shared `file.rs` additions (VM_STATUS and VM_STATUS_MIRROR from `supports_vm_status`, +//! INDEPENDENT_QUEUES from `queue_limits`, SOFT_FAULTS suppressed by `soft_faults`), the feature +//! word is exactly 0x3e (0xbe with `g15_early_tiling`). Never advertise SOFT_FAULTS, +//! FEW_PRIMITIVES or RENDER_RAY_QUERY on G15S. + +use kernel::prelude::*; + +use super::wire as uapi; + +use super::{hw, Gpu, GPU_GENERATION}; +use crate::module_parameters; + +/// Operations per VM_BIND batch at most: batches are atomic and may be large (sparse residency +/// updates); this bounds the kernel's staging allocation. +pub(crate) const MAX_BINDS: u32 = 1 << 20; + +/// Host deadline of a published prefix without progress, ms (at least 100). +pub(crate) fn deadline_ms() -> u32 { + (*module_parameters::g15_deadline_ms.value()).max(100) +} + +/// Compute no-progress budget before the kill, ms. +pub(crate) fn compute_deadline_ms() -> u32 { + (*module_parameters::g15_compute_deadline_ms.value()).max(100) +} + +/// Repoll while a ready prefix waits for record consumption, us. +pub(crate) fn repoll_us() -> u32 { + (*module_parameters::g15_repoll_us.value()).clamp(5, 100) +} + +/// Completion repoll interval, us. +pub(crate) fn resnap_us() -> u32 { + (*module_parameters::g15_resnap_us.value()).clamp(25, 1000) +} + +/// The explicit firmware idle standby time, us (0: not set). +pub(crate) fn idle_standby_override_us() -> u32 { + *module_parameters::g15_idle_standby_us.value() +} + +/// Firmware idle minimum confidence, permille. +pub(crate) fn idle_min_conf() -> u32 { + (*module_parameters::g15_idle_min_conf.value()).clamp(1, 1000) +} + +/// Whether firmware halts are answered with the handshake (else fail-stop). +pub(crate) fn recovery() -> bool { + *module_parameters::g15_recovery.value() != 0 +} + +/// Whether the CDM/VDM stream gates are on. +pub(crate) fn stream_gate() -> bool { + *module_parameters::g15_stream_gate.value() != 0 +} + +/// The 8-byte TEXT digest prefix of an unpinned firmware build to admit (0: none). +pub(crate) fn accept_text_prefix() -> u64 { + *module_parameters::g15_accept_text_prefix.value() +} + +/// The parameter for the PB growth maximum the firmware is given, in blocks, clamped to +/// 64..=4095. [`super::pools::Pools::soft_max`] raises it to at least the initial 1024 blocks; +/// the blocks above it, up to the 4095 posted at most, answer the synchronous requests of +/// partial renders. It is read-only after boot and does not change the 512 MiB heap. +pub(crate) fn pb_max_blocks() -> u32 { + (*module_parameters::g15_pb_max_blocks.value()).clamp(64, super::hw::tuning::PB_MAX_BLOCKS) +} + +/// Whether FRAGMENT_BARRIERS is advertised and mapped to early tiling. +pub(crate) fn early_tiling() -> bool { + *module_parameters::g15_early_tiling.value() != 0 +} + +/// Fills the generation part of GET_PARAMS group 0. +pub(crate) fn fill(gpu: &Gpu, params: &mut uapi::drm_asahi_params_global) -> Result { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS as u64; + if early_tiling() { + params.features |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS as u64; + } + params.gpu_generation = GPU_GENERATION; + params.gpu_variant = gpu.cfg.gpu_variant as u32; + params.gpu_revision = gpu.cfg.gpu_revision; + params.chip_id = gpu.cfg.chip_id; + params.num_dies = 1; + params.num_clusters_total = gpu.cfg.num_clusters; + params.num_cores_per_cluster = gpu.cfg.num_cores_per_cluster; + params.max_frequency_khz = gpu.max_frequency_khz; + params.usc_generation = 3; + params.gpu_hal_generation = uapi::drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY; + params.max_attachments = hw::sizes::MAX_ATTACHMENTS as u32; + for (i, mask) in gpu + .id + .core_masks + .iter() + .enumerate() + .take(gpu.cfg.num_clusters as usize) + { + *(params.core_masks.get_mut(i).ok_or(EIO)?) = (*mask).into(); + } + Ok(()) +} + +/// QUEUE_LIMITS: a policy bound on logical queues (they own no firmware state) and the +/// number of prefixes in flight. +pub(crate) fn queue_limits() -> uapi::drm_asahi_queue_limits { + uapi::drm_asahi_queue_limits { + max_queues: hw::sizes::MAX_QUEUES, + max_in_flight_per_queue: hw::sizes::FLIGHTS as u32, + ..Default::default() + } +} + +/// Shared status/queue features in the native G15 ABI's assignments. +pub(crate) fn feature_flags() -> u64 { + uapi::SHARED_FEATURE_FLAGS +} diff --git a/drivers/gpu/drm/asahi/g15/poll.rs b/drivers/gpu/drm/asahi/g15/poll.rs new file mode 100644 index 0000000000000..751f19865436f --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/poll.rs @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The poll work: drain, retire, growth, hang checks, publication; and the repoll timers. +//! +//! Order of one pass, under the device lock: drain the receive rings; retire flights (stamps, +//! queue DONE/READ, channel consumption, stage-event counts, idle bits, fault banks; renew the +//! progress deadline), with up to four trailing 5 us polls after an EP 0x20 event; answer PB +//! growth ([`Firmware::answer_growth`]); check hangs and recover ([`Firmware::check_hangs`], +//! which calls [`Firmware::progress`]); publish ([`Firmware::publish`]). Then drop the lock +//! and complete the retired jobs ([`super::job::complete_all`]). Wakeups: the EP 0x20 event +//! kick, then an hrtimer re-poll every `g15_resnap_us` (100) while anything is published, or +//! `g15_repoll_us` (15) while a ready prefix waits for record consumption, with a 1-jiffy +//! delayed-work backup. + +use core::sync::atomic::Ordering; + +use kernel::{ + bindings, c_str, debugfs, fmt, impl_has_hr_timer, + prelude::*, + sync::{Arc, ArcBorrow}, + time::{ + delay, + hrtimer::{ + HrTimerCallback, HrTimerCallbackContext, HrTimerPointer, HrTimerRestart, RelativeMode, + }, + Delta, Jiffies, Monotonic, + }, + workqueue::RawWorkItem, +}; + +use super::channel::Channel; +use super::engine::{complete, now_ns, rmb, Flight, Lap}; +use super::fw::graph::qstate; +use super::hw::sizes; +use super::job::{self, Completions, Job}; +use super::regs::EngineState; +use super::{params, Firmware, Shared, POLL_WORK}; +use crate::module_parameters; + +impl_has_hr_timer! { + impl HasHrTimer for Shared { + mode: RelativeMode, field: self.resnap + } +} + +impl HrTimerCallback for Shared { + type Pointer<'a> = Arc; + + /// Runs the poll now. + fn run(this: ArcBorrow<'_, Self>, _ctx: HrTimerCallbackContext<'_, Self>) -> HrTimerRestart { + let this: Arc = this.into(); + this.poll_now(); + HrTimerRestart::NoRestart + } +} + +/// The `asahi-g15/stats` debugfs file contents. +fn stats_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let state = shared.state.lock(); + match state.as_deref() { + Some(firmware) => firmware.write_stats(f), + None => Ok(()), + } +} + +/// The `asahi-g15/pstate` file contents. +fn pstate_show(shared: &Arc, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let state = shared.state.lock(); + match state.as_deref() { + Some(firmware) => firmware.write_pstate(f), + None => Ok(()), + } +} + +impl Shared { + /// One poll pass (the poll work item). + pub(crate) fn run_poll(self: Arc) { + let mut done = Completions::new(); + { + let mut state = self.state.lock(); + if let Some(firmware) = state.as_deref_mut() { + if firmware.engine.shared.is_none() { + firmware.engine.shared = Some(self.clone()); + } + let evented = self.evented.swap(false, Ordering::AcqRel); + match firmware.poll(evented, &mut done) { + Ok(true) => self.rearm(firmware.engine.chan_wait), + Ok(false) => {} + Err(error) => self.fail_stop(error), + } + } + } + job::complete_all(done); + } + + /// Hands a job to the engine and publishes what can be published. Err: the job was not + /// accepted (the engine failed: EIO) and the caller fails it; once accepted, its fence is + /// signalled by the engine, also on failure. + pub(crate) fn submit(self: &Arc, job: Arc) -> Result { + let mut done = Completions::new(); + let accepted = { + let mut state = self.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + if firmware.engine.shared.is_none() { + firmware.engine.shared = Some(self.clone()); + } + let accepted = firmware.enqueue(job); + if accepted.is_ok() { + if let Err(error) = firmware.publish(&mut done) { + self.fail_stop(error); + } + if !firmware.engine.flights.is_empty() { + self.rearm(firmware.engine.chan_wait); + } + } + accepted + }; + job::complete_all(done); + accepted + } + + /// Queues the poll work now, or brings a pending one forward. Callable from any context. + pub(crate) fn poll_now(self: &Arc) { + self.mod_poll(0); + } + + /// Queues the poll on `system_highpri_wq` after `delay` jiffies, or moves an already + /// pending one to that expiry. + fn mod_poll(self: &Arc, delay: Jiffies) { + // SAFETY: The closure returns false only when `mod_delayed_work_on` found the work + // already pending (it then only modified its timer), so the queue keeps the reference + // it took when it was queued; otherwise the work is queued and owns this reference. + let _ = unsafe { + as RawWorkItem>::__enqueue(self.clone(), |work| { + let dwork = kernel::container_of!(work, bindings::delayed_work, work); + !bindings::mod_delayed_work_on( + bindings::wq_misc_consts_WORK_CPU_UNBOUND as kernel::ffi::c_int, + bindings::system_highpri_wq, + dwork, + delay, + ) + }) + }; + } + + /// Re-checks a published command soon: hrtimer, with a one-jiffy backup; sooner while a + /// ready prefix waits for a channel record's consumption. + pub(crate) fn rearm(self: &Arc, chan_wait: bool) { + let mut us = params::resnap_us(); + if chan_wait && *module_parameters::g15_repoll_us.value() != 0 { + us = params::repoll_us(); + } + { + let mut handle = self.resnap_handle.lock(); + // Dropping the previous handle cancels a still pending expiry before the timer + // is started again. + drop(handle.take()); + *handle = Some(self.clone().start(Delta::from_micros(us as i64))); + } + self.mod_poll(1); + } + + /// Starts what the engine needs besides the firmware state: the `asahi-g15/stats` and + /// `asahi-g15/pstate` files. + pub(crate) fn start_engine(self: &Arc) -> Result { + let dir = debugfs::Dir::new(c_str!("asahi-g15")); + let file = KBox::pin_init( + dir.read_callback_file(c_str!("stats"), self.clone(), &stats_show), + GFP_KERNEL, + )?; + let pstate = KBox::pin_init( + dir.read_callback_file(c_str!("pstate"), self.clone(), &pstate_show), + GFP_KERNEL, + )?; + let mut state = self.state.lock(); + let firmware = state.as_deref_mut().ok_or(ENODEV)?; + firmware.engine.shared = Some(self.clone()); + firmware.engine.debugfs = Some(file); + firmware.engine.debugfs_pstate = Some(pstate); + Ok(()) + } +} + +/// Stamp cell `obs` reached its own target, bounded by the newest published target `tail`. +pub(crate) fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { + obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) +} + +/// Cursor `c` of a ring of `m` entries passed end `h` of entries from `b`, within tail `t`. +pub(crate) fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool { + let d = |x: u32| x.wrapping_add(m).wrapping_sub(b) % m; + let (dc, dh, dt) = (d(c), d(h), d(t)); + dh <= dc && dc <= dt +} + +impl Firmware { + /// One poll pass under the device lock. Returns whether to re-arm the poll timers. + pub(crate) fn poll(&mut self, evented: bool, done: &mut Completions) -> Result { + if self.recovery.failed().is_some() { + return Ok(false); + } + // With nothing published only an armed hang request whose prefix ended or a firmware + // error event needs this pass: one that arrived after its prefix's receipts is answered + // here (the firmware halted and waits for the resume) instead of being taken for a + // failure of the next prefix. Drain the receive rings before deciding, published or + // not, so the error count is current and idle log/trace/stats traffic is consumed. With + // prefixes published the retirement loop below drains first. + if self.engine.flights.is_empty() { + self.drain_rings()?; + if self.recovery_idle() { + return Ok(false); + } + } + let mut lap = Lap::start(); + let start = lap.on().then(now_ns); + self.retire(evented, done)?; + lap.lap(&mut self.engine.prof.retire_ns); + self.answer_growth()?; + self.check_hangs(done)?; + let published = self.publish(done); + let s = &mut self.engine.stats; + if self.engine.flights.is_empty() && s.idle_from_ns != 0 { + // Idle with nothing ready is not a gap: the clients are late. + s.starve_from_ns = s.idle_from_ns; + s.idle_from_ns = 0; + } + if let Some(start) = start { + self.engine.prof.polls += 1; + self.engine.prof.poll_ns += now_ns().wrapping_sub(start); + } + published?; + Ok(!self.engine.flights.is_empty() || self.recovery.needs_repoll() || self.engine.frozen) + } + + /// Drains the rings and retires finished flights, with up to four trailing 5 us polls + /// after an event. A receipt fault is left in `Recovery::receipt_fault` for the hang + /// checks. + pub(crate) fn retire(&mut self, evented: bool, done: &mut Completions) -> Result { + // After an event, a few short trailing polls catch the final receipts. + let mut tail = if evented { 4 } else { 0 }; + let mut ret; + + loop { + let mut lap = Lap::start(); + self.drain_rings()?; + lap.lap(&mut self.engine.prof.drain_ns); + ret = self.retired(); + while let Ok(true) = ret { + // An armed kill that ended it, or a doomed flight. + let outcome = self.retire_outcome(); + self.retire_oldest(outcome, done); + ret = self.retired(); + } + if ret.is_err() || tail == 0 || self.engine.flights.is_empty() { + break; + } + tail -= 1; + delay::udelay(Delta::from_micros(5)); + } + self.recovery.receipt_fault = ret.is_err(); + Ok(()) + } + + fn r32(o: &super::object::FwObject, offset: usize) -> Result { + o.r32(offset) + } + + /// Retirement of the oldest published prefix: Ok(true) retired, Ok(false) not yet, + /// Err(EIO) fault. The last published one needs every receipt exactly: final stamps, + /// DONE/READ at the tail heads, consumed pipes, new firing bits and idle engines. + pub(crate) fn retired(&self) -> Result { + let Some(f) = self.engine.oldest() else { + return Ok(false); + }; + rmb(); + if self.engine.flights.len() > 1 { + return self.flight_retired(f); + } + let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine); + let ch = &self.channels; + // A render: both stamp pairs, both queues, both pipes. + if f.renders != 0 { + let pending = Self::r32(&r.user_stamp[0], 0)? != e.ta_target + || Self::r32(&r.fw_stamp[0], 0)? != e.ta_target + || Self::r32(&r.user_stamp[1], 0)? != e.fs_target + || Self::r32(&r.fw_stamp[1], 0)? != e.fs_target + || Self::r32(&r.qstate[0], qstate::DONE)? != e.ta_head + || Self::r32(&r.qstate[0], qstate::READ)? != e.ta_head + || Self::r32(&r.qstate[1], qstate::DONE)? != e.fs_head + || Self::r32(&r.qstate[1], qstate::READ)? != e.fs_head + || ch.read(Channel::Ta)? != ch.producer(Channel::Ta) + || ch.read(Channel::Fs)? != ch.producer(Channel::Fs); + if pending { + return Ok(false); + } + } + if f.computes != 0 { + let pending = Self::r32(&g.user_stamp, 0)? != e.target + || Self::r32(&g.fw_stamp, 0)? != e.target + || Self::r32(&g.qstate, qstate::DONE)? != e.head + || Self::r32(&g.qstate, qstate::READ)? != e.head + || ch.read(Channel::Compute)? != ch.producer(Channel::Compute) + || self.events.compute_events == e.events; + if pending { + return Ok(false); + } + } + // New firing bits: at least one per participating stage (1, 2 or 3). + let stages = if f.computes != 0 { 3 } else { 2 }; + if f.renders != 0 && self.events.stage_events.wrapping_sub(e.stage_seen) < stages { + return Ok(false); + } + let state = self.session.regs.engine_state()?; + if state.fault() != 0 { + self.log_fault(&state); + return Err(EIO); + } + Ok(state.busy == 0) + } + + /// The oldest published prefix while later ones may run: its own stamp targets reached + /// within the published interval, DONE past its saved ring ends, its channel records + /// consumed. Engine idle is not required; faults still are checked. + fn flight_retired(&self, f: &Flight) -> Result { + let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine); + let ch = &self.channels; + let cap = sizes::QUEUE_CAP; + let rec_passed = |c: Channel, rec: u32| -> Result { + Ok(ring_passed( + ch.read(c)?, + (rec + 255) % 256, + rec, + ch.producer(c), + 256, + )) + }; + + if f.computes != 0 { + let pending = !stamp_reached(Self::r32(&g.user_stamp, 0)?, f.target, e.target) + || !stamp_reached(Self::r32(&g.fw_stamp, 0)?, f.target, e.target) + || !ring_passed( + Self::r32(&g.qstate, qstate::DONE)?, + f.c_start, + f.head, + e.head, + cap, + ) + || !rec_passed(Channel::Compute, f.rec[2])?; + if pending { + return Ok(false); + } + } + if f.renders != 0 { + let pending = !stamp_reached(Self::r32(&r.user_stamp[0], 0)?, f.ta_target, e.ta_target) + || !stamp_reached(Self::r32(&r.fw_stamp[0], 0)?, f.ta_target, e.ta_target) + || !stamp_reached(Self::r32(&r.user_stamp[1], 0)?, f.fs_target, e.fs_target) + || !stamp_reached(Self::r32(&r.fw_stamp[1], 0)?, f.fs_target, e.fs_target) + || !ring_passed( + Self::r32(&r.qstate[0], qstate::DONE)?, + f.ta_start, + f.ta_head, + e.ta_head, + cap, + ) + || !ring_passed( + Self::r32(&r.qstate[1], qstate::DONE)?, + f.fs_start, + f.fs_head, + e.fs_head, + cap, + ) + || !rec_passed(Channel::Ta, f.rec[0])? + || !rec_passed(Channel::Fs, f.rec[1])?; + if pending { + return Ok(false); + } + } + let state = self.session.regs.engine_state()?; + if state.fault() != 0 { + self.log_fault(&state); + return Err(EIO); + } + Ok(true) + } + + /// Logs the non-zero fault banks of `state`. + pub(crate) fn log_fault(&self, state: &EngineState) { + for bank in 0..2 { + let (f, va) = (state.status[bank], state.address[bank]); + if f == 0 { + continue; + } + let access = if (f >> 4) & 1 != 0 { "read" } else { "write" }; + let reason = if (f >> 1) & 7 != 0 { + "reason!=unmapped" + } else { + "unmapped" + }; + dev_info!( + self.graph.dev.as_ref(), + "G15: fault bank {}: status {:#x} va {:#x} ({} {} level {} unit {:#x} ctx {})\n", + bank, + f, + va, + access, + reason, + (f >> 7) & 3, + (f >> 9) & 0xff, + (f >> 17) & 0x3f + ); + } + } + + /// The oldest published prefix retired: its storage is released, its job completes when + /// this was its last published prefix and nothing of it is left to publish. A failed + /// (hang-killed, doomed, recovered) prefix fails its job, which drops its unpublished + /// commands. Recovery calls this for every published prefix after a recovered failure. + pub(crate) fn retire_oldest(&mut self, result: Result, done: &mut Completions) { + if self.engine.flights.is_empty() { + return; + } + let Ok(mut f) = self.engine.flights.remove(0) else { + return; + }; + if result.is_ok() { + self.stat_retire(&f); + } + self.slots.release(&mut f.claims); // after the stats read its timestamp cells + if f.rebuild { + f.rebuild = false; + self.engine.rebuild_live = false; + } + if self.engine.flights.is_empty() { + // Drained: a gap until the next publication. + self.engine.stats.idle_from_ns = now_ns(); + } + // The next prefix starts executing about now. + self.renew_deadlines(); + let job = f.job.clone(); + let p = &job.progress; + p.next.store(p.next() + f.count as u32, Ordering::Relaxed); + if let Err(error) = result { + complete(done, &job, Err(error)); + if p.is_pending() { + // Drop its unpublished commands. + p.pending.store(false, Ordering::Relaxed); + self.engine.pending.retain(|j| !Arc::ptr_eq(j, &job)); + p.count.store(p.sent(), Ordering::Relaxed); + } + } + if p.next() < p.sent() || p.sent() < p.count() { + return; // more of it published or still to publish + } + let s = &mut self.engine.stats; + if result.is_ok() { + s.jobs += 1; + s.job_cmds += p.count() as u64; + s.job_max = s.job_max.max(p.count() as u64); + } + complete(done, &job, Ok(())); + // The rest of a compute horizon, after the first job (fence order). + for j2 in f.group.iter() { + let p2 = &j2.progress; + p2.next.store(p2.count(), Ordering::Relaxed); + match result { + Err(error) => complete(done, j2, Err(error)), + Ok(()) => { + s.jobs += 1; + s.job_cmds += p2.count() as u64; + s.job_max = s.job_max.max(p2.count() as u64); + } + } + complete(done, j2, Ok(())); + } + } + + /// Progress of the oldest prefix: a FW stamp of an engine on which it still has unreached + /// work moved since the last poll (later prefixes' work on other engines does not count). + /// Renews the deadline; the first call for a prefix only records its position. + /// [`Self::check_hangs`] calls it after restoring an armed request, while no request is + /// armed. + pub(crate) fn progress(&mut self) { + let Some(f) = self.engine.oldest() else { + return; + }; + let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine); + let (mut c, mut ta, mut fs) = (0, 0, 0); + + if f.computes != 0 { + c = g.fw_stamp.r32(0).unwrap_or(0); + if stamp_reached(c, f.target, e.target) { + c = 0; // its computes are done: later work does not count + } + } + if f.renders != 0 { + ta = r.fw_stamp[0].r32(0).unwrap_or(0); + fs = r.fw_stamp[1].r32(0).unwrap_or(0); + if stamp_reached(ta, f.ta_target, e.ta_target) { + ta = 0; + } + if stamp_reached(fs, f.fs_target, e.fs_target) { + fs = 0; + } + } + let moved = e.progress_flight == f.published_ns && [c, ta, fs] != e.progress; + let published = f.published_ns; + if moved { + self.renew_deadlines(); + } + self.engine.progress_flight = published; + self.engine.progress = [c, ta, fs]; + } + + /// A retired prefix: publication to retirement (including any wait behind its + /// predecessor), event times, internal stage times of its members. + fn stat_retire(&mut self, f: &Flight) { + let now = now_ns(); + let d = now.wrapping_sub(f.published_ns); + let kind = if f.computes != 0 && f.renders != 0 { + 2 + } else if f.renders != 0 { + 1 + } else { + 0 + }; + let (mut lo, mut hi) = (u64::MAX, 0u64); + let (mut first, mut last) = (u64::MAX, 0u64); + let scrub = self.slots.scrub.clone(); + let s = &mut self.engine.stats; + + // Add one stage interval read from internal timestamp cells (start, end). + let mut stage = |o: &super::object::FwObject, off: usize, sum: &mut u64| { + let rd = |x: usize| o.r32(x).unwrap_or(0) as u64; + let start = rd(off + 4) << 32 | rd(off); + let end = rd(off + 12) << 32 | rd(off + 8); + if start == 0 || end < start { + return; + } + *sum += end - start; + lo = lo.min(start); + hi = hi.max(end); + }; + for k in 0..f.renders as usize { + let p = &scrub.passes[f.claims.pass[k] as usize]; + stage(&p.its, 0, &mut s.ta_ticks); + stage(&p.its, 0x10, &mut s.fs_ticks); + } + for k in 0..(f.computes as usize).min(f.claims.slot_count as usize) { + let c = &scrub.compute[f.claims.slot[k] as usize]; + stage(&c.its, 0, &mut s.cdm_ticks); + } + if hi > lo { + s.span_ticks += hi - lo; + } + for b in 0..3 { + let t = self.events.event_ns[b]; + let skip = if b == 2 { + f.computes == 0 + } else { + f.renders == 0 + }; + if skip { + continue; + } + first = first.min(t); + last = last.max(t); + } + if first > f.published_ns && last <= now { + s.first_ns[kind] += first - f.published_ns; + s.last_ns[kind] += last - f.published_ns; + } + s.count[0] += f.computes as u64; + s.count[1] += f.renders as u64; + if kind == 2 { + s.count[2] += f.count as u64; + } + s.prefixes[kind] += 1; + s.busy_ns[kind] += d; + s.max_ns[kind] = s.max_ns[kind].max(d); + } + + /// Engine state report for failures and hang requests (without the initdata Runtime + /// readiness and Flags words). + pub(crate) fn report(&self, what: &str) { + let (g, r, e) = (&self.graph.compute, &self.graph.render, &self.engine); + let ch = &self.channels; + let rd = |o: &super::object::FwObject, off| o.r32(off).unwrap_or(0xdeadbeef); + let state = self.session.regs.engine_state().unwrap_or_default(); + let f = e.oldest(); + let ms = f.map_or(0, |f| now_ns().wrapping_sub(f.published_ns) / 1_000_000); + + dev_info!( + self.graph.dev.as_ref(), + "G15: {}: ctx={} prefixes={} n={} target={:#x} user={:#x} fw={:#x} done={} read={} write={} head={} ch={}/{} events={}/{} errors={} busy={} fault={:#x} {}ms\n", + what, + f.map_or(0, |f| f.job.vm.slot), + e.flights.len(), + e.ordinal, + e.target, + rd(&g.user_stamp, 0), + rd(&g.fw_stamp, 0), + rd(&g.qstate, qstate::DONE), + rd(&g.qstate, qstate::READ), + rd(&g.qstate, qstate::WRITE), + e.head, + ch.read(Channel::Compute).unwrap_or(0xdeadbeef), + ch.write_cell(Channel::Compute).unwrap_or(0xdeadbeef), + self.events.compute_events, + e.events, + self.events.errors, + state.busy, + state.fault(), + ms + ); + if f.is_some_and(|f| f.renders != 0) { + dev_info!( + self.graph.dev.as_ref(), + "G15: render: draw={} ta {:#x}/{:#x}/{:#x} done={} read={} write={} head={} ch0={}/{}; fs {:#x}/{:#x}/{:#x} done={} read={} write={} head={} ch1={}/{}; stage events={}/{}\n", + e.draw, + e.ta_target, + rd(&r.user_stamp[0], 0), + rd(&r.fw_stamp[0], 0), + rd(&r.qstate[0], qstate::DONE), + rd(&r.qstate[0], qstate::READ), + rd(&r.qstate[0], qstate::WRITE), + e.ta_head, + ch.read(Channel::Ta).unwrap_or(0xdeadbeef), + ch.write_cell(Channel::Ta).unwrap_or(0xdeadbeef), + e.fs_target, + rd(&r.user_stamp[1], 0), + rd(&r.fw_stamp[1], 0), + rd(&r.qstate[1], qstate::DONE), + rd(&r.qstate[1], qstate::READ), + rd(&r.qstate[1], qstate::WRITE), + e.fs_head, + ch.read(Channel::Fs).unwrap_or(0xdeadbeef), + ch.write_cell(Channel::Fs).unwrap_or(0xdeadbeef), + self.events.stage_events, + e.stage_seen + ); + } + } + + /// The `asahi-g15/stats` contents. Scripts parse the keys: keep them stable. + pub(crate) fn write_stats(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = &self.engine.stats; + let slots = &self.slots.stats; + let (recoveries, hang_kills) = self.recovery_counts(); + let us = |ns: u64| ns / 1000; + + for (e, name) in ["compute", "render", "mixed"].iter().enumerate() { + writeln!( + f, + "{}: count {} prefixes {} busy_us {} max_us {} first_us {} last_us {}", + name, + s.count[e], + s.prefixes[e], + us(s.busy_ns[e]), + us(s.max_ns[e]), + us(s.first_ns[e]), + us(s.last_ns[e]) + )?; + } + writeln!( + f, + "gaps {} gap_us {} yields {} overlaps {}", + s.gaps, + us(s.gap_ns), + s.yields, + s.overlaps + )?; + writeln!( + f, + "jobs {} job_cmds {} job_max {} early {} recoveries {} hang_kills {} clustered {}", + s.jobs, s.job_cmds, s.job_max, s.early, recoveries, hang_kills, s.clustered + )?; + writeln!( + f, + "ct_sync {} slot_sync {} starved_us {} prep_us {} grouped {} pass_sync {}", + slots.ct_sync, + slots.slot_sync, + us(s.starved_ns), + us(s.prep_ns), + s.grouped, + slots.pass_sync + )?; + writeln!(f, "type0: fs2c {} c2ta {}", s.fs2c0, s.c2ta0)?; + write!(f, "horizon:")?; + for n in s.hz.iter() { + write!(f, " {}", n)?; + } + writeln!( + f, + " stop {}/{}/{}/{} behind {} hold {}/{}/{} ahead {}", + s.hz_stop[0], + s.hz_stop[1], + s.hz_stop[2], + s.hz_stop[3], + s.hz_behind, + s.hold[0], + s.hold[1], + s.hold[2], + s.hz_ahead + )?; + self.pools.write_stats(f)?; + writeln!( + f, + "stage: ta_us {} fs_us {} cdm_us {} span_us {}", + s.ta_ticks / 24, + s.fs_ticks / 24, + s.cdm_ticks / 24, + s.span_ticks / 24 + )?; + // The phase timers of debug_flags bit 49, and the scrub flush waits. + let p = &self.engine.prof; + writeln!( + f, + "prof: prefixes {} select_us {} encode_us {} claim_us {} write_us {} post_us {} polls {} drain_us {} retire_us {} poll_us {} flushes {} flush_us {}", + p.prefixes, + us(p.select_ns), + us(p.encode_ns), + us(p.claim_ns), + us(p.write_ns), + us(p.post_ns), + p.polls, + us(p.drain_ns), + us(p.retire_ns), + us(p.poll_ns), + slots.flushes, + us(slots.flush_ns) + ) + } +} diff --git a/drivers/gpu/drm/asahi/g15/pools.rs b/drivers/gpu/drm/asahi/g15/pools.rs new file mode 100644 index 0000000000000..c635ed11c4a0c --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/pools.rs @@ -0,0 +1,836 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Render UMA/PB allocators, their spare set and parameter-buffer growth. +//! +//! Record layouts: [`super::fw::buffer`]; the ledger and request checks: [`growth`] +//! (host-tested). +//! +//! - One 512 MiB PB heap at [`super::hw::client::PB_HEAP`] (4096 blocks of 128 KiB), backed on +//! demand: the initial blocks when the GPU starts, then chunks of at most +//! [`growth::CHUNK_BLOCKS`] (4 MiB) as growth needs them ([`Firmware::back_pb`]), never freed +//! or shrunk. The backing ([`PbChunk`]) is mapped only into user VMs, at the fixed client GVA, +//! each VM's before its prefixes are published ([`Firmware::map_pb`]); nothing names its pages +//! through the host window or context 0 (the records hold client GVAs only). Initial +//! [`super::hw::tuning::PB_BLOCKS`] (1024), at most [`super::hw::tuning::PB_MAX_BLOCKS`] +//! (4095) posted, so the producer never returns to 0; the firmware is given +//! [`Pools::soft_max`] (default 3072) as its growth maximum. Truthful K = 4096 in every +//! profile (PB reconstruction, also in recovery, synchronizes 8 * K bytes of the block list: +//! K = 0x26cd overran the 0x8000-byte list and crashed the second recovery). +//! - Growth: the event drain queues type-7 requests; [`Firmware::answer_growth`] answers each +//! one in the poll pass: type 7, PB slot 0, the context of a live published render prefix, +//! counter >= next expected, epoch == current halt count; write records `(G - 0x1000000000) +//! >> 15` at producer % K, barrier, block_ctl+4, barrier, then DEVCTL 7 with +4 = 1 echoing +//! slot, context, counter, position and epoch; +10 blocks per request. Above the firmware's +//! growth maximum the requests are answered as before, for the synchronous growth of partial +//! renders (Mesa's partial background/EOT programs). At the ledger's maximum the reply is +4 +//! = 0, a refusal, twice per initialization; later requests are left unanswered. A refusal +//! gives no pages and can leave the TA stalled; the poll pass returns, and the host deadline +//! and its recovery path apply, without a guarantee that they end the stall. Invalid requests +//! are logged and left unanswered, so the host deadline applies. Recovery drops pending +//! requests (epoch gate). +//! - Managers ASC-cached with an uncached spare set allocated with the graph; the first +//! recovered render abort swaps to the spare and never rewrites the cached set (the ASC may +//! still hold dirty lines of it); later aborts rewrite the spare in place. block_ctl stays +//! uncached when growth is on (a live host producer). +//! +//! The engine uses: [`Firmware::init_render_pools`] (render graph init: `false`; the priming +//! prefix after an abort: `true` when [`super::recovery::Recovery::reinit`] is set), the VAs +//! for its encoders ([`Pools::allocator_refs`]), [`Firmware::publish_pb`] after the render +//! threshold, and [`Pools::write_stats`] for the debugfs `pb:` line. + +mod growth; + +use core::fmt; +use core::sync::atomic::Ordering; +use kernel::{ + device, + prelude::*, + sync::{aref::ARef, Arc}, +}; + +use super::fw::buffer::{self, block_ctl, counter, misc, pb_manager, uma_manager, uma_pools}; +use super::fw::channels::DevCtl; +use super::fw::initdata::control; +use super::hw::{client, pte, sizes, tuning}; +use super::object::{self, Class, FwObject, HeldPages, Objects, Placement, Request}; +use super::vm::VmState; +use super::Firmware; +use crate::driver::AsahiDrmDevice as AsahiDevice; +use crate::{gem, mmu}; + +pub(crate) use growth::Ledger; + +static_assert!(buffer::MAX_BLOCKS == sizes::PB_BLOCKS); +static_assert!(buffer::MAX_PAGES == sizes::PB_PAGES); +static_assert!(buffer::BLOCK_LIST_RECORDS == sizes::PB_LIST_RECORDS); +static_assert!(buffer::GROW_STEP == sizes::PB_GROW_STEP); +static_assert!(buffer::MAX_BLOCKS as u64 * buffer::BLOCK_BYTES == client::PB_HEAP_SIZE); +static_assert!(buffer::PAGE_LIST_SIZE as u64 == client::PB_PAGE_LIST_SIZE); +static_assert!(uma_manager::SEEDS == sizes::FREELIST_SEED); +static_assert!(uma_pools::POOL_SIZE == sizes::POOL_SIZE); +// The ledger keeps the driver's PB sizes: neither is capped by [`growth::MAX_POSTED`]. +static_assert!(tuning::PB_MAX_BLOCKS <= growth::MAX_POSTED); +static_assert!(buffer::MIN_BLOCKS <= tuning::PB_BLOCKS); +static_assert!(tuning::PB_BLOCKS <= tuning::PB_MAX_BLOCKS); +// The PB alias is GPU-only (not cached and firmware-visible): dropping one is a plain unmap, +// with no firmware cache flush (`KernelMapping::drop`). +static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent()); + +/// Physical backing of PB blocks `[first, first + blocks)`: a zeroed GEM kernel object, pinned +/// and DMA-mapped ([`object::gpu_backing`]), aliased only into user VMs at the blocks' fixed +/// client GVA ([`Firmware::map_pb`]). Retained until reboot, like every firmware object; each +/// alias also holds the object. +struct PbChunk { + first: u32, + blocks: u32, + _pages: HeldPages, + backing: gem::ObjectRef, +} + +impl PbChunk { + fn new(dev: &AsahiDevice, first: u32, blocks: u32) -> Result { + if blocks == 0 || first.checked_add(blocks).ok_or(EINVAL)? > buffer::MAX_BLOCKS { + return Err(EINVAL); + } + let (backing, pages) = + object::gpu_backing(dev, blocks as usize * buffer::BLOCK_BYTES as usize)?; + Ok(Self { + first, + blocks, + _pages: pages, + backing, + }) + } + + /// The block after the chunk's last. + fn end(&self) -> u32 { + self.first + self.blocks + } + + /// Maps the whole chunk, exactly its object's size, at its fixed client GVA in `vm`. + fn map(&self, vm: &mmu::Vm) -> Result { + vm.map_at( + client::PB_HEAP + self.first as u64 * buffer::BLOCK_BYTES, + self.blocks as usize * buffer::BLOCK_BYTES as usize, + &self.backing.gem, + mmu::Prot::from_pte(pte::LEAF_CACHED), + false, + ) + } +} + +/// The allocator state a rebuild after an abort moves to the spare. +struct AllocatorSet { + /// TA (0) and FS (1) UMA managers and their counters. + uma_manager: [Arc; 2], + uma_counter: [Arc; 2], + block_ctl: Arc, + pb_counter: Arc, + misc: Arc, + scene_list: Arc, + pb_manager: Arc, +} + +/// One render engine's UMA pools: eight GPU-only 4 MiB pools, the freelist seeded with them and +/// the pool table. +struct UmaEngine { + pools: KVec>, + freelist: Arc, + pool_table: Arc, +} + +/// Firmware VAs of the render allocators in use (after a rebuild possibly the spare set), as +/// the TA/FS images, the scene and InitBM records name them. +#[derive(Copy, Clone, Default)] +pub(crate) struct AllocatorRefs { + /// UMA managers of TA (0) and FS (1). + pub(crate) uma_mgr: [u64; 2], + /// PB manager (InitBM +0x14, TA/FS work and sequences), scene list (scene +0x0/+0x8 at + /// `+ 4 * (draw % 48)`, TA work) and misc record (scene +0x40). + pub(crate) pb_mgr: u64, + pub(crate) scene_list: u64, + pub(crate) mgr_misc: u64, + /// Initial PB blocks of the current initialization (InitBM +0x10, TA register #33). + pub(crate) init_blocks: u32, +} + +/// PB growth statistics. +#[derive(Default)] +pub(crate) struct PbStats { + pub(crate) requests: u64, + pub(crate) grown: u64, + pub(crate) refused: u64, + pub(crate) bad: u64, +} + +/// Render allocator and PB state: the allocators, the PB ledger and the spare set. +pub(crate) struct Pools { + /// The GPU device, for logging (also used by recovery). + pub(crate) dev: ARef, + /// The DRM device, for the PB backing objects. + drm: ARef, + uma: [UmaEngine; 2], + /// The allocator set in use. + live: AllocatorSet, + /// The never-exposed uncached set the first rebuild moves to; afterwards the replaced + /// cached set, retained (only with `manager_cached`). + spare: Option, + /// The spare set has not been used yet. + spare_ready: bool, + pb_pages: Arc, + pb_blocks: Arc, + /// The PB backing, contiguous from block 0, in block order: the initial blocks, then the + /// growth chunks. Its capacity, reserved up front, is the most the ledger can need + /// ([`growth::max_chunks`]). + chunks: KVec, + /// The PB growth ledger since the last (re)initialization. + pub(crate) ledger: Ledger, + /// C1 names the current initial pages. + c1_written: bool, + /// The unanswered-at-the-maximum warning was given since the last initialization. + exhausted_logged: bool, + pub(crate) stats: PbStats, +} + +/// The ASC-cached twin of an allocator manager's class. +const fn manager_class(class: Class) -> Class { + if !tuning::MANAGER_CACHED { + return class; + } + match class { + Class::Shared => Class::SharedCached, + _ => Class::FwCached, + } +} + +/// Allocates a zeroed firmware object anywhere in the firmware window: `client` names the +/// fixed client page and leaf of its alias in every VM, `cpu` is false for GPU-only storage. +fn alloc( + objects: &mut Objects, + size: u64, + class: Class, + client: Option<(u64, u64)>, + offset: u64, + cpu: bool, +) -> Result> { + objects.alloc(Request { + size: size as usize, + class, + placement: Placement::Anywhere, + offset: offset as usize, + client, + lower: None, + cpu, + }) +} + +/// Writes `count` little-endian records of `W` bytes, record `i` at `offset + i * W`, in chunks. +fn write_records( + object: &FwObject, + offset: usize, + count: usize, + record: impl Fn(usize) -> [u8; W], +) -> Result { + const CHUNK: usize = 512; + let mut buf = [0u8; CHUNK]; + let per_chunk = CHUNK / W; + let mut i = 0; + while i < count { + let n = per_chunk.min(count - i); + for j in 0..n { + buf[j * W..(j + 1) * W].copy_from_slice(&record(i + j)); + } + object.write(offset + i * W, &buf[..n * W])?; + i += n; + } + Ok(()) +} + +/// A u64 record. +fn le64(value: u64) -> [u8; 8] { + value.to_le_bytes() +} + +impl UmaEngine { + /// Allocates engine `e`'s UMA pools, freelist and pool table, then its manager and counter, + /// in this order. + fn alloc(objects: &mut Objects, e: u64) -> Result<(Self, Arc, Arc)> { + let mut pools = KVec::with_capacity(uma_pools::POOLS, GFP_KERNEL)?; + for k in 0..uma_pools::POOLS as u64 { + pools.push( + alloc( + objects, + uma_pools::POOL_SIZE, + Class::Shared, + Some((client::uma(e, k), pte::LEAF_NC)), + 0, + false, + )?, + GFP_KERNEL, + )?; + } + let freelist = alloc( + objects, + uma_pools::FREELIST_SIZE as u64, + Class::Shared, + Some((client::uma_freelist(e), pte::LEAF_NC)), + 0, + true, + )?; + let pool_table = alloc( + objects, + uma_pools::POOL_TABLE_SIZE as u64, + Class::Shared, + Some((client::uma_table_page(e), pte::LEAF_NC)), + uma_pools::POOL_TABLE_OFFSET as u64, + true, + )?; + let manager = alloc( + objects, + uma_manager::SIZE as u64, + manager_class(Class::Shared), + None, + 0, + true, + )?; + let counter = alloc(objects, counter::SIZE as u64, Class::Shared, None, 0, true)?; + Ok(( + Self { + pools, + freelist, + pool_table, + }, + manager, + counter, + )) + } +} + +/// The client GVA of an object with a client alias. +fn gva(object: &FwObject) -> Result { + object.gva.ok_or(EINVAL) +} + +impl Pools { + /// Allocates the render allocator objects: per engine the UMA pools, freelist, pool table, + /// manager and counter; block control, PB counter, misc, scene list and PB manager; the + /// uncached spare set; the PB page list, block list and heap. Contents are written by + /// [`Firmware::init_render_pools`]. + pub(crate) fn new(dev: &AsahiDevice, objects: &mut Objects) -> Result { + let (uma0, manager0, counter0) = UmaEngine::alloc(objects, 0)?; + let (uma1, manager1, counter1) = UmaEngine::alloc(objects, 1)?; + let small = |objects: &mut Objects, size: usize, class| { + alloc(objects, size as u64, class, None, 0, true) + }; + // A live producer with growth: firmware index 2. + let block_ctl_class = if tuning::PB_GROW { + Class::Fw + } else { + manager_class(Class::Fw) + }; + let block_ctl = small(objects, block_ctl::SIZE, block_ctl_class)?; + let pb_counter = small(objects, counter::SIZE, Class::Fw)?; + let misc = small(objects, misc::SIZE, manager_class(Class::Fw))?; + let scene_list = small(objects, buffer::SCENE_LIST_SIZE, Class::Shared)?; + let pb_manager = small(objects, pb_manager::SIZE, manager_class(Class::Fw))?; + let live = AllocatorSet { + uma_manager: [manager0, manager1], + uma_counter: [counter0, counter1], + block_ctl, + pb_counter, + misc, + scene_list, + pb_manager, + }; + // The uncached spare set for the first rebuild after an abort (manager_cached). + let spare = if tuning::MANAGER_CACHED { + let m0 = small(objects, uma_manager::SIZE, Class::Shared)?; + let c0 = small(objects, counter::SIZE, Class::Shared)?; + let m1 = small(objects, uma_manager::SIZE, Class::Shared)?; + let c1 = small(objects, counter::SIZE, Class::Shared)?; + Some(AllocatorSet { + uma_manager: [m0, m1], + uma_counter: [c0, c1], + block_ctl: small(objects, block_ctl::SIZE, Class::Fw)?, + pb_counter: small(objects, counter::SIZE, Class::Fw)?, + misc: small(objects, misc::SIZE, Class::Fw)?, + scene_list: small(objects, buffer::SCENE_LIST_SIZE, Class::Shared)?, + pb_manager: small(objects, pb_manager::SIZE, Class::Fw)?, + }) + } else { + None + }; + let pb_pages = alloc( + objects, + buffer::PAGE_LIST_SIZE as u64, + Class::Shared, + Some((client::PB_PAGE_LIST, pte::LEAF_PB_LIST)), + 0, + true, + )?; + let pb_blocks = alloc( + objects, + buffer::BLOCK_LIST_SIZE as u64, + Class::Shared, + Some((client::PB_BLOCK_LIST, pte::LEAF_PB_LIST)), + 0, + true, + )?; + // Backing for the initial blocks; growth adds the rest. + let ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let mut chunks = + KVec::with_capacity(growth::max_chunks(ledger.initial, ledger.max), GFP_KERNEL)?; + chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?; + Ok(Self { + dev: dev.as_ref().as_ref().into(), + drm: dev.into(), + uma: [uma0, uma1], + live, + spare_ready: spare.is_some(), + spare, + pb_pages, + pb_blocks, + chunks, + ledger, + c1_written: false, + exhausted_logged: false, + stats: PbStats::default(), + }) + } + + /// First rebuild after an abort with cached managers: the cached set may still have dirty + /// lines in the ASC, so it is retained untouched and the graph moves to the never-exposed + /// uncached spare (and its own counters and scene list), which [`Self::init`] then + /// initializes. + fn take_spare(&mut self) { + if let Some(spare) = self.spare.as_mut() { + core::mem::swap(&mut self.live, spare); + } + self.spare_ready = false; + dev_info!( + self.dev, + "render allocators moved to a fresh uncached set; the cached set is retained\n" + ); + } + + /// Writes the initial allocator state: UMA pools, freelists and managers; the PB lists, + /// manager and counters. The firmware owns them once published; rewritten (zeroed first) + /// only after a recovered render abort, followed by a fresh InitBM. + fn init(&mut self) -> Result { + for e in 0..2 { + let uma = &self.uma[e]; + let manager = &self.live.uma_manager[e]; + let count = &self.live.uma_counter[e]; + manager.fill(0, 0, uma_manager::SIZE)?; + count.fill(0, 0, counter::SIZE)?; + // Freelist: the eight 4 MiB pools in 0x1000 units. + let per_pool = (uma_pools::POOL_SIZE / uma_pools::SEED_BYTES) as usize; + let mut bases = [0u64; uma_pools::POOLS]; + for (k, pool) in uma.pools.iter().enumerate() { + bases[k] = gva(pool)?; + } + write_records(&uma.freelist, 0, uma_pools::POOLS * per_pool, |i| { + le64(bases[i / per_pool] + (i % per_pool) as u64 * uma_pools::SEED_BYTES) + })?; + for (k, base) in bases.iter().enumerate() { + uma.pool_table + .w64(uma_pools::POOL_TABLE_STRIDE * k, uma_pools::row(*base))?; + } + manager.write( + 0, + &buffer::uma_manager_image( + e as u32 + 1, + e as u32, + gva(&uma.freelist)?, + gva(&uma.pool_table)?, + count.va, + ), + )?; + count.w32(0, 1)?; + } + + // PB: 16384 pages of 0x8000 in 4096 blocks of 0x20000; with growth the first initial + // blocks of them, at most 4095 posted, and page limits at the firmware's growth maximum. + // In every profile the block list declares its real 4096-record capacity. + self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + self.exhausted_logged = false; + if tuning::PB_GROW { + // C1 names the initial pages again. + self.c1_written = false; + } + let set = &self.live; + set.pb_manager.fill(0, 0, pb_manager::SIZE)?; + set.block_ctl.fill(0, 0, block_ctl::SIZE)?; + set.misc.fill(0, 0, misc::SIZE)?; + set.pb_counter.fill(0, 0, counter::SIZE)?; + set.scene_list.fill(0, 0, buffer::SCENE_LIST_SIZE)?; + if tuning::PB_GROW { + self.pb_pages.fill(0, 0, buffer::PAGE_LIST_SIZE)?; + self.pb_blocks.fill(0, 0, buffer::BLOCK_LIST_SIZE)?; + } + let heap = client::PB_HEAP; + let initial = self.ledger.initial; + if initial > self.backed() { + return Err(EINVAL); + } + write_records( + &self.pb_pages, + 0, + (buffer::PAGES_PER_BLOCK * initial) as usize, + |i| buffer::page_record(heap, i as u32).to_le_bytes(), + )?; + // The high word of every block record is zero (zero-filled, or only ever written 0). + write_records(&self.pb_blocks, 0, initial as usize, |i| { + le64(buffer::block_record(heap, i as u32) as u64) + })?; + set.pb_manager.write( + 0, + // The maximum the firmware sees is the asynchronous growth limit, below the mapped + // heap: the blocks above it answer the synchronous requests a partial render makes + // (the firmware cannot finish a scene at its maximum without them; refused, its TA + // stalls and no kill ends it: ticket malik-na/linux-aurora#69). + &buffer::pb_manager_image( + initial, + self.soft_max(), + gva(&self.pb_pages)?, + gva(&self.pb_blocks)?, + set.block_ctl.va, + set.pb_counter.va, + ), + )?; + set.block_ctl.write(0, &buffer::block_ctl_image(initial))?; + set.misc.w32(misc::ENABLE, 1)?; + set.pb_counter.w32(0, 1)?; + crate::mem::sync(); + Ok(()) + } + + /// The allocator addresses the render encoders name (the live set: after the first + /// rebuild, the spare). + pub(crate) fn allocator_refs(&self) -> AllocatorRefs { + let set = &self.live; + AllocatorRefs { + uma_mgr: [set.uma_manager[0].va, set.uma_manager[1].va], + pb_mgr: set.pb_manager.va, + scene_list: set.scene_list.va, + mgr_misc: set.misc.va, + init_blocks: self.ledger.initial, + } + } + + /// The firmware's growth maximum in blocks (`g15_pb_max_blocks`), at least the initial size + /// and at most the mapped heap. + pub(crate) fn soft_max(&self) -> u32 { + super::params::pb_max_blocks().clamp(self.ledger.initial, self.ledger.max) + } + + /// The blocks with backing, from block 0. + pub(crate) fn backed(&self) -> u32 { + self.chunks.last().map_or(0, PbChunk::end) + } + + /// The debugfs line `pb: blocks initial/posted/firmware-max/max backed blocks/chunks + /// requests grown refused bad`. + pub(crate) fn write_stats(&self, f: &mut impl fmt::Write) -> fmt::Result { + writeln!( + f, + "pb: blocks {}/{}/{}/{} backed {}/{} requests {} grown {} refused {} bad {}", + self.ledger.initial, + self.ledger.posted, + self.soft_max(), + self.ledger.max, + self.backed(), + self.chunks.len(), + self.stats.requests, + self.stats.grown, + self.stats.refused, + self.stats.bad + ) + } +} + +impl Firmware { + /// Maps the PB backing into `vm` up to its last chunk, before anything of a prefix of the + /// VM is published (the start of `publish_prefix_into`). Render work runs in one VM at a + /// time (`Engine::publish`), so the VM about to run is the only one through which the GPU + /// can reach a posted block next; a VM created or idle while the backing grew catches up + /// here, also a closed VM whose accepted jobs still run. + /// + /// Each chunk's alias joins the VM's driver mappings, held until the VM is torn down, and + /// the VM's count moves past the chunk only then. When that fails, the alias just made is + /// dropped again: nothing of the VM runs before its count covers the chunk, and the leaf is + /// GPU-only, so the drop is a plain unmap. The chunks mapped before stay counted. + /// + /// The one-VM rule is checked first: a flight of another VM, or a count past the chunks, + /// fails with EPROTO before anything is mapped. + pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result { + let mut mapped = vm.pb_chunks.load(Ordering::Relaxed); + if mapped > self.pools.chunks.len() + || self + .engine + .flights + .iter() + .any(|f| !core::ptr::eq(&*f.job.vm, vm)) + { + dev_err!( + self.dev(), + "PB backing: ctx {} maps {} of {} chunks with {} flights of another VM\n", + vm.slot, + mapped, + self.pools.chunks.len(), + self.engine + .flights + .iter() + .filter(|f| !core::ptr::eq(&*f.job.vm, vm)) + .count() + ); + return Err(super::boot::eproto()); + } + while let Some(chunk) = self.pools.chunks.get(mapped) { + let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; + aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; + vm.vm.append_driver_mappings(aliases)?; + mapped += 1; + vm.pb_chunks.store(mapped, Ordering::Relaxed); + } + Ok(()) + } + + /// The VM whose render work runs: every flight's (`Engine::publish` puts a flight behind + /// another only for the same VM), and it must be the asking context's. EPROTO otherwise. + fn running_vm(&self, ctx: u32) -> Result> { + let flights = &self.engine.flights; + let vm = flights + .first() + .ok_or_else(super::boot::eproto)? + .job + .vm + .clone(); + if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) { + dev_err!( + self.dev(), + "PB growth: ctx {} is not the one VM in flight (ctx {}, {} flights)\n", + ctx, + vm.slot, + flights.len() + ); + return Err(super::boot::eproto()); + } + Ok(vm) + } + + /// Backs the PB up to block `end` in the VM that runs, the asking context `ctx`, before an + /// answer posts those blocks (nothing new when they are backed already): chunks of at most + /// [`growth::CHUNK_BLOCKS`] blocks are allocated ([`object::gpu_backing`]), mapped into that + /// VM and kept, one at a time. The VM first catches up ([`Self::map_pb`], which also checks + /// the one-VM rule), so its count equals the chunks. A chunk is kept + /// only once its alias in the VM is held, and the VM's count moves with it; when the alias + /// cannot be held, the alias and the chunk are dropped, neither ever reachable (the leaf is + /// GPU-only: a plain unmap). An error leaves the chunks backed so far for the next request, + /// and nothing is posted. The allocation sleeps (shmem pages with `__GFP_RETRY_MAYFAIL`) + /// while the firmware waits for the answer; how long is not bounded here. + fn back_pb(&mut self, end: u32, ctx: u32) -> Result { + let vm = self.running_vm(ctx)?; + self.map_pb(&vm)?; + while self.pools.backed() < end { + let (first, blocks) = + growth::next_chunk(self.pools.backed(), self.pools.ledger.max).ok_or(ENOSPC)?; + if self.pools.chunks.len() == self.pools.chunks.capacity() { + return Err(ENOSPC); + } + let chunk = PbChunk::new(&self.pools.drm, first, blocks)?; + let mut aliases = KVec::with_capacity(1, GFP_KERNEL)?; + aliases.push(chunk.map(&vm.vm)?, GFP_KERNEL)?; + vm.vm.append_driver_mappings(aliases)?; + // Within the capacity checked above. + self.pools + .chunks + .push_within_capacity(chunk) + .map_err(|_| EIO)?; + vm.pb_chunks + .store(self.pools.chunks.len(), Ordering::Relaxed); + } + Ok(()) + } + + /// (Re)initializes the render UMA managers, PB lists and PB manager. `rebuild`: the priming + /// prefix after a recovered abort: the first one moves to the uncached spare set, later + /// ones rewrite it in place. + pub(crate) fn init_render_pools(&mut self, rebuild: bool) -> Result { + if rebuild && self.pools.spare_ready { + self.pools.take_spare(); + } + self.pools.init() + } + + /// The PB counters of a publication with renders, after the render threshold: the PB + /// counter names the prefix's last draw, and C1 = {0x07400000, 4 * initial blocks} is + /// written once per (re)initialization. An error leaves the prefix unpublished. + pub(crate) fn publish_pb(&mut self, draw: u32) -> Result { + self.pools.live.pb_counter.w32(0, draw)?; + if !self.pools.c1_written { + self.write_c1(&buffer::c1_record(self.pools.ledger.initial))?; + self.pools.c1_written = true; + } + Ok(()) + } + + /// Answers the firmware's PB-growth requests, outside the event drain. + /// A request names PB slot 0, the context of a published render prefix and the next + /// request counter; anything else is logged and left unanswered (its TA then stays stalled + /// and the deadline applies). With growth up to ten more blocks of the mapped heap are + /// written to unposted block-list records, then the live block_ctl+4 producer advances and + /// DEVCTL 7 reports success; at the ledger's maximum the reply is the refusal (+4 = 0), + /// which gives the firmware no new pages, and after two refusals requests are left + /// unanswered. The firmware imports the blocks itself: P is never rewritten. Only the + /// requests queued before the pass are taken. A failed reply is logged; a request that + /// cannot be processed at all (a firmware object access failed) is logged and left + /// unanswered like an invalid one. Never fails: the poll pass continues. + pub(crate) fn answer_growth(&mut self) -> Result { + // Only the requests queued before this pass: a reply's wait drains the rings and may + // queue the next request, which the next pass answers (the device lock is released in + // between). + let queued = self.events.grow.len(); + for _ in 0..queued { + let Ok(request) = self.events.grow.remove(0) else { + break; + }; + self.pools.stats.requests += 1; + if let Err(e) = self.answer_request(&request) { + self.pools.stats.bad += 1; + dev_err!( + self.dev(), + "PB growth request {} ctx {} not answered: {}\n", + request[buffer::grow_request::COUNTER], + request[buffer::grow_request::CONTEXT], + e.to_errno() + ); + } + } + Ok(()) + } + + /// Processes one growth request. + fn answer_request(&mut self, request: &[u32; 14]) -> Result { + let ctl = self.init.control.clone().ok_or(ENODEV)?; + let epoch = + ctl.r32(control::HALT_COUNT)? as u64 | (ctl.r32(control::HALT_COUNT + 4)? as u64) << 32; + let ctx = request[buffer::grow_request::CONTEXT]; + let live = self + .engine + .flights + .iter() + .enumerate() + .any(|(i, f)| self.receipts(i).renders != 0 && f.job.vm.slot == ctx); + // At the maximum, after the refusals, the request is left unanswered: the poll pass + // returns, and the host deadline and its recovery path apply to a stalled TA. + if self.pools.ledger.exhausted() { + self.pools.stats.bad += 1; + if !self.pools.exhausted_logged { + self.pools.exhausted_logged = true; + dev_warn!( + self.dev(), + "PB growth request {} ctx {} left unanswered: {} refusals at the maximum ({} blocks); the host deadline applies\n", + request[buffer::grow_request::COUNTER], + ctx, + self.pools.ledger.refusals, + self.pools.ledger.max + ); + } + return Ok(()); + } + // A request from before a halt has no authority over the PB now. + if !self + .pools + .ledger + .admits(request, tuning::PB_GROW, live, epoch) + { + self.pools.stats.bad += 1; + dev_warn!( + self.dev(), + "PB growth request not answered: ctx {} slot {} counter {} (next {}) epoch {:#x} (now {:#x}) live {} pb_grow {}\n", + ctx, + request[buffer::grow_request::SLOT], + request[buffer::grow_request::COUNTER], + self.pools.ledger.next_request, + growth::request_epoch(request), + epoch, + live as u32, + tuning::PB_GROW as u32 + ); + return Ok(()); + } + // Every block the answer posts is backed in the VM that runs before anything of the + // answer is written; the one-VM rule is checked for every answer. If that fails, nothing + // is posted or answered (answer_growth logs it; the deadline applies), and the backing + // made so far stays for the next request. + let end = self.pools.ledger.posted + self.pools.ledger.next_count(); + self.back_pb(end, ctx)?; + let (first, n) = self.pools.ledger.accept(request); + let heap = client::PB_HEAP; + for b in first..first + n { + self.pools.pb_blocks.w64( + buffer::block_record_offset(b), + buffer::block_record(heap, b) as u64, + )?; + } + if n != 0 { + // Records before their producer. + crate::mem::sync(); + let producer = self.pools.ledger.post(n); + self.pools + .live + .block_ctl + .w32(block_ctl::PRODUCER, producer)?; + // Producer before the reply. + crate::mem::sync(); + self.pools.stats.grown += n as u64; + } else { + self.pools.stats.refused += 1; + } + let reply = DevCtl { + words: buffer::grow_reply_words(request, n != 0), + }; + let ret = self.devctl(&reply, true); + let ledger = self.pools.ledger; + let soft = self.pools.soft_max(); + // Every answer past the firmware's maximum is logged with its TA position: a scene + // that keeps asking at one position is a runaway, one that advances is large. + let past_soft = n != 0 && ledger.posted > soft; + let info = self.pools.ledger.log_answer(request, n, ret.is_err()) || past_soft; + let (counter, position) = ( + request[buffer::grow_request::COUNTER], + request[buffer::grow_request::POSITION], + ); + let ret = match ret { + Ok(()) => 0, + Err(e) => e.to_errno(), + }; + if info { + dev_info!( + self.dev(), + "PB growth request {} ctx {} position {}: +{} blocks ({} of {}), DEVCTL 7: {}\n", + counter, + ctx, + position, + n, + ledger.posted, + ledger.max, + ret + ); + } else { + dev_dbg!( + self.dev(), + "PB growth request {} ctx {} position {}: +{} blocks ({} of {}), DEVCTL 7: {}\n", + counter, + ctx, + position, + n, + ledger.posted, + ledger.max, + ret + ); + } + Ok(()) + } +} diff --git a/drivers/gpu/drm/asahi/g15/pools/growth.rs b/drivers/gpu/drm/asahi/g15/pools/growth.rs new file mode 100644 index 0000000000000..20b298e641459 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/pools/growth.rs @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The PB growth ledger and the growth-request checks. +//! +//! The firmware asks for more PB with an event of type 7 naming PB slot 0, the context of a +//! published render, its request counter (P+0x60 before its increment), the TA position and the +//! halt epoch; the host appends at most ten blocks of the mapped heap to the block list and +//! advances the block_ctl+4 producer (modulo K), then replies with DEVCTL 7. At the ledger's +//! maximum the reply refuses; after [`MAX_REFUSALS`] refusals since the last initialization, +//! later requests are left unanswered. That bounds the poll pass. It does not make the scene +//! finish: a refused or unanswered request can leave the TA stalled, and the host deadline and +//! its recovery path then apply, without a guarantee that they end the stall. A request from +//! before a halt (epoch) or not matching a published render prefix is left unanswered. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. + +use crate::g15::fw::buffer::{grow_request, BLOCK_LIST_RECORDS, GROW_STEP, MAX_BLOCKS, MIN_BLOCKS}; + +/// The PB ledger since the last (re)initialization. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Ledger { + /// Initial blocks (the C1 pages, InitBM blocks and TA register #33 name them). + pub(crate) initial: u32, + /// The most blocks posted, at most [`MAX_POSTED`]. The PB manager's P+0x70/+0x74 hold + /// 4 * the firmware's growth maximum (`Pools::soft_max()`) in pages, which may be lower. + pub(crate) max: u32, + /// Blocks posted to the block list, cumulative (the block_ctl+4 producer is this modulo K). + pub(crate) posted: u32, + /// The next expected request counter. + pub(crate) next_request: u32, + /// A refusal at the maximum was logged since the last initialization. + pub(crate) refusal_logged: bool, + /// Refusals answered at the maximum since the last initialization. After + /// [`MAX_REFUSALS`] further requests are left unanswered: a firmware that keeps asking at + /// the maximum would otherwise be answered forever, under the device lock (ticket + /// malik-na/linux-aurora#69). Unanswered, the poll pass returns, and the host deadline and + /// its recovery path apply. + pub(crate) refusals: u32, +} + +/// Refusals answered at the maximum before requests are left to the deadline. +pub(crate) const MAX_REFUSALS: u32 = 2; + +/// Blocks of each PB backing chunk after the initial one (4 MiB): growth backs at most this +/// much at a time while the firmware waits for its answer. +pub(crate) const CHUNK_BLOCKS: u32 = 32; + +/// The backing chunk that follows `backed` blocks of a ledger with maximum `max`: its first block +/// and size, at most [`CHUNK_BLOCKS`] and never past `max`; none from `max` on. +pub(crate) const fn next_chunk(backed: u32, max: u32) -> Option<(u32, u32)> { + if backed >= max { + None + } else if max - backed < CHUNK_BLOCKS { + Some((backed, max - backed)) + } else { + Some((backed, CHUNK_BLOCKS)) + } +} + +/// The chunks that back a ledger with `initial` and `max` blocks at most: the initial one and +/// [`next_chunk`]'s from `initial` to `max`. +pub(crate) const fn max_chunks(initial: u32, max: u32) -> usize { + 1 + max.saturating_sub(initial).div_ceil(CHUNK_BLOCKS) as usize +} + +/// The most blocks a ledger posts: the mapped heap, and below K, so that the block_ctl+4 +/// producer (posted modulo K) is never 0, the value of an empty block list. +pub(crate) const MAX_POSTED: u32 = if MAX_BLOCKS < BLOCK_LIST_RECORDS { + MAX_BLOCKS +} else { + BLOCK_LIST_RECORDS - 1 +}; + +impl Ledger { + /// A fresh ledger. With growth, the initial size is clamped to 64..=[`MAX_POSTED`] blocks + /// and the maximum to initial..=[`MAX_POSTED`]; without, both are [`MAX_POSTED`]. + pub(crate) const fn new(grow: bool, blocks: u32, max_blocks: u32) -> Self { + let (initial, max) = if grow { + let nb = clamp(blocks, MIN_BLOCKS, MAX_POSTED); + (nb, clamp(max_blocks, nb, MAX_POSTED)) + } else { + (MAX_POSTED, MAX_POSTED) + }; + Self { + initial, + max, + posted: initial, + next_request: 0, + refusal_logged: false, + refusals: 0, + } + } + + /// The live block_ctl+4 producer. + pub(crate) const fn producer(&self) -> u32 { + self.posted % BLOCK_LIST_RECORDS + } + + /// Whether the refusals at the maximum are used up: further requests go unanswered. + pub(crate) const fn exhausted(&self) -> bool { + self.refusals >= MAX_REFUSALS + } + + /// Whether `request` may be answered: growth on, an event of type 7 for PB slot 0 from a + /// context with a published render prefix (`live`), a counter not below the next expected + /// one, the current halt epoch, and refusals at the maximum not used up. + pub(crate) fn admits(&self, request: &[u32; 14], grow: bool, live: bool, epoch: u64) -> bool { + !self.exhausted() + && grow + && request[grow_request::TYPE] == 7 + && request[grow_request::SLOT] == 0 + && live + && request[grow_request::COUNTER] >= self.next_request + && request_epoch(request) == epoch + } + + /// The blocks the next accepted request posts: at most [`GROW_STEP`], none at the maximum. + pub(crate) fn next_count(&self) -> u32 { + GROW_STEP.min(self.max - self.posted) + } + + /// Accepts an admitted request: the next expected counter follows it, and the returned + /// blocks `[first, first + count)` are to be written to the block list, `count` 0 meaning a + /// refusal at the maximum. + pub(crate) fn accept(&mut self, request: &[u32; 14]) -> (u32, u32) { + self.next_request = request[grow_request::COUNTER].wrapping_add(1); + let count = self.next_count(); + if count == 0 { + self.refusals += 1; + } + (self.posted, count) + } + + /// Records `count` blocks as posted after their records were written and returns the new + /// producer. + pub(crate) fn post(&mut self, count: u32) -> u32 { + self.posted += count; + self.producer() + } + + /// Whether the answer is logged at info level: a failed reply, the first two requests, or + /// the first refusal at the maximum since the initialization. Records the refusal. + pub(crate) fn log_answer( + &mut self, + request: &[u32; 14], + count: u32, + reply_failed: bool, + ) -> bool { + let info = reply_failed + || request[grow_request::COUNTER] < 2 + || (count == 0 && self.posted == self.max && !self.refusal_logged); + self.refusal_logged |= count == 0; + info + } +} + +/// The halt epoch a request names (u64 at words 5..6). +pub(crate) const fn request_epoch(request: &[u32; 14]) -> u64 { + request[grow_request::EPOCH] as u64 | (request[grow_request::EPOCH + 1] as u64) << 32 +} + +const fn clamp(v: u32, lo: u32, hi: u32) -> u32 { + if v < lo { + lo + } else if v > hi { + hi + } else { + v + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn request(ctx: u32, counter: u32, epoch: u64) -> [u32; 14] { + let mut q = [0u32; 14]; + q[0] = 7; + q[1] = ctx; + q[2] = 0; + q[3] = counter; + q[4] = 0x100; + q[5] = epoch as u32; + q[6] = (epoch >> 32) as u32; + q + } + + #[test] + fn clamps() { + let l = Ledger::new(true, 1024, 2048); + assert_eq!( + (l.initial, l.max, l.posted, l.next_request), + (1024, 2048, 1024, 0) + ); + assert_eq!(l.producer(), 1024); + assert_eq!(Ledger::new(true, 10, 20).initial, 64); + assert_eq!(Ledger::new(true, 10, 20).max, 64); + assert_eq!(Ledger::new(true, 5000, 100).initial, 4095); + assert_eq!(Ledger::new(true, 512, 1024).max, 1024); + assert_eq!(Ledger::new(true, 512, 100).max, 512); + let fixed = Ledger::new(false, 512, 1024); + assert_eq!((fixed.initial, fixed.max, fixed.posted), (4095, 4095, 4095)); + // The values the driver passes (tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS) are kept. + let l = Ledger::new(true, 1024, 4095); + assert_eq!( + (l.initial, l.max, l.posted, l.producer()), + (1024, 4095, 1024, 1024) + ); + } + + #[test] + fn producer_never_returns_to_zero() { + assert_eq!(MAX_POSTED, 4095); + assert!(MAX_POSTED <= MAX_BLOCKS && MAX_POSTED < BLOCK_LIST_RECORDS); + let values = [ + 0, + 1, + MIN_BLOCKS - 1, + MIN_BLOCKS, + MIN_BLOCKS + 1, + 1023, + 1024, + 1025, + 3071, + 3072, + 3073, + MAX_POSTED - 1, + MAX_POSTED, + MAX_BLOCKS, + MAX_BLOCKS + 1, + BLOCK_LIST_RECORDS, + 2 * BLOCK_LIST_RECORDS, + u32::MAX, + ]; + for grow in [true, false] { + for blocks in values { + for max_blocks in values { + let mut l = Ledger::new(grow, blocks, max_blocks); + assert!(MIN_BLOCKS <= l.initial && l.initial <= l.max && l.max <= MAX_POSTED); + assert_eq!(l.posted, l.initial); + assert!((1..=MAX_POSTED).contains(&l.producer())); + // Every producer of a growth to the maximum stays in 1..=4095. + let mut counter = 0; + loop { + let (first, n) = l.accept(&request(5, counter, 1)); + assert_eq!(first, l.posted); + if n == 0 { + break; + } + let producer = l.post(n); + assert!((1..=MAX_POSTED).contains(&producer)); + assert_eq!(producer, l.posted); + counter += 1; + } + assert_eq!(l.posted, l.max); + assert_eq!(l.refusals, 1); + } + } + } + } + + #[test] + fn admission() { + let l = Ledger::new(true, 1024, 2048); + let q = request(5, 0, 3); + assert!(l.admits(&q, true, true, 3)); + assert!(!l.admits(&q, false, true, 3)); + assert!(!l.admits(&q, true, false, 3)); + // Request of an earlier halt epoch (dropped at recovery, never answered). + assert!(!l.admits(&q, true, true, 4)); + let mut bad = q; + bad[0] = 6; + assert!(!l.admits(&bad, true, true, 3)); + let mut bad = q; + bad[2] = 1; + assert!(!l.admits(&bad, true, true, 3)); + // 64-bit epochs. + let q = request(5, 0, 0x1_0000_0002); + assert!(l.admits(&q, true, true, 0x1_0000_0002)); + assert!(!l.admits(&q, true, true, 2)); + } + + #[test] + fn growth_to_the_maximum() { + // 512 initial and 1024 maximum blocks. + let mut l = Ledger::new(true, 512, 1024); + let mut grown = 0; + let mut counter = 0; + loop { + let q = request(5, counter, 1); + assert!(l.admits(&q, true, true, 1)); + let (first, n) = l.accept(&q); + assert_eq!(first, 512 + grown); + if n == 0 { + break; + } + assert!(n <= 10); + assert_eq!(l.post(n), (512 + grown + n) % 4096); + grown += n; + counter += 1; + } + assert_eq!(l.posted, 1024); + assert_eq!(grown, 512); + // 51 requests of 10 blocks and one of 2. + assert_eq!(counter, 52); + // A stale (repeated) counter is refused, a later one accepted. + assert!(!l.admits(&request(5, counter - 1, 1), true, true, 1)); + assert!(l.admits(&request(5, counter + 3, 1), true, true, 1)); + // One refusal was answered above; one more is, then requests go unanswered. + assert_eq!(l.refusals, 1); + assert!(!l.exhausted()); + let q = request(5, counter + 3, 1); + assert_eq!(l.accept(&q), (1024, 0)); + assert!(l.exhausted()); + assert!(!l.admits(&request(5, counter + 4, 1), true, true, 1)); + // A fresh ledger starts over. + assert!(!Ledger::new(true, 512, 1024).exhausted()); + } + + /// Backing for every ledger the constructor makes, the driver's included: the chunks from + /// the initial one are contiguous, at most [`CHUNK_BLOCKS`] each, end at the maximum and + /// number at most [`max_chunks`]; growth with the backing made first never posts a block + /// past it. + #[test] + fn backing_chunks() { + assert_eq!(max_chunks(1024, 4095), 1 + 96); + assert_eq!(max_chunks(MIN_BLOCKS, MAX_POSTED), 1 + 126); + assert_eq!(max_chunks(MAX_POSTED, MAX_POSTED), 1); + assert_eq!(next_chunk(1024, 4095), Some((1024, 32))); + assert_eq!(next_chunk(4064, 4095), Some((4064, 31))); + assert_eq!(next_chunk(4095, 4095), None); + assert_eq!(next_chunk(4096, 4095), None); + let values = [ + 0, 1, MIN_BLOCKS, 65, 1023, 1024, 1025, 3072, 4094, MAX_POSTED, 5000, + ]; + for grow in [true, false] { + for blocks in values { + for max_blocks in values { + let mut l = Ledger::new(grow, blocks, max_blocks); + let (mut backed, mut chunks) = (l.initial, 1); + let mut counter = 0; + loop { + let end = l.posted + l.next_count(); + while backed < end { + let (first, n) = next_chunk(backed, l.max).unwrap(); + assert_eq!(first, backed); + assert!(n >= 1 && n <= CHUNK_BLOCKS); + backed += n; + chunks += 1; + } + let (first, n) = l.accept(&request(5, counter, 1)); + assert!(first + n <= backed); + if n == 0 { + break; + } + l.post(n); + counter += 1; + } + assert_eq!((l.posted, backed), (l.max, l.max)); + assert!(chunks <= max_chunks(l.initial, l.max)); + assert_eq!(next_chunk(backed, l.max), None); + } + } + } + } + + #[test] + fn producer_wraps_at_k() { + let mut l = Ledger::new(true, 2048, 2048); + assert_eq!(l.producer(), 2048); + l.posted = 4095; + assert_eq!(l.post(1), 0); + assert_eq!(l.post(10), 10); + } + + #[test] + fn logging() { + let mut l = Ledger::new(true, 64, 64); + // The first two requests are logged. + assert!(l.log_answer(&request(1, 0, 0), 10, false)); + assert!(l.log_answer(&request(1, 1, 0), 10, false)); + assert!(!l.log_answer(&request(1, 2, 0), 10, false)); + // A failed reply is logged. + assert!(l.log_answer(&request(1, 5, 0), 10, true)); + // The first refusal at the maximum is logged once. + assert!(l.log_answer(&request(1, 6, 0), 0, false)); + assert!(!l.log_answer(&request(1, 7, 0), 0, false)); + assert!(l.refusal_logged); + } +} diff --git a/drivers/gpu/drm/asahi/g15/power.rs b/drivers/gpu/drm/asahi/g15/power.rs new file mode 100644 index 0000000000000..58b6636fde383 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/power.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The firmware idle policy. +//! +//! The words and their rationale are in [`idle`]. Initdata writes them into the Globals image +//! before the firmware first sees it: +//! +//! - +0x7a8 standby = `g15_idle_standby_us` when set (clamped 100..100000), else the GPU node's +//! `apple,idleoff-standby-timer`, else the tuned 2500 ([`idle::standby`]); +//! - +0x7c4 minimum confidence = `g15_idle_min_conf` (500 permille as f32); +//! - +0x30 predictive = 0 ([`super::hw::tuning::IDLE_PREDICTIVE`]): the finite delay, the GPU +//! powers down after the +0x9a0 delay (template 2, about 2 ms); +//! - +0x2c idle decision enabled (template 1, unchanged). +//! +//! DVFS is firmware-run on the ADT tables. The state it runs at is read back by +//! `asahi-g15/pstate` ([`pstate`], [`super::Firmware::write_pstate`]): the newest power-state +//! record of the stats ring, and HardwareDataA's state words. + +mod idle; +pub(crate) mod pstate; +pub(crate) use idle::{permille_f32, standby}; + +use kernel::{fmt, prelude::*}; + +use super::board; +use super::fw::initdata::{globals, hwdata}; +use super::hw::rings; +use super::Firmware; + +static_assert!(idle::ENABLE == globals::IDLE_ENABLE); +static_assert!(idle::PREDICTIVE == globals::IDLE_PREDICTIVE); +static_assert!(idle::STANDBY_US == globals::IDLE_STANDBY_US); +static_assert!(idle::MIN_CONF == globals::IDLE_MIN_CONF); +static_assert!(idle::DELAY == globals::IDLE_DELAY); + +static_assert!(pstate::STATES == board::DVFS_STATES); +static_assert!(pstate::STATS_RECORD * pstate::STATS_SLOTS as usize == rings::STATS_RING_SIZE); +static_assert!(pstate::STATS_SLOTS == super::fw::channels::RING_SLOTS); + +impl Firmware { + /// HardwareDataA's state words, as the firmware last wrote them. + fn pstate_words(&self) -> Result { + let hw = self.init.hwdata.as_ref().ok_or(ENODEV)?; + let word = |offset: usize| hw.r32(hwdata::HWDATA_A + offset); + Ok(pstate::HwWords { + actual: word(pstate::HWDATA_A_ACTUAL)?, + target: word(pstate::HWDATA_A_TARGET)?, + current: word(pstate::HWDATA_A_CURRENT)?, + freq_mhz: word(pstate::HWDATA_A_FREQ_MHZ)?, + }) + } + + /// The `asahi-g15/pstate` file ([`pstate::write`]): the stats ring's records so far, those + /// it holds unconsumed (read, not consumed), HardwareDataA's words and table A of the + /// device tree's operating points. Read-only: nothing firmware-visible is written. + pub(crate) fn write_pstate(&self, f: &mut impl fmt::Write) -> fmt::Result { + let Ok(hw) = self.pstate_words() else { + return Ok(()); + }; + let seen = self + .channels + .peek_stats() + .unwrap_or(self.channels.stats_seen); + let table = pstate::Table::from_mhz(board::table_a_mhz(&self.session.adt)); + pstate::write(f, &seen, &hw, &table) + } +} diff --git a/drivers/gpu/drm/asahi/g15/power/idle.rs b/drivers/gpu/drm/asahi/g15/power/idle.rs new file mode 100644 index 0000000000000..0db36fd859e50 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/power/idle.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The firmware idle-policy words. +//! +//! Globals inputs of the firmware's idle decision: +//! +//! - +0x7a8 standby time, us (board template 700): the GPU stays up about this long after its +//! last work; tuned 2500 (with 700 us the GPU powers down between works ~100 us apart and +//! pays ~300 us at the next pickup). +//! - +0x7c4 minimum confidence of the predictive model, f32 (template 0x3f333333 = 0.7); tuned +//! 0.5. +//! - +0x30 predictive branch (template 1); 0 = the ordinary finite delay: the GPU powers down +//! once idle for the +0x9a0 delay (template 2, about 2 ms). The predictive branch makes 2-3 % +//! of works under sustained low-duty submission pay a ~300 us power-up. +//! - +0x2c enables the idle decision (template 1, unchanged). +//! +//! The standby time comes from [`standby`]: the explicit `g15_idle_standby_us` when set, else the +//! device tree's `apple,idleoff-standby-timer` where the machine has one, else the tuned value. +//! +//! Core-only (no `kernel` imports), so that it can be tested on the host. + +/// Globals offsets (mirrors of `fw::initdata::globals`). +pub(crate) const ENABLE: usize = 0x2c; +pub(crate) const PREDICTIVE: usize = 0x30; +pub(crate) const STANDBY_US: usize = 0x7a8; +pub(crate) const MIN_CONF: usize = 0x7c4; +pub(crate) const DELAY: usize = 0x9a0; + +/// Where the standby time came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum StandbySource { + /// The `g15_idle_standby_us` module parameter. + Parameter, + /// The GPU node's `apple,idleoff-standby-timer`. + DeviceTree, + /// The tuned value. + Tuning, +} + +impl StandbySource { + pub(crate) const fn name(self) -> &'static str { + match self { + Self::Parameter => "the module parameter", + Self::DeviceTree => "the device tree", + Self::Tuning => "the tuning", + } + } +} + +/// Bounds of an explicit or device-tree standby time, us. +pub(crate) const STANDBY_MIN_US: u32 = 100; +pub(crate) const STANDBY_MAX_US: u32 = 100000; + +/// Chooses the standby time, us: `param` (0: not set, else clamped to the bounds) wins, then +/// `dt` (the device-tree value, if the machine has one and it is within the bounds), else +/// `tuned`. Returns the value and its source; the flag tells a device-tree value outside the +/// bounds was ignored. +pub(crate) fn standby(param: u32, dt: Option, tuned: u32) -> (u32, StandbySource, bool) { + let rejected = matches!(dt, Some(v) if !(STANDBY_MIN_US..=STANDBY_MAX_US).contains(&v)); + if param != 0 { + return ( + param.clamp(STANDBY_MIN_US, STANDBY_MAX_US), + StandbySource::Parameter, + rejected, + ); + } + match dt { + Some(v) if !rejected => (v, StandbySource::DeviceTree, false), + _ => (tuned, StandbySource::Tuning, rejected), + } +} + +/// IEEE-754 single bits of `permille / 1000`, `permille` clamped to 1..=1000 (0 cannot be +/// expressed: it reads 0.001). +pub(crate) const fn permille_f32(permille: u32) -> u32 { + let p = if permille < 1 { + 1 + } else if permille > 1000 { + 1000 + } else { + permille + }; + // p/1000 * 2^k in [0.5, 1). + let mut k = 0u32; + while p * (1u32 << k) < 500 { + k += 1; + } + if p == 1000 { + return 0x3f800000; + } + // p/1000 = m/2^24 * 2^(1-k-1), m in [2^23, 2^24). + ((126 - k) << 23) | ((((p as u64) << (24 + k)) / 1000) as u32 - (1u32 << 23)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn f32_bits(x: f32) -> u32 { + x.to_bits() + } + + #[test] + fn standby_choice() { + use StandbySource::*; + assert_eq!(standby(0, Some(1234), 2500), (1234, DeviceTree, false)); + assert_eq!(standby(0, None, 2500), (2500, Tuning, false)); + assert_eq!(standby(900, Some(1234), 2500), (900, Parameter, false)); + assert_eq!(standby(900, None, 2500), (900, Parameter, false)); + assert_eq!(standby(5, None, 2500), (100, Parameter, false)); + assert_eq!(standby(10_000_000, None, 2500), (100000, Parameter, false)); + assert_eq!(standby(0, Some(0), 2500), (2500, Tuning, true)); + assert_eq!(standby(0, Some(100001), 2500), (2500, Tuning, true)); + assert_eq!(standby(0, Some(100), 2500), (100, DeviceTree, false)); + assert_eq!(standby(0, Some(100000), 2500), (100000, DeviceTree, false)); + assert_eq!(standby(700, Some(0), 2500), (700, Parameter, true)); + } + + #[test] + fn permille() { + assert_eq!(permille_f32(500), 0x3f000000); + assert_eq!(permille_f32(700), 0x3f333333); + assert_eq!(permille_f32(1000), 0x3f800000); + assert_eq!(permille_f32(2000), 0x3f800000); + assert_eq!(permille_f32(0), permille_f32(1)); + assert_eq!(permille_f32(250), 0x3e800000); + // Within one ulp (truncation) of the nearest float everywhere. + for p in 1..=1000u32 { + let want = f32_bits(p as f32 / 1000.0); + let got = permille_f32(p); + assert!(want.abs_diff(got) <= 1, "{p}: {got:#x} vs {want:#x}"); + } + } +} diff --git a/drivers/gpu/drm/asahi/g15/power/pstate.rs b/drivers/gpu/drm/asahi/g15/power/pstate.rs new file mode 100644 index 0000000000000..411fe22413893 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/power/pstate.rs @@ -0,0 +1,376 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The GPU's current performance state, as the firmware reports it. +//! +//! Two places carry it: +//! +//! - The stats ring: 256 records of 0x40 bytes, a u32 tag at +0 and, in a power-state record +//! (tag 0x0b), the performance-state index at +0x20. +//! - HardwareDataA (hardware record +0x4580): actual (+0x2c), target (+0x30) and current +//! (+0x38) state indices and the frequency in MHz as an IEEE single (+0x58). +//! +//! An index counts the firmware's DVFS states, 0 (off) to 8; its frequency is the state's +//! table-A operating point ([`super::super::board`]), taken from the device tree. +//! +//! [`write`] prints the `asahi-g15/pstate` file. Its first line is what a sampler reads: +//! `pstate freq_khz source `, from the newest power-state record +//! seen, or HardwareDataA's current state when the stats ring has carried none. +//! +//! Core-only (no `kernel` imports), so that it can be tested on the host. + +use core::fmt; + +/// DVFS states of the firmware tables (= `board::DVFS_STATES`). +pub(crate) const STATES: usize = 9; + +/// HardwareDataA words. +pub(crate) const HWDATA_A_ACTUAL: usize = 0x2c; +pub(crate) const HWDATA_A_TARGET: usize = 0x30; +pub(crate) const HWDATA_A_CURRENT: usize = 0x38; +pub(crate) const HWDATA_A_FREQ_MHZ: usize = 0x58; + +/// Stats ring records. +pub(crate) const STATS_RECORD: usize = 0x40; +pub(crate) const STATS_SLOTS: u32 = 256; +pub(crate) const STATS_TAG: usize = 0x00; +pub(crate) const TAG_POWER_STATE: u32 = 0x0b; +pub(crate) const POWER_STATE_INDEX: usize = 0x20; + +/// Tags counted one by one; higher tags are counted together. +pub(crate) const TAGS: usize = 16; + +/// What the stats ring has carried. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct StatsSeen { + /// Records consumed. + pub(crate) records: u64, + /// Records per tag; the last slot counts every tag from `TAGS - 1` up. + pub(crate) tags: [u64; TAGS], + /// The index of the newest power-state record. + pub(crate) last: Option, +} + +impl StatsSeen { + /// Notes one record: its tag and the word at [`POWER_STATE_INDEX`]. + pub(crate) fn note(&mut self, tag: u32, index: u32) { + self.records = self.records.wrapping_add(1); + let slot = (tag as usize).min(TAGS - 1); + self.tags[slot] = self.tags[slot].wrapping_add(1); + if tag == TAG_POWER_STATE { + self.last = Some(index); + } + } +} + +/// The ring slots from the consumer `read` to the producer `write`, oldest first; `None` when +/// either index is out of the ring. +pub(crate) fn pending(read: u32, write: u32) -> Option> { + if read >= STATS_SLOTS || write >= STATS_SLOTS { + return None; + } + let count = (write + STATS_SLOTS - read) % STATS_SLOTS; + Some((0..count).map(move |i| (read + i) % STATS_SLOTS)) +} + +/// The HardwareDataA words. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct HwWords { + pub(crate) actual: u32, + pub(crate) target: u32, + pub(crate) current: u32, + /// IEEE single bits, MHz. + pub(crate) freq_mhz: u32, +} + +/// The table-A frequency of each firmware state, kHz. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Table { + pub(crate) khz: [u32; STATES], +} + +impl Table { + /// From the table-A operating points in MHz. + pub(crate) fn from_mhz(mhz: [u32; STATES]) -> Self { + Self { + khz: mhz.map(|m| m.saturating_mul(1000)), + } + } + + /// The frequency of state `index`, kHz; `None` past the table. + pub(crate) fn khz(&self, index: u32) -> Option { + self.khz.get(index as usize).copied() + } +} + +/// An IEEE single number of MHz in kHz, rounded to nearest; `None` for a negative number, an +/// infinity, a NaN or a value past `u32::MAX` kHz. +pub(crate) fn f32_mhz_to_khz(bits: u32) -> Option { + let exp = (bits >> 23) & 0xff; + let frac = bits & 0x7f_ffff; + if bits & 0x7fff_ffff == 0 { + return Some(0); + } + if bits >> 31 != 0 || exp == 0xff { + return None; + } + // value = m * 2^(e - 150); a subnormal has no implicit bit and e = 1. + let (m, e) = if exp == 0 { + (frac as u64, 1i32) + } else { + ((frac | 0x80_0000) as u64, exp as i32) + }; + let scaled = m * 1000; // < 2^34 + let shift = e - 150; + let khz = if shift >= 0 { + if shift > 31 { + return None; + } + scaled << shift + } else { + let s = (-shift) as u32; + if s >= 64 { + 0 + } else { + let q = scaled >> s; + let rem = scaled & ((1u64 << s) - 1); + let half = 1u64 << (s - 1); + // Ties to even. + if rem > half || (rem == half && q & 1 == 1) { + q + 1 + } else { + q + } + } + }; + u32::try_from(khz).ok() +} + +/// Where the reported state comes from. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Source { + Stats, + HwData, +} + +impl Source { + fn name(self) -> &'static str { + match self { + Source::Stats => "stats", + Source::HwData => "hwdata", + } + } +} + +/// The reported state: the newest power-state record, else HardwareDataA's current state. +pub(crate) fn current(seen: &StatsSeen, hw: &HwWords) -> (u32, Source) { + match seen.last { + Some(index) => (index, Source::Stats), + None => (hw.current, Source::HwData), + } +} + +/// A frequency for the file: kHz, or `unknown`. +struct Khz(Option); + +impl fmt::Display for Khz { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + Some(khz) => write!(f, "{}", khz), + None => f.write_str("unknown"), + } + } +} + +/// The `asahi-g15/pstate` file: +/// +/// ```text +/// pstate freq_khz source +/// stats: records power_states last tags : ... +/// hwdata: actual target current freq_mhz_f32 freq_khz +/// table_khz: x 9 +/// ``` +pub(crate) fn write( + f: &mut impl fmt::Write, + seen: &StatsSeen, + hw: &HwWords, + table: &Table, +) -> fmt::Result { + let (index, source) = current(seen, hw); + writeln!( + f, + "pstate {} freq_khz {} source {}", + index, + Khz(table.khz(index)), + source.name() + )?; + write!( + f, + "stats: records {} power_states {} last ", + seen.records, seen.tags[TAG_POWER_STATE as usize] + )?; + match seen.last { + Some(last) => write!(f, "{}", last)?, + None => f.write_str("none")?, + } + f.write_str(" tags")?; + for (tag, n) in seen.tags.iter().enumerate() { + if *n != 0 { + write!(f, " {}:{}", tag, n)?; + } + } + writeln!(f)?; + writeln!( + f, + "hwdata: actual {} target {} current {} freq_mhz_f32 {:#010x} freq_khz {}", + hw.actual, + hw.target, + hw.current, + hw.freq_mhz, + Khz(f32_mhz_to_khz(hw.freq_mhz)) + )?; + f.write_str("table_khz:")?; + for khz in table.khz { + write!(f, " {}", khz)?; + } + writeln!(f) +} + +#[cfg(test)] +mod tests { + use super::*; + + // A made-up table, not any machine's. + fn table() -> Table { + Table::from_mhz([0, 100, 200, 300, 400, 500, 600, 700, 800]) + } + + #[test] + fn table_is_khz_and_bounded() { + let t = table(); + assert_eq!(t.khz(0), Some(0)); + assert_eq!(t.khz(8), Some(800_000)); + assert_eq!(t.khz(9), None); + assert_eq!(t.khz(u32::MAX), None); + } + + #[test] + fn notes_tags_and_the_newest_power_state() { + let mut s = StatsSeen::default(); + assert_eq!(s.last, None); + s.note(4, 7); + assert_eq!(s.last, None); + s.note(TAG_POWER_STATE, 3); + s.note(TAG_POWER_STATE, 8); + s.note(9, 1); + s.note(200, 0); + assert_eq!(s.records, 5); + assert_eq!(s.last, Some(8)); + assert_eq!(s.tags[4], 1); + assert_eq!(s.tags[TAG_POWER_STATE as usize], 2); + assert_eq!(s.tags[9], 1); + assert_eq!(s.tags[TAGS - 1], 1); + } + + #[test] + fn pending_walks_from_read_to_write_with_wrap() { + let v: Vec = pending(3, 3).unwrap().collect(); + assert!(v.is_empty()); + let v: Vec = pending(3, 6).unwrap().collect(); + assert_eq!(v, [3, 4, 5]); + let v: Vec = pending(254, 1).unwrap().collect(); + assert_eq!(v, [254, 255, 0]); + assert!(pending(256, 0).is_none()); + assert!(pending(0, 256).is_none()); + } + + #[test] + fn f32_mhz_in_khz() { + let k = |x: f32| f32_mhz_to_khz(x.to_bits()); + assert_eq!(k(0.0), Some(0)); + assert_eq!(f32_mhz_to_khz(0x8000_0000), Some(0)); // -0 + assert_eq!(k(1.0), Some(1000)); + assert_eq!(k(800.0), Some(800_000)); + assert_eq!(k(1234.5), Some(1_234_500)); + assert_eq!(k(0.0015), Some(2)); // 1.5 kHz, nearest of the single + assert_eq!(k(0.0004), Some(0)); + assert_eq!(k(4_000_000.0), Some(4_000_000_000)); + assert_eq!(k(5_000_000.0), None); // past u32 kHz + assert_eq!(k(-1.0), None); + assert_eq!(k(f32::INFINITY), None); + assert_eq!(k(f32::NAN), None); + assert_eq!(f32_mhz_to_khz(1), Some(0)); // smallest subnormal + // Exact halves (an odd mantissa x 2^-4 MHz is n + 0.5 kHz) round to even. + for m in [0x80_0001u32, 0x80_0003, 0x80_0005, 0xff_ffff] { + let bits = (146 << 23) | (m & 0x7f_ffff); + let want = (m as f64 * 62.5).round_ties_even() as u32; + assert_eq!(f32_mhz_to_khz(bits), Some(want), "{m:#x}"); + } + // Against the host's arithmetic over a sweep. + for i in 0..20_000u32 { + let x = i as f32 * 0.37; + let want = (x as f64 * 1000.0).round_ties_even() as u32; + assert_eq!(k(x), Some(want), "{x}"); + } + } + + #[test] + fn current_prefers_the_stats_ring() { + let hw = HwWords { + current: 5, + ..Default::default() + }; + let mut s = StatsSeen::default(); + assert_eq!(current(&s, &hw), (5, Source::HwData)); + s.note(TAG_POWER_STATE, 8); + assert_eq!(current(&s, &hw), (8, Source::Stats)); + s.note(TAG_POWER_STATE, 0); + assert_eq!(current(&s, &hw), (0, Source::Stats)); + } + + #[test] + fn file_from_the_stats_ring() { + let mut s = StatsSeen::default(); + s.note(4, 0); + s.note(TAG_POWER_STATE, 8); + let hw = HwWords { + actual: 8, + target: 8, + current: 6, + freq_mhz: 800.0f32.to_bits(), + }; + let mut out = String::new(); + write(&mut out, &s, &hw, &table()).unwrap(); + assert_eq!( + out, + "pstate 8 freq_khz 800000 source stats\n\ + stats: records 2 power_states 1 last 8 tags 4:1 11:1\n\ + hwdata: actual 8 target 8 current 6 freq_mhz_f32 0x44480000 freq_khz 800000\n\ + table_khz: 0 100000 200000 300000 400000 500000 600000 700000 800000\n" + ); + } + + #[test] + fn file_from_hwdata_and_an_index_past_the_table() { + let hw = HwWords { + actual: 1, + target: 2, + current: 12, + freq_mhz: f32::NAN.to_bits(), + }; + let mut out = String::new(); + write(&mut out, &StatsSeen::default(), &hw, &table()).unwrap(); + let mut lines = out.lines(); + assert_eq!( + lines.next(), + Some("pstate 12 freq_khz unknown source hwdata") + ); + assert_eq!( + lines.next(), + Some("stats: records 0 power_states 0 last none tags") + ); + assert_eq!( + lines.next(), + Some("hwdata: actual 1 target 2 current 12 freq_mhz_f32 0x7fc00000 freq_khz unknown") + ); + } +} diff --git a/drivers/gpu/drm/asahi/g15/queue.rs b/drivers/gpu/drm/asahi/g15/queue.rs new file mode 100644 index 0000000000000..cc87f1ccbe084 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/queue.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Logical queues: `crate::queue::Queue` for G15. +//! +//! Logical queues own no firmware state: they are independent scheduling entities over the one +//! shared graph, executed in one total order across queues and VMs. A device-wide host credit +//! bounds live logical queues to the advertised MAX_QUEUES; it reserves no hardware graph or +//! execution context. The native ABI hardware-reservation wording remains a compatibility limit. +//! QUEUE_CREATE validates +//! `usc_exec_base` (4 GiB aligned, the window inside the user range) and refuses a failed VM or +//! GPU (EIO); HIGH/REALTIME are refused by the shared `file.rs`. Ray-query scratch layouts are +//! refused by the shared default `reserve_render_scs` (RENDER_RAY_QUERY is never advertised). +//! +//! QUEUE_DESTROY (dropping the queue) drains: the scheduler entity is torn down first, which +//! waits (killable) until every accepted job has been handed to the engine; the engine then runs +//! them to completion and their fences signal in order. A closing process waits at most a second +//! for jobs still waiting for inputs and cancels them (ECANCELED, in fence order). +//! +//! SUBMIT checks, in order: the stream's structure and the sync count (EINVAL), the VM's state +//! (a failed VM refuses with EIO), the timestamp handles, the EXACT_PRIOR ordinals against the +//! commands accepted on this queue before the ioctl, a job reference on the VM, then every +//! command's spans and stream gates ([`super::command`]). The accepted job is pushed to the +//! queue's scheduler and its fence published in every output sync; a refused SUBMIT publishes a +//! signalled error fence instead. + +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use kernel::{dma_fence::Fence, prelude::*, sync::Arc, uapi, xarray}; + +use super::{ + command::{self, USC_WINDOW}, + fence::{Syncs, Timeline}, + hw, + job::{Job, Scheduler}, + params, + vm::{self, VmState}, + Gpu, Shared, +}; +use crate::{file, mmu}; + +/// Per-queue state the engine reads. +pub(crate) struct QueueState { + /// Host priority: `DRM_ASAHI_PRIORITY_REALTIME - priority` as `file.rs` passes it (MEDIUM 2, + /// LOW 3; `file.rs` refuses HIGH and REALTIME). Smaller is served first. + pub(crate) priority: u32, + /// USC executable window base. + pub(crate) usc_base: u64, + /// Engine service turn of this queue (least recently served first). + pub(crate) turn: AtomicU64, + /// Owned-job counts at the queue's last 32 arrivals (compute horizon ahead), recorded by the + /// engine when a job arrives. + pub(crate) owned: [AtomicU32; 32], + pub(crate) owned_index: AtomicU32, +} + +/// One device-wide logical-queue credit, released after the queue scheduler drains. +struct QueueCredit { + shared: Arc, +} + +impl QueueCredit { + fn acquire(shared: &Arc) -> Result { + shared + .logical_queues + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + (count < hw::sizes::MAX_QUEUES).then(|| count + 1) + }) + .map_err(|_| ENOMEM)?; + Ok(Self { + shared: shared.clone(), + }) + } +} + +impl Drop for QueueCredit { + fn drop(&mut self) { + let previous = self.shared.logical_queues.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous != 0); + } +} + +/// A G15 logical queue. +pub(crate) struct Queue { + // Dropped first: QUEUE_DESTROY drains the scheduler before anything else goes. + scheduler: Scheduler, + pub(crate) state: Arc, + /// Commands accepted per engine (render, compute) since creation, for EXACT_PRIOR. + pub(crate) accepted: [u64; 2], + vm: Arc, + shared: Arc, + timeline: Arc, + // Last field: all preceding fields (especially scheduler) drop before capacity returns. + _credit: QueueCredit, +} + +impl Drop for Queue { + fn drop(&mut self) { + // The registry's references to closed VMs (this queue's VM among them, if its handle is + // gone) go now; this VM's state lives on until the scheduler below has drained. + self.vm.prune_registry(); + } +} + +impl Queue { + /// Creates a logical queue. `priority` is `DRM_ASAHI_PRIORITY_REALTIME - priority`, as + /// `file.rs` passes it. + pub(crate) fn new(gpu: &Gpu, vm: mmu::Vm, priority: u32, usc_exec_base: u64) -> Result { + if priority > uapi::drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME { + return Err(EINVAL); + } + let range = hw::USER_RANGE; + if usc_exec_base & (USC_WINDOW - 1) != 0 + || usc_exec_base < range.start + || usc_exec_base + .checked_add(USC_WINDOW) + .is_none_or(|end| end > range.end) + { + return Err(EINVAL); + } + if gpu.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + let vm_state = vm::lookup(gpu, &vm)?; + if vm_state.status.get() != 0 { + return Err(EIO); + } + let credit = QueueCredit::acquire(&gpu.shared)?; + let timeline = Timeline::new()?; + let scheduler = Scheduler::new(&gpu.shared.dev)?; + let state = Arc::new( + QueueState { + priority, + usc_base: usc_exec_base, + turn: AtomicU64::new(0), + owned: [const { AtomicU32::new(0) }; 32], + owned_index: AtomicU32::new(0), + }, + GFP_KERNEL, + )?; + Ok(Self { + scheduler, + state, + accepted: [0; 2], + vm: vm_state, + shared: gpu.shared.clone(), + timeline, + _credit: credit, + }) + } + + /// Validates a SUBMIT and pushes its job; returns the job's fence. + fn submit_job( + &mut self, + id: u64, + syncs: &Syncs, + sync_count: usize, + bytes: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result { + if sync_count > command::MAX_SYNCS { + return Err(EINVAL); + } + command::check_stream(bytes, params::early_tiling())?; + if self.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + if self.vm.status.get() != 0 { + return Err(EIO); + } + let prepared = command::prepare(bytes, self.accepted, objects)?; + // The job reference first, so that the mappings checked below stay retained. + let guard = self.vm.vm.retain_first_job()?; + let submission = prepared.validate(&command::Context { + vm: &self.vm.vm, + usc_base: self.state.usc_base, + })?; + let mut inputs = KVec::with_capacity(syncs.inputs().len(), GFP_KERNEL)?; + for sync in syncs.inputs() { + if let Some(fence) = &sync.fence { + inputs.push(fence.clone(), GFP_KERNEL)?; + } + } + let fence = self.timeline.next()?; + let job = Job::new( + id, + self.state.clone(), + self.vm.clone(), + submission.commands, + fence, + submission.timestamps, + guard, + )?; + let published = job.fence.fence(); + self.scheduler.push(job, self.shared.clone(), inputs)?; + // Accepted: the whole SUBMIT counts for later EXACT_PRIOR ordinals. + self.accepted[command::RENDER] += submission.counts[command::RENDER]; + self.accepted[command::COMPUTE] += submission.counts[command::COMPUTE]; + Ok(published) + } +} + +impl crate::queue::Queue for Queue { + fn submit( + &mut self, + _reg_data: &crate::driver::AsahiRegData<'_>, + id: u64, + syncs: KVec, + in_sync_count: usize, + cmdbuf_raw: &[u8], + objects: Pin<&xarray::XArray>>, + ) -> Result { + let sync_count = syncs.len(); + let syncs = Syncs::new(syncs, in_sync_count); + match self.submit_job(id, &syncs, sync_count, cmdbuf_raw, objects) { + Ok(fence) => { + syncs.accept(&fence); + Ok(()) + } + Err(error) => { + // A refused submission leaves the VM usable unless the device is lost. + self.vm.status.record_if_device_loss(error); + syncs.refuse(&self.timeline, error); + Err(error) + } + } + } +} diff --git a/drivers/gpu/drm/asahi/g15/recovery.rs b/drivers/gpu/drm/asahi/g15/recovery.rs new file mode 100644 index 0000000000000..0827385e92c5f --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/recovery.rs @@ -0,0 +1,688 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hang detection, kills, the halt/resume handshake and fail-stop. +//! +//! The pure parts are in [`logic`] (host-tested); [`seams`] adapts the engine, register and VM +//! state they decide on. +//! +//! - Host deadline `g15_deadline_ms` (5000) from the oldest prefix's start or last progress +//! (a firmware stamp moving on an engine where that prefix still has work), renewed at every +//! retirement. At expiry with `HANG_PREEMPT`: arm the kill for the oldest prefix (compute +//! divisor +0x1c while its computes are unfinished, else fragment +0x24). +//! - Compute deadline `g15_compute_deadline_ms` (1000: a compute may run for up to 1 s): the +//! oldest prefix's first unfinished compute has its predecessors done, its FS wait reached, +//! its record consumed and the work ring READ past it, with no progress: class 2 -> kill; +//! class 0 -> fail-stop. Measured from the started compute: the total order never blames a +//! job for another VM's older work. +//! - Kill: Globals divisor = 1, DEVCTL 0x13 posted without waiting; publication frozen; 2 s +//! budget: (a) the prefix retires -> ETIMEDOUT, VM lost, later same-VM prefixes EIO; render +//! allocators re-initialized only for a fragment kill; waited disarm before publishing +//! anything; (b) an error event -> handshake, then disarm; (c) timeout -> fail-stop. +//! - Handshake on an error event or receipt fault: drop pending growth requests; wait up to +//! 200 x ~1-1.5 ms for Control+0x4590 = 1 while draining; write halted = 0, barrier, +//! resume = 1; wait <= 500 ms for the firmware to clear resume; wait <= 1000 polls for a full +//! drain, answering up to 4 further halts the same way. Then fail every published prefix +//! (EIO on fault, ETIMEDOUT on timeout) and latch the VM error; if any had renders, set +//! fs_resync and reinit so the next first TA waits on the FS stamp actually reached and an +//! exclusive priming prefix rebuilds the render allocators. A firmware error with nothing +//! published is answered and blames nobody. +//! - Fail-stop (latched until reboot, every VM EIO, published owners retained): publication +//! error after a queue WRITE advanced; recovery off, RTKit crash, no halt, resume not +//! consumed, not drained, more than 4 further halts, failed disarm; deadline without event +//! and no effective kill; class-0 compute hang; unanswerable TA stall. +//! +//! # The poll pass (poll.rs) +//! +//! 1. Failed ([`Recovery::failed`]): nothing. +//! 2. Nothing published and [`Firmware::recovery_idle`]: nothing, no repoll. +//! 3. Drain and retire. For each retired oldest prefix, [`Firmware::retire_outcome`] gives its +//! result (the kill outcome (a), or EIO for a doomed prefix); a receipt fault sets +//! [`Recovery::receipt_fault`]. +//! 4. [`Firmware::answer_growth`] (pools.rs). +//! 5. [`Firmware::check_hangs`]: waited disarm after an ended kill, progress, compute hang, +//! host deadline, recovery or fail-stop. `Err` = fail-stop done (flights failed, VMs failed, +//! pending drained): return it so `Shared::fail_stop` latches. +//! 6. Publish (frozen while [`Recovery::armed`]); repoll while anything is published or armed. + +pub(crate) mod logic; +mod seams; + +use kernel::{ + bindings, device, + prelude::*, + sync::Arc, + time::{ClockSource, Monotonic}, +}; + +use logic::{Failure, HangAction, HangInputs}; + +use super::fw::channels::{devctl, DevCtl, RING_SLOTS}; +use super::fw::initdata::{control, globals}; +use super::hw::{sizes, tuning}; +use super::job::{Completions, Job}; +use super::{params, Firmware}; + +static_assert!(logic::COMPUTE_KILL == globals::COMPUTE_KILL); +static_assert!(logic::FRAGMENT_KILL == globals::FRAGMENT_KILL); +static_assert!(logic::COMPUTE_MONITOR == globals::COMPUTE_MONITOR); +static_assert!(logic::QUEUE_CAP == sizes::QUEUE_CAP); +static_assert!(logic::CHANNEL_SLOTS == RING_SLOTS); +static_assert!(logic::TA == super::channel::Channel::Ta as usize); +static_assert!(logic::FS == super::channel::Channel::Fs as usize); +static_assert!(logic::COMPUTE == super::channel::Channel::Compute as usize); + +/// Monotonic time, ns. +pub(crate) fn now_ns() -> u64 { + ::ktime_get() as u64 +} + +/// One recovery poll interval, 1-1.5 ms. +fn poll_sleep() { + // SAFETY: Sleeping is allowed here: the poll work and the scheduler's run path hold only + // the device mutex. + unsafe { bindings::usleep_range_state(1000, 1500, bindings::TASK_UNINTERRUPTIBLE) }; +} + +/// Orders firmware-visible stores and loads: a full `dsb sy`, at least as strong as `wmb()`, +/// `mb()` and `smp_rmb()`. +fn barrier() { + crate::mem::sync(); +} + +/// A result as logged: 0 or a negative errno. +fn errno(result: &Result) -> i32 { + match result { + Ok(()) => 0, + Err(error) => error.to_errno(), + } +} + +/// Queues `job`'s completion once. Without memory for the entry the job stays referenced +/// forever and never completes (logged). +pub(crate) fn push_completion(done: &mut Completions, job: &Arc, result: Result) { + if done.iter().any(|(queued, _)| Arc::ptr_eq(queued, job)) { + return; + } + if done.push((job.clone(), result), GFP_KERNEL).is_err() { + pr_err!( + "G15: no memory to complete job {}; it is retained\n", + job.id + ); + core::mem::forget(job.clone()); + } +} + +/// Hang and recovery state. +pub(crate) struct Recovery { + /// Host deadline and compute no-progress deadline, monotonic ns; [`logic::EXPIRED`] (0) + /// has passed; renewed by [`Firmware::renew_deadlines`]. + pub(crate) deadline_ns: u64, + pub(crate) compute_deadline_ns: u64, + /// An armed kill: the Globals divisor offset (0x1c compute, 0x24 fragment) and when it was + /// armed. Publication is frozen while it is set (`Engine::frozen` mirrors it). + pub(crate) armed: Option<(usize, u64)>, + /// The prefix the kill was armed for retired: the disarm is pending. + pub(crate) kill_ended: bool, + /// After a recovered render abort: the next first TA waits on the FS stamp reached, and an + /// exclusive priming prefix rebuilds the render allocators. The engine clears both when + /// it publishes renders. + pub(crate) fs_resync: bool, + pub(crate) reinit: bool, + /// Firmware error events already answered. + pub(crate) errors_seen: u32, + /// The retirement check of this poll pass saw a fault; set by the poll pass, consumed by + /// [`Firmware::check_hangs`]. + pub(crate) receipt_fault: bool, + /// The latched backend failure: nothing is accepted or published again. + pub(crate) failed: Option, + /// Statistics (debugfs keys `recoveries`, `hang_kills`). + pub(crate) recoveries: u64, + pub(crate) hang_kills: u64, +} + +impl Recovery { + pub(crate) fn new() -> Self { + Self { + deadline_ns: logic::EXPIRED, + compute_deadline_ns: logic::EXPIRED, + armed: None, + kill_ended: false, + fs_resync: false, + reinit: false, + errors_seen: 0, + receipt_fault: false, + failed: None, + recoveries: 0, + hang_kills: 0, + } + } + + /// The latched backend failure, if any. + pub(crate) fn failed(&self) -> Option { + self.failed + } + + /// The poll must run again soon besides any published prefix: a kill is armed. + pub(crate) fn needs_repoll(&self) -> bool { + self.armed.is_some() + } +} + +impl Firmware { + /// The GPU device, for logging. + pub(crate) fn dev(&self) -> &device::Device { + &self.pools.dev + } + + /// Renews the host and compute deadlines from now: at the publication of a prefix with + /// nothing in flight, at every retirement, and on progress. + pub(crate) fn renew_deadlines(&mut self) { + let (deadline, compute) = logic::renew( + now_ns(), + params::deadline_ms(), + params::compute_deadline_ms(), + ); + self.recovery.deadline_ns = deadline; + self.recovery.compute_deadline_ns = compute; + } + + /// Firmware error events arrived that recovery has not answered. + fn new_errors(&self) -> bool { + self.events.errors != self.recovery.errors_seen + } + + /// Nothing for the poll pass to do when nothing is published: no kill armed (whose prefix + /// ended) and no firmware error event to answer (the pass is skipped without a repoll). + pub(crate) fn recovery_idle(&self) -> bool { + self.recovery.armed.is_none() && !self.new_errors() + } + + /// `(recoveries, hang_kills)` for `asahi-g15/stats`. + pub(crate) fn recovery_counts(&self) -> (u64, u64) { + (self.recovery.recoveries, self.recovery.hang_kills) + } + + /// The result of the oldest published prefix, which just retired. + /// Retired after an armed request (the first such retirement): the firmware ended it. + /// Its job fails with ETIMEDOUT and its client's VM is lost, like a hang reset; later + /// prefixes (same VM) may have been ended by the request too before it is restored: they + /// fail with EIO. A compute request ends compute work only: the prefix's renders completed + /// with their own receipts and the render allocators stay; only a fragment kill of a + /// prefix with renders re-initializes them. Otherwise a doomed prefix fails with EIO. + pub(crate) fn retire_outcome(&mut self) -> Result { + let doomed = self.engine.flights.first().is_some_and(|f| f.doomed); + let offset = match self.recovery.armed { + Some((offset, _)) if !self.recovery.kill_ended => offset, + _ => return if doomed { Err(EIO) } else { Ok(()) }, + }; + self.recovery.kill_ended = true; + self.recovery.hang_kills += 1; + if self.receipts(0).renders != 0 && offset == logic::FRAGMENT_KILL { + // Aborted scene: PB state untrusted. + self.recovery.reinit = true; + } + for flight in self.engine.flights.iter_mut().skip(1) { + flight.doomed = true; + } + let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + let kind = if offset == logic::FRAGMENT_KILL { + "fragment" + } else { + "compute" + }; + dev_warn!( + self.dev(), + "hung submission of ctx {} ended by the firmware's {} request\n", + ctx, + kind + ); + Err(ETIMEDOUT) + } + + /// The hang and recovery branches of the poll pass, after retirement and growth. `Err` + /// means the backend fail-stopped here: published prefixes failed and retained, every VM + /// failed, pending jobs failed; the caller latches [`crate::g15::Shared::fail_stop`] and + /// publishes nothing. + pub(crate) fn check_hangs(&mut self, done: &mut Completions) -> Result { + if let Some(error) = self.recovery.failed { + return Err(error); + } + let fault = core::mem::take(&mut self.recovery.receipt_fault); + // The request's prefix ended: restore the divisor (and wait for its import) before + // anything is published, unless a firmware error arrived meanwhile: the firmware may + // have halted, and the recovery below answers it first, then restores the divisor. + if self.recovery.kill_ended + && !fault + && self.transport_error().is_none() + && !self.new_errors() + && self.preempt_disarm().is_err() + { + // Not restored: fail below. + self.recovery.deadline_ns = logic::EXPIRED; + } + let flight = !self.engine.flights.is_empty(); + if flight && self.recovery.armed.is_none() { + self.progress(); + } + let now = now_ns(); + let inputs = HangInputs { + flight, + armed: self.recovery.armed.is_some(), + fault, + transport: self.transport_error().is_some(), + new_errors: self.new_errors(), + compute_deadline_expired: logic::expired(now, self.recovery.compute_deadline_ns), + deadline_expired: logic::expired(now, self.recovery.deadline_ns), + class: tuning::COMPUTE_QUEUE_CLASS, + recovery: params::recovery(), + hang_preempt: tuning::HANG_PREEMPT, + }; + let decision = logic::decide(&inputs, || self.compute_hang_now()); + if decision.compute_hang { + self.report("compute hang"); + } + let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + if decision.class0_hang { + dev_warn!( + self.dev(), + "class-0 compute hang of ctx {} (not preemptible): failing\n", + ctx + ); + // Take the fail-stop branch. + self.recovery.deadline_ns = logic::EXPIRED; + } + match decision.action { + HangAction::Idle | HangAction::Wait => Ok(()), + HangAction::ArmCompute => { + // The started compute made no progress for its budget, with no firmware + // event: ask the firmware to end it; publication stays frozen. + dev_warn!( + self.dev(), + "compute of ctx {} made no progress for {} ms: asking the firmware to end it\n", + ctx, + params::compute_deadline_ms() + ); + let _ = self.kill_arm(logic::COMPUTE_KILL); + Ok(()) + } + HangAction::ArmDeadline => { + // Host deadline, no firmware event: let the firmware kill it. + self.report("hung, requesting preemption"); + let offset = self.kill_offset_now(); + let _ = self.kill_arm(offset); + Ok(()) + } + HangAction::Fail { recover } => self.fail_or_recover(recover, fault, done), + } + } + + /// The failure branch: a firmware error event or a receipt fault with a live transport + /// waits for the firmware's own halt and answers it; every published prefix fails with it. + /// A hang in the render-only allocator rebuild prefix recovers like any render hang and + /// re-primes. An armed request is restored after the resume, before + /// anything is published; if it cannot be, the GPU stops. Anything else fail-stops. + fn fail_or_recover(&mut self, recover: bool, fault: bool, done: &mut Completions) -> Result { + let flight = !self.engine.flights.is_empty(); + let ctx = self.engine.flights.first().map_or(0, |f| f.job.vm.slot); + self.report(if flight { + "prefix FAILED" + } else { + "firmware error with no prefix published" + }); + let render = (0..self.engine.flights.len()).any(|i| self.receipts(i).renders != 0); + if recover { + let (error, mut recovered) = self.recover(); + if recovered && self.preempt_disarm().is_err() { + recovered = false; + } + if recovered { + self.recovery.errors_seen = self.events.errors; + self.recovery.recoveries += 1; + if render { + self.recovery.fs_resync = true; + self.recovery.reinit = true; + } + dev_warn!( + self.dev(), + "recovered: ctx {} submission failed with {}\n", + ctx, + error.to_errno() + ); + // The failing client's VM is lost; other clients continue. + while !self.engine.flights.is_empty() { + self.retire_oldest(Err(error), done); + } + return Ok(()); + } + } + let error = if fault { + EIO + } else { + self.transport_error().unwrap_or(ETIMEDOUT) + }; + self.fail_stop(error, done); + Err(error) + } + + /// Stops the backend until reboot: restores an armed divisor if possible, latches the + /// failure, fails and retains every published prefix, fails every live VM with EIO and + /// fails the pending jobs (retaining those partly published). The caller returns the error + /// so that [`crate::g15::Shared::fail_stop`] latches the device; a job the caller holds + /// outside `engine.pending` (the job whose publication failed) is the caller's to fail and + /// retain. + pub(crate) fn fail_stop(&mut self, error: Error, done: &mut Completions) { + let _ = self.preempt_disarm(); + if self.recovery.failed.is_none() { + self.recovery.failed = Some(error); + } + // Published and uncertain: never release their owners. + self.fail_flights(error, done); + // No recovery: every VM is lost, also those with no failed job. + self.fail_all_vms(EIO); + let pending = core::mem::take(&mut self.engine.pending); + for job in pending.iter() { + push_completion(done, job, Err(EIO)); + // A partly published job already ran commands: its owners stay retained. + if self.job_partly_published(job) { + self.retain(job); + } + } + } + + /// Fails every published prefix: their jobs fail (each once) and, with every owner the + /// firmware may still reference, are retained. The prefixes' storage is never released: + /// the flights are leaked. + pub(crate) fn fail_flights(&mut self, error: Error, done: &mut Completions) { + let flights = core::mem::take(&mut self.engine.flights); + for flight in flights.iter() { + for job in Iterator::chain(core::iter::once(&flight.job), flight.group.iter()) { + push_completion(done, job, Err(error)); + self.retain(job); + } + } + core::mem::forget(flights); + } + + /// Keeps `job` until reboot: its owners (VM mappings, storage) stay alive because the + /// firmware may still reference them. + pub(crate) fn retain(&mut self, job: &Arc) { + if self + .engine + .retained + .iter() + .any(|kept| Arc::ptr_eq(kept, job)) + { + return; + } + if self.engine.retained.push(job.clone(), GFP_KERNEL).is_err() { + core::mem::forget(job.clone()); + } + } + + /// The started-compute hang test on the oldest prefix. + fn compute_hang_now(&self) -> bool { + if self.engine.flights.is_empty() { + return false; + } + match self.observe() { + Ok(observed) => logic::compute_hang(&self.receipts(0), &self.tail(), &observed), + Err(_) => false, + } + } + + /// The divisor the host deadline arms for the oldest prefix. + fn kill_offset_now(&self) -> usize { + let observed = self.observe().unwrap_or_default(); + logic::kill_offset(&self.receipts(0), &self.tail(), &observed) + } + + /// Firmware DONE/READ reached the newest published ends, channels consumed, engines idle. + fn drained(&mut self) -> bool { + let tail = self.tail(); + match self.observe() { + Ok(observed) if logic::queues_drained(&tail, &observed) => { + matches!(self.engine_state(), Ok(state) if !state.busy) + } + _ => false, + } + } + + fn globals_w32(&self, offset: usize, value: u32) -> Result { + self.init.globals.as_ref().ok_or(ENODEV)?.w32(offset, value) + } + + fn control_r32(&self, offset: usize) -> Result { + self.init.control.as_ref().ok_or(ENODEV)?.r32(offset) + } + + fn control_w32(&self, offset: usize, value: u32) -> Result { + self.init.control.as_ref().ok_or(ENODEV)?.w32(offset, value) + } + + /// Writes a Globals divisor and waits for its import. + pub(crate) fn preempt_set(&mut self, offset: usize, divisor: u32) -> Result { + let ret = self.globals_w32(offset, divisor).and_then(|()| { + barrier(); + self.devctl(&DevCtl::op(devctl::IMPORT_GLOBALS), true) + }); + dev_info!( + self.dev(), + "preemption divisor Globals+{:#x}={}: {}\n", + offset, + divisor, + errno(&ret) + ); + ret + } + + /// The class-2 compute profile: the unchanged-work compute monitor off (Globals +0x9b0 = 0, + /// waited import) before the compute queue's first exposure (the periodic request +0x1c is + /// 0 in the image). Called once by the backend start after the graphs are initialized. + pub(crate) fn compute_monitor_off(&mut self) -> Result { + if tuning::COMPUTE_QUEUE_CLASS == 2 { + self.preempt_set(logic::COMPUTE_MONITOR, 0) + } else { + Ok(()) + } + } + + /// Arms the hang request: divisor 1 at Globals `offset`, DEVCTL 0x13 posted without + /// waiting (the firmware may halt before it consumes it), publication frozen until the + /// outcome, whose budget is 2 s (none if the post failed). + fn kill_arm(&mut self, offset: usize) -> Result { + let ret = self.globals_w32(offset, 1).and_then(|()| { + barrier(); + self.devctl(&DevCtl::op(devctl::IMPORT_GLOBALS), false) + }); + let now = now_ns(); + self.recovery.armed = Some((offset, now)); + self.recovery.kill_ended = false; + self.engine.frozen = true; + self.recovery.deadline_ns = logic::kill_deadline(now, ret.is_ok()); + dev_info!( + self.dev(), + "hang request Globals+{:#x}=1: {}\n", + offset, + errno(&ret) + ); + ret + } + + /// Restores the armed divisor to 0 and waits for the import before anything is published + /// again (an earlier arming record is consumed first). On failure the request stays armed + /// and publication frozen. + fn preempt_disarm(&mut self) -> Result { + let Some((offset, _)) = self.recovery.armed else { + return Ok(()); + }; + self.preempt_set(offset, 0)?; + self.recovery.armed = None; + self.recovery.kill_ended = false; + self.engine.frozen = false; + Ok(()) + } + + /// Logs the recovery handshake state: Control halt count, halted, resume and flag; + /// HardwareData fault information; up to 16 pending records. + fn recover_snapshot(&self, phase: &str) { + let ctl = |offset| self.control_r32(offset).unwrap_or(0); + let fault_info = |offset| { + self.init + .hwdata + .as_ref() + .and_then(|hw| hw.r32(self.cfg.layout.hwdata_view + offset).ok()) + .unwrap_or(0) + }; + dev_info!( + self.dev(), + "recovery {}: halt_count={:#x} halted={} resume={} flag={:#x} faultinfo status={} do_not_halt={} did_not_halt={} total={}\n", + phase, + ctl(control::HALT_COUNT), + ctl(control::HALTED), + ctl(control::RESUME), + ctl(control::FLAG), + fault_info(0), + fault_info(4), + fault_info(8), + fault_info(0x18) + ); + let mut shown = 0; + for i in 0..128 { + if shown == 16 { + break; + } + let info = ctl(control::PENDING + 8 * i); + if info & 1 == 0 { + continue; + } + shown += 1; + dev_info!( + self.dev(), + " pending[{}]: info {:#x} slot {} wait {:#x}\n", + i, + info, + (info >> 4) & 0x7f, + ctl(control::PENDING + 8 * i + 4) + ); + } + } + + /// Acknowledges a firmware halt: halted = 0, then resume = 1, ordered; nothing else. Then + /// waits up to 500 polls for the firmware to clear resume. Returns whether it did, and + /// the polls waited. + fn acknowledge_halt(&mut self) -> Result<(bool, u32)> { + self.control_w32(control::HALTED, 0)?; + barrier(); + self.control_w32(control::RESUME, 1)?; + barrier(); + let mut t = 0; + while t < logic::RESUME_POLLS && self.control_r32(control::RESUME)? != 0 { + poll_sleep(); + t += 1; + } + Ok((self.control_r32(control::RESUME)? == 0, t)) + } + + /// The firmware recovery handshake, with a published prefix active (or a firmware error + /// with nothing published) and a firmware error event seen (or a receipt fault). Returns + /// the error for the failed prefixes and whether the firmware halted, consumed the host's + /// resume and drained; otherwise the caller fail-stops (no halt, transport failure, or a + /// queue the firmware did not drain). + fn recover(&mut self) -> (Error, bool) { + // Growth requests of the failed epoch are never answered. + self.events.grow.clear(); + barrier(); + let fault = self.engine_state().map_or(0, |state| state.fault); + let last = self.events.last_error; + let error = match logic::failure(fault, last.map(|e| e.kind)) { + Failure::Fault => EIO, + Failure::Timeout => ETIMEDOUT, + }; + let (kind, slot, raw) = last.map_or((0, -1, [0; 14]), |e| (e.kind, e.slot, e.raw)); + let what = if error == EIO { "fault" } else { "timeout" }; + dev_warn!( + self.dev(), + "GPU {}: event type {} slot {} raw {:08x?}; recovering\n", + what, + kind, + slot, + raw + ); + self.recover_snapshot("error"); + let recovered = match self.handshake() { + Ok(recovered) => recovered, + Err(e) => { + dev_err!( + self.dev(), + "recovery: firmware state unavailable: {}\n", + e.to_errno() + ); + false + } + }; + (error, recovered) + } + + /// The halt, resume and drain steps of [`Self::recover`]. + fn handshake(&mut self) -> Result { + // The event precedes the halt publication: wait for the real halt. + let mut t = 0; + while t < logic::HALT_POLLS && self.control_r32(control::HALTED)? != 1 { + let _ = self.drain_rings(); + poll_sleep(); + t += 1; + } + if self.control_r32(control::HALTED)? != 1 || self.transport_error().is_some() { + dev_err!(self.dev(), "recovery: firmware did not halt ({} ms)\n", t); + return Ok(false); + } + self.recover_snapshot("halted"); + let (resumed, t) = self.acknowledge_halt()?; + if !resumed || self.transport_error().is_some() { + dev_err!(self.dev(), "recovery: resume not consumed ({} ms)\n", t); + return Ok(false); + } + // The firmware clears resume before finishing: wait for the queues to drain. Work of + // the failed prefixes that runs after the resume can fail in turn (a compute that + // waited on the aborted FS hitting a firmware compute timeout): its own halt is + // answered the same way, a bounded number of times, and the drain continues. + let mut halts = 0; + let mut t = 0; + while t < logic::DRAIN_POLLS && !self.drained() { + let _ = self.drain_rings(); + if self.control_r32(control::HALTED)? == 1 + && self.transport_error().is_none() + && halts < logic::FURTHER_HALTS + { + halts += 1; + self.recover_snapshot("halted again"); + if !self.acknowledge_halt()?.0 { + dev_err!( + self.dev(), + "recovery: resume after a further halt not consumed\n" + ); + return Ok(false); + } + // Restart the drain wait, counting this poll. + t = 1; + continue; + } + poll_sleep(); + t += 1; + } + self.recover_snapshot("resumed"); + self.report("recovery"); + if !self.drained() + || self.control_r32(control::HALTED)? != 0 + || self.transport_error().is_some() + { + dev_err!( + self.dev(), + "recovery: failed prefix not drained ({} ms)\n", + t + ); + return Ok(false); + } + Ok(true) + } +} diff --git a/drivers/gpu/drm/asahi/g15/recovery/logic.rs b/drivers/gpu/drm/asahi/g15/recovery/logic.rs new file mode 100644 index 0000000000000..6d36d1d6550d5 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/recovery/logic.rs @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Pure hang-detection and recovery decisions. +//! +//! The arithmetic and branch structure of the poll pass, free of locks, objects and clocks so +//! host tests can exercise them: receipt comparisons, the started-compute hang test, the kill +//! divisor choice and the queue-drain test of the recovery handshake, the deadlines and the +//! hang branches. Times are monotonic nanoseconds; a deadline `d` has passed when `now > d`. +//! +//! Core-only (no `kernel` imports): this file is compiled by the host test harness as well. + +/// Entries of a firmware work ring (= `hw::sizes::QUEUE_CAP`). +pub(crate) const QUEUE_CAP: u32 = 0x500; +/// Records of a submit channel ring (= `fw::channels::RING_SLOTS`). +pub(crate) const CHANNEL_SLOTS: u32 = 256; +/// Channel indices. +pub(crate) const TA: usize = 0; +pub(crate) const FS: usize = 1; +pub(crate) const COMPUTE: usize = 2; + +/// Globals divisors of the firmware's periodic context-store request: +/// compute (+0x1c, ends a running class-2 compute) and fragment (+0x24). Mirrors +/// `fw::initdata::globals::{COMPUTE_KILL, FRAGMENT_KILL}`. +pub(crate) const COMPUTE_KILL: usize = 0x1c; +pub(crate) const FRAGMENT_KILL: usize = 0x24; +/// Globals unchanged-work compute monitor (`fw::initdata::globals::COMPUTE_MONITOR`). +pub(crate) const COMPUTE_MONITOR: usize = 0x9b0; + +/// Outcome budget of an armed kill: the prefix retires or an error event arrives within it. +pub(crate) const KILL_BUDGET_MS: u64 = 2000; +/// Recovery handshake bounds: polls for the firmware's halt, polls for the resume to be +/// consumed, polls for the drain, and further halts answered during the drain. +/// Every poll sleeps 1-1.5 ms. +pub(crate) const HALT_POLLS: u32 = 200; +pub(crate) const RESUME_POLLS: u32 = 500; +pub(crate) const DRAIN_POLLS: u32 = 1000; +pub(crate) const FURTHER_HALTS: u32 = 4; + +/// Stamp cell `obs` reached the own target `own`, bounded by the newest published target +/// `tail`. Stamps are seeded so that a written stamp is never 0 and has a zero low byte. +pub(crate) const fn stamp_reached(obs: u32, own: u32, tail: u32) -> bool { + obs != 0 && obs & 0xff == 0 && obs.wrapping_sub(own) <= tail.wrapping_sub(own) +} + +/// Cursor `c` of a ring of `m` entries passed the end `h` of entries starting at `b`, within +/// the tail `t`. +/// +/// `c` is a firmware-written cursor: the u32 arithmetic wraps (a garbage cursor must not trip +/// `CONFIG_RUST_OVERFLOW_CHECKS`). +pub(crate) const fn ring_passed(c: u32, b: u32, h: u32, t: u32, m: u32) -> bool { + let dc = c.wrapping_add(m).wrapping_sub(b) % m; + let dh = h.wrapping_add(m).wrapping_sub(b) % m; + let dt = t.wrapping_add(m).wrapping_sub(b) % m; + dh <= dc && dc <= dt +} + +/// The receipts of one published prefix, filled at publication. +#[derive(Copy, Clone, Default, Debug, PartialEq, Eq)] +pub(crate) struct Receipts { + /// Compute and render members of the prefix. + pub(crate) computes: u32, + pub(crate) renders: u32, + /// Compute: final FW stamp target, work-ring end after the prefix and before it. + pub(crate) target: u32, + pub(crate) head: u32, + pub(crate) c_start: u32, + /// Render: final TA/FS stamp targets, ring ends after and before the prefix. + pub(crate) ta_target: u32, + pub(crate) fs_target: u32, + pub(crate) ta_head: u32, + pub(crate) fs_head: u32, + pub(crate) ta_start: u32, + pub(crate) fs_start: u32, + /// Host channel producers (TA, FS, compute) after the prefix's records. + pub(crate) rec: [u32; 3], + /// Compute member j's FW stamp target, and the FS target it waits on directly (0: none). + pub(crate) cstamp: [u32; 16], + pub(crate) cwait: [u32; 16], +} + +/// The newest published receipts. +#[derive(Copy, Clone, Default, Debug, PartialEq, Eq)] +pub(crate) struct Tail { + pub(crate) target: u32, + pub(crate) head: u32, + pub(crate) ta_target: u32, + pub(crate) fs_target: u32, + pub(crate) ta_head: u32, + pub(crate) fs_head: u32, + pub(crate) producer: [u32; 3], +} + +/// Firmware-written words read in one observation (single aligned loads of uncached cells). +#[derive(Copy, Clone, Default, Debug, PartialEq, Eq)] +pub(crate) struct Observed { + /// FW stamps: compute (the compute graph's `fw_stamp`), TA and FS (the render graph's + /// `fw_stamp[0..1]`). + pub(crate) compute_stamp: u32, + pub(crate) ta_stamp: u32, + pub(crate) fs_stamp: u32, + /// Work-ring DONE (qstate +0) and READ (qstate +0x30) of the compute, TA and FS queues. + pub(crate) compute_done: u32, + pub(crate) compute_read: u32, + pub(crate) ta_done: u32, + pub(crate) ta_read: u32, + pub(crate) fs_done: u32, + pub(crate) fs_read: u32, + /// Channel consumers (the read cell of the TA, FS and compute channel state blocks). + pub(crate) consumer: [u32; 3], +} + +/// Engine receipts from the SGX status window: busy is bit 0 of C020 | C120; fault is the D8C0 +/// status of fault banks 0 and 1. +#[derive(Copy, Clone, Default, Debug, PartialEq, Eq)] +pub(crate) struct EngineState { + pub(crate) busy: bool, + pub(crate) fault: u64, +} + +/// The oldest prefix `f`'s first unfinished compute has started and makes no progress: its +/// predecessors in the prefix are done (earlier compute members by the reached FW stamp, a +/// render it waits on directly by the FS stamp), the firmware consumed the prefix's compute +/// record and its work-ring READ passed the prefix's entries (the private start timestamp is +/// written only at completion). Later members, renders waiting on it included, may be +/// unfinished. +pub(crate) fn compute_hang(f: &Receipts, t: &Tail, o: &Observed) -> bool { + let fw = o.compute_stamp; + if f.computes == 0 || stamp_reached(fw, f.target, t.target) { + return false; // no compute, or its computes are done + } + let computes = (f.computes as usize).min(f.cstamp.len()); + let mut j = 0; + while j < computes && stamp_reached(fw, f.cstamp[j], t.target) { + j += 1; + } + if j == computes { + return false; + } + if f.cwait[j] != 0 && !stamp_reached(o.fs_stamp, f.cwait[j], t.fs_target) { + return false; // still waiting on a render of the prefix + } + if o.consumer[COMPUTE] != f.rec[COMPUTE] { + return false; // its record not consumed yet + } + ring_passed(o.compute_read, f.c_start, f.head, t.head, QUEUE_CAP) +} + +/// The divisor the host deadline arms: the compute request while the oldest prefix's computes +/// are not done, else the fragment request for a prefix with renders. +pub(crate) fn kill_offset(f: &Receipts, t: &Tail, o: &Observed) -> usize { + if f.computes != 0 && !stamp_reached(o.compute_stamp, f.target, t.target) { + COMPUTE_KILL + } else if f.renders != 0 { + FRAGMENT_KILL + } else { + COMPUTE_KILL + } +} + +/// Firmware DONE/READ reached the newest published ends of every queue and every channel +/// record was consumed (the caller adds the engine-idle test). +pub(crate) fn queues_drained(t: &Tail, o: &Observed) -> bool { + o.ta_done == t.ta_head + && o.ta_read == t.ta_head + && o.fs_done == t.fs_head + && o.fs_read == t.fs_head + && o.consumer[TA] == t.producer[TA] + && o.consumer[FS] == t.producer[FS] + && o.compute_done == t.head + && o.compute_read == t.head + && o.consumer[COMPUTE] == t.producer[COMPUTE] +} + +/// What a firmware error means for the failed prefixes. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Failure { + /// A GPU fault (EIO). + Fault, + /// A firmware timeout (ETIMEDOUT). + Timeout, +} + +/// Classifies a recovery: a pending fault status or a type-0 event is a fault. `event_type` is +/// the last error event's type; with none seen, type 0 applies. +pub(crate) fn failure(fault: u64, event_type: Option) -> Failure { + if fault != 0 || event_type.unwrap_or(0) == 0 { + Failure::Fault + } else { + Failure::Timeout + } +} + +const NSEC_PER_MSEC: u64 = 1_000_000; + +/// Host and compute deadlines `deadline_ms`/`compute_deadline_ms` from `now` (both budgets +/// are at least 100 ms). +pub(crate) fn renew(now: u64, deadline_ms: u32, compute_deadline_ms: u32) -> (u64, u64) { + ( + now + deadline_ms.max(100) as u64 * NSEC_PER_MSEC, + now + compute_deadline_ms.max(100) as u64 * NSEC_PER_MSEC, + ) +} + +/// Host deadline after arming a kill: the outcome budget, or at once (`EXPIRED`) when the +/// request could not be posted. +pub(crate) fn kill_deadline(now: u64, posted: bool) -> u64 { + if posted { + now + KILL_BUDGET_MS * NSEC_PER_MSEC + } else { + EXPIRED + } +} + +/// A deadline that has passed. Monotonic ns since boot never wrap, so 0 is in the past for +/// every later check. +pub(crate) const EXPIRED: u64 = 0; + +/// `time_after(now, deadline)`. +pub(crate) const fn expired(now: u64, deadline: u64) -> bool { + now > deadline +} + +/// The state the hang branches of the poll pass decide on. +#[derive(Copy, Clone, Default, Debug)] +pub(crate) struct HangInputs { + /// A prefix is published. + pub(crate) flight: bool, + /// A kill request is armed. + pub(crate) armed: bool, + /// The retirement check saw a fault. + pub(crate) fault: bool, + /// The RTKit transport failed. + pub(crate) transport: bool, + /// Firmware error events not yet answered. + pub(crate) new_errors: bool, + /// The compute no-progress and host deadlines passed. + pub(crate) compute_deadline_expired: bool, + pub(crate) deadline_expired: bool, + /// The compute queue's scheduling class (2: killable), recovery and hang preemption + /// enabled. + pub(crate) class: u32, + pub(crate) recovery: bool, + pub(crate) hang_preempt: bool, +} + +/// What the hang branches do. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum HangAction { + /// Nothing published or armed and no error: go straight to publication. + Idle, + /// Nothing to do in this pass; poll again. + Wait, + /// The oldest prefix's started compute made no progress for its budget, class 2: arm the + /// compute request (+0x1c); publication stays frozen. + ArmCompute, + /// The host deadline passed with no event: arm the request [`kill_offset`] chooses. + ArmDeadline, + /// Failure: answer the firmware halt first when `recover`, else fail-stop. + Fail { recover: bool }, +} + +/// The decision of the hang branches. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) struct Decision { + pub(crate) action: HangAction, + /// The started-compute hang test matched (reported as "compute hang"). + pub(crate) compute_hang: bool, + /// ... on the class-0 queue, which cannot be ended: the host deadline is expired so the + /// failure branch fail-stops now. The caller persists the expired deadline. + pub(crate) class0_hang: bool, +} + +/// The hang branches of the poll pass, after the waited disarm and the progress update. +/// `compute_hang` evaluates [`compute_hang`] for the oldest prefix; it is called only once its +/// deadline passed (and again for a class-0 queue at the host deadline), never on the fast +/// path. +pub(crate) fn decide(i: &HangInputs, mut compute_hang: impl FnMut() -> bool) -> Decision { + let mut d = Decision { + action: HangAction::Wait, + compute_hang: false, + class0_hang: false, + }; + let quiet = !i.fault && !i.transport && !i.new_errors; + if !i.flight && !i.armed && quiet { + d.action = HangAction::Idle; + return d; + } + let mut deadline_expired = i.deadline_expired; + if i.flight && quiet && !i.armed && i.compute_deadline_expired && compute_hang() { + d.compute_hang = true; + if i.class != 0 && i.recovery { + d.action = HangAction::ArmCompute; + return d; + } + d.class0_hang = true; + deadline_expired = true; + } + if i.flight + && quiet + && deadline_expired + && !i.armed + && (i.class != 0 || !compute_hang()) + && i.hang_preempt + && i.recovery + { + d.action = HangAction::ArmDeadline; + } else if !quiet || ((i.flight || i.armed) && deadline_expired) { + d.action = HangAction::Fail { + recover: i.recovery && !i.transport && (i.new_errors || i.fault), + }; + } + d +} + +#[cfg(test)] +mod tests { + use super::*; + + const C0: u32 = 0xc1000000; + + fn stamp(n: u32) -> u32 { + C0 + (n << 8) + } + + #[test] + fn stamps() { + assert!(!stamp_reached(0, stamp(1), stamp(3))); + assert!(!stamp_reached(stamp(1) + 1, stamp(1), stamp(3))); + assert!(stamp_reached(stamp(1), stamp(1), stamp(3))); + assert!(stamp_reached(stamp(3), stamp(1), stamp(3))); + assert!(!stamp_reached(stamp(0), stamp(1), stamp(3))); + assert!(!stamp_reached(stamp(4), stamp(1), stamp(3))); + // Across the u32 wrap. + let own = 0xffffff00u32; + assert!(stamp_reached(0x00000100, own, 0x00000200)); + assert!(!stamp_reached(0xfffffe00, own, 0x00000200)); + } + + #[test] + fn rings() { + // Entries 10..14 of a 0x500 ring, tail at 20. + assert!(!ring_passed(12, 10, 14, 20, QUEUE_CAP)); + assert!(ring_passed(14, 10, 14, 20, QUEUE_CAP)); + assert!(ring_passed(20, 10, 14, 20, QUEUE_CAP)); + assert!(!ring_passed(21, 10, 14, 20, QUEUE_CAP)); + // Wrapped. + assert!(ring_passed(2, 0x4fe, 1, 5, QUEUE_CAP)); + assert!(!ring_passed(0x4ff, 0x4fe, 1, 5, QUEUE_CAP)); + // A channel record: consumed when the cursor passed rec - 1 .. rec. + let rec = 7; + assert!(!ring_passed(6, (rec + 255) % 256, rec, 9, CHANNEL_SLOTS)); + assert!(ring_passed(7, (rec + 255) % 256, rec, 9, CHANNEL_SLOTS)); + assert!(ring_passed(0, 255, 0, 3, CHANNEL_SLOTS)); + } + + /// A compute-only prefix of three computes, stamps 1..3, ring entries 10..13 (with the + /// compute->compute barriers), compute record producer 5. + fn prefix() -> (Receipts, Tail, Observed) { + let mut f = Receipts { + computes: 3, + target: stamp(3), + head: 15, + c_start: 10, + rec: [0, 0, 5], + ..Default::default() + }; + f.cstamp[..3].copy_from_slice(&[stamp(1), stamp(2), stamp(3)]); + let t = Tail { + target: stamp(3), + head: 15, + producer: [0, 0, 5], + ..Default::default() + }; + let o = Observed { + compute_stamp: stamp(0), + compute_read: 15, + consumer: [0, 0, 5], + ..Default::default() + }; + (f, t, o) + } + + #[test] + fn compute_hangs() { + let (mut f, t, mut o) = prefix(); + // Started (record consumed, READ past the prefix), first compute unfinished. + assert!(compute_hang(&f, &t, &o)); + // Second compute running after the first finished. + o.compute_stamp = stamp(1); + assert!(compute_hang(&f, &t, &o)); + // All done: no hang. + o.compute_stamp = stamp(3); + assert!(!compute_hang(&f, &t, &o)); + o.compute_stamp = stamp(1); + // Record not consumed yet. + o.consumer[COMPUTE] = 4; + assert!(!compute_hang(&f, &t, &o)); + o.consumer[COMPUTE] = 5; + // READ not past the prefix's entries. + o.compute_read = 12; + assert!(!compute_hang(&f, &t, &o)); + o.compute_read = 15; + // The running compute waits on an FS of the prefix that has not completed. + f.cwait[1] = 0x3d000300; + o.fs_stamp = 0x3d000200; + let mut t2 = t; + t2.fs_target = 0x3d000400; + assert!(!compute_hang(&f, &t2, &o)); + o.fs_stamp = 0x3d000300; + assert!(compute_hang(&f, &t2, &o)); + // No compute in the prefix. + f.computes = 0; + assert!(!compute_hang(&f, &t, &o)); + } + + #[test] + fn kill_offsets() { + let (mut f, t, mut o) = prefix(); + assert_eq!(kill_offset(&f, &t, &o), COMPUTE_KILL); + o.compute_stamp = stamp(3); + f.renders = 1; + assert_eq!(kill_offset(&f, &t, &o), FRAGMENT_KILL); + f.renders = 0; + assert_eq!(kill_offset(&f, &t, &o), COMPUTE_KILL); + f.computes = 0; + f.renders = 2; + assert_eq!(kill_offset(&f, &t, &o), FRAGMENT_KILL); + } + + #[test] + fn drained() { + let t = Tail { + head: 15, + ta_head: 7, + fs_head: 9, + producer: [3, 4, 5], + ..Default::default() + }; + let mut o = Observed { + compute_done: 15, + compute_read: 15, + ta_done: 7, + ta_read: 7, + fs_done: 9, + fs_read: 9, + consumer: [3, 4, 5], + ..Default::default() + }; + assert!(queues_drained(&t, &o)); + o.fs_read = 8; + assert!(!queues_drained(&t, &o)); + o.fs_read = 9; + o.consumer[COMPUTE] = 4; + assert!(!queues_drained(&t, &o)); + } + + #[test] + fn failures_and_deadlines() { + assert_eq!(failure(0, Some(4)), Failure::Timeout); + assert_eq!(failure(1, Some(4)), Failure::Fault); + assert_eq!(failure(0, Some(0)), Failure::Fault); + assert_eq!(failure(0, None), Failure::Fault); + assert_eq!( + renew(1000, 5000, 1000), + (1000 + 5_000_000_000, 1000 + 1_000_000_000) + ); + assert_eq!(renew(0, 10, 0), (100_000_000, 100_000_000)); + assert_eq!(kill_deadline(7, true), 7 + 2_000_000_000); + assert!(expired(1, kill_deadline(7, false))); + assert!(!expired(5, 5)); + assert!(expired(6, 5)); + } + + fn base() -> HangInputs { + HangInputs { + flight: true, + class: 2, + recovery: true, + hang_preempt: true, + ..Default::default() + } + } + + #[test] + fn decisions() { + let never = || -> bool { panic!("compute_hang evaluated on the fast path") }; + // Idle: nothing published, nothing armed, no error. + let i = HangInputs { + flight: false, + ..base() + }; + assert_eq!(decide(&i, never).action, HangAction::Idle); + // Published, within the deadlines: wait (no compute-hang evaluation). + assert_eq!(decide(&base(), never).action, HangAction::Wait); + // Compute deadline passed but no started compute hangs: wait. + let i = HangInputs { + compute_deadline_expired: true, + ..base() + }; + assert_eq!(decide(&i, || false).action, HangAction::Wait); + // ... with a hang on class 2: arm the compute request. + let d = decide(&i, || true); + assert_eq!(d.action, HangAction::ArmCompute); + assert!(d.compute_hang && !d.class0_hang); + // ... with recovery off: no request; the host deadline is expired, the failure + // branch fail-stops (no event, so no handshake). + let i0 = HangInputs { + recovery: false, + ..i + }; + let d = decide(&i0, || true); + assert_eq!(d.action, HangAction::Fail { recover: false }); + assert!(d.class0_hang); + // ... on class 0: fail-stop at once. + let i0 = HangInputs { class: 0, ..i }; + let d = decide(&i0, || true); + assert_eq!(d.action, HangAction::Fail { recover: false }); + assert!(d.compute_hang && d.class0_hang); + // Host deadline with no event: arm the request. + let i = HangInputs { + deadline_expired: true, + ..base() + }; + assert_eq!(decide(&i, never).action, HangAction::ArmDeadline); + // ... without hang preemption: fail-stop. + let i2 = HangInputs { + hang_preempt: false, + ..i + }; + assert_eq!( + decide(&i2, never).action, + HangAction::Fail { recover: false } + ); + // ... class 0 with a compute hang: no request (it cannot end it). + let i2 = HangInputs { class: 0, ..i }; + assert_eq!( + decide(&i2, || true).action, + HangAction::Fail { recover: false } + ); + assert_eq!(decide(&i2, || false).action, HangAction::ArmDeadline); + // Armed, budget running: wait; budget over: fail-stop. + let i = HangInputs { + armed: true, + ..base() + }; + assert_eq!(decide(&i, never).action, HangAction::Wait); + let i = HangInputs { + armed: true, + deadline_expired: true, + ..base() + }; + assert_eq!( + decide(&i, never).action, + HangAction::Fail { recover: false } + ); + // Armed with nothing published (the ended prefix retired, disarm pending): wait. + let i = HangInputs { + flight: false, + armed: true, + ..base() + }; + assert_eq!(decide(&i, never).action, HangAction::Wait); + // An error event: handshake, also with nothing published. + for flight in [true, false] { + let i = HangInputs { + flight, + new_errors: true, + ..base() + }; + assert_eq!(decide(&i, never).action, HangAction::Fail { recover: true }); + } + // A receipt fault: handshake. + let i = HangInputs { + fault: true, + ..base() + }; + assert_eq!(decide(&i, never).action, HangAction::Fail { recover: true }); + // Transport failure: fail-stop, never a handshake. + let i = HangInputs { + transport: true, + new_errors: true, + ..base() + }; + assert_eq!( + decide(&i, never).action, + HangAction::Fail { recover: false } + ); + // Recovery off: an error fail-stops. + let i = HangInputs { + new_errors: true, + recovery: false, + ..base() + }; + assert_eq!( + decide(&i, never).action, + HangAction::Fail { recover: false } + ); + } +} diff --git a/drivers/gpu/drm/asahi/g15/recovery/seams.rs b/drivers/gpu/drm/asahi/g15/recovery/seams.rs new file mode 100644 index 0000000000000..eb66a14e496ab --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/recovery/seams.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The register, engine, channel and VM state recovery reads, as [`super::logic`]'s types, and +//! failing every VM. +//! +//! The other operations recovery, PB growth and the idle policy use are +//! `Firmware::{transport_error, drain_rings, devctl}` (channel.rs), `write_c1` (initdata.rs), +//! `Firmware::{progress, retire_oldest, report}` (poll.rs) and `job_partly_published` +//! (engine.rs). + +use kernel::prelude::*; + +use super::logic::{EngineState, Observed, Receipts, Tail}; +use crate::g15::channel::Channel; +use crate::g15::fw::graph::qstate; +use crate::g15::Firmware; + +impl Firmware { + // --- MMIO --- + + /// Engine busy and fault status (`Regs::engine_state`): busy = bit 0 of C020 | C120, fault + /// = D8C0 of fault banks 0 and 1 with the bank selector D800 saved and restored. Every + /// nonzero bank is logged with its decoded status (`log_fault`). + pub(crate) fn engine_state(&mut self) -> Result { + let state = self.session.regs.engine_state()?; + if state.fault() != 0 { + self.log_fault(&state); + } + Ok(EngineState { + busy: state.busy & 1 != 0, + fault: state.fault(), + }) + } + + // --- Engine --- + + /// Receipts of the published prefix `index` (0 = oldest): the `engine::Flight` fields + /// filled at publication. + pub(crate) fn receipts(&self, index: usize) -> Receipts { + let Some(f) = self.engine.flights.get(index) else { + return Receipts::default(); + }; + Receipts { + computes: f.computes, + renders: f.renders, + target: f.target, + head: f.head, + c_start: f.c_start, + ta_target: f.ta_target, + fs_target: f.fs_target, + ta_head: f.ta_head, + fs_head: f.fs_head, + ta_start: f.ta_start, + fs_start: f.fs_start, + rec: f.rec, + cstamp: f.cstamp, + cwait: f.cwait, + } + } + + /// The newest published receipts: the `engine::Engine` fields and the channel producers. + pub(crate) fn tail(&self) -> Tail { + let e = &self.engine; + Tail { + target: e.target, + head: e.head, + ta_target: e.ta_target, + fs_target: e.fs_target, + ta_head: e.ta_head, + fs_head: e.fs_head, + producer: [ + self.channels.producer(Channel::Ta), + self.channels.producer(Channel::Fs), + self.channels.producer(Channel::Compute), + ], + } + } + + /// One observation of the firmware-written receipt words: the compute/TA/FS FW stamps, the + /// compute/TA/FS work-ring DONE and READ cursors and the three submit channels' consumers. + pub(crate) fn observe(&self) -> Result { + let (g, r, ch) = (&self.graph.compute, &self.graph.render, &self.channels); + Ok(Observed { + compute_stamp: g.fw_stamp.r32(0)?, + ta_stamp: r.fw_stamp[0].r32(0)?, + fs_stamp: r.fw_stamp[1].r32(0)?, + compute_done: g.qstate.r32(qstate::DONE)?, + compute_read: g.qstate.r32(qstate::READ)?, + ta_done: r.qstate[0].r32(qstate::DONE)?, + ta_read: r.qstate[0].r32(qstate::READ)?, + fs_done: r.qstate[1].r32(qstate::DONE)?, + fs_read: r.qstate[1].r32(qstate::READ)?, + consumer: [ + ch.read(Channel::Ta)?, + ch.read(Channel::Fs)?, + ch.read(Channel::Compute)?, + ], + }) + } + + // --- VMs --- + + /// Latches `error` on every live VM (`Registry::fail_all`). + pub(crate) fn fail_all_vms(&mut self, error: Error) { + self.vms.fail_all(error) + } +} diff --git a/drivers/gpu/drm/asahi/g15/regs.rs b/drivers/gpu/drm/asahi/g15/regs.rs new file mode 100644 index 0000000000000..d5435e9f1c7ea --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/regs.rs @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! G15 MMIO: SGX identification, power, core and status windows, and ASC control. +//! +//! Windows inside `sgx` (accessed through the shared [`crate::regs::Resources`] runtime-offset +//! accessors): id `+0xd04000` (`+0` version, `+0x10` counts), open `+0xd14000` (write +//! 0x70001), core `+0xe00000` (`+0x1500` core mask, `+0x1000` pstate), status `+0xc000` +//! (`+0x20`/`+0x120` busy, `+0x1800` bank select, `+0x18c0` fault status, `+0x18c8` fault +//! address >> 6). ASC `+0x44` bit 4 is RUN (mapped from `asc` here; the shared layer does not +//! map it). No blind reads of unpowered blocks: read only after the AGX vote. +//! +//! The status window needs 64-bit stores (the fault bank select), which the shared accessors do +//! not offer, so it is mapped here, as is `asc`. This module also holds the [`Mapping`] type the +//! UAT regions use and the memory barriers. + +use kernel::{ + bindings, c_str, + device::Core, + io::{Io, Mmio, MmioRaw, Region}, + platform, + prelude::*, +}; + + + +/// Write memory barrier (`wmb()`): `dsb st`. +#[inline(always)] +pub(crate) fn wmb() { + // SAFETY: A barrier has no memory-safety requirements. + unsafe { core::arch::asm!("dsb st", options(nostack, preserves_flags)) }; +} + +/// Read memory barrier (`rmb()`): `dsb ld`. +#[inline(always)] +pub(crate) fn rmb() { + // SAFETY: A barrier has no memory-safety requirements. + unsafe { core::arch::asm!("dsb ld", options(nostack, preserves_flags)) }; +} + +/// Full memory barrier (`mb()`): `dsb sy`. +#[inline(always)] +pub(crate) fn mb() { + // SAFETY: A barrier has no memory-safety requirements. + unsafe { core::arch::asm!("dsb sy", options(nostack, preserves_flags)) }; +} + +/// How a [`Mapping`] was created. +#[derive(Copy, Clone, PartialEq, Eq)] +enum Kind { + /// `ioremap`/`ioremap_np`: device memory (registers). + Device, + /// `memremap(MEMREMAP_WC)`: normal non-cacheable memory. + WriteCombined, + /// `memremap(MEMREMAP_WB)`: System RAM through the kernel's own mapping. + WriteBack, +} + +/// A CPU mapping of a physical range that is not System RAM, accessed with the `readl`/`writel` +/// family (the accessors of [`Mmio`], with their barriers). +/// +/// # Invariants +/// +/// `raw` describes a live mapping of `raw.maxsize()` bytes created as `kind` says, which is +/// unmapped only on drop. +pub(crate) struct Mapping { + raw: MmioRaw>, + phys: u64, + kind: Kind, +} + +// SAFETY: The mapping is plain memory or registers; all accesses are single volatile accesses +// through `Mmio`, and callers serialize the protocols built on them. +unsafe impl Send for Mapping {} +// SAFETY: See `Send`. +unsafe impl Sync for Mapping {} + +impl Mapping { + /// Maps device registers (non-posted if `nonposted`). + pub(crate) fn device(phys: u64, size: usize, nonposted: bool) -> Result { + if size == 0 { + return Err(EINVAL); + } + // SAFETY: Mapping a physical range has no Rust-visible side effects; the range belongs to + // this device's `reg` resources. + let addr = unsafe { + if nonposted { + bindings::ioremap_np(phys, size) + } else { + bindings::ioremap(phys, size) + } + }; + if addr.is_null() { + return Err(ENOMEM); + } + Ok(Self { + raw: MmioRaw::new_region(addr as usize, size)?, + phys, + kind: Kind::Device, + }) + } + + /// Maps memory that is not System RAM write-combined. + pub(crate) fn write_combined(phys: u64, size: usize) -> Result { + if size == 0 { + return Err(EINVAL); + } + // SAFETY: memremap() of a range outside System RAM creates a new mapping; the range is + // an ADT-described GPU region that the caller admitted. + let addr = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WC as _) }; + if addr.is_null() { + return Err(ENOMEM); + } + Ok(Self { + raw: MmioRaw::new_region(addr as usize, size)?, + phys, + kind: Kind::WriteCombined, + }) + } + + /// Maps a boot-loader region the firmware shares with the host: write-combined when it is + /// a carveout outside System RAM (the T6030 regions), through the kernel's cacheable + /// mapping when it is a reserved-memory region inside System RAM (the T6031 regions, which + /// the earlier driver of that SoC used the same way). + pub(crate) fn region(phys: u64, size: usize) -> Result { + if size == 0 { + return Err(EINVAL); + } + let mode = super::boot::glue::memory_mode(phys, size as u64)?; + if mode == bindings::MEMREMAP_WC as u32 { + return Self::write_combined(phys, size); + } + // SAFETY: memremap() of System RAM with MEMREMAP_WB returns its existing mapping; the + // range is a reserved-memory region of the GPU node that nothing else uses. + let addr = unsafe { bindings::memremap(phys, size, bindings::MEMREMAP_WB as _) }; + if addr.is_null() { + return Err(ENOMEM); + } + Ok(Self { + raw: MmioRaw::new_region(addr as usize, size)?, + phys, + kind: Kind::WriteBack, + }) + } + + /// The `readl`/`writel` accessors of this mapping. + pub(crate) fn io(&self) -> Mmio<'_, Region<0>> { + // SAFETY: By the type invariant `raw` describes a live mapping of `maxsize` bytes. + unsafe { Mmio::from_raw(self.raw) } + } + + /// Physical address of byte 0. + pub(crate) fn phys(&self) -> u64 { + self.phys + } +} + +impl Drop for Mapping { + fn drop(&mut self) { + // SAFETY: By the type invariant the mapping is live and was created as `kind` says. + unsafe { + match self.kind { + Kind::Device => bindings::iounmap(self.raw.addr() as *mut _), + Kind::WriteCombined | Kind::WriteBack => { + bindings::memunmap(self.raw.addr() as *mut _) + } + } + } + } +} + +/// Most clusters of a supported SoC. +pub(crate) const MAX_CLUSTERS: usize = 4; + +/// Identification read at start. +#[derive(Copy, Clone, Default)] +pub(crate) struct Identity { + pub(crate) version: u32, + pub(crate) counts: u32, + /// Raw enabled-core mask (`+0xe01500`, and `+0xe01504` above 32 core slots) and its + /// per-cluster split ([`super::hw::CLUSTER_BITS`] each, cluster 0 lowest), one entry per + /// cluster of the SoC. + pub(crate) cores: u64, + pub(crate) core_masks: [u32; MAX_CLUSTERS], +} + +/// Engine receipts: busy bit and the two fault banks. +#[derive(Copy, Clone, Default)] +pub(crate) struct EngineState { + /// Bit 0 of `+0xc020 | +0xc120`. + pub(crate) busy: u64, + /// `+0xd8c0` under fault-bank selectors 0 and 1. + pub(crate) status: [u64; 2], + /// `+0xd8c8 << 6` under fault-bank selectors 0 and 1. + pub(crate) address: [u64; 2], +} + +impl EngineState { + /// Fault status of both banks, OR-ed together. + pub(crate) fn fault(&self) -> u64 { + self.status[0] | self.status[1] + } +} + +/// G15 register access. +pub(crate) struct Regs { + /// The shared SGX mapping. + sgx: Mapping, + /// ASC control (`asc`, 0x4000). + asc: Mapping, + /// SGX status window (`sgx + 0xc000`, 0x4000). + status: Mapping, +} + +impl Regs { + /// Maps the ASC control page and the SGX status window. + pub(crate) fn new(pdev: &platform::Device>) -> Result { + let window = |name: &CStr, offset: u64| -> Result { + let r = pdev.resource_by_name(name).ok_or(EINVAL)?; + let nonposted = r + .flags() + .contains(kernel::io::resource::Flags::IORESOURCE_MEM_NONPOSTED); + if offset + WINDOW as u64 > r.size() { + return Err(EINVAL); + } + Mapping::device(r.start() + offset, WINDOW, nonposted) + }; + Ok(Self { + asc: window(c_str!("asc"), 0)?, + status: window(c_str!("sgx"), SGX_STATUS as u64)?, + sgx: { + let r = pdev.resource_by_name(c_str!("sgx")).ok_or(EINVAL)?; + let size = usize::try_from(r.size())?; + Mapping::device(r.start(), size, r.flags().contains(kernel::io::resource::Flags::IORESOURCE_MEM_NONPOSTED))? + }, + }) + } + + /// Reads the identification words and the enabled-core mask (once the GPU is powered). A + /// SoC with more than 32 core slots carries the mask in two words. + pub(crate) fn identity(&self, cfg: &super::Config) -> Result { + let version = self.sgx.io().try_read32(SGX_ID)?; + let counts = self.sgx.io().try_read32(SGX_ID + 0x10)?; + let mut cores = u64::from(self.sgx.io().try_read32(SGX_CORE + 0x1500)?); + if cfg.core_slots() > 32 { + cores |= u64::from(self.sgx.io().try_read32(SGX_CORE + 0x1504)?) << 32; + } + let bits = super::hw::CLUSTER_BITS; + let mask = (1u64 << bits) - 1; + let mut core_masks = [0u32; MAX_CLUSTERS]; + for (cluster, slot) in core_masks + .iter_mut() + .enumerate() + .take((cfg.num_clusters as usize).min(MAX_CLUSTERS)) + { + *slot = ((cores >> (cluster as u32 * bits)) & mask) as u32; + } + Ok(Identity { + version, + counts, + cores, + core_masks, + }) + } + + /// Opens the SGX for the firmware. + pub(crate) fn open(&self) -> Result { + self.sgx.io().try_write32(0x70001, SGX_OPEN) + } + + /// ASC control word (`asc + 0x44`). + pub(crate) fn asc_control(&self) -> Result { + self.asc.io().try_read32(ASC_CONTROL) + } + + /// Writes the ASC control word (`asc + 0x44`). + pub(crate) fn set_asc_control(&self, value: u32) -> Result { + self.asc.io().try_write32(value, ASC_CONTROL) + } + + /// Engine receipts, under the device lock: busy is bit 0 of `C020 | C120`; the fault status + /// is `D8C0` under fault-bank selectors 0 and 1 (`D800`), with the saved selector restored. + /// 64-bit accesses. + pub(crate) fn engine_state(&self) -> Result { + let r = self.status.io(); + let mut state = EngineState { + busy: (r.try_read64(0x20)? | r.try_read64(0x120)?) & 1, + ..Default::default() + }; + let sel = r.try_read64(0x1800)?; + for bank in 0..2 { + r.try_write64(bank as u64, 0x1800)?; + state.status[bank] = r.try_read64(0x18c0)?; + state.address[bank] = r.try_read64(0x18c8)? << 6; + } + r.try_write64(sel, 0x1800)?; + Ok(state) + } +} + +/// Size of each MMIO window mapped here. +const WINDOW: usize = 0x4000; + +pub(crate) const SGX_ID: usize = 0xd04000; +pub(crate) const SGX_OPEN: usize = 0xd14000; +pub(crate) const SGX_CORE: usize = 0xe00000; +pub(crate) const SGX_STATUS: usize = 0xc000; +pub(crate) const ASC_CONTROL: usize = 0x44; +pub(crate) const ASC_RUN: u32 = 1 << 4; diff --git a/drivers/gpu/drm/asahi/g15/slots.rs b/drivers/gpu/drm/asahi/g15/slots.rs new file mode 100644 index 0000000000000..aae6d8804ee7f --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/slots.rs @@ -0,0 +1,880 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Compute slots, render pass slots and clustering buffers, and their background scrub. +//! +//! States CLEAN/USED/DIRTY/CLEARING/STALE. A retired slot becomes DIRTY and is reset on a +//! dedicated unbound, reclaim-safe workqueue (work images, sequences, TA/FS scratch, stamps, +//! dirty TPC/tilemap spans, aux/heap/scene-user, the 3 preemption areas); a claim resets +//! synchronously only if the slot is not CLEAN. Slot 0 keeps the maximum TPC/tilemap; a 17th +//! TPC slot would collide with the clustering arena. +//! +//! The slot objects never change after allocation and live in [`Scrub`], shared by the engine +//! (under the device lock) and the scrub work (which never takes the device lock); the states +//! and the dirty spans are under the scrub spinlock. A claim may flush the scrub work under the +//! device lock: the scrub work runs on its own `WQ_UNBOUND | WQ_MEM_RECLAIM` queue because +//! flushing a system-queue work from the reclaim-safe scheduler work WARNs. + +use kernel::{ + bindings, c_str, impl_has_work, new_spinlock, new_work, + prelude::*, + sync::{Arc, SpinLock}, + workqueue::{self, HasWork, Work, WorkItem}, +}; + +use super::encode::{Geometry, PassRefs}; +use super::engine::wmb; +use super::fw::work::{barrier, compute, fs, ta}; +use super::graph::{alloc, cmd_fw, cmd_shared, Client}; +use super::hw::{client, sizes, tuning}; +use super::object::{Class, FwObject, Objects}; + +/// Reset state of a slot's mutable storage. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum SlotState { + /// All zero (scrubbed) and free. + Clean, + /// Held by a prefix being prepared or published. + Used, + /// Retired, waiting for the scrub. + Dirty, + /// Being scrubbed by the scrub work. + Clearing, + /// Retired without slot rotation: scrubbed at its next claim. + Stale, +} + +/// One compute command slot. +pub(crate) struct ComputeSlot { + pub(crate) its: Arc, + /// Part of the slot's firmware layout; the host does not use it. + _uts: Arc, + pub(crate) preempt: Arc, + pub(crate) work: Arc, + pub(crate) tail: Arc, + pub(crate) seq: Arc, + pub(crate) scratch: KVec>, + /// Mixed prefix: the wait on the previous render's FS or the previous compute. + pub(crate) barrier: Arc, +} + +/// A compute slot's objects before its barrier exists (the barriers are allocated after the +/// render graph). +struct ComputeParts { + its: Arc, + uts: Arc, + preempt: Arc, + work: Arc, + tail: Arc, + seq: Arc, + scratch: KVec>, +} + +impl ComputeParts { + /// Allocates compute slot `k`'s objects. + fn new(o: &mut Objects, k: u64) -> Result { + let sh = Class::Shared; + let its = alloc(o, 16, sh, None, 0, false)?; + let uts = alloc(o, 16, Class::Fw, None, 0, false)?; + let preempt = alloc( + o, + 0x10000, + sh, + Client::nc(client::PREEMPT + k * client::PREEMPT_STRIDE), + 0, + false, + )?; + let work = alloc( + o, + compute::WORK_SIZE, + cmd_shared(), + Client::nc(client::WORK_PAGE + k * client::CWORK_STRIDE), + client::WORK_OFF as usize, + true, + )?; + let tail = alloc(o, 0x10000, sh, None, 0, false)?; + let seq = alloc(o, compute::SEQ_OBJECT, cmd_fw(), None, 0, false)?; + let mut scratch = KVec::with_capacity(5, GFP_KERNEL)?; + for _ in 0..5 { + scratch.push(alloc(o, 0x10000, sh, None, 0, false)?, GFP_KERNEL)?; + } + Ok(Self { + its, + uts, + preempt, + work, + tail, + seq, + scratch, + }) + } + + fn with_barrier(self, barrier: Arc) -> ComputeSlot { + ComputeSlot { + its: self.its, + _uts: self.uts, + preempt: self.preempt, + work: self.work, + tail: self.tail, + seq: self.seq, + scratch: self.scratch, + barrier, + } + } +} + +impl ComputeSlot { + /// Per-use reset of a retired slot's runtime storage: timestamp cells, tail and sequence + /// scratch zeroed, preemption poisoned with 0xcc. + fn scrub(&self) -> Result { + self.its.fill(0, 0, 16)?; + self.tail.fill(0, 0, 0x10000)?; + for s in self.scratch.iter() { + s.fill(0, 0, 0x10000)?; + } + self.preempt.fill(0, 0xcc, 0x10000) + } +} + +/// One render pass slot. +pub(crate) struct PassSlot { + pub(crate) aux: Arc, + pub(crate) pre: KVec>, + pub(crate) tpc: Arc, + pub(crate) tilemap: Arc, + pub(crate) heap: Arc, + pub(crate) scene_user: Arc, + pub(crate) fs_scratch: Arc, + pub(crate) fs_work: Arc, + pub(crate) ta_work: Arc, + pub(crate) ta_seq: Arc, + pub(crate) fs_seq: Arc, + pub(crate) ta_scratch: Arc, + pub(crate) scene: Arc, + pub(crate) its: Arc, + /// Part of the slot's firmware layout; the host does not use it. + _uts: Arc, + pub(crate) own_barrier: Arc, + pub(crate) prev_barrier: Arc, + pub(crate) fs_dep: Arc, +} + +impl PassSlot { + /// Allocates pass slot `i`'s objects, with their client leaf classes. + fn new(o: &mut Objects, i: u64, pre_records: u32) -> Result { + let sh = Class::Shared; + let (ps, ws) = (i * client::PASS_STRIDE, i * client::WORK_STRIDE); + let tpc = if i != 0 { + client::TPC_SLOT_SIZE + } else { + client::TPC_SIZE + }; + let tilemap = if i != 0 { + client::TILEMAP_SLOT_SIZE + } else { + client::TILEMAP_SIZE + }; + let r = pre_records as usize; + + let aux = alloc( + o, + 0x20000, + sh, + Client::nc(client::AUX + i * client::AUX_STRIDE), + 0, + false, + )?; + let mut pre = KVec::with_capacity(3, GFP_KERNEL)?; + pre.push( + alloc( + o, + r * 0x540, + sh, + Client::cached(client::PRE0_PAGE + ps), + 0x3580, + false, + )?, + GFP_KERNEL, + )?; + pre.push( + alloc( + o, + r * 0x280, + sh, + Client::cached(client::PRE1_PAGE + ps), + 0x3b00, + false, + )?, + GFP_KERNEL, + )?; + pre.push( + alloc( + o, + r * 0x40, + sh, + Client::cached(client::PRE2_PAGE + ps), + 0x3f80, + false, + )?, + GFP_KERNEL, + )?; + let tpc = alloc( + o, + tpc as usize, + sh, + Client::cached(client::TPC + i * client::TPC_SLOT_STRIDE), + 0, + false, + )?; + let tilemap = alloc( + o, + tilemap as usize, + sh, + Client::cached(client::TILEMAP + i * client::TPC_SLOT_STRIDE), + 0, + false, + )?; + let heap = alloc( + o, + 0x1000, + sh, + Client::cached(client::HEAP_META_PAGE + ps), + 0x3000, + false, + )?; + let scene_user = alloc( + o, + 0x10000, + sh, + Client::cached(client::SCENE_USER + ps), + 0, + false, + )?; + let fs_scratch = alloc( + o, + 0x10000, + sh, + Client::cached(client::FS_SCRATCH + ps), + 0, + false, + )?; + let fs_work = alloc( + o, + fs::WORK_SIZE, + cmd_shared(), + Client::nc(client::FS_WORK_PAGE + ws), + client::FS_WORK_OFF as usize, + true, + )?; + let ta_work = alloc( + o, + ta::WORK_SIZE, + cmd_shared(), + Client::nc(client::TA_WORK_PAGE + ws), + client::TA_WORK_OFF as usize, + true, + )?; + Ok(Self { + aux, + pre, + tpc, + tilemap, + heap, + scene_user, + fs_scratch, + fs_work, + ta_work, + ta_seq: alloc(o, ta::SEQ_OBJECT, cmd_fw(), None, 0, false)?, + fs_seq: alloc(o, fs::SEQ_OBJECT, cmd_fw(), None, 0, false)?, + ta_scratch: alloc(o, 0x10000, sh, None, 0, false)?, + scene: alloc(o, super::fw::work::scene::SIZE, sh, None, 0, false)?, + // Internal TA start/end, FS start/end; legacy user cells likewise. + its: alloc(o, 32, sh, None, 0, false)?, + _uts: alloc(o, 32, Class::Fw, None, 0, false)?, + own_barrier: alloc(o, barrier::SIZE, cmd_fw(), None, 0, false)?, + prev_barrier: alloc(o, barrier::SIZE, cmd_fw(), None, 0, false)?, + fs_dep: alloc(o, barrier::SIZE, cmd_fw(), None, 0, false)?, + }) + } + + /// Per-use reset of a pass slot's storage: work images, microsequences, scratch, stage + /// stamps, the TPC/tilemap spans its last use wrote, auxiliary, heap metadata and + /// scene-user (`PASS_FULL_RESET`) and the preemption save areas. The slot is neither + /// published nor claimed; the dirty spans are passed in. + fn scrub(&self, pre_records: u32, tpc_dirty: u64, tilemap_dirty: u64) -> Result { + let r = pre_records as usize; + + self.ta_work.fill(0, 0, ta::WORK_SIZE)?; + self.fs_work.fill(0, 0, fs::WORK_SIZE)?; + self.ta_seq.fill(0, 0, ta::SEQ_OBJECT)?; + self.fs_seq.fill(0, 0, fs::SEQ_OBJECT)?; + self.ta_scratch.fill(0, 0, 0x10000)?; + self.fs_scratch.fill(0, 0, 0x10000)?; + self.its.fill(0, 0, 32)?; + self.tpc.fill(0, 0, tpc_dirty as usize)?; + self.tilemap.fill(0, 0, tilemap_dirty as usize)?; + if tuning::PASS_FULL_RESET { + self.aux.fill(0, 0, 0x20000)?; + self.heap.fill(0, 0, 0x1000)?; + self.scene_user.fill(0, 0, 0x10000)?; + } + // Preemption save areas start zeroed on every use. + self.pre[0].fill(0, 0, r * 0x540)?; + self.pre[1].fill(0, 0, r * 0x280)?; + self.pre[2].fill(0, 0, r * 0x40) + } + + /// The addresses the render encoders name. + pub(crate) fn refs(&self) -> PassRefs { + PassRefs { + ta_work: self.ta_work.va, + ta_work_gva: self.ta_work.gva.unwrap_or(0), + fs_work: self.fs_work.va, + fs_work_gva: self.fs_work.gva.unwrap_or(0), + ta_seq: self.ta_seq.va, + fs_seq: self.fs_seq.va, + scene: self.scene.va, + tpc_gva: self.tpc.gva.unwrap_or(0), + tilemap_gva: self.tilemap.gva.unwrap_or(0), + heap_gva: self.heap.gva.unwrap_or(0), + scene_user_gva: self.scene_user.gva.unwrap_or(0), + pre_gva: [ + self.pre[0].gva.unwrap_or(0), + self.pre[1].gva.unwrap_or(0), + self.pre[2].gva.unwrap_or(0), + ], + fs_scratch_gva: self.fs_scratch.gva.unwrap_or(0), + aux_gva: self.aux.gva.unwrap_or(0), + its: self.its.va, + ta_scratch: self.ta_scratch.va, + } + } +} + +/// Reset states and dirty spans, under the scrub spinlock. +struct States { + compute: [SlotState; sizes::COMPUTE_SLOTS], + pass: [SlotState; sizes::RENDER_SLOTS], + /// The TPC/tilemap spans a pass slot's last use may have written. + tpc_dirty: [u64; sizes::RENDER_SLOTS], + tilemap_dirty: [u64; sizes::RENDER_SLOTS], + ct: [SlotState; sizes::CT_BUFS], + /// The span a clustering buffer's pass may have written. + ct_dirty: [u64; sizes::CT_BUFS], +} + +/// An owned `WQ_UNBOUND | WQ_MEM_RECLAIM` workqueue with one active work (allocated ordered, +/// which is the same for its single work item). +struct ClearQueue(*mut bindings::workqueue_struct); + +// SAFETY: The workqueue API is thread-safe; this only holds the pointer. +unsafe impl Send for ClearQueue {} +// SAFETY: The workqueue API is thread-safe; this only holds the pointer. +unsafe impl Sync for ClearQueue {} + +impl ClearQueue { + fn new() -> Result { + // SAFETY: The name is a valid NUL-terminated string the C side copies. + let ptr = unsafe { + bindings::alloc_ordered_workqueue( + c_str!("asahi-g15-clear").as_char_ptr(), + bindings::wq_flags_WQ_MEM_RECLAIM, + ) + }; + if ptr.is_null() { + return Err(ENOMEM); + } + Ok(Self(ptr)) + } + + fn queue(&self) -> &workqueue::Queue { + // SAFETY: The pointer is a live workqueue until `self` is dropped. + unsafe { workqueue::Queue::from_raw(self.0) } + } +} + +impl Drop for ClearQueue { + fn drop(&mut self) { + // SAFETY: The pointer came from `alloc_ordered_workqueue` and is not used afterwards; + // `destroy_workqueue` drains it first. + unsafe { bindings::destroy_workqueue(self.0) }; + } +} + +/// The slot objects and the background scrub. +#[pin_data] +pub(crate) struct Scrub { + pub(crate) compute: KVec, + pub(crate) passes: KVec, + /// Clustering buffers: a GPU-cached client view of 64 tilemap segments and four metadata + /// regions, after the ordinary TPC/tilemap slots. + pub(crate) cts: KVec>, + /// Preemption records per buffer: 2, 3 with clustering. + pub(crate) pre_records: u32, + #[pin] + states: SpinLock, + #[pin] + work: Work, + queue: ClearQueue, +} + +impl_has_work! { + impl HasWork for Scrub { self.work } +} + +impl WorkItem for Scrub { + type Pointer = Arc; + + fn run(this: Arc) { + this.clear_all(); + } +} + +impl Scrub { + /// Resets every DIRTY pass slot, compute slot and clustering buffer, outside the device + /// lock. + fn clear_all(&self) { + loop { + let taken = { + let mut st = self.states.lock(); + let k = st.pass.iter().position(|s| *s == SlotState::Dirty); + k.map(|k| { + st.pass[k] = SlotState::Clearing; + let dirty = (st.tpc_dirty[k], st.tilemap_dirty[k]); + st.tpc_dirty[k] = 0; + st.tilemap_dirty[k] = 0; + (k, dirty) + }) + }; + let Some((k, (tpc, tilemap))) = taken else { + break; + }; + let _ = self.passes[k].scrub(self.pre_records, tpc, tilemap); + wmb(); // reset before the slot can be claimed + self.states.lock().pass[k] = SlotState::Clean; + } + loop { + let taken = { + let mut st = self.states.lock(); + let k = st.compute.iter().position(|s| *s == SlotState::Dirty); + if let Some(k) = k { + st.compute[k] = SlotState::Clearing; + } + k + }; + let Some(k) = taken else { + break; + }; + let _ = self.compute[k].scrub(); + wmb(); // scrubbed before the slot can be claimed + self.states.lock().compute[k] = SlotState::Clean; + } + loop { + let taken = { + let mut st = self.states.lock(); + let k = st.ct.iter().position(|s| *s == SlotState::Dirty); + k.map(|k| { + st.ct[k] = SlotState::Clearing; + (k, st.ct_dirty[k]) + }) + }; + let Some((k, dirty)) = taken else { + return; + }; + let _ = self.cts[k].fill(0, 0, dirty as usize); + wmb(); // zeroes complete before the buffer can be claimed + let mut st = self.states.lock(); + st.ct_dirty[k] = 0; + st.ct[k] = SlotState::Clean; + } + } + + /// Waits for the scrub work. + fn flush(self: &Arc) { + // SAFETY: `self` is a live `Arc`, so its `work` field is a valid, initialized + // `work_struct` for the duration of the call. + unsafe { + let work = >::raw_get_work(Arc::as_ptr(self).cast_mut()); + bindings::flush_work(Work::raw_get(work)); + } + } + + /// Queues the scrub work on its own queue. + fn kick(self: &Arc) { + // Already queued if this fails, which is just as good. + let _ = self.queue.queue().enqueue(self.clone()); + } +} + +/// The storage claims of a prefix: compute slots, pass slots and clustering buffers. +#[derive(Copy, Clone)] +pub(crate) struct Claims { + /// Compute slot of compute member j. + pub(crate) slot: [u32; sizes::COMPUTE_SLOTS], + pub(crate) slot_count: u32, + /// Pass slot of render member j, and the set of them. + pub(crate) pass: [u32; sizes::RENDER_SLOTS], + pub(crate) pass_mask: u32, + /// Clustering buffers claimed. + pub(crate) ct: [u32; sizes::RENDER_SLOTS], + pub(crate) ct_count: u32, + /// The compute slots rotate (`tuning::SLOT_ROTATE` at the first member). + pub(crate) rotating: bool, +} + +impl Claims { + pub(crate) const fn new() -> Self { + Self { + slot: [0; sizes::COMPUTE_SLOTS], + slot_count: 0, + pass: [0; sizes::RENDER_SLOTS], + pass_mask: 0, + ct: [0; sizes::RENDER_SLOTS], + ct_count: 0, + rotating: false, + } + } +} + +/// Slot accounting statistics (`asahi-g15/stats`). +#[derive(Copy, Clone, Default)] +pub(crate) struct SlotStats { + /// Clustering buffers zeroed at claim, pass slots reset at claim, compute slots scrubbed + /// at claim. + pub(crate) ct_sync: u64, + pub(crate) pass_sync: u64, + pub(crate) slot_sync: u64, + /// Claims that waited for the scrub work, and how long (timed only with debug_flags bit 49). + pub(crate) flushes: u64, + pub(crate) flush_ns: u64, +} + +/// Slot and storage state. +pub(crate) struct Slots { + pub(crate) scrub: Arc, + /// Next compute slot to try. + pub(crate) cursor: u32, + /// Pass slots held by published or prepared prefixes. + pub(crate) pass_busy: u32, + /// Next small pass slot (1..slots-1) to try. + pub(crate) pass_cursor: u32, + pub(crate) stats: SlotStats, +} + +impl Slots { + /// Waits for the scrub work, counted. + fn flush_scrub(&mut self, scrub: &Arc) { + let mut lap = super::engine::Lap::start(); + scrub.flush(); + self.stats.flushes += 1; + lap.lap(&mut self.stats.flush_ns); + } + + /// Allocates the compute slots, the pass slots, the compute slots' barriers, the clustering + /// buffers and the scrub workqueue. + pub(crate) fn new(objects: &mut Objects) -> Result { + // Clustered tiling needs three records in each preemption buffer (two per-cluster plus + // an aggregate), each buffer then spanning two pages from the same payload offset. + let pre_records = if tuning::VERTEX_CLUSTERING { 3 } else { 2 }; + let mut compute = KVec::with_capacity(sizes::COMPUTE_SLOTS, GFP_KERNEL)?; + let mut passes = KVec::with_capacity(sizes::RENDER_SLOTS, GFP_KERNEL)?; + let mut cts = KVec::with_capacity(sizes::CT_BUFS, GFP_KERNEL)?; + + // Allocation order: the compute slots, the pass slots, then the compute slots' barriers + // and the clustering buffers. + let mut parts = KVec::with_capacity(sizes::COMPUTE_SLOTS, GFP_KERNEL)?; + for k in 0..sizes::COMPUTE_SLOTS as u64 { + parts.push(ComputeParts::new(objects, k)?, GFP_KERNEL)?; + } + for i in 0..sizes::RENDER_SLOTS as u64 { + passes.push(PassSlot::new(objects, i, pre_records)?, GFP_KERNEL)?; + } + for part in parts.drain_all() { + let barrier = alloc(objects, barrier::SIZE, cmd_fw(), None, 0, false)?; + compute.push(part.with_barrier(barrier), GFP_KERNEL)?; + } + // Clustering storage: GPU-cached client view and shared upper alias. + if pre_records == 3 { + for k in 0..sizes::CT_BUFS as u64 { + let ct = alloc( + objects, + client::CT_SIZE as usize, + Class::Shared, + Client::cached(client::CT + k * client::CT_STRIDE), + 0, + false, + )?; + cts.push(ct, GFP_KERNEL)?; + } + } + let scrub = Arc::pin_init( + try_pin_init!(Scrub { + compute, + passes, + cts, + pre_records, + states <- new_spinlock!( + States { + compute: [SlotState::Clean; sizes::COMPUTE_SLOTS], + pass: [SlotState::Clean; sizes::RENDER_SLOTS], + tpc_dirty: [0; sizes::RENDER_SLOTS], + tilemap_dirty: [0; sizes::RENDER_SLOTS], + ct: [SlotState::Clean; sizes::CT_BUFS], + ct_dirty: [0; sizes::CT_BUFS], + }, + "g15::Scrub::states" + ), + work <- new_work!("g15::Scrub::work"), + queue: ClearQueue::new()?, + }), + GFP_KERNEL, + )?; + Ok(Self { + scrub, + cursor: 0, + pass_busy: 0, + pass_cursor: 0, + stats: SlotStats::default(), + }) + } + + /// Scrubs every compute slot and marks it CLEAN. Pass slots and clustering buffers start as + /// fresh zero pages. + pub(crate) fn init(&self) -> Result { + for s in self.scrub.compute.iter() { + s.scrub()?; + } + let mut st = self.scrub.states.lock(); + st.compute = [SlotState::Clean; sizes::COMPUTE_SLOTS]; + Ok(()) + } + + /// Compute slots a new prefix can still claim: not held by a published one. + pub(crate) fn free_compute(&self) -> u32 { + let st = self.scrub.states.lock(); + st.compute.iter().filter(|s| **s != SlotState::Used).count() as u32 + } + + /// A scrubbed compute slot for compute member `j` of the prefix being prepared: the first + /// clean slot from the cursor, which follows the previous prefix's slots, so they can be + /// scrubbed while this one runs; with none clean, a retired slot is scrubbed here. Slots + /// of published prefixes are never taken. Without rotation member j takes slot j when it + /// is free. None: all held. + pub(crate) fn claim_compute(&mut self, f: &mut Claims, j: u32) -> Result> { + let scrub = self.scrub.clone(); + let slots = sizes::COMPUTE_SLOTS; + let mut found = None; + let mut sync = false; + + if j == 0 { + f.rotating = tuning::SLOT_ROTATE; + } + if !f.rotating && (j as usize) < slots { + let k = j as usize; + let mut st = scrub.states.lock(); + while st.compute[k] == SlotState::Clearing { + drop(st); + self.flush_scrub(&scrub); + st = scrub.states.lock(); + } + if st.compute[k] != SlotState::Used { + sync = st.compute[k] != SlotState::Clean; + st.compute[k] = SlotState::Used; + found = Some(k); + } + } + for _attempt in 0..2 { + if found.is_some() { + break; + } + let mut clearing = false; + { + let mut st = scrub.states.lock(); + for i in 0..slots { + let k = (self.cursor as usize + i) % slots; + if st.compute[k] == SlotState::Clean { + found = Some(k); + break; + } + } + if found.is_none() { + for i in 0..slots { + let k = (self.cursor as usize + i) % slots; + let s = st.compute[k]; + if found.is_none() && (s == SlotState::Dirty || s == SlotState::Stale) { + found = Some(k); + } + clearing |= s == SlotState::Clearing; + } + } + if let Some(k) = found { + sync = st.compute[k] != SlotState::Clean; + st.compute[k] = SlotState::Used; + } + } + if found.is_none() && !clearing { + break; // all held by published prefixes + } + if found.is_none() { + self.flush_scrub(&scrub); // the rest are being scrubbed + } + } + let Some(k) = found else { + return Ok(None); + }; + if sync { + scrub.compute[k].scrub()?; + self.stats.slot_sync += 1; + } + self.cursor = ((k + 1) % slots) as u32; + f.slot[f.slot_count as usize] = k as u32; + f.slot_count += 1; + Ok(Some(k)) + } + + /// A pass slot for a render of the given geometry, not held and not already picked: the + /// next small slot after the previous prefix's, else slot 0 (which holds any admitted + /// geometry); None if none is free. + pub(crate) fn pick_pass(&self, geo: &Geometry, picked: u32) -> Option { + let busy = self.pass_busy | picked; + let slots = sizes::RENDER_SLOTS as u32; + let passes = &self.scrub.passes; + let fits = |k: usize| { + geo.tpc_bytes <= passes[k].tpc.size as u64 + && geo.tilemap_bytes <= passes[k].tilemap.size as u64 + }; + + if slots > 1 { + for j in 0..slots - 1 { + let k = 1 + (self.pass_cursor + j) % (slots - 1); + if busy & (1 << k) == 0 && fits(k as usize) { + return Some(k as usize); + } + } + } + if busy & 1 == 0 && fits(0) { + return Some(0); + } + None + } + + /// Claims pass slot `k` for a render of the prefix being prepared: reset now unless the + /// background reset already did. + pub(crate) fn claim_pass(&mut self, k: usize) -> Result { + let scrub = self.scrub.clone(); + let (sync, tpc, tilemap) = { + let mut st = scrub.states.lock(); + while st.pass[k] == SlotState::Clearing { + drop(st); + self.flush_scrub(&scrub); + st = scrub.states.lock(); + } + let sync = st.pass[k] != SlotState::Clean; + let dirty = (st.tpc_dirty[k], st.tilemap_dirty[k]); + if sync { + st.tpc_dirty[k] = 0; + st.tilemap_dirty[k] = 0; + } + st.pass[k] = SlotState::Used; + (sync, dirty.0, dirty.1) + }; + if sync { + scrub.passes[k].scrub(scrub.pre_records, tpc, tilemap)?; + self.stats.pass_sync += 1; + } + Ok(()) + } + + /// Records the TPC/tilemap spans a claimed pass slot's render may write. + pub(crate) fn set_pass_dirty(&mut self, k: usize, tpc: u64, tilemap: u64) { + let mut st = self.scrub.states.lock(); + st.tpc_dirty[k] = tpc; + st.tilemap_dirty[k] = tilemap; + } + + /// A zeroed clustering buffer of at least `bytes` for a pass of the prefix being + /// prepared, or None: none provisioned or too small. + pub(crate) fn claim_ct(&mut self, f: &mut Claims, bytes: u64) -> Result> { + let scrub = self.scrub.clone(); + let mut found = None; + let mut clear = 0; + + if scrub.cts.is_empty() || bytes > scrub.cts[0].size as u64 { + return Ok(None); + } + for _attempt in 0..2 { + { + let mut st = scrub.states.lock(); + found = st.ct.iter().position(|s| *s == SlotState::Clean); + // None clean: take a retired one and zero it here. + if found.is_none() { + found = st.ct.iter().position(|s| *s == SlotState::Dirty); + } + if let Some(k) = found { + clear = if st.ct[k] == SlotState::Dirty { + st.ct_dirty[k] + } else { + 0 + }; + st.ct[k] = SlotState::Used; + } + } + if found.is_some() { + break; + } + self.flush_scrub(&scrub); // the rest are being cleared + } + let Some(k) = found else { + return Ok(None); + }; + if clear != 0 { + scrub.cts[k].fill(0, 0, clear as usize)?; + self.stats.ct_sync += 1; + } + scrub.states.lock().ct_dirty[k] = bytes; + f.ct[f.ct_count as usize] = k as u32; + f.ct_count += 1; + Ok(Some(k)) + } + + /// The storage of a retired (or never published) prefix: compute slots and clustering + /// buffers are reset in the background while later prefixes run; its pass slots become + /// free again. + pub(crate) fn release(&mut self, f: &mut Claims) { + let mut mask = f.pass_mask; + + self.pass_busy &= !f.pass_mask; + f.pass_mask = 0; + if f.ct_count == 0 && f.slot_count == 0 && mask == 0 { + return; + } + { + let mut st = self.scrub.states.lock(); + let mut k = 0; + while mask != 0 { + // Only a slot this prefix used is dirty; one it claimed but never wrote (a + // publication that failed before its records) keeps its state. + if mask & 1 != 0 && st.pass[k] == SlotState::Used { + st.pass[k] = SlotState::Dirty; + } + k += 1; + mask >>= 1; + } + for c in &f.ct[..f.ct_count as usize] { + st.ct[*c as usize] = SlotState::Dirty; + } + // Without rotation the next claim of a slot scrubs it (STALE). + for s in &f.slot[..f.slot_count as usize] { + st.compute[*s as usize] = if f.rotating { + SlotState::Dirty + } else { + SlotState::Stale + }; + } + } + f.ct_count = 0; + f.slot_count = 0; + if tuning::CT_ASYNC { + self.scrub.kick(); + } + } +} diff --git a/drivers/gpu/drm/asahi/g15/timestamp.rs b/drivers/gpu/drm/asahi/g15/timestamp.rs new file mode 100644 index 0000000000000..5a11f5550c059 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/timestamp.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The firmware timestamp arena. +//! +//! GEM_BIND_OBJECT with USAGE_TIMESTAMPS maps the BO range into the 64 MiB arena at +//! [`super::hw::va::TIMESTAMPS`] in the host window with the firmware/GPU shared class +//! ([`super::hw::pte::SHARED`]); the firmware writes nanoseconds directly from the start/end +//! microsequence records (four distinct values for a render). The arena is a range of the +//! adopted UAT's `kernel_vm`, so the mapping is a shared-layer `mmu::KernelMapping`, placed in +//! the arena by the shared VA allocator (ENOSPC when full). Its leaves are cleaned to the point +//! of coherency by the shared page-table code; a page table it adds reaches the firmware's view +//! through the host-window sync ([`super::object::Objects::sync_window`]) before the VA is +//! returned. +//! +//! A job retains the mappings it names, and failed jobs are retained until +//! reboot, so an arena alias the firmware may still write is never unmapped. Unmapping clears +//! leaves only: the tables stay linked. + +use core::ops::Range; +use core::sync::atomic::Ordering; +use kernel::prelude::*; + +use super::{hw, Gpu}; +use crate::{gem, mmu}; + +/// The arena in the host window. +const ARENA: Range = hw::va::TIMESTAMPS..hw::va::TIMESTAMPS + hw::va::TIMESTAMPS_SIZE; + +/// Maps a timestamp buffer into the arena (`gpu::Gpu::map_timestamp_buffer`). +pub(crate) fn map( + gpu: &Gpu, + mut bo: gem::ObjectRef, + range: Range, +) -> Result { + if gpu.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + let size = range.end.checked_sub(range.start).ok_or(EINVAL)?; + if size == 0 || (range.start | size) & mmu::UAT_PGMSK != 0 || range.end > bo.size() { + return Err(EINVAL); + } + let mapping = bo.map_range_into_range( + gpu.uat.kernel_vm(), + range, + ARENA, + mmu::UAT_PGSZ as u64, + mmu::Prot::from_pte(hw::pte::SHARED), + false, + )?; + // The firmware walks the host window through its own second-level table. + let synced = gpu + .shared + .state + .lock() + .as_deref_mut() + .ok_or(ENODEV) + .and_then(|firmware| firmware.objects.sync_window(mapping.iova_range())); + // On failure the mapping is dropped (unmapped) without the device lock. + synced?; + Ok(mapping) +} diff --git a/drivers/gpu/drm/asahi/g15/uat.rs b/drivers/gpu/drm/asahi/g15/uat.rs new file mode 100644 index 0000000000000..854ff34e729a3 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/uat.rs @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! The adopted UAT: PPL handoff, TTBAT context 0, the host window and TLB maintenance. +//! +//! The firmware's kernel root (the TABLES region, ADT `gpu-region`) and the shared L2 page are +//! not System RAM, so the shared page-table code cannot walk them. The design: +//! +//! 1. Map handoff, TTBAT, kernel root page and shared L2 (write-combined carveouts, or the +//! kernel's mapping of reserved System RAM, [`Mapping::region`]). PPL exchange: write the +//! AP magic, `cur_slot = 0xffffffff`, zero the 65 flush entries and `+0x640`, then poll the +//! firmware magic (1 s). +//! 2. `mmu::Uat::new_adopted` with `kernel_root = TABLES`: user VMs and the kernel lower VM get +//! driver-owned RAM roots; `kernel_vm` manages the host window under a RAM shadow root. +//! 3. Under the PPL lock: zero root entries 2..63 of the firmware kernel root and all of the +//! shared L2, TLBI all, root[2] = shared L2 | 3, zero TTBAT 2..127, TTBAT context 0 = +//! (kernel lower root | 1, TABLES | 1). Root entries 0..1 belong to the firmware. +//! 4. [`Window::sync_range`]: after any mapping in `kernel_vm`, copy the shadow root's window L2 +//! entries (they name RAM L3 tables) into the shared L2, before the firmware uses the new +//! addresses. Tables are never freed on unmap. +//! +//! Point of coherency: the tables are RAM pages the shared code writes through the cacheable +//! linear map, and the UAT walker does not snoop the CPU caches, so every table the firmware can +//! walk is cleaned to the point of coherency before it becomes reachable, and changed leaves are +//! cleaned before the new addresses are published: [`Window::sync_range`] for the host window, +//! [`clean_lower`] for the context-0 lower tree. (The cleaning in `pgtable.rs` makes these +//! redundant but harmless.) +//! +//! User VMs are bound to exclusive TTBAT slots for their lifetime through `mmu::Uat::bind` +//! (`vm.rs`); the shared layer writes TTBAT `(root | 1 | slot << 48, TABLES | 1 | +//! slot << 48)` under the PPL lock and invalidates the slot's ASID. + +use core::ops::Range; +use core::sync::atomic::Ordering; + +use kernel::{ + bindings, device, + io::Io, + prelude::*, + sync::Arc, + time::{delay::fsleep, Delta, Instant, Monotonic}, +}; + +use super::hw; +use super::regs::{mb, rmb, wmb, Mapping}; +use crate::driver::AsahiDrmDevice as AsahiDevice; +use crate::{mmu, pgtable}; + +/// Entries of a UAT table page. +const ENTRIES: usize = 2048; +/// Entries of the top-level table used with a 42-bit input range. +const ROOT_ENTRIES: usize = 64; +/// Input address mask of a 42-bit UAT. +const IAS_MASK: u64 = (1 << 42) - 1; +/// Valid table/leaf descriptor bits. +const VALID: u64 = 3; +/// Root entry of the host window (2). +const WINDOW_INDEX: usize = ((hw::HOST & IAS_MASK) >> 36) as usize; + +/// Root, L2 and L3 indices of `va`. +const fn indices(va: u64) -> [usize; 3] { + let va = va & IAS_MASK; + [ + ((va >> 36) & 63) as usize, + ((va >> 25) & 2047) as usize, + ((va >> 14) & 2047) as usize, + ] +} + +/// Creates the adopted UAT over the ADT regions (step 2 above). +pub(crate) fn adopt( + dev: &AsahiDevice, + cfg: &'static super::Config, + adt: &super::adt::GpuAdt, +) -> Result> { + let geometry = mmu::UatGeometry { + ias: cfg.uat_ias, + oas: cfg.uat_oas, + }; + let regions = mmu::AdoptedRegions { + handoff: adt.handoff.clone(), + ttbs: adt.ttbs.clone(), + kernel_root: adt.tables.start, + }; + Ok(Arc::new( + mmu::Uat::new_adopted(dev, geometry, ®ions)?, + GFP_KERNEL, + )?) +} + +/// Runs `f` on the entries of the RAM table page at `pa`. +fn with_table( + pa: u64, + count: usize, + f: impl FnOnce(&[core::sync::atomic::AtomicU64]) -> R, +) -> Result { + // SAFETY: Every table walked here is a page the shared page-table code allocated in RAM + // (`Page::alloc_page`) and never frees while the firmware runs (tables are not freed on + // unmap, and these VMs are retained until reboot). + unsafe { pgtable::with_root_entries(pa, count, f) } +} + +/// Cleans `len` bytes of the RAM table page at `pa`, from entry `first`, to the point of +/// coherency. +fn clean_entries(pa: u64, first: usize, count: usize) -> Result { + if first + count > ENTRIES { + return Err(EINVAL); + } + with_table(pa, ENTRIES, |entries| { + let start = entries[first..].as_ptr() as usize; + // SAFETY: `start..start + count * 8` lies inside the borrowed table page; cleaning a + // cacheable range has no other effect. + unsafe { + bindings::dcache_clean_poc(start as _, (start + count * 8) as _); + } + }) +} + +/// Cleans a whole RAM table page to the point of coherency. +fn clean_page(pa: u64) -> Result { + clean_entries(pa, 0, ENTRIES) +} + +/// Reads entry `index` of the RAM table page at `pa`. +fn entry(pa: u64, index: usize) -> Result { + with_table(pa, index + 1, |entries| { + entries[index].load(Ordering::Acquire) + }) +} + +/// Reads the leaf word that translates `va` under the RAM root `root`, or `None` if a level is +/// missing. +pub(crate) fn read_leaf(root: u64, va: u64) -> Result> { + let [i0, i1, i2] = indices(va); + let l1 = entry(root, i0)?; + if l1 & VALID != VALID { + return Ok(None); + } + let l2 = entry(l1 & hw::PA_MASK, i1)?; + if l2 & VALID != VALID { + return Ok(None); + } + Ok(Some(entry(l2 & hw::PA_MASK, i2)?)) +} + +/// Cleans every table of the context-0 lower tree (the kernel lower VM) to the point of +/// coherency: its root, then each second-level and leaf table. Called before TTBAT context 0 +/// names the root and after every lower mapping. +pub(crate) fn clean_lower(uat: &mmu::Uat) -> Result { + let root = uat.kernel_lower_root(); + let mut tables = KVec::new(); + for i in 0..ROOT_ENTRIES { + let e = entry(root, i)?; + if e & VALID == VALID { + tables.push(e & hw::PA_MASK, GFP_KERNEL)?; + } + } + let mut leaves = KVec::new(); + for &l2 in tables.iter() { + with_table(l2, ENTRIES, |entries| -> Result { + for e in entries { + let e = e.load(Ordering::Acquire); + if e & VALID == VALID { + leaves.push(e & hw::PA_MASK, GFP_KERNEL)?; + } + } + Ok(()) + })??; + } + // Leaves first, then the tables naming them, then the root. + for &pa in Iterator::chain(leaves.iter(), tables.iter()) { + clean_page(pa)?; + } + clean_page(root)?; + mb(); + Ok(()) +} + +/// Invalidates every TLB entry of the GPU UAT (`TLBI VMALLE1OS`). +pub(crate) fn tlbi_all() { + crate::mem::sync(); + crate::mem::tlbi_all(); + crate::mem::sync(); +} + +/// The firmware's view of the host window (root entry 2 -> shared L2) and the PPL handoff. +pub(crate) struct Window { + /// PPL handoff page. + handoff: Mapping, + /// TTBAT page. + ttbs: Mapping, + /// First page of the firmware kernel root tables. + root: Mapping, + /// The shared L2 page. + shared_l2: Mapping, + /// The shared L2 entries last written, and a snapshot buffer of the shadow L2. + image: KVec, + snapshot: KVec, +} + +impl Window { + /// Maps the UAT handoff regions ([`Mapping::region`]). + pub(crate) fn new(adt: &super::adt::GpuAdt) -> Result { + Ok(Self { + handoff: Mapping::region(adt.handoff.start, hw::HANDOFF_SIZE as usize)?, + ttbs: Mapping::region(adt.ttbs.start, hw::TTBS_SIZE as usize)?, + root: Mapping::region(adt.tables.start, hw::PAGE as usize)?, + shared_l2: Mapping::region(adt.shared_l2.start, hw::SHARED_L2_SIZE as usize)?, + image: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, + snapshot: KVec::from_elem(0u64, ENTRIES, GFP_KERNEL)?, + }) + } + + /// Cold-start check: all 128 TTBAT words are zero. + pub(crate) fn ttbat_is_clear(&self) -> Result { + let t = self.ttbs.io(); + for i in 0..128 { + if t.try_read64(i * 8)? != 0 { + return Ok(false); + } + } + Ok(true) + } + + /// Takes the PPL lock: AP interest, then wait while the firmware holds it, yielding the turn + /// when the firmware asks for it; 100 ms. + pub(crate) fn ppl_lock(&self) -> Result { + let h = self.handoff.io(); + let start = Instant::::now(); + let expired = || start.elapsed() > Delta::from_millis(100); + h.try_write8(1, 0x10)?; + wmb(); + while h.try_read8(0x11)? != 0 { + if h.try_read32(0x14)? != 0 { + h.try_write8(0, 0x10)?; + wmb(); + while h.try_read32(0x14)? != 0 { + if expired() { + return Err(ETIMEDOUT); + } + } + h.try_write8(1, 0x10)?; + wmb(); + } + if expired() { + h.try_write8(0, 0x10)?; + return Err(ETIMEDOUT); + } + core::hint::spin_loop(); + } + rmb(); + Ok(()) + } + + /// Releases the PPL lock. + pub(crate) fn ppl_unlock(&self) -> Result { + let h = self.handoff.io(); + wmb(); + h.try_write32(1, 0x14)?; + h.try_write8(0, 0x10)?; + wmb(); + Ok(()) + } + + /// The PPL exchange and the exclusive startup epoch: host window and TTBAT context 0 (step 3 + /// above). + pub(crate) fn bootstrap(&mut self, dev: &device::Device, uat: &mmu::Uat) -> Result { + let h = self.handoff.io(); + // AP side of the PPL exchange; never overwrite firmware interest/magic. + h.try_write64(hw::PPL_MAGIC, 0x00)?; + h.try_write32(0xffffffff, 0x18)?; + for i in 0..=64 { + for w in 0..3 { + h.try_write64(0, 0x20 + 24 * i + 8 * w)?; + } + } + h.try_write64(0, 0x640)?; + wmb(); + // Wait up to 1 s for the firmware magic. + let start = Instant::::now(); + loop { + let magic = h.try_read64(0x08)?; + if magic == hw::PPL_MAGIC { + break; + } + if start.elapsed() > Delta::from_millis(1000) { + let magic = h.try_read64(0x08)?; + if magic == hw::PPL_MAGIC { + break; + } + dev_err!(dev, "PPL firmware magic missing: {:#x}\n", magic); + return Err(ETIMEDOUT); + } + fsleep(Delta::from_micros(250)); + } + // The context-0 lower root is the kernel lower VM's RAM root, cleaned before TTBAT names + // it. + let lower = uat.kernel_lower_root(); + clean_lower(uat)?; + self.ppl_lock()?; + let result = self.epoch(lower); + self.ppl_unlock()?; + result?; + self.image.fill(0); + let r = self.root.io(); + dev_info!( + dev, + "MMU: initialized, root[0..2]={:#x} {:#x} {:#x}\n", + r.try_read64(0)?, + r.try_read64(8)?, + r.try_read64(16)? + ); + Ok(()) + } + + /// The writes of the startup epoch, under the PPL lock. + fn epoch(&self, lower: u64) -> Result { + let root = self.root.io(); + let l2 = self.shared_l2.io(); + let ttbs = self.ttbs.io(); + // Exclusive startup epoch: host entries 2.. of the upper root go first. + for i in 2..ROOT_ENTRIES { + root.try_write64(0, i * 8)?; + } + for i in 0..ENTRIES { + l2.try_write64(0, i * 8)?; + } + tlbi_all(); + root.try_write64(self.shared_l2.phys() | hw::pte::TABLE, WINDOW_INDEX * 8)?; + // TTBAT context0: owned lower root, upper root = pagetables base. + for i in 2..128 { + ttbs.try_write64(0, i * 8)?; + } + ttbs.try_write64(lower | 1, 0)?; + ttbs.try_write64(self.root.phys() | 1, 8)?; + Ok(()) + } + + /// The RAM L2 table of the shadow window, if anything is mapped in the window. + fn shadow_l2(uat: &mmu::Uat) -> Result> { + let e = entry(uat.kernel_window_root(), WINDOW_INDEX)?; + Ok((e & VALID == VALID).then_some(e & hw::PA_MASK)) + } + + /// Copies the shadow window's L2 entries into the shared L2 and cleans the leaves of `range` + /// to the point of coherency, in leaf tables the firmware can already reach. New leaf tables + /// are always cleaned whole before they are linked. + pub(crate) fn sync_range(&mut self, uat: &mmu::Uat, range: Range) -> Result { + if range.start < hw::HOST || range.end > hw::HOST + hw::HOST_SIZE || range.is_empty() { + return Err(EINVAL); + } + self.sync_inner(uat, Some(range)) + } + + fn sync_inner(&mut self, uat: &mmu::Uat, range: Option>) -> Result { + let Some(l2) = Self::shadow_l2(uat)? else { + return Ok(()); + }; + let snapshot = &mut self.snapshot; + with_table(l2, ENTRIES, |entries| { + for (out, e) in snapshot.iter_mut().zip(entries) { + *out = e.load(Ordering::Acquire); + } + })?; + let shared = self.shared_l2.io(); + for i in 0..ENTRIES { + let e = self.snapshot[i]; + if e & VALID != VALID { + if self.image[i] != 0 { + // Tables are never freed, so a linked entry cannot disappear. + return Err(EIO); + } + continue; + } + let l3 = e & hw::PA_MASK; + if self.image[i] != e { + // A new leaf table: complete it in memory before the firmware can walk into it. + clean_page(l3)?; + mb(); + shared.try_write64(e, i * 8)?; + self.image[i] = e; + continue; + } + match &range { + None => clean_page(l3)?, + Some(r) => { + let base = hw::HOST + ((i as u64) << 25); + let start = r.start.max(base); + let end = r.end.min(base + (1 << 25)); + if start < end { + let first = ((start - base) >> 14) as usize; + let last = ((end - 1 - base) >> 14) as usize; + clean_entries(l3, first, last - first + 1)?; + } + } + } + } + mb(); + Ok(()) + } + + /// Leaf tables linked into the shared L2. + pub(crate) fn l3_count(&self) -> usize { + self.image.iter().filter(|&&e| e != 0).count() + } +} diff --git a/drivers/gpu/drm/asahi/g15/vm.rs b/drivers/gpu/drm/asahi/g15/vm.rs new file mode 100644 index 0000000000000..41cc7e438a6d6 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/vm.rs @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! User VMs on G15. +//! +//! A user VM is a shared-layer [`mmu::Vm`] created by the adopted UAT, with tracked status +//! ([`mmu::Vm::with_tracked_status`]: VM_STATUS, mirror, job guards, deferred close; no residency +//! reclaim). At creation: +//! +//! - install the client aliases of the firmware objects marked for every user VM +//! ([`object::Objects::clients`]) as driver mappings at their fixed GVAs, and reserve +//! [`super::hw::client::CLIENT_APERTURE`], the pass-slot range and the PB page list +//! (`vm.install_driver_mappings`): user binds there fail (EINVAL). The aperture holds the +//! whole PB range, whose backing is mapped later, before the VM's first prefix is published +//! (`Firmware::map_pb`, counted in [`VmState::pb_chunks`]); +//! - bind it to an exclusive TTBAT slot ([`mmu::Uat::bind`]) for as long as its [`VmState`] +//! lives; at most [`super::hw::sizes::MAX_VMS`] such VMs (ENOSPC beyond, checked before +//! binding so the slot allocator never blocks). +//! +//! The [`VmState`] of a VM is shared by the [`Registry`] (while the client's VM handle exists), +//! the VM's queues and its jobs. It cannot be owned by the [`mmu::Vm`] itself: the slot binding +//! holds a `Vm` reference. The registry drops its reference once the shared lifetime reports the +//! client's handle closed ([`mmu::Vm::is_closed`]), at the next VM creation; the slot returns when +//! the last queue or job of the VM is gone (failed jobs are retained until reboot, and their +//! VM's slot with them). +//! +//! User page tables are cleaned to the point of coherency by the shared `pgtable.rs` (enabled by +//! the adopted UAT); user objects are prepared at creation (`gpu::Gpu::prepare_new_objects`). +//! User leaves use the shared GPU-shared protections (AttrIndex 2) for NC and WRITEBACK objects +//! alike. + +use core::ops::Range; +use core::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; +use kernel::{ + drm::gem::BaseObject, + new_mutex, + prelude::*, + sync::{aref::ARef, Arc, Mutex}, +}; + +use super::{hw, object, Gpu}; +use crate::{debug::*, g17::status::VmStatus, gem, mmu, module_parameters}; + +const DEBUG_CLASS: DebugFlags = DebugFlags::Mmu; + +/// Descriptor bits the shared page-table code adds to every user leaf: valid page, non-global. +const USER_LEAF_BITS: u64 = 0x803; + +/// A firmware object aliased at a fixed GVA in every user VM. +struct ClientAlias { + /// The object (a GEM kernel object); its first `size` bytes are mapped. + gem: ARef, + /// Page-aligned client GVA of the object's first page. + gva: u64, + /// Mapped size, whole pages. + size: usize, + /// The exact client leaf word ([`hw::pte::LEAF_NC`], [`hw::pte::LEAF_CACHED`] or + /// [`hw::pte::LEAF_PB_LIST`]). + leaf: u64, +} + +impl ClientAlias { + /// The client aliases of the firmware object allocator, validated: whole pages of the + /// object, inside a range reserved for the kernel, and a leaf word that round-trips exactly + /// through the shared page-table code. + fn collect(objects: &object::Objects) -> Result> { + let clients = objects.clients(); + let mut aliases = KVec::with_capacity(clients.len(), GFP_KERNEL)?; + for object in clients { + let Some((gva, leaf)) = object.client() else { + continue; + }; + let alias = ClientAlias { + gem: object.gem(), + gva, + size: object.bytes(), + leaf, + }; + alias.check()?; + aliases.push(alias, GFP_KERNEL)?; + } + Ok(aliases) + } + + fn check(&self) -> Result { + let end = self.gva.checked_add(self.size as u64).ok_or(EINVAL)?; + let leaf = mmu::Prot::from_pte(self.leaf).leaf_bits() | USER_LEAF_BITS; + if self.size == 0 + || (self.gva | self.size as u64) & mmu::UAT_PGMSK as u64 != 0 + || self.size > self.gem.size() + || leaf != self.leaf + || !reserved_ranges() + .iter() + .any(|range| range.start <= self.gva && end <= range.end) + { + pr_err!( + "G15: refusing client alias {:#x}+{:#x} leaf {:#x}\n", + self.gva, + self.size, + self.leaf + ); + return Err(EINVAL); + } + Ok(()) + } +} + +/// G15 state of one user VM, shared by the registry, its queues and its jobs. +pub(crate) struct VmState { + /// The `mmu::Vm` identifier (`Gpu::ids.vm`). + pub(crate) id: u64, + /// The VM's TTBAT slot (ASID), for the work images' context. + pub(crate) slot: u32, + /// The sticky status (shared with `file.rs`). + pub(crate) status: Arc, + /// The VM: span checks, stream reads and job guards. + pub(crate) vm: mmu::Vm, + /// The exclusive slot binding, released before the live count drops. + bind: Option, + registry: Arc, + /// PB backing chunks mapped into the VM, from the first (`Firmware::map_pb`); changed only + /// under the device lock. + pub(crate) pb_chunks: AtomicUsize, +} + +impl VmState { + /// Drops the registry's references to closed VMs (the queues call this when destroyed). + pub(crate) fn prune_registry(&self) { + self.registry.prune(); + } +} + +impl Drop for VmState { + fn drop(&mut self) { + mod_pr_debug!("G15: VM {} releases context {}\n", self.id, self.slot); + drop(self.bind.take()); + self.registry.live.fetch_sub(1, Ordering::AcqRel); + } +} + +/// Live-VM accounting and the VMs whose client handle is open. +#[pin_data] +pub(crate) struct Registry { + /// VMs whose client handle may still be open, oldest first. + #[pin] + vms: Mutex>>, + /// Live [`VmState`]s (each holds a TTBAT slot) plus reservations being created. + live: AtomicU32, +} + +/// A reserved place among the live VMs, released unless a [`VmState`] takes it over. +struct Reservation<'a>(Option<&'a Registry>); + +impl Reservation<'_> { + fn into_state(mut self) { + self.0 = None; + } +} + +impl Drop for Reservation<'_> { + fn drop(&mut self) { + if let Some(registry) = self.0 { + registry.live.fetch_sub(1, Ordering::AcqRel); + } + } +} + +impl Registry { + pub(crate) fn new() -> Result> { + Arc::pin_init( + pin_init!(Self { + vms <- new_mutex!(KVec::new(), "g15::Registry::vms"), + live: AtomicU32::new(0), + }), + GFP_KERNEL, + ) + } + + /// Reserves a place for a new VM after dropping the registry's references to closed VMs. + fn reserve(&self) -> Result> { + self.prune(); + // The shared slot allocator hands out a single slot with robust isolation. + let limit = if *module_parameters::robust_isolation.value() != 0 { + 1 + } else { + hw::sizes::MAX_VMS + }; + self.live + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |live| { + (live < limit).then_some(live + 1) + }) + .map_err(|_| ENOSPC)?; + Ok(Reservation(Some(self))) + } + + /// Drops the registry's references to VMs whose client handle is closed. Called on VM close + /// ([`closed`]), on queue destruction and before a new VM is created: a closed VM's page + /// tables, bound objects and TTBAT slot go as soon as its last queue or job is gone. + fn prune(&self) { + let mut closed = KVec::new(); + { + let mut vms = self.vms.lock(); + let mut index = 0; + while index < vms.len() { + if vms[index].vm.is_closed() && closed.push(vms[index].clone(), GFP_KERNEL).is_ok() + { + // Cannot fail: `index` is in range. + let _ = vms.remove(index); + } else { + index += 1; + } + } + } + // VM teardown may take GPUVM and reservation locks: not under the registry lock. + drop(closed); + } + + /// The driver mappings and reserved ranges of a new VM. + fn driver_mappings( + vm: &mmu::Vm, + aliases: KVec, + ) -> Result<(KVec, KVec>)> { + let mut mappings = KVec::with_capacity(aliases.len(), GFP_KERNEL)?; + for alias in aliases { + mappings.push( + vm.map_at( + alias.gva, + alias.size, + &alias.gem, + mmu::Prot::from_pte(alias.leaf), + false, + )?, + GFP_KERNEL, + )?; + } + let ranges = reserved_ranges(); + let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; + for range in ranges { + reserved.push(range, GFP_KERNEL)?; + } + Ok((mappings, reserved)) + } + + /// The state of a VM whose client handle is open. + pub(crate) fn find(&self, vm: &mmu::Vm) -> Result> { + let id = vm.id(); + self.vms + .lock() + .iter() + .find(|state| state.id == id) + .cloned() + .ok_or(ENOENT) + } + + /// Latches `error` on every VM whose client handle is open. The fail-stop path calls it under + /// the device lock. + pub(crate) fn fail_all(&self, error: Error) { + for state in self.vms.lock().iter() { + state.status.record(error); + } + } +} + +/// User VA ranges that belong to the kernel in every VM: the client aperture, the render pass +/// slots and the PB page list (the last two need 32-bit compact pointers). Every client alias +/// lies inside one of them ([`ClientAlias::check`]). +pub(crate) fn reserved_ranges() -> [Range; 3] { + use hw::client::*; + [ + CLIENT_APERTURE, + PRE0_PAGE..PRE0_PAGE + hw::sizes::RENDER_SLOTS as u64 * PASS_STRIDE, + PB_PAGE_LIST..PB_PAGE_LIST + PB_PAGE_LIST_SIZE, + ] +} + +/// Creates a user VM (`gpu::Gpu::new_vm`). +pub(crate) fn new_vm(gpu: &Gpu, kernel_range: Range) -> Result { + if gpu.shared.crashed.load(Ordering::Acquire) { + return Err(EIO); + } + // Snapshot the registry and the client aliases; map them without the device lock. + let (registry, aliases) = { + let state = gpu.shared.state.lock(); + let firmware = state.as_ref().ok_or(ENODEV)?; + ( + firmware.vms.clone(), + ClientAlias::collect(&firmware.objects)?, + ) + }; + let reservation = registry.reserve()?; + let id = gpu.ids.vm.next(); + let vm = gpu.uat.new_vm(id, kernel_range)?.with_tracked_status()?; + let status = vm.status().ok_or(EINVAL)?.clone(); + let (mappings, reserved) = Registry::driver_mappings(&vm, aliases)?; + vm.install_driver_mappings(mappings, reserved)?; + // The context names the root only now: everything above is already visible to the walker. + let bind = gpu.uat.bind(&vm)?; + let slot = bind.slot(); + // From here on the state (even one that fails to allocate) gives the place back. + reservation.into_state(); + let state = Arc::new( + VmState { + id, + slot, + status, + vm: vm.clone(), + bind: Some(bind), + registry: registry.clone(), + pb_chunks: AtomicUsize::new(0), + }, + GFP_KERNEL, + )?; + registry.vms.lock().push(state, GFP_KERNEL)?; + mod_pr_debug!("G15: VM {} context {}\n", id, slot); + Ok(vm) +} + +/// A client's VM handle is gone (`gpu::Gpu::vm_closed`): drop the registry's references to +/// closed VMs now, outside the device lock, instead of at the next VM creation. +pub(crate) fn closed(gpu: &Gpu) { + let registry = gpu + .shared + .state + .lock() + .as_ref() + .map(|firmware| firmware.vms.clone()); + if let Some(registry) = registry { + registry.prune(); + } +} + +/// The state of a VM for a new queue. +pub(crate) fn lookup(gpu: &Gpu, vm: &mmu::Vm) -> Result> { + let registry = gpu.shared.state.lock().as_ref().ok_or(ENODEV)?.vms.clone(); + registry.find(vm) +} diff --git a/drivers/gpu/drm/asahi/g15/wire.rs b/drivers/gpu/drm/asahi/g15/wire.rs new file mode 100644 index 0000000000000..6fbb360e11a61 --- /dev/null +++ b/drivers/gpu/drm/asahi/g15/wire.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Private G15 wire for Mesa 751 / public SF `asahi_m3_drm.h`. +//! +//! G15 modules that decode commands or advertise features do +//! `use crate::g15::wire as uapi`. Normal and G17 code keep `kernel::uapi`. +//! This module is not installed UAPI and is not a bindgen input. +//! `include/uapi/drm/asahi_drm.h` stays the canonical header: render 448 bytes +//! with `prior_vdm` at 320, compute 200 bytes with `scs_layout` at 128. +//! +//! Native feature numbers, matching that Mesa header. Canonical bits 10..15 +//! are a different assignment; Mesa rejects any of them (`0xfc00`). +//! +//! | bit | native name | canonical name at the same bit | +//! | 0 | SOFT_FAULTS | SOFT_FAULTS | +//! | 1 | VM_STATUS | VM_STATUS | +//! | 2 | INDEPENDENT_QUEUES | INDEPENDENT_QUEUES | +//! | 3 | PER_COMMAND_INPUT_SYNCS | SCHEDULED_QUEUES | +//! | 4 | VM_STATUS_MIRROR | RENDER_TIMESTAMP_COPIES | +//! | 5 | EXACT_PRIOR_BARRIERS | COMPUTE_TIMESTAMP_COPIES | +//! | 6 | FEW_PRIMITIVES | FRAGMENT_DEPENDENCY | +//! | 7 | FRAGMENT_BARRIERS | FRAGMENT_DEPENDENCY_COMPUTE | +//! | 8 | RENDER_RAY_QUERY | COMPUTE_WIDE_VISIBILITY | + +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +use core::mem::{offset_of, size_of}; + +use kernel::uapi; + +use crate::util::AnyBitPattern; + +/// Canonical bits 10..15. A G15 feature word with any of these set is the +/// maintained assignment, which Mesa 751 refuses for chip 0x6030 and 0x6031. +pub(crate) const LEGACY_FEATURE_MASK: u64 = 0xfc00; + +// Bindgen names, so `use crate::g15::wire as uapi` still matches params.rs. +// Scoped to these nine items; the rest of the module stays under the lint. +#[allow(non_upper_case_globals)] +pub(crate) const drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS: u64 = 1 << 0; +#[allow(non_upper_case_globals)] +pub(crate) const drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS: u64 = 1 << 1; +#[allow(non_upper_case_globals)] +pub(crate) const drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES: u64 = 1 << 2; +#[allow(non_upper_case_globals)] +pub(crate) const drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS: u64 = 1 << 3; +#[allow(non_upper_case_globals)] +pub(crate) const drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR: u64 = 1 << 4; +#[allow(non_upper_case_globals)] +pub(crate) const drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS: u64 = 1 << 5; +#[allow(non_upper_case_globals)] +pub(crate) const drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES: u64 = 1 << 6; +#[allow(non_upper_case_globals)] +pub(crate) const drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS: u64 = 1 << 7; +#[allow(non_upper_case_globals)] +pub(crate) const drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY: u64 = 1 << 8; + +/// What `Gpu::feature_flags` returns for G15. This replaces the shared default +/// (canonical VM_STATUS | VM_STATUS_MIRROR | INDEPENDENT_QUEUES | optional +/// SOFT_FAULTS). Mirror is native bit 4. Soft faults, few-primitives and ray +/// query are not advertised. Command bits 3, 5 and 7 stay in `params::fill` +/// via this module's names, so this constant does not include them. +pub(crate) const SHARED_FEATURE_FLAGS: u64 = drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS + | drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES + | drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR; + +/// Command-feature bits `params::fill` ORs. Bit 7 only while early tiling is on. +pub(crate) const fn command_feature_flags(early_tiling: bool) -> u64 { + let mut flags = drm_asahi_feature_DRM_ASAHI_FEATURE_PER_COMMAND_INPUT_SYNCS + | drm_asahi_feature_DRM_ASAHI_FEATURE_EXACT_PRIOR_BARRIERS; + if early_tiling { + flags |= drm_asahi_feature_DRM_ASAHI_FEATURE_FRAGMENT_BARRIERS; + } + flags +} + +/// Full G15 GET_PARAMS word: `0x3e`, or `0xbe` with early tiling. +pub(crate) const fn feature_word(early_tiling: bool) -> u64 { + SHARED_FEATURE_FLAGS | command_feature_flags(early_tiling) +} + +/// Ioctl types and constants whose layout and values match the canonical header. +/// Render flags used by G15 are the same numbers on both headers (1<<0, 1<<1, +/// 1<<2, 1<<4, 1<<18, 1<<20). Feature constants are not reexported. +pub(crate) use uapi::{ + drm_asahi_attachment, drm_asahi_bg_eot, drm_asahi_cmd_header, drm_asahi_cmd_type_DRM_ASAHI_CMD_COMPUTE, + drm_asahi_cmd_type_DRM_ASAHI_CMD_RENDER, drm_asahi_cmd_type_DRM_ASAHI_SET_COMPUTE_ATTACHMENTS, + drm_asahi_cmd_type_DRM_ASAHI_SET_FRAGMENT_ATTACHMENTS, + drm_asahi_cmd_type_DRM_ASAHI_SET_VERTEX_ATTACHMENTS, drm_asahi_gpu_hal_generation_DRM_ASAHI_GPU_HAL_LEGACY, + drm_asahi_helper_program, drm_asahi_params_global, drm_asahi_queue_limits, + drm_asahi_render_flags_DRM_ASAHI_RENDER_DBIAS_IS_INT, + drm_asahi_render_flags_DRM_ASAHI_RENDER_FRAGMENT_BARRIERS, + drm_asahi_render_flags_DRM_ASAHI_RENDER_NO_VERTEX_CLUSTERING, + drm_asahi_render_flags_DRM_ASAHI_RENDER_PROCESS_EMPTY_TILES, + drm_asahi_render_flags_DRM_ASAHI_RENDER_RSRC_SPEC_HI, + drm_asahi_render_flags_DRM_ASAHI_RENDER_VERTEX_SCRATCH, drm_asahi_timestamp, + drm_asahi_timestamps, drm_asahi_zls_buffer, DRM_ASAHI_BARRIER_NONE, +}; + +/// Native render command. 384 bytes. The first 256 match the canonical prefix +/// through `bg_partial_rsrc_spec_hi` (offset 252). `prior_vdm` is at 256 and +/// `prior_cdm` at 264. `ray_query` is at 272 and is not advertised: a nonzero +/// byte from there on is rejected by `command::check_stream`. A 256-byte packet +/// zero-fills the priors; a 272-byte packet zero-fills `ray_query`. +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub(crate) struct drm_asahi_cmd_render { + pub flags: u32, + pub isp_zls_pixels: u32, + pub vdm_ctrl_stream_base: u64, + pub vertex_helper: drm_asahi_helper_program, + pub fragment_helper: drm_asahi_helper_program, + pub isp_scissor_base: u64, + pub isp_dbias_base: u64, + pub isp_oclqry_base: u64, + pub depth: drm_asahi_zls_buffer, + pub stencil: drm_asahi_zls_buffer, + pub zls_ctrl: u64, + pub ppp_multisamplectl: u64, + pub sampler_heap: u64, + pub ppp_ctrl: u32, + pub width_px: u16, + pub height_px: u16, + pub layers: u16, + pub sampler_count: u16, + pub utile_width_px: u8, + pub utile_height_px: u8, + pub samples: u8, + pub sample_size_B: u8, + pub isp_merge_upper_x: u32, + pub isp_merge_upper_y: u32, + pub bg: drm_asahi_bg_eot, + pub eot: drm_asahi_bg_eot, + pub partial_bg: drm_asahi_bg_eot, + pub partial_eot: drm_asahi_bg_eot, + pub isp_bgobjdepth: u32, + pub isp_bgobjvals: u32, + pub ts_vtx: drm_asahi_timestamps, + pub ts_frag: drm_asahi_timestamps, + pub bg_eot_rsrc_spec_hi: u32, + pub bg_eot_partial_rsrc_spec_hi: u32, + pub bg_rsrc_spec_hi: u32, + pub bg_partial_rsrc_spec_hi: u32, + pub prior_vdm: u64, + pub prior_cdm: u64, + pub ray_query: [uapi::drm_asahi_render_ray_query_stage; 2], +} + +/// Native compute command. 136 bytes. The 64-byte prefix ends at `ts`. +/// `scs_layout` is at 64, `prior_vdm` at 72, `prior_cdm` at 80. G15 rejects a +/// nonzero `scs_layout` and the intersection tail; those fields still sit at +/// the native offsets so a 136-byte packet is not read as canonical scratch. +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub(crate) struct drm_asahi_cmd_compute { + pub flags: u32, + pub sampler_count: u32, + pub cdm_ctrl_stream_base: u64, + pub cdm_ctrl_stream_end: u64, + pub sampler_heap: u64, + pub helper: drm_asahi_helper_program, + pub ts: drm_asahi_timestamps, + pub scs_layout: u64, + pub prior_vdm: u64, + pub prior_cdm: u64, + pub cdm_auxiliary_allocation: u32, + pub reserved: u32, + pub intersection_pool: u64, + pub intersection_pool_bytes: u64, + pub intersection_first: u32, + pub intersection_count: u32, + pub intersection_default_program: u64, + pub intersection_default_control: u64, +} + +// SAFETY: Every field is an integer or a `repr(C)` UAPI struct of integers. +// `command::payload` zero-fills a short record and rejects a nonzero tail. +unsafe impl AnyBitPattern for drm_asahi_cmd_render {} +// SAFETY: Same as `drm_asahi_cmd_render`. +unsafe impl AnyBitPattern for drm_asahi_cmd_compute {} + +const _: () = { + assert!(size_of::() == 448); + assert!(offset_of!(uapi::drm_asahi_cmd_render, prior_vdm) == 320); + assert!(size_of::() == 200); + assert!(offset_of!(uapi::drm_asahi_cmd_compute, scs_layout) == 128); + + assert!(size_of::() == 384); + assert!(offset_of!(drm_asahi_cmd_render, bg_partial_rsrc_spec_hi) == 252); + assert!(offset_of!(drm_asahi_cmd_render, prior_vdm) == 256); + assert!(offset_of!(drm_asahi_cmd_render, prior_cdm) == 264); + assert!(offset_of!(drm_asahi_cmd_render, ray_query) == 272); + + assert!(size_of::() == 136); + assert!(offset_of!(drm_asahi_cmd_compute, scs_layout) == 64); + assert!(offset_of!(drm_asahi_cmd_compute, prior_vdm) == 72); + assert!(offset_of!(drm_asahi_cmd_compute, prior_cdm) == 80); + assert!(offset_of!(drm_asahi_cmd_compute, cdm_auxiliary_allocation) == 88); + assert!(offset_of!(drm_asahi_cmd_compute, intersection_default_control) == 128); + + assert!(SHARED_FEATURE_FLAGS == 0x16); + assert!(command_feature_flags(false) == 0x28); + assert!(feature_word(false) == 0x3e); + assert!(feature_word(true) == 0xbe); + assert!(feature_word(true) & LEGACY_FEATURE_MASK == 0); + assert!(drm_asahi_feature_DRM_ASAHI_FEATURE_RENDER_RAY_QUERY == 1 << 8); + assert!(drm_asahi_feature_DRM_ASAHI_FEATURE_FEW_PRIMITIVES == 1 << 6); + assert!(drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS == 1); +}; diff --git a/drivers/gpu/drm/asahi/g17/channel.rs b/drivers/gpu/drm/asahi/g17/channel.rs index 8e4740b76a166..d15bac5b3618e 100644 --- a/drivers/gpu/drm/asahi/g17/channel.rs +++ b/drivers/gpu/drm/asahi/g17/channel.rs @@ -99,7 +99,11 @@ impl Rings { ReportRing::Trace => init.status_read_words::(role, offset)?[0], ReportRing::Statistics => init.status_read_words::<1>(role, offset)?[0], }; - if tag >= u32::BITS || allowed_types & (1 << tag) == 0 { + // Statistics records carry nothing the host uses; one of a type it does not + // know is skipped like the others (the firmware emits types up to 32). + if matches!(ring, ReportRing::Trace) + && (tag >= u32::BITS || allowed_types & (1 << tag) == 0) + { return Err(EIO); } consumer = (consumer + 1) & (capacity - 1); diff --git a/drivers/gpu/drm/asahi/g17/freelist.rs b/drivers/gpu/drm/asahi/g17/freelist.rs index c9c9ebda7f6bc..6e01f6df4322d 100644 --- a/drivers/gpu/drm/asahi/g17/freelist.rs +++ b/drivers/gpu/drm/asahi/g17/freelist.rs @@ -483,6 +483,7 @@ impl ComputePool { state_va: state.gpu_va(), }), )?; + control.write(FreeListControl::EVENT_FLAG_ENABLE, 1u32)?; fence(Ordering::SeqCst); Ok(Self { pages: Some(pages), @@ -539,6 +540,8 @@ impl ComputePool { }); header.unk_54[..8].copy_from_slice(&bytes); self.control.write(0, header)?; + self.control + .write(FreeListControl::EVENT_FLAG_ENABLE, 1u32)?; fence(Ordering::SeqCst); self.pages = Some(pages); Ok(()) diff --git a/drivers/gpu/drm/asahi/g17/fw/compute.rs b/drivers/gpu/drm/asahi/g17/fw/compute.rs index 0633da7f43541..a3e2899d26502 100644 --- a/drivers/gpu/drm/asahi/g17/fw/compute.rs +++ b/drivers/gpu/drm/asahi/g17/fw/compute.rs @@ -20,8 +20,16 @@ use super::{ pub(crate) const COMPUTE_DESCRIPTOR_SIZE: usize = 0x4000; /// QoS class of compute kicks. pub(crate) const COMPUTE_QOS_CLASS: u8 = 0x08; -/// Event mask of compute kicks. -pub(crate) const COMPUTE_KICK_EVENT_MASK: [u64; 4] = [0, 0, 4, 0]; +/// Event mask of a compute kick: one bit at `priority * 64 + context_id` +/// (AGXG17X AGXCLChannelSKSM::submitBuffer). +pub(crate) fn compute_kick_event_mask(priority: u8, context_id: u32) -> Result<[u64; 4]> { + if priority >= 3 || context_id >= 64 { + return Err(EINVAL); + } + let mut mask = [0u64; 4]; + mask[usize::from(priority)] = 1 << context_id; + Ok(mask) +} const COMPUTE_REGISTERS: usize = 41; const COMPUTE_REGISTERS_MAX: usize = COMPUTE_REGISTERS + 1; diff --git a/drivers/gpu/drm/asahi/g17/fw/initdata.rs b/drivers/gpu/drm/asahi/g17/fw/initdata.rs index 1f25bd0fc1c0d..36c5dcd3ea48f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/fw/initdata.rs @@ -109,6 +109,8 @@ impl LevelDescriptor { /// Addresses and geometry common to both roots. pub(crate) struct RootArgs { + /// Firmware interface version ([`INTERFACE_VERSION`] for G17P). + pub(crate) version: [u16; 4], /// Region A. pub(crate) region_a_va: u64, /// Region C. @@ -166,7 +168,7 @@ static_assert!(size_of::() == 0xb8); impl Root { fn new(role: Role, args: &RootArgs, status_b_va: u64) -> Self { Self { - version: INTERFACE_VERSION, + version: args.version, region_a_va: args.region_a_va, unk_10: 0, main_config_va: args.main_config_va, @@ -572,6 +574,15 @@ impl MainConfig { self.unk_300 = 4; self.power_config_va = args.bundle_va + bundle::POWER_CONFIG as u64; } + + /// Initializes the g17c secondary configuration; its firmware finds the + /// [`PowerConfig`] 0x10 bytes after the G17P field. + pub(crate) fn init_secondary_g17c(&mut self, args: &MainConfigArgs) { + self.init_secondary(args); + let va = self.power_config_va; + self.power_config_va = 0; + self.unk_479[8..16].copy_from_slice(&va.to_le_bytes()); + } } // --------------------------------------------------------------------------- @@ -593,6 +604,10 @@ pub(crate) struct RegisterWindow { pub(crate) va: u64, /// Size in bytes; need not be a multiple of the page size. pub(crate) size: u32, + /// Equal elements the window holds, at `size / elements` strides of the GPU address. + pub(crate) elements: u32, + /// Physical distance between elements; 0 when they are contiguous. + pub(crate) phys_stride: u64, /// Value of [`RegisterMapEntry::unk_18`]. pub(crate) unk_18: u64, /// Value of [`RegisterMapEntry::unk_20`]. @@ -651,7 +666,7 @@ impl RegisterMapEntry { phys: window.phys, va: window.va, total_size: window.size, - element_size: window.size, + element_size: window.size / window.elements, unk_18: window.unk_18, unk_20: window.unk_20, unk_24: 0, @@ -832,13 +847,41 @@ pub(crate) struct HwDataArgs<'a> { pub(crate) qos_va: u64, } +/// Register-map slots of the g17c hardware data; everything after the map moves by +/// 7 slots (0x118 bytes). +pub(crate) const G17C_REGISTER_MAP_SLOTS: usize = 60; +/// The g17c chip identifier is a u64 (AGXG17X AGXFirmware::initFirmwareData), which moves +/// the fields after it to G17P + 0x11c. +pub(crate) const G17C_CHIP_ID_HI: usize = 4; +/// Bytes the g17c hardware data inserts after [`HwData::unk_2554`] (to G17P + 0x124). +pub(crate) const G17C_HW_DATA_GAP_A: usize = 8; +/// Bytes the g17c hardware data inserts after [`HwData::unk_25b4`] (to G17P + 0x128). +pub(crate) const G17C_HW_DATA_GAP_B: usize = 4; +/// Bytes the g17c hardware data inserts after [`HwData::ksm_pause_timeout_us`] (to G17P + 0x12c): +/// one u32 the firmware skips while it is zero. +pub(crate) const G17C_HW_DATA_GAP_C: usize = 4; +/// The g17c hardware data. +pub(crate) type HwDataG17c = HwData< + G17C_REGISTER_MAP_SLOTS, + G17C_CHIP_ID_HI, + G17C_HW_DATA_GAP_A, + G17C_HW_DATA_GAP_B, + G17C_HW_DATA_GAP_C, +>; + /// Size of the QoS object named by the hardware data. pub(crate) const QOS_SIZE: usize = 0xe40; /// The hardware data, at the base of the bundle and shared by both /// instances. #[repr(C, packed)] -pub(crate) struct HwData { +pub(crate) struct HwData< + const SLOTS: usize = REGISTER_MAP_SLOTS, + const CHIP_HI: usize = 0, + const GAP_A: usize = 0, + const GAP_B: usize = 0, + const GAP_C: usize = 0, +> { /// Required: 0x6f_0000_0000. pub(crate) unk_0: u64, /// Required: 0xffc0_0000. @@ -866,10 +909,12 @@ pub(crate) struct HwData { pub(crate) unk_5d8: [u16; 12], pub(crate) unk_5f0: [u8; 0x50], /// Register windows the firmware maps. - pub(crate) register_map: [RegisterMapEntry; REGISTER_MAP_SLOTS], + pub(crate) register_map: [RegisterMapEntry; SLOTS], pub(crate) unk_e88: [u8; 8], /// Chip identifier. pub(crate) chip_id: u32, + /// High half of the g17c chip identifier ([`G17C_CHIP_ID_HI`]), zero. + pub(crate) chip_id_hi: [u8; CHIP_HI], /// Required: 1. pub(crate) unk_e94: u32, /// Required: 1. @@ -964,6 +1009,8 @@ pub(crate) struct HwData { pub(crate) unk_2550: u32, /// Required: 7. pub(crate) unk_2554: u32, + /// Bytes the g17c hardware data adds here ([`G17C_HW_DATA_GAP_A`]). + pub(crate) unk_g17c_a: [u8; GAP_A], pub(crate) unk_2558: [u8; 0x10], /// Required: 5. pub(crate) unk_2568: u32, @@ -980,9 +1027,14 @@ pub(crate) struct HwData { pub(crate) unk_25b0: [u8; 4], /// Required: 0xffff_ffff in every entry, one per work channel. pub(crate) unk_25b4: [u32; WORK_CHANNELS], + /// Bytes the g17c hardware data adds here ([`G17C_HW_DATA_GAP_B`]). + pub(crate) unk_g17c_b: [u8; GAP_B], pub(crate) unk_25e4: [u8; 0x10], - /// Required: 1. - pub(crate) unk_25f4: u32, + /// Limit of the firmware's KSM scheduling-pause poll, in microseconds + /// (agfKSMKickQueuesSchedulingPause). AGXG17X AGXArmFirmware::initFirmwareData: 150. + pub(crate) ksm_pause_timeout_us: u32, + /// Bytes the g17c hardware data adds here ([`G17C_HW_DATA_GAP_C`]). + pub(crate) unk_g17c_c: [u8; GAP_C], pub(crate) unk_25f8: [u8; 8], /// Required: 1. pub(crate) unk_2600: u32, @@ -1021,12 +1073,35 @@ static_assert!(core::mem::offset_of!(HwData, unk_258e) == 0x258e); static_assert!(core::mem::offset_of!(HwData, completion_rings) == 0x2614); static_assert!(core::mem::offset_of!(HwData, qos_va) == 0x26ac); static_assert!(size_of::() == 0x3db4); +static_assert!(core::mem::offset_of!(HwDataG17c, chip_id) == 0xfa8); +static_assert!(core::mem::offset_of!(HwDataG17c, unk_ed0) == 0xfec); +static_assert!(core::mem::offset_of!(HwDataG17c, unk_f04) == 0x1020); +static_assert!(core::mem::offset_of!(HwDataG17c, perf_a) == 0x10e0); +static_assert!(core::mem::offset_of!(HwDataG17c, perf_b_freq_mhz) == 0x1924); +static_assert!(core::mem::offset_of!(HwDataG17c, perf_a_states) == 0x1ae4); +static_assert!(core::mem::offset_of!(HwDataG17c, perf_b) == 0x1df4); +static_assert!(core::mem::offset_of!(HwDataG17c, unk_2544) == 0x2660); +static_assert!(core::mem::offset_of!(HwDataG17c, unk_2558) == 0x267c); +static_assert!(core::mem::offset_of!(HwDataG17c, unk_25b4) == 0x26d8); +static_assert!(core::mem::offset_of!(HwDataG17c, ksm_pause_timeout_us) == 0x271c); +static_assert!(core::mem::offset_of!(HwDataG17c, unk_g17c_c) == 0x2720); +static_assert!(core::mem::offset_of!(HwDataG17c, completion_rings) == 0x2740); +static_assert!(core::mem::offset_of!(HwDataG17c, qos_va) == 0x27d8); +static_assert!(bundle::HW_DATA + size_of::() <= bundle::VIEWS[2]); static_assert!(bundle::HW_DATA + size_of::() <= bundle::VIEWS[2]); // SAFETY: `HwData` consists of integers and arrays of integers only. -unsafe impl Zeroable for HwData {} +unsafe impl + Zeroable for HwData +{ +} + +impl + HwData +{ + /// The g17c layout, whose constants differ from G17P's in places. + const G17C: bool = CHIP_HI == G17C_CHIP_ID_HI; -impl HwData { const UNK_E0: [u16; 24] = [ 0x2008, 0, 0, 0, 0, 0x2008, 0, 0, 0, 0, 0x2008, 0, 0x24cb, 0, 0x2cfa, 0xc98a, 0x24cb, 0xf4f6, 0xe917, 0x1877, 0x24cb, 0x38d9, 0, 0xbdab, @@ -1056,7 +1131,7 @@ impl HwData { self.unk_3e0 = Self::UNK_3E0; self.unk_5d8 = Self::UNK_5D8; - let mut used = [false; REGISTER_MAP_SLOTS]; + let mut used = [false; SLOTS]; let mut claim = |slot: usize| -> Result { match used.get_mut(slot) { Some(used) if !*used => { @@ -1080,7 +1155,12 @@ impl HwData { self.unk_e98 = 1; self.unk_ea0 = 1; self.unk_ea4 = 1; - self.unk_eb8 = 1; + // AGXG17X stores the words 0, 0, 0, 1 from 0xfcc: 0 here, 1 in the word after. + if Self::G17C { + self.unk_ebc = 1u32.to_le_bytes(); + } else { + self.unk_eb8 = 1; + } self.unk_ec0 = 1; self.unk_ec8 = 1; self.unk_ed0 = 0x5dc0; @@ -1095,7 +1175,7 @@ impl HwData { self.unk_f6c = 1; self.unk_f88 = 6; self.unk_f8c = 1; - self.unk_fac = 1; + self.unk_fac = if Self::G17C { 0 } else { 1 }; self.unk_fb8 = 0x1e; self.unk_fbc = 4; self.unk_fc0 = 6; @@ -1110,8 +1190,14 @@ impl HwData { self.perf_a_states[index] = perf.states_a[index].into(); self.perf_b_states[index] = perf.states_b[index].into(); } - self.perf_a_relative = perf.relative_a; - self.perf_b_relative = perf.relative_b; + if Self::G17C { + // AGXG17X populateLinearPowerTransferTable: f32 percentages. + self.perf_a_relative = perf.relative_a.map(f32_bits); + self.perf_b_relative = perf.relative_b.map(f32_bits); + } else { + self.perf_a_relative = perf.relative_a; + self.perf_b_relative = perf.relative_b; + } self.unk_1848 = [0x3f82_8f5c; 11]; self.unk_2544 = 4; self.unk_2548 = 6; @@ -1122,10 +1208,11 @@ impl HwData { self.unk_2570 = 1; self.unk_258e = WORK_CHANNELS as u16; - self.unk_2590 = [0, 1, 1, 1]; + self.unk_2590 = if Self::G17C { [0, 0, 1, 0] } else { [0, 1, 1, 1] }; self.unk_25a0 = [0, 1, 1, 1]; self.unk_25b4 = [u32::MAX; WORK_CHANNELS]; - self.unk_25f4 = 1; + // G17P's value of 1 us is shorter than the G17C KSM's pause acknowledgement. + self.ksm_pause_timeout_us = if Self::G17C { 150 } else { 1 }; self.unk_2600 = 1; self.unk_2608 = 1; self.unk_2610 = 0x100; @@ -1143,6 +1230,11 @@ impl HwData { self.unk_26b4 = 1; self.unk_26b8 = 1; self.unk_26e8 = 1; + if Self::G17C { + // AGXG17X AGXArmFirmware::initFirmwareData: 1 at 0x27ec and a u64 1 at 0x281c. + self.unk_26bc[4..8].copy_from_slice(&1u32.to_le_bytes()); + self.unk_26ec[4..8].copy_from_slice(&1u32.to_le_bytes()); + } self.unk_3d98 = 0xffff_ffff; self.unk_3db0 = 0xffff_ffff; Ok(()) @@ -1475,6 +1567,12 @@ impl PrimaryStatusB { /// Offset of the [`PowerState`]. pub(crate) const POWER_STATE: usize = 0xe440; + /// g17c: the primary's flag (u32), raised before it reads the secondary's flag; it lies + /// inside G17P's [`PowerState`], which leaves those bytes zero. + pub(crate) const G17C_FLAG: usize = 0xe71c; + /// g17c: pointer (u64) to the secondary's flag ([`PowerState::G17C_FLAG`]). + const G17C_PEER_FLAG: usize = 0xe720; + /// Initializes the primary status B; `fwctl_va` is the zeroed /// firmware-control page ([`FWCTL_SIZE`] bytes). pub(crate) fn init(&mut self, fwctl_va: u64) { @@ -1485,6 +1583,17 @@ impl PrimaryStatusB { self.unk_e434 = 1; self.power.init(); } + + /// Initializes the g17c primary status B. Its recovery area is laid out differently, + /// so only the status block, the power state and the pointer to the secondary's flag + /// (`peer_flag_va`) are set. + pub(crate) fn init_g17c(&mut self, peer_flag_va: u64) { + clear(self); + self.status = StatusBlock::new(); + self.power.init(); + let at = Self::G17C_PEER_FLAG - Self::POWER_STATE; + self.power.0[at..at + 8].copy_from_slice(&peer_flag_va.to_le_bytes()); + } } @@ -1538,6 +1647,21 @@ type Words = &'static [(usize, u32, usize)]; fn fill_words(bytes: &mut [u8], words: Words) { bytes.fill(0); + put_words(bytes, words); +} + +/// IEEE-754 single-precision bits of `value`, exact below 2^24. +const fn f32_bits(value: u32) -> u32 { + if value == 0 { + return 0; + } + let exponent = 31 - value.leading_zeros(); + let mantissa = (value << (23 - exponent)) & 0x7f_ffff; + (127 + exponent) << 23 | mantissa +} + +/// Stores `words` into `bytes` without clearing it first. +fn put_words(bytes: &mut [u8], words: Words) { for &(offset, value, repeat) in words { for index in 0..repeat { let at = offset + 4 * index; @@ -1620,6 +1744,26 @@ const POWER_CONFIG_HEAD_WORDS: Words = &[ (0x1c98, 0xff7fff7f, 74), ]; +/// The die temperature limiter, relative to its place in the tail: G17P 0x550, and 0x34 +/// bytes later on g17c (secondary firmware: the limiter at PowerConfig + 0x231c). +const LIMITER_WORDS: Words = &[ + (0x0, TEMP_SENSOR_MASK as u32, 1), + (0x8, TEMP_RELEASE, 1), + (0x10, 4, 1), + (0x24, 0x3f800000, 1), + (0x2c, TEMP_KI, 1), + (0x38, 0x47800000, 1), + (0x3c, TEMP_KP, 1), + (0x48, 0x28, 1), + (0x4c, 0x3e8, 2), + (0x5c, TEMP_TARGET, 1), + (0x90, 0x3e8, 1), +]; +/// Tail offset of [`LIMITER_WORDS`] for G17P. +const LIMITER_G17P: usize = 0x550; +/// Tail offset of [`LIMITER_WORDS`] for g17c. +const LIMITER_G17C: usize = 0x584; + const POWER_CONFIG_TAIL_WORDS: Words = &[ (0x0, 0xff7fff7f, 181), (0x2d4, 0x7fff7f, 1), @@ -1632,17 +1776,6 @@ const POWER_CONFIG_TAIL_WORDS: Words = &[ (0x504, 0x3e8, 1), (0x50c, 0x3f4ccccd, 1), (0x510, 0x3e4ccccd, 1), - (0x550, TEMP_SENSOR_MASK as u32, 1), - (0x558, TEMP_RELEASE, 1), - (0x560, 4, 1), - (0x574, 0x3f800000, 1), - (0x57c, TEMP_KI, 1), - (0x588, 0x47800000, 1), - (0x58c, TEMP_KP, 1), - (0x598, 0x28, 1), - (0x59c, 0x3e8, 2), - (0x5ac, TEMP_TARGET, 1), - (0x5e0, 0x3e8, 1), (0x7fc, 0x42000006, 1), (0xa10, 0xb8520000, 1), (0xa14, 0x3f7e, 1), @@ -1826,7 +1959,13 @@ pub(crate) struct PowerState(pub(crate) [u8; 0x6c0]); unsafe impl Zeroable for PowerState {} impl PowerState { - fn init(&mut self) { + /// g17c: the secondary raises this flag (u32) before it reads the primary's flag + /// through [`Self::G17C_PEER_FLAG`]; AGXG17X starts it at 1. + pub(crate) const G17C_FLAG: usize = 0x784; + /// g17c: pointer (u64) to the primary's flag in its status B. + pub(crate) const G17C_PEER_FLAG: usize = 0x788; + + pub(crate) fn init(&mut self) { fill_words(&mut self.0, POWER_STATE_WORDS); } } @@ -1871,8 +2010,15 @@ pub(crate) struct PowerConfigTail(pub(crate) [u8; 0x1000]); unsafe impl Zeroable for PowerConfigTail {} impl PowerConfigTail { - fn init(&mut self) { + fn init(&mut self, limiter: usize) { fill_words(&mut self.0, POWER_CONFIG_TAIL_WORDS); + put_words(&mut self.0[limiter..], LIMITER_WORDS); + if limiter == LIMITER_G17C { + // The g17c secondary reads PowerConfig+0x2d78 (tail 0xfb8) next to the 110 C + // target at a kick (trace 0x20e); macOS has 11200 there (112.00 C), where the + // G17P tail has 65536.0. + put_words(&mut self.0, &[(0xfb8, 11_200, 1)]); + } } } @@ -1915,7 +2061,13 @@ impl PowerConfig { /// Initializes the power configuration. pub(crate) fn init(&mut self) { self.head.init(); - self.tail.init(); + self.tail.init(LIMITER_G17P); + } + + /// Initializes the g17c power configuration. + pub(crate) fn init_g17c(&mut self) { + self.head.init(); + self.tail.init(LIMITER_G17C); } } @@ -2051,6 +2203,10 @@ impl FreeListControl { /// Offset of the firmware-completed counter. It must be read bytewise. pub(crate) const COMPLETED: usize = 0x54; + /// Offset, past the structure, of the u32 that lets the firmware raise this pool's KSM + /// event flag (bit `unk_10` * 64 + context): AGXG17X sets 1 for compute pools + /// (AGXUSCPrivMemFList::populateInitialFirmwareState). + pub(crate) const EVENT_FLAG_ENABLE: usize = 0x7c; /// Returns the control object of the free list `args` describes. pub(crate) const fn new(args: &FreeListArgs) -> Self { diff --git a/drivers/gpu/drm/asahi/g17/fw/kick.rs b/drivers/gpu/drm/asahi/g17/fw/kick.rs index e07fbfb485a10..c5f05f774222f 100644 --- a/drivers/gpu/drm/asahi/g17/fw/kick.rs +++ b/drivers/gpu/drm/asahi/g17/fw/kick.rs @@ -355,6 +355,9 @@ pub(crate) struct KickArgs<'a> { /// queue. A queue's first kick names `timestamp.prev()` on render queues /// and [`KickTimestamp::ZERO`] on compute queues. pub(crate) parent: KickTimestamp, + /// The entry lists `parent`. macOS (AGXG17X encodeDependencyList) lists it on + /// every compute kick; on a queue's first kick it names the stamp before the kick. + pub(crate) implicit_parent: bool, /// Kicks of other queues this kick must wait for. pub(crate) barriers: &'a [KickDependency], /// GPU address of the work descriptor (firmware alias). @@ -404,7 +407,10 @@ pub(crate) struct KickEntry { pub(crate) register_arrays: [u64; 4], pub(crate) unk_170: u64, pub(crate) unk_178: u64, - pub(crate) pad_180: [u8; KICK_ENTRY_STRIDE - 0x180], + /// Type of each listed dependency, two bits each; 1 for every barrier and + /// the implicit parent (AGXG17X encodeDependencyList). + pub(crate) dependency_types: u64, + pub(crate) pad_188: [u8; KICK_ENTRY_STRIDE - 0x188], } static_assert!(size_of::() == KICK_ENTRY_STRIDE); @@ -461,12 +467,15 @@ impl KickEntry { for (slot, barrier) in dependencies.iter_mut().zip(args.barriers) { *slot = barrier.word(); } - dependencies[barriers] = KickDependency { - qid: args.qid, - timestamp: args.parent, + let mut count = barriers; + if args.implicit_parent { + dependencies[barriers] = KickDependency { + qid: args.qid, + timestamp: args.parent, + } + .word(); + count += 1; } - .word(); - let count = barriers + 1; let mask = if count == KICK_DEPENDENCIES { u32::MAX } else { @@ -488,62 +497,93 @@ impl KickEntry { register_arrays: args.register_arrays.map(RegisterArrayBinding::word), unk_170: 0, unk_178: Self::UNK_178, - pad_180: [0; KICK_ENTRY_STRIDE - 0x180], + dependency_types: (0..count).fold(0u64, |types, index| types | 1 << (2 * index)), + pad_188: [0; KICK_ENTRY_STRIDE - 0x188], }) } } -/// The complete write-only installation of a fragment/tiling queue pair. -/// Firmware must see both queues configured before either outer command. -pub(crate) struct RenderRegistration { - writes: [(usize, u64); 8], +/// The hardware installation of one kick queue, as AGXG17X +/// KSMKickQueue::configureHardware writes it for every tiling, fragment and +/// compute queue: the configuration tag and payload through the indirect port, +/// then the queue's validity bit, high mask word before low. +pub(crate) struct KickRegistration { + writes: [(usize, u64); 4], } -impl RenderRegistration { +impl KickRegistration { const TAG_PORT: usize = 0x4090; const DATA_PORT: usize = 0x4098; const VALID_LOW: usize = 0x21068; const VALID_HIGH: usize = 0x21070; - const CONFIG_TAG: u64 = 0x0000_0400_0002_1008; + /// AGXG17X: 0x0020_0800_0002_1008 | qid << 44 (the G17P form was + /// 0x0000_0400_0002_1008 | qid << 43). + const CONFIG_TAG: u64 = 0x0020_0800_0002_1008; + const QID_SHIFT: u32 = 44; const ADDRESS_MASK: u64 = 0x0000_07ff_ffff_ffe0; - /// Validates both queues without touching MMIO. The resulting transaction - /// writes fragment configuration and validity before tiling configuration - /// and validity. Each validity mask is stored high before low. Both queues - /// are registered at priority class `priority`. - pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64), priority: u8) -> Result { - if fragment.0 == tiling.0 || priority >= Policy::PRIORITIES { + /// Validates the queue without touching MMIO. `va` is the kick ring's + /// lower address; the queue is registered at priority class `priority`. + pub(crate) fn new( + qid: u8, + va: u64, + engine: super::queue::DataMaster, + priority: u8, + ) -> Result { + if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 || priority >= Policy::PRIORITIES { return Err(EINVAL); } - let mut writes = [(0, 0); 8]; - for (slot, ((qid, va), engine)) in [ - (fragment, super::queue::DataMaster::Fragment), - (tiling, super::queue::DataMaster::Tiling), - ] - .into_iter() - .enumerate() - { - if qid > QID_MAX || va & !Self::ADDRESS_MASK != 0 { - return Err(EINVAL); - } - let completion = (3 * (4 - u64::from(priority)) + 0x3fe).wrapping_shl(54); - let payload = ((engine as u64 + 1) << 60) | va | completion; - let (low, high) = if qid < 64 { - (1u64 << qid, 0) - } else { - (0, 1u64 << (qid - 64)) - }; - writes[slot * 4..slot * 4 + 4].copy_from_slice(&[ - (Self::TAG_PORT, Self::CONFIG_TAG | (u64::from(qid) << 43)), + let completion = (3 * (4 - u64::from(priority)) + 0x3fe).wrapping_shl(54); + let payload = ((engine as u64 + 1) << 60) | va | completion; + let (low, high) = if qid < 64 { + (1u64 << qid, 0) + } else { + (0, 1u64 << (qid - 64)) + }; + Ok(Self { + writes: [ + (Self::TAG_PORT, Self::CONFIG_TAG | (u64::from(qid) << Self::QID_SHIFT)), (Self::DATA_PORT, payload), (Self::VALID_HIGH, high), (Self::VALID_LOW, low), - ]); + ], + }) + } + + pub(crate) fn writes(&self) -> &[(usize, u64)] { + &self.writes + } +} + +/// The complete write-only installation of a fragment/tiling queue pair. +/// Firmware must see both queues configured before either outer command. +pub(crate) struct RenderRegistration { + writes: [(usize, u64); 8], +} + +impl RenderRegistration { + /// Validates both queues without touching MMIO. The resulting transaction + /// writes fragment configuration and validity before tiling configuration + /// and validity. Both queues are registered at priority class `priority`. + pub(crate) fn new(fragment: (u8, u64), tiling: (u8, u64), priority: u8) -> Result { + if fragment.0 == tiling.0 { + return Err(EINVAL); } + let fragment = KickRegistration::new( + fragment.0, + fragment.1, + super::queue::DataMaster::Fragment, + priority, + )?; + let tiling = + KickRegistration::new(tiling.0, tiling.1, super::queue::DataMaster::Tiling, priority)?; + let mut writes = [(0, 0); 8]; + writes[..4].copy_from_slice(fragment.writes()); + writes[4..].copy_from_slice(tiling.writes()); Ok(Self { writes }) } - pub(crate) fn writes(&self) -> &[(usize, u64); 8] { + pub(crate) fn writes(&self) -> &[(usize, u64)] { &self.writes } } diff --git a/drivers/gpu/drm/asahi/g17/fw/queue.rs b/drivers/gpu/drm/asahi/g17/fw/queue.rs index 48b110a4193e3..cf13849089c15 100644 --- a/drivers/gpu/drm/asahi/g17/fw/queue.rs +++ b/drivers/gpu/drm/asahi/g17/fw/queue.rs @@ -574,8 +574,8 @@ impl KickAnnounce { } } -/// Names the kick of a compute queue that precedes the announced one -/// (tag 16). +/// Names the announced kick of a compute queue (tag 16); the firmware then +/// releases that kick's slot (AGXG17X trace: its stamp is the starting kick's). /// /// A record slot larger than this structure must be zero beyond it. #[repr(C, packed)] @@ -585,8 +585,8 @@ pub(crate) struct KickPredecessor { pub(crate) tag: u32, /// Kicked queue. pub(crate) qid: u16, - /// Timestamp of the queue's kick preceding the announced one. - pub(crate) previous: u64, + /// Timestamp of the announced kick. + pub(crate) kick: u64, pub(crate) unk_0e: u32, } @@ -595,16 +595,15 @@ static_assert!(size_of::() == 0x12); impl KickPredecessor { const UNK_0E: u32 = 0xfe; - /// Names `previous` as the kick of queue `qid` preceding the announced - /// one. - pub(crate) fn new(qid: u8, previous: KickTimestamp) -> Result { + /// Names `kick`, the announced kick of queue `qid`. + pub(crate) fn new(qid: u8, kick: KickTimestamp) -> Result { if qid > QID_MAX { return Err(EINVAL); } Ok(Self { tag: CommandTag::KickPredecessor as u32, qid: qid as u16, - previous: previous.get(), + kick: kick.get(), unk_0e: Self::UNK_0E, }) } diff --git a/drivers/gpu/drm/asahi/g17/initdata.rs b/drivers/gpu/drm/asahi/g17/initdata.rs index d0dcc85c7f09e..abadbff285649 100644 --- a/drivers/gpu/drm/asahi/g17/initdata.rs +++ b/drivers/gpu/drm/asahi/g17/initdata.rs @@ -8,7 +8,7 @@ use super::{ fw::initdata as abi, object::{CpuMap, KernelObject, Placement}, - Config, Role, + Abi, Config, Role, }; use crate::{driver::AsahiDrmDevice, hw::t8140, mmu}; use core::ops::Range; @@ -148,6 +148,8 @@ pub(super) struct InitData { cluster: usize, bundle: usize, region_c: usize, + /// The g17c layout. + g17c: bool, qos: usize, pb: usize, page_pool: usize, @@ -228,6 +230,7 @@ impl InitData { cluster: 0, bundle: 0, region_c: 0, + g17c: false, qos: 0, pb: 0, page_pool: 0, @@ -695,6 +698,7 @@ impl InitData { render: &mmu::Vm, metrics: &KernelObject, ) -> Result { + self.g17c = matches!(cfg.abi, Abi::G17c); let range = uat.geometry().kernel_range(); let env = BuildEnv { dev, @@ -704,7 +708,7 @@ impl InitData { base: range.start, dynamic: range.start + t8140::dynamic::KERNEL_OFFSET..range.end, }; - self.map_register_windows(&env)?; + self.map_register_windows(&env, cfg)?; let tables = self.build_descriptor_tables(&env)?; self.build_render_free_lists(&env)?; let first_block = self.build_render_pool(&env)?; @@ -713,19 +717,36 @@ impl InitData { self.initialize_firmware_objects(&env, cfg, &tables, &bundle) } - fn map_register_windows(&mut self, env: &BuildEnv<'_>) -> Result { - for window in &t8140::REGISTER_WINDOWS { - let (phys, va, size) = window.mapping().ok_or(EINVAL)?; - self.aliases.push( - env.vm.map_io( - env.dev.as_ref().as_ref(), - va, - phys.try_into()?, - size.try_into()?, - mmu::PROT_FW_MMIO_RW, - )?, - GFP_KERNEL, - )?; + fn map_register_windows(&mut self, env: &BuildEnv<'_>, cfg: &Config) -> Result { + for window in cfg.register_windows { + // Each element is mapped on its own: elements are contiguous in the GPU address + // space but may be apart physically. + let element = window.size / window.elements; + let stride = match window.phys_stride { + 0 => element as u64, + stride => stride, + }; + for index in 0..window.elements as u64 { + let part = abi::RegisterWindow { + phys: window.phys + index * stride, + va: window.va + index * element as u64, + size: element, + elements: 1, + phys_stride: 0, + ..*window + }; + let (phys, va, size) = part.mapping().ok_or(EINVAL)?; + self.aliases.push( + env.vm.map_io( + env.dev.as_ref().as_ref(), + va, + phys.try_into()?, + size.try_into()?, + mmu::PROT_FW_MMIO_RW, + )?, + GFP_KERNEL, + )?; + } } Ok(()) } @@ -988,18 +1009,27 @@ impl InitData { bundle: &BundleObjects, ) -> Result { let base = env.base; - let opps = Self::opps(env.dev)?; + let opps = Self::opps(env.dev, cfg.opps)?; let hw_args = abi::HwDataArgs { chip_id: cfg.chip_id, - windows: &t8140::REGISTER_WINDOWS, - reserved_slots: &t8140::RESERVED_REGISTER_SLOTS, + windows: cfg.register_windows, + reserved_slots: cfg.reserved_slots, opps: &opps, - perf: &t8140::PERF, + perf: cfg.perf, completion_rings: bundle.completion_rings, qos_va: self.object(bundle.qos)?.gpu_va(), }; - self.object_mut(self.bundle)? - .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?; + match cfg.abi { + Abi::G17p => self + .object_mut(self.bundle)? + .initialize::(abi::bundle::HW_DATA, |hw| hw.init(&hw_args))?, + Abi::G17c => self + .object_mut(self.bundle)? + .initialize::( + abi::bundle::HW_DATA, + |hw| hw.init(&hw_args), + )?, + } self.object_mut(self.region_c)? .initialize::(0, |region| { region.init(); @@ -1010,9 +1040,36 @@ impl InitData { self.object_mut(self.cluster)? .write(abi::private::SECONDARY_STATUS_A, abi::StatusBlock::new())?; let fwctl_va = self.object(tables.fwctl)?.gpu_va(); + let status_b_va = base + high::PRIVATE_CLUSTER + + (abi::private::PRIMARY_STATE + abi::private::STATUS_B) as u64; + // g17c: the firmware-control page, unused otherwise, holds the secondary's power + // state (AGXG17X: its own 0x790-byte object); the primary and the secondary each + // point at the other's flag. + let secondary_power_va = match cfg.abi { + Abi::G17p => { + status_b_va + abi::PrimaryStatusB::POWER_STATE as u64 + } + Abi::G17c => { + let page = self.object_mut(tables.fwctl)?; + page.initialize::(0, |power| { + power.init(); + Ok(()) + })?; + page.write(abi::PowerState::G17C_FLAG, 1u32)?; + page.write( + abi::PowerState::G17C_PEER_FLAG, + status_b_va + abi::PrimaryStatusB::G17C_FLAG as u64, + )?; + fwctl_va + } + }; self.object_mut(self.cluster)? .initialize::(abi::private::STATUS_B, |status| { - status.init(fwctl_va); + match cfg.abi { + Abi::G17p => status.init(fwctl_va), + Abi::G17c => status + .init_g17c(secondary_power_va + abi::PowerState::G17C_FLAG as u64), + } Ok(()) })?; self.object_mut(self.cluster)? @@ -1025,7 +1082,10 @@ impl InitData { )?; self.object_mut(self.bundle)? .initialize::(abi::bundle::POWER_CONFIG, |power| { - power.init(); + match cfg.abi { + Abi::G17p => power.init(), + Abi::G17c => power.init_g17c(), + } Ok(()) })?; let secondary_status = base @@ -1064,7 +1124,10 @@ impl InitData { .initialize::(main_offset, |main| { match role { Role::Primary => main.init_primary(&args, &views), - Role::Secondary => main.init_secondary(&args), + Role::Secondary => match cfg.abi { + Abi::G17p => main.init_secondary(&args), + Abi::G17c => main.init_secondary_g17c(&args), + }, }; Ok(()) })?; @@ -1077,6 +1140,7 @@ impl InitData { abi::OpeningRecord::new(abi_role), )?; let root = abi::RootArgs { + version: cfg.interface_version, region_a_va: self.object(bundle.region_a)?.gpu_va(), region_c_va: self.object(self.region_c)?.gpu_va(), main_config_va: main_va, @@ -1093,8 +1157,7 @@ impl InitData { abi::SECONDARY_ROOT_OFFSET, abi::SecondaryRoot::new( &root, - base + high::PRIVATE_CLUSTER - + (abi::private::STATUS_B + abi::PrimaryStatusB::POWER_STATE) as u64, + secondary_power_va, state_va + abi::private::STATUS_B as u64, ), )?, @@ -1121,6 +1184,7 @@ impl InitData { &mut self, vm: &mmu::Vm, metrics: &KernelObject, + cfg: &Config, ) -> Result> { let mut mappings = KVec::with_capacity(24, GFP_KERNEL)?; mappings.push( @@ -1150,12 +1214,17 @@ impl InitData { )?; } mappings.push( - metrics.map_alias(vm, t8140::CONFIG.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, + metrics.map_alias(vm, cfg.pm_metrics.va, mmu::PROT_GPU_FW_SHARED_RW)?, GFP_KERNEL, )?; Ok(mappings) } + /// The firmware objects use the g17c layout. + pub(super) fn g17c(&self) -> bool { + self.g17c + } + /// Writes the effort input in firmware order, then cleans its cache line to coherency. pub(super) fn publish_effort(&self, effort: u32) -> Result { let region = self.object(self.region_c)?; @@ -1164,6 +1233,12 @@ impl InitData { let line = region.pointer(abi::RegionC::EFFORT, 4)?; override_word.store(0, Ordering::Relaxed); effort_word.store(effort, Ordering::Relaxed); + if self.g17c { + // The g17c secondary reads Region C +0xe8 and +0xec (gfx1 0x15bcc), never +0xe4: + // the G17P pair moves up by 4, zero at +0xec and the effort at +0xe8. + region.word(abi::RegionC::UNK_E8 + 4)?.store(0, Ordering::Relaxed); + override_word.store(effort, Ordering::Relaxed); + } // SAFETY: line points into the retained kernel mapping; cleaning this host-owned cache // line and the following system barrier publish the two naturally aligned stores. unsafe { @@ -1180,7 +1255,7 @@ impl InitData { .load(Ordering::Relaxed)) } - fn opps(dev: &AsahiDrmDevice) -> Result> { + fn opps(dev: &AsahiDrmDevice, expected: usize) -> Result> { let node = dev.as_ref().as_ref().of_node().ok_or(EIO)?; let table = node .parse_phandle(c_str!("operating-points-v2"), 0) @@ -1195,7 +1270,7 @@ impl InitData { GFP_KERNEL, )?; } - if opps.len() != t8140::OPPS { + if opps.len() != expected { return Err(EINVAL); } Ok(opps) diff --git a/drivers/gpu/drm/asahi/g17/mod.rs b/drivers/gpu/drm/asahi/g17/mod.rs index 6ca0398abd2e2..6e6dee5878b9e 100644 --- a/drivers/gpu/drm/asahi/g17/mod.rs +++ b/drivers/gpu/drm/asahi/g17/mod.rs @@ -24,6 +24,8 @@ pub(crate) use command::Attachments; mod completion; pub(crate) mod context; mod dependency; +#[cfg(CONFIG_DRM_ASAHI_G15)] +pub(crate) use dependency::inherited_error; mod event; mod feed; mod fence; @@ -60,7 +62,8 @@ use kernel::{ io::{ IoBase, IoSysMap, - Region, // + Region, + SysMem, // }, impl_has_delayed_work, impl_has_work, new_condvar, new_delayed_work, new_mutex, new_work, platform, @@ -137,8 +140,19 @@ pub(crate) struct FixedBuffer { pub(crate) size: usize, } +/// Firmware interface generation, which selects the layouts of the structures that differ. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum Abi { + /// G17P firmware (t8140). + G17p, + /// G17C firmware (t6050). + G17c, +} + /// Static configuration of a SoC with a G17 GPU. pub(crate) struct Config { + /// Firmware interface generation. + pub(crate) abi: Abi, /// SoC identifier. pub(crate) chip_id: u32, /// Expected GPU identification. @@ -151,8 +165,21 @@ pub(crate) struct Config { pub(crate) secondary_root_offset: u64, /// Crash log buffer of each [`Role`]. pub(crate) crash_buffers: [FixedBuffer; 2], + /// The firmware keeps each crash log in its own memory and requests it by physical address, + /// so the host maps that memory instead of its [`Self::crash_buffers`]. + pub(crate) crash_log_phys: bool, /// Parameter buffer page metrics shared with the firmware. pub(crate) pm_metrics: FixedBuffer, + /// Firmware MMIO register windows, by register-map slot. + pub(crate) register_windows: &'static [fw::initdata::RegisterWindow], + /// Register-map slots that are reserved but have no window. + pub(crate) reserved_slots: &'static [usize], + /// Performance-table configuration. + pub(crate) perf: &'static fw::initdata::PerfConfig, + /// Operating points expected in the device tree (performance-state count). + pub(crate) opps: usize, + /// Firmware interface version written to the root; the firmware rejects any other. + pub(crate) interface_version: [u16; 4], } /// A firmware coprocessor instance. @@ -205,7 +232,24 @@ const CRASH_BUFFER_FILL: [u8; 2] = [0xa5, 0x5a]; /// A crash log buffer handed to RTKit. pub(crate) struct CrashLog { iova: usize, - map: shmem::VMapOwned, + map: CrashMap, +} + +/// Memory behind a [`CrashLog`]. +enum CrashMap { + /// The host's crash log buffer. + Object(shmem::VMapOwned), + /// Firmware memory at the requested physical address, from `memremap`. + Firmware { ptr: *mut u8, size: usize }, +} + +impl Drop for CrashMap { + fn drop(&mut self) { + if let Self::Firmware { ptr, .. } = *self { + // SAFETY: `ptr` came from `memremap` and is not used after this. + unsafe { kernel::bindings::memunmap(ptr.cast()) }; + } + } } impl rtkit::Buffer for CrashLog { @@ -214,7 +258,13 @@ impl rtkit::Buffer for CrashLog { } fn buf(&mut self) -> Result> { - Ok(IoSysMap::from((&self.map).as_view())) + Ok(match &self.map { + CrashMap::Object(map) => IoSysMap::from(map.as_view()), + // SAFETY: `ptr` maps `size` bytes, 4-byte aligned, for as long as `self` lives. + CrashMap::Firmware { ptr, size } => IoSysMap::Sys(unsafe { + SysMem::new(Region::ptr_from_raw_parts_mut(*ptr, *size)) + }), + }) } } @@ -224,6 +274,7 @@ struct RoleState { role: Role, shared: Arc, crash_log: FixedBuffer, + crash_log_phys: bool, crash_map: shmem::VMapOwned, crashed: AtomicBool, #[pin] @@ -307,9 +358,25 @@ impl rtkit::Operations for RoleOps { { return Err(EINVAL); } + if data.crash_log_phys { + // SAFETY: Maps firmware memory the firmware points the host at; nothing else uses it. + let ptr = unsafe { + kernel::bindings::memremap(iova as _, size, kernel::bindings::MEMREMAP_WB as _) + }; + if ptr.is_null() { + return Err(ENOMEM); + } + return Ok(CrashLog { + iova, + map: CrashMap::Firmware { + ptr: ptr.cast(), + size, + }, + }); + } Ok(CrashLog { iova, - map: data.crash_map.owner().owned_vmap()?, + map: CrashMap::Object(data.crash_map.owner().owned_vmap()?), }) } } @@ -356,6 +423,7 @@ impl Coprocessor { va: buffer.va, size: buffer.size, }, + crash_log_phys: cfg.crash_log_phys, crash_map, crashed: AtomicBool::new(false), ack <- Opaque::ffi_init(|ptr| { @@ -562,7 +630,9 @@ impl power::Control for Firmware { self.primary.state.shared.crashed.load(Ordering::Acquire) } fn powered(&self) -> bool { - self.bridge.regs.g17_configuration_powered() + // 0xe01000 reads 0 on G17C whatever the power state, so there a retired KeepPowered + // counts as powered. + self.init.g17c() || self.bridge.regs.g17_configuration_powered() } fn counters(&self) -> Result<[u32; 3]> { self.init.control_counters() @@ -1133,7 +1203,11 @@ impl gpu::Gpu for Gpu { .uat .new_vm(self.ids.vm.next(), kernel_range)? .with_status()?; - let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics)?; + let vm = match self.cfg.abi { + Abi::G17p => vm, + Abi::G17c => vm.without_context_views(), + }; + let mappings = firmware.init.user_aliases(&vm, &firmware._pm_metrics, self.cfg)?; let ranges = hw::t8140::resources::client_reserved_ranges(self.cfg.pm_metrics.va).ok_or(EINVAL)?; let mut reserved = KVec::with_capacity(ranges.len(), GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/g17/queue/compute.rs b/drivers/gpu/drm/asahi/g17/queue/compute.rs index b78598cfbb760..d0fc389103500 100644 --- a/drivers/gpu/drm/asahi/g17/queue/compute.rs +++ b/drivers/gpu/drm/asahi/g17/queue/compute.rs @@ -1583,7 +1583,7 @@ impl Queue { ) -> Result { use crate::g17::fw::{ channels::WorkSlot, - compute::{ComputeArgs, ComputeDescriptor, COMPUTE_KICK_EVENT_MASK, COMPUTE_QOS_CLASS}, + compute::{compute_kick_event_mask, ComputeArgs, ComputeDescriptor, COMPUTE_QOS_CLASS}, kick::{KickArgs, KickQos}, }; let (epoch, addresses) = self.publication_preflight(&packet, host)?; @@ -1663,11 +1663,12 @@ impl Queue { let config = self.queue_config(&identity, first, args.context_id, policy, &packet)?; let announce = KickAnnounce::new(qid, DataMaster::Compute, kick_timestamp, policy.priority())?; - let predecessor = KickPredecessor::new(qid, kick_parent)?; + let predecessor = KickPredecessor::new(qid, kick_timestamp)?; let kick_args = KickArgs { qid, timestamp: kick_timestamp, parent: kick_parent, + implicit_parent: true, barriers: dependencies, descriptor_va: descriptor_high, queue_va: self.graph.graph.gpu_va(), @@ -1676,7 +1677,9 @@ impl Queue { class: COMPUTE_QOS_CLASS, }, mcache: None, - event_mask: COMPUTE_KICK_EVENT_MASK, + // The firmware raises this flag for the pool (its control sets + // FreeListControl::EVENT_FLAG_ENABLE; unk_10 = 2 = the priority word). + event_mask: compute_kick_event_mask(policy.priority(), args.context_id.into())?, register_arrays: ComputeDescriptor::register_bindings(descriptor_low, *scratch)?, scs_enabled: scratch.enabled(), priority: policy.priority(), diff --git a/drivers/gpu/drm/asahi/g17/queue/render.rs b/drivers/gpu/drm/asahi/g17/queue/render.rs index 99be455b48e40..320a73f714316 100644 --- a/drivers/gpu/drm/asahi/g17/queue/render.rs +++ b/drivers/gpu/drm/asahi/g17/queue/render.rs @@ -596,6 +596,8 @@ impl Pair { qid: qids[stage], timestamp: kicks[stage], parent: parents[stage], + // Render keeps its parent on the first kick; not yet checked against AGXG17X. + implicit_parent: true, barriers, descriptor_va: self.memory.graph.descriptor_va(stage, ordinal), queue_va: queues[stage], diff --git a/drivers/gpu/drm/asahi/g17/runtime.rs b/drivers/gpu/drm/asahi/g17/runtime.rs index ffaff6d80dbf9..0012df635a750 100644 --- a/drivers/gpu/drm/asahi/g17/runtime.rs +++ b/drivers/gpu/drm/asahi/g17/runtime.rs @@ -910,6 +910,11 @@ impl super::Firmware { { return Err(EAGAIN); } + // Hold the GPU powered (KeepPowered) across compute kicks: the firmware's KSM + // scheduling pause is acknowledged only while the GPU is held on. + if let Err(error) = self.acquire_registration_power() { + pr_warn!("G17 compute power hold failed: {:?}\n", error); + } let queues = &mut self.queues; let queue = queues .compute diff --git a/drivers/gpu/drm/asahi/g17/runtime/render.rs b/drivers/gpu/drm/asahi/g17/runtime/render.rs index a47abf2045526..5e3422bd341a3 100644 --- a/drivers/gpu/drm/asahi/g17/runtime/render.rs +++ b/drivers/gpu/drm/asahi/g17/runtime/render.rs @@ -409,7 +409,7 @@ impl crate::g17::queue::render::Host for Host<'_> { .firmware .bridge .regs - .install_render_pair(registration)? + .install_kick_queues(registration.writes())? { return Ok(()); } diff --git a/drivers/gpu/drm/asahi/gem.rs b/drivers/gpu/drm/asahi/gem.rs index 2266d18e9c85f..3bcf8a9692dcc 100644 --- a/drivers/gpu/drm/asahi/gem.rs +++ b/drivers/gpu/drm/asahi/gem.rs @@ -385,3 +385,57 @@ pub(crate) fn madvise(gem: &Object, dontneed: bool) -> Result { } Ok(u32::from(gem.madvise(i32::from(dontneed)))) } + +/// Populates and pins a new user object through its DMA mapping, before any CPU view exists. +/// +/// On a device that is not DMA-coherent, the DMA mapping cleans the cacheable kernel alias of the +/// freshly zeroed pages to the point of coherency, so that no dirty line can later overwrite data +/// written through a write-combined view. The pin keeps reclaim from creating a new cacheable +/// alias. Only used when the GPU asks for it ([`crate::gpu::Gpu::prepare_new_objects`]). +/// +/// The DMA mapping only cleans if the device is not marked DMA-coherent (G15's synthesized node +/// has no `dma-coherent`). Should it ever be marked so, the pages are cleaned here instead. +#[cfg(CONFIG_DRM_ASAHI_G15)] +pub(crate) fn prepare_new_object(gem: &Object) -> Result { + // SAFETY: G15 objects belong to the retained bound platform device; user object + // creation occurs only after registration and before a file can outlive its device. + let dev: &kernel::device::Device = gem.dev().as_ref().as_ref(); + let sgt = held_sg_table(gem, unsafe { dev.as_bound() })?; + // SAFETY: The device outlives its GEM objects; the flag is fixed when the device is probed. + if !unsafe { asahi_g15_dma_coherent(dev.as_raw().cast()) } { + return Ok(()); + } + static WARNED: core::sync::atomic::AtomicBool = core::sync::atomic::AtomicBool::new(false); + if !WARNED.swap(true, Ordering::Relaxed) { + dev_warn!( + dev, + "GPU marked DMA-coherent: cleaning new objects to the point of coherency explicitly\n" + ); + } + let page_size = kernel::page::PAGE_SIZE; + for segment in sgt.iter() { + // Without an IOMMU in front of the GPU, its DMA addresses are physical addresses. + let start = segment.dma_address() as kernel::addr::PhysicalAddr; + let len = segment.dma_len() as usize; + let mut offset = 0; + while offset < len { + let phys = start + offset as kernel::addr::PhysicalAddr; + // SAFETY: The pinned SG table keeps these shmem pages allocated. + let page = unsafe { kernel::page::Page::borrow_phys(&phys) }.ok_or(EIO)?; + page.with_pointer_into_page(0, page_size, |p| { + crate::pgtable::clean_poc(p, page_size); + Ok(()) + })?; + offset += page_size; + } + } + crate::pgtable::poc_barrier(); + Ok(()) +} + + +#[cfg(CONFIG_DRM_ASAHI_G15)] +extern "C" { fn asahi_g15_dma_coherent(dev: *mut core::ffi::c_void) -> bool; } + +#[cfg(not(CONFIG_DRM_ASAHI_G15))] +pub(crate) fn prepare_new_object(_gem: &Object) -> Result { Err(EINVAL) } diff --git a/drivers/gpu/drm/asahi/gpu.rs b/drivers/gpu/drm/asahi/gpu.rs index ed54c350d7dbc..90bb25451019e 100644 --- a/drivers/gpu/drm/asahi/gpu.rs +++ b/drivers/gpu/drm/asahi/gpu.rs @@ -11,6 +11,7 @@ //! are unified, so that the top level of the driver itself (in `driver`) does not have to concern //! itself with version dependence. +use crate::module_parameters; use core::any::Any; use core::ops::Range; use core::slice; @@ -292,6 +293,21 @@ pub(crate) trait Gpu: Send + Sync { fn queue_limits(&self) -> Option { None } + /// Shared GET_PARAMS additions, using this backend's feature-number assignments. + fn feature_flags(&self) -> u64 { + let mut flags = 0; + if self.supports_vm_status() { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS as u64 + | uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_VM_STATUS_MIRROR as u64; + } + if self.queue_limits().is_some() { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_INDEPENDENT_QUEUES as u64; + } + if self.soft_faults() && *module_parameters::fault_control.value() == 0xb { + flags |= uapi::drm_asahi_feature_DRM_ASAHI_FEATURE_SOFT_FAULTS as u64; + } + flags + } /// Temporary capacity hint and validity in milliseconds for subsequent syncobj waits. fn syncobj_wait_hint(&self) -> Option<(u32, u32)> { None @@ -300,6 +316,28 @@ pub(crate) trait Gpu: Send + Sync { fn base_clock_hz(&self) -> u32; /// Return the GPU address space geometry. fn uat_geometry(&self) -> mmu::UatGeometry; + /// Backend-specific user address aperture; other GPUs keep their geometry's range. + fn user_range(&self) -> Range { + self.uat_geometry().user_usable_range() + } + /// Use tracked command publication and sticky VM status on this backend. + fn tracked_submission(&self) -> bool { + false + } + /// Whether faults may be reported as recoverable to userspace. + fn soft_faults(&self) -> bool { + true + } + /// Populate and clean fresh object backing before any WC CPU mapping. + fn prepare_new_objects(&self) -> bool { + false + } + /// Optional backend-specific upper bound for a VM_BIND batch. + fn bind_batch_limit(&self) -> Option { + None + } + /// A userspace VM handle was closed; release backend-only registry references. + fn vm_closed(&self) {} /// Create a new `Vm` with the given range reserved for kernel-managed objects. fn new_vm(&self, kernel_range: Range) -> Result; /// Create a new user command queue. diff --git a/drivers/gpu/drm/asahi/hw/mod.rs b/drivers/gpu/drm/asahi/hw/mod.rs index deffe3fe47d0b..0fa2ca423ac91 100644 --- a/drivers/gpu/drm/asahi/hw/mod.rs +++ b/drivers/gpu/drm/asahi/hw/mod.rs @@ -11,6 +11,7 @@ use kernel::prelude::*; const MAX_POWERZONES: usize = 5; +pub(crate) mod t6050; pub(crate) mod t600x; pub(crate) mod t602x; pub(crate) mod t8103; diff --git a/drivers/gpu/drm/asahi/hw/t6050.rs b/drivers/gpu/drm/asahi/hw/t6050.rs new file mode 100644 index 0000000000000..04167db081c46 --- /dev/null +++ b/drivers/gpu/drm/asahi/hw/t6050.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT + +//! Hardware configuration for t6050 platforms (M5 Max), GPU generation G17, +//! variant C (g17c firmware, HAL300 command stream). +//! +//! The register map is the one macOS hands the firmware on this machine. Still +//! taken from t8140: the perf curves, the secondary root offset and the +//! parameter-buffer metrics buffer. See kernel/GPU.md. + +use crate::g17::fw::initdata::{PerfConfig, RegisterWindow}; +use crate::g17::{Abi, Config, FixedBuffer, Identity}; +use crate::hw::{t8140, GpuVariant}; + +/// Static configuration of the T6050 GPU. +pub(crate) const CONFIG: Config = Config { + abi: Abi::G17c, + chip_id: CHIP_ID, + identity: Identity { + // ID register 0x0b041100: family 0x0b, variant 0x04 (probe). + family: 0xb, + variant: 0x4, + num_dies: 1, + gpu_variant: GpuVariant::C, + // AGX3 shaders and the HAL300 command stream (Mesa's asahi_drm.h: + // DRM_ASAHI_GPU_HAL_300 for G17G/G17S/G17C). + usc_generation: 3, + hal_generation: 300, + }, + uat_ias: 42, + uat_oas: 42, + // Placeholder (t8140); consumed after the version gate. + secondary_root_offset: 0x40000, + // Crash-log buffers: the firmware requests these during RTKit init, before + // the version gate, at the physical addresses of its GFX / GFX1 + // __DATA_SHARED_RO segments (masked low 40 bits). Where iBoot loads them on + // mBoot-20457.60.65 (macOS 27.3 beta 1): 0x1000286c000 / 0x10002928000. + crash_buffers: [ + FixedBuffer { + va: 0x286_c000, + size: 0x8000, + }, + FixedBuffer { + va: 0x292_8000, + size: 0x8000, + }, + ], + // The requested iovas are the physical addresses of those segments, where the + // firmware writes its crash logs. + crash_log_phys: true, + // Placeholder (t8140); consumed after the version gate. + pm_metrics: t8140::CONFIG.pm_metrics, + register_windows: ®ISTER_WINDOWS, + reserved_slots: &RESERVED_REGISTER_SLOTS, + perf: &PERF, + opps: 14, + // g17c firmware interface version (GFX 0x218a4, GFX1 0x26ce4). Accepted by + // the firmware (gpu4: no mismatch; crash is downstream on G17P-layout structs). + interface_version: [0x04c0, 0x8728, 0x4339, 0x0c8e], +}; + +/// Chip identifier written to the hardware data. +pub(crate) const CHIP_ID: u32 = 0x6050; + +/// Base of the firmware MMIO aperture in the kernel window. +const MMIO_APERTURE: u64 = 0xffff_fc21_8000_0000; +/// GPU register block (sgx). +const SGX: u64 = 0x23_0000_0000; + +const fn window(slot: usize, phys: u64, offset: u64, size: u32, unk_18: u64) -> RegisterWindow { + RegisterWindow { + slot, + phys, + va: MMIO_APERTURE + offset, + size, + elements: 1, + phys_stride: 0, + unk_18, + unk_20: 2, + } +} + +/// A window of `elements` blocks of `element` bytes, `phys_stride` apart physically. +const fn array(slot: usize, phys: u64, offset: u64, element: u32, elements: u32, phys_stride: u64) -> RegisterWindow { + RegisterWindow { + elements, + phys_stride, + ..window(slot, phys, offset, element * elements, 0) + } +} + +/// A window macOS describes with an address but type 0 (`unk_20` 0). +const fn disabled(slot: usize, phys: u64, offset: u64, size: u32) -> RegisterWindow { + RegisterWindow { + unk_20: 0, + ..window(slot, phys, offset, size, 0) + } +} + +/// Register windows the firmware maps, by register-map slot, as the macOS +/// AGXG17X kext fills them on this machine (workflow wf_a0538809: its static +/// sgx table, its per-block filler functions and the ADT). Slots macOS leaves +/// empty stay empty here. +pub(crate) const REGISTER_WINDOWS: [RegisterWindow; 30] = [ + // GPCAICSWInt: the AIC software-interrupt page. + window(0, 0x2_8041_4000, 0xc_0000, 0x4000, 0), + // MCache: one block per GPU cluster. + array(3, 0x22_2010_4000, 0x10_0000, 0x2_4000, 4, 0x200_0000), + disabled(4, 0x2_8046_8000, 0x1a_0000, 0x4000), + // Push telemetry dashboard and its config space; the read space is disabled. + window(9, 0x2_8425_0000, 0x1a_8000, 0x4000, 0), + disabled(10, 0x2_8424_0000, 0x1b_0000, 0x4000), + window(11, 0x2_8425_8000, 0x1b_8000, 0x4000, 0), + window(12, 0x5_0965_c000, 0x1c_0000, 0x4000, 0), + disabled(14, 0x23_8030_0000, 0x1c_8000, 0x8000), + window(15, 0x2_8cdc_0000, 0x1d_0000, 0x2_4000, 0), + // RGX: the sgx base. + window(17, SGX, 0x8_0000, 0x2_1500, 0), + window(18, SGX + 0xeb_4000, 0x20_0000, 0x4000, 0), + // AFR: four blocks 0x20_0000 apart. + array(20, SGX + 0x300_0000, 0x30_0000, 0x4_0000, 4, 0x20_0000), + window(22, SGX + 0x100_0000, 0xb_0000, 0x8000, 0), + window(23, SGX + 0xf0_0000, 0x20_8000, 0x1_0000, 0), + // DVFM of the GPU: one block per cluster. + array(24, SGX + 0xec_0000, 0x22_0000, 0x1_0000, 4, 0), + window(31, SGX + 0xd0_4000, 0x0_0000, 0x8000, 0xd0_4000), + window(33, SGX + 0xd5_0000, 0x1_0000, 0x1_0000, 0xd5_0000), + window(34, SGX + 0xd1_0000, 0x2_0000, 0x4000, 0xd1_0000), + window(36, SGX + 0xd4_0000, 0x2_8000, 0x4000, 0xd4_0000), + window(37, SGX + 0xd6_0000, 0x3_0000, 0x2_0000, 0xd6_0000), + window(38, SGX + 0xd4_4000, 0x5_4000, 0x4000, 0xd4_4000), + window(39, SGX + 0xd4_c000, 0x5_c000, 0x200, 0xd4_c000), + window(40, SGX + 0xe0_0000, 0x6_0000, 0x4000, 0xe0_0000), + window(42, SGX + 0xe4_0000, 0x6_8000, 0x4000, 0xe4_0000), + // Metrology sensors. + array(44, SGX + 0xe0_8000, 0x26_0000, 0x8000, 2, 0), + // Throttle state counters. + window(46, SGX + 0xe3_3800, 0x27_3800, 0x4000, 0), + window(48, SGX + 0xe6_0000, 0x7_0000, 0x58, 0xe6_0000), + window(52, SGX + 0xe4_8000, 0x28_0000, 0x8000, 0), + // AFRDCPMS ("TMB AFR DCPMS"). + window(53, SGX + 0x2_3d00, 0xc_bd00, 0x200, 0), + window(58, SGX + 0xe7_3800, 0x29_3800, 0x4000, 0), +]; + +/// Register-map slots macOS names with type 2 but gives no window. +pub(crate) const RESERVED_REGISTER_SLOTS: [usize; 17] = [ + 2, 5, 6, 7, 8, 21, 25, 26, 27, 32, 35, 41, 43, 45, 47, 49, 54, +]; + +/// Placeholder perf config. The M5 Max has 14 OPPs / 13 active states, but the +/// shared `PERF_STATES` is still 11, so only the first 11 states are mapped here +/// (enough to build valid init data and reach the version gate). `relative_*` +/// are t8140's (macOS-measured, not statically recoverable for T6050); SRAM +/// floor is the T6050 value. DVFS tuning only, after the version gate. +pub(crate) const PERF: PerfConfig = PerfConfig { + states_a: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + states_b: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + relative_a: [0, 15, 22, 29, 42, 56, 74, 86, 90, 92, 100], + relative_b: [0, 0, 13, 24, 40, 52, 63, 73, 83, 88, 100], + sram_min_mv: 775, +}; diff --git a/drivers/gpu/drm/asahi/hw/t8140.rs b/drivers/gpu/drm/asahi/hw/t8140.rs index 143ba720d06f2..93242350a9e05 100644 --- a/drivers/gpu/drm/asahi/hw/t8140.rs +++ b/drivers/gpu/drm/asahi/hw/t8140.rs @@ -3,6 +3,7 @@ //! Hardware configuration for t8140 platforms (A18 Pro). use crate::g17::{ + Abi, Config, FixedBuffer, Identity, // @@ -11,6 +12,7 @@ use crate::hw::GpuVariant; /// Static configuration of the T8140 GPU. pub(crate) const CONFIG: Config = Config { + abi: Abi::G17p, chip_id: CHIP_ID, identity: Identity { family: 0xa, @@ -36,15 +38,21 @@ pub(crate) const CONFIG: Config = Config { size: 0x8000, }, ], + crash_log_phys: false, // Read by the firmware through the kernel lower root, whose top-level entries it only // samples when it starts. pm_metrics: FixedBuffer { va: 0x10_0008_0000, size: 0x8000, }, + register_windows: ®ISTER_WINDOWS, + reserved_slots: &RESERVED_REGISTER_SLOTS, + perf: &PERF, + opps: OPPS, + interface_version: INTERFACE_VERSION, }; -use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, PERF_STATES}; +use crate::g17::fw::initdata::{PerfConfig, RegisterWindow, INTERFACE_VERSION, PERF_STATES}; /// Chip identifier written to the hardware data. pub(crate) const CHIP_ID: u32 = 0x8140; @@ -65,6 +73,8 @@ const fn window( phys, va: MMIO_APERTURE + offset, size, + elements: 1, + phys_stride: 0, unk_18, unk_20, } diff --git a/drivers/gpu/drm/asahi/mmu.rs b/drivers/gpu/drm/asahi/mmu.rs index 4ea797d398a8b..2488a9f7ade27 100644 --- a/drivers/gpu/drm/asahi/mmu.rs +++ b/drivers/gpu/drm/asahi/mmu.rs @@ -1475,6 +1475,8 @@ unsafe impl Send for UatShared {} #[pin_data] struct UatInner { firmware: UatFirmware, + #[cfg(CONFIG_DRM_ASAHI_G15)] + clean_poc: bool, #[pin] shared: Mutex, #[pin] @@ -1661,6 +1663,15 @@ impl Vm { UatPageTable::new(geometry.ias, geometry.oas)? }; + #[cfg(CONFIG_DRM_ASAHI_G15)] + let page_table = { + let mut page_table = page_table; + if uat_inner.clean_poc { + page_table.enable_clean_poc()?; + } + page_table + }; + let (va_range, gpuvm_range) = if is_kernel { (geometry.kernel_range(), kernel_range.clone()) } else { @@ -1737,6 +1748,69 @@ impl Vm { Ok(self) } + /// Drop the G17P job context views; the HAL300 firmware has none. + pub(crate) fn without_context_views(mut self) -> Self { + self.context_bindings = None; + self + } + + /// Attach a status and job lifetime tracking before sharing a newly created user VM with + /// queues, without the G17 canonical context views, shared-object footprints and idle + /// residency reclaim of [`Self::with_status`]. Used by G15, whose user objects stay pinned. + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn with_tracked_status(mut self) -> Result { + self.status = Some(Arc::pin_init(crate::g17::status::VmStatus::new(), GFP_KERNEL)?); + self.lifetime = Some(lifetime::VmLifetime::new()?); + Ok(self) + } + + /// The identifier this VM was created with (G15 keeps per-VM state keyed by it). + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn id(&self) -> u64 { + self.id + } + + /// Copies `buf.len()` bytes of GPU-readable memory at `va` of this VM into `buf`, for admission + /// checks of streams the GPU will execute (G15 stream gates). + /// + /// Every page must be mapped GPU-readable and be RAM; otherwise EFAULT. The lines are cleaned + /// and invalidated to the point of coherency first (never only invalidated: CPU-written lines + /// may be dirty), so the copy sees what the GPU or a write-combined CPU view stored. The + /// mappings cannot change during the copy (GPUVM exec lock). + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn read_user(&self, va: u64, buf: &mut [u8]) -> Result { + let mut inner = lock_vm(&self.gpuvm, None, false)?; + let mut done = 0usize; + while done < buf.len() { + let address = va.checked_add(done as u64).ok_or(EFAULT)?; + let page = address & !(UAT_PGMSK as u64); + let within = (address - page) as usize; + let count = (buf.len() - done).min(UAT_PGSZ - within); + if page < inner.va_range.start || page >= inner.va_range.end { + return Err(EFAULT); + } + let leaf = inner.page_table.leaf_entry(page).map_err(|_| EFAULT)?; + if !Prot::from_pte(leaf).allows_gpu(true, false) { + return Err(EFAULT); + } + let phys = inner.page_table.translate_iova(page).map_err(|_| EFAULT)?; + // SAFETY: `phys` is checked to be a RAM page; it stays mapped (and its backing pinned) + // while the exec lock is held, and is only read here. + let ram = unsafe { kernel::page::Page::borrow_phys(&phys) }.ok_or(EFAULT)?; + let target = &mut buf[done..done + count]; + ram.with_pointer_into_page(within, count, |p| { + pgtable::clean_invalidate_poc(p, count); + pgtable::poc_barrier(); + // SAFETY: with_pointer_into_page() checked that `count` bytes at `p` are inside the + // page, and `target` holds exactly `count` bytes. + unsafe { core::ptr::copy_nonoverlapping(p, target.as_mut_ptr(), count) }; + Ok(()) + })?; + done += count; + } + Ok(()) + } + /// Status shared by every queue using this VM, if enabled by its GPU. pub(crate) fn status(&self) -> Option<&Arc> { self.status.as_ref() @@ -2348,7 +2422,149 @@ impl Drop for VmInner { } } +/// Physical regions of a UAT whose handoff the generation has already established itself +/// ([`Uat::new_adopted`]). None of them needs to be in the kernel linear map. +#[cfg(CONFIG_DRM_ASAHI_G15)] +pub(crate) struct AdoptedRegions { + /// The handoff region (PPL magic, lock and flush requests). + pub(crate) handoff: Range, + /// The TTBAT. + pub(crate) ttbs: Range, + /// The firmware kernel (upper) root, which every user context names as its upper half. + pub(crate) kernel_root: PhysicalAddr, +} + impl Uat { + #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Map a region given by its physical range, write-combined. + fn map_physical(range: &Range, min_size: usize) -> Result { + let size: usize = (range.end.checked_sub(range.start).ok_or(EINVAL)?).try_into()?; + if size < min_size || (range.start | range.end) & (UAT_PGMSK as PhysicalAddr) != 0 { + return Err(EINVAL); + } + // SAFETY: All-zero is a valid `struct resource` (no name, parent or children). + let mut raw: kernel::bindings::resource = unsafe { core::mem::zeroed() }; + raw.start = range.start; + raw.end = range.end - 1; + raw.flags = kernel::bindings::IORESOURCE_MEM as _; + // SAFETY: `Resource` is a transparent wrapper around `Opaque`, and + // `raw` describes a valid memory range. + let res: io::resource::Resource = + unsafe { core::mem::transmute(kernel::types::Opaque::new(raw)) }; + // A carveout outside System RAM is mapped write-combined; a reserved-memory region + // inside System RAM (mapped by the kernel) through the kernel's own mapping, as + // `map_region` does for the other generations. + let mode = crate::g15::memory_mode(range.start, size as u64)?; + let ram = mode == kernel::bindings::MEMREMAP_WB as u32; + let flags = if ram { + io::mem::MemFlag::WB + } else { + io::mem::MemFlag::WC + }; + // SAFETY: The UAT regions are only accessed through this mapping by the CPU; the GPU + // reaches them through its own translation. + let map = unsafe { io::mem::Mem::try_new(res, flags.into()) }?; + Ok(UatRegion { + base: range.start, + map, + }) + } + + #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Creates a `Uat` for a handoff that the caller establishes itself (G15). + /// + /// This only maps the handoff and the TTBAT (write-combined); neither is written here. The + /// caller performs the PPL magic exchange and installs TTBAT context 0 before any user `Vm` is + /// bound, naming [`Self::kernel_lower_vm`] as its lower half and `regions.kernel_root` as its + /// upper half. Bound user `Vm`s also name `regions.kernel_root` as their upper half. + /// + /// The firmware kernel root is not in the kernel linear map, so the driver cannot walk it. + /// [`Self::kernel_vm`] instead manages the driver window (the kernel range) below a + /// driver-owned shadow root that no context names: the caller links the window's second-level + /// entries ([`Self::kernel_window_root`]) into the firmware's own table for that window after + /// mapping and before the firmware uses the new addresses. Page tables are never freed on + /// unmap, so linked entries stay valid. + pub(crate) fn new_adopted( + dev: &driver::AsahiDrmDevice, + geometry: UatGeometry, + regions: &AdoptedRegions, + ) -> Result { + let handoff_rgn = Self::map_physical(®ions.handoff, HANDOFF_SIZE)?; + let ttbs_rgn = Self::map_physical(®ions.ttbs, SLOTS_SIZE)?; + + // SAFETY: The Handoff struct layout matches the firmware's view of memory at this + // address, and the region is at least large enough per the check above. + let handoff = unsafe { &(handoff_rgn.map.ptr() as *mut Handoff).as_ref().unwrap() }; + + let inner = Arc::pin_init( + try_pin_init!(UatInner { + firmware: UatFirmware::Handoff, + // The G15 UAT walker does not snoop the CPU caches. + clean_poc: true, + handoff_flush <- pin_init::pin_init_array_from_fn(|i| { + new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") + }), + shared <- new_mutex!( + UatShared { + kernel_ttb1: regions.kernel_root, + execution_contexts: !EXECUTION_CONTEXT_MASK, + context_generations: [0; UAT_NUM_CTX], + secondary_ttb1: None, + map_kernel_to_user: true, + handoff_rgn, + ttbs_rgn, + }, + "uat_shared" + ), + }), + GFP_KERNEL, + )?; + + // The shadow root is retained for the lifetime of the driver window it describes. + let shadow = kernel::page::Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; + let shadow_root = kernel::page::Page::into_phys(shadow); + + let kernel_lower_vm = + Vm::new(dev.into(), inner.clone(), geometry.user_range(), geometry, None, 1)?; + let kernel_vm = Vm::new( + dev.into(), + inner.clone(), + geometry.kernel_range(), + geometry, + Some(shadow_root), + 0, + )?; + + Ok(Self { + dev: dev.into(), + geometry, + kernel_vm, + kernel_lower_vm, + inner, + slots: slotalloc::SlotAllocator::new( + UAT_USER_CTX as u32, + (), + |_inner, _slot| Some(SlotInner()), + c_str!("Uat::SlotAllocator"), + static_lock_class!(), + static_lock_class!(), + )?, + }) + } + + #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Root of the kernel lower `Vm`, for TTBAT context 0 of an adopted handoff. + pub(crate) fn kernel_lower_root(&self) -> PhysicalAddr { + self.kernel_lower_vm.ttb() + } + + #[cfg(CONFIG_DRM_ASAHI_G15)] + /// Root whose driver-window entry names the second-level table of [`Self::kernel_vm`]'s + /// driver window. With [`Self::new_adopted`] this is the shadow root that no context names. + pub(crate) fn kernel_window_root(&self) -> PhysicalAddr { + self.kernel_vm.ttb() + } + /// Map a bootloader-preallocated memory region fn map_region( dev: &device::Device, @@ -2688,6 +2904,8 @@ impl Uat { Arc::pin_init( try_pin_init!(UatInner { firmware, + #[cfg(CONFIG_DRM_ASAHI_G15)] + clean_poc: false, handoff_flush <- pin_init::pin_init_array_from_fn(|i| { new_mutex!(HandoffFlush(&handoff.flush[i]), "handoff_flush") }), diff --git a/drivers/gpu/drm/asahi/mmu/context.rs b/drivers/gpu/drm/asahi/mmu/context.rs index d459a3a91afce..9267f3a25def7 100644 --- a/drivers/gpu/drm/asahi/mmu/context.rs +++ b/drivers/gpu/drm/asahi/mmu/context.rs @@ -200,7 +200,9 @@ impl Vm { /// Requires a job mapping guard before validating or installing these views. pub(crate) fn install_job_context_aliases(&self) -> Result { - let context = self.context_bindings.as_ref().ok_or(EINVAL)?; + let Some(context) = self.context_bindings.as_ref() else { + return Ok(()); + }; let (generation, snapshot) = { let bindings = context.bindings.lock(); let mut snapshot = KVec::with_capacity(CONTEXT_VIEWS.len(), GFP_KERNEL)?; diff --git a/drivers/gpu/drm/asahi/mmu/lifetime.rs b/drivers/gpu/drm/asahi/mmu/lifetime.rs index 98006b9459c98..4b881d6f1ebfd 100644 --- a/drivers/gpu/drm/asahi/mmu/lifetime.rs +++ b/drivers/gpu/drm/asahi/mmu/lifetime.rs @@ -333,6 +333,12 @@ impl Vm { drop(previous); } + /// Whether userspace has permanently closed this VM's map admission. + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn is_closed(&self) -> bool { + self.lifetime.as_ref().is_some_and(|lifetime| lifetime.state.lock().closed) + } + /// Tests mappings and reservations which userspace must never replace. pub(crate) fn driver_range_overlaps(&self, range: Range) -> bool { let Some(lifetime) = self.lifetime.as_ref() else { diff --git a/drivers/gpu/drm/asahi/pgtable.rs b/drivers/gpu/drm/asahi/pgtable.rs index af5fdbb5dc27b..1655c1183438b 100644 --- a/drivers/gpu/drm/asahi/pgtable.rs +++ b/drivers/gpu/drm/asahi/pgtable.rs @@ -53,6 +53,75 @@ const UAT_LEVELS: usize = 3; /// Number of address bits below the index of a top-level page table entry. pub(crate) const UAT_TOP_LEVEL_SHIFT: usize = UAT_PGBIT + (UAT_LEVELS - 1) * UAT_LVBIT; +/// Size of the smallest CPU data cache line (`CTR_EL0.DminLine`). +#[inline(always)] +fn dcache_line_size() -> usize { + let ctr: u64; + // SAFETY: CTR_EL0 is readable at EL1 and reading it has no side effects. + unsafe { + core::arch::asm!( + "mrs {ctr}, ctr_el0", + ctr = out(reg) ctr, + options(nomem, nostack, preserves_flags) + ) + }; + 4 << ((ctr >> 16) & 0xf) +} + +/// Cleans the CPU data cache lines covering `len` bytes at `ptr` to the point of coherency, for an +/// agent that does not snoop the CPU caches. [`poc_barrier`] completes the maintenance. +pub(crate) fn clean_poc(ptr: *const u8, len: usize) { + if len == 0 { + return; + } + let line = dcache_line_size(); + let end = ptr as usize + len; + let mut addr = ptr as usize & !(line - 1); + while addr < end { + // SAFETY: Cleaning a cache line of a mapped kernel address writes dirty data back without + // changing memory contents. The asm is a compiler barrier, so earlier stores are emitted + // first, and the architecture orders them with the clean of the same line. + unsafe { + core::arch::asm!( + "dc cvac, {addr}", + addr = in(reg) addr, + options(nostack, preserves_flags) + ) + }; + addr += line; + } +} + +/// Cleans and invalidates the CPU data cache lines covering `len` bytes at `ptr` to the point of +/// coherency, so that later CPU reads observe what a non-snooping agent wrote. Lines the CPU wrote +/// are written back first. [`poc_barrier`] completes the maintenance. +#[cfg(CONFIG_DRM_ASAHI_G15)] +pub(crate) fn clean_invalidate_poc(ptr: *const u8, len: usize) { + if len == 0 { + return; + } + let line = dcache_line_size(); + let end = ptr as usize + len; + let mut addr = ptr as usize & !(line - 1); + while addr < end { + // SAFETY: Clean+invalidate of a mapped kernel address never discards dirty data. + unsafe { + core::arch::asm!( + "dc civac, {addr}", + addr = in(reg) addr, + options(nostack, preserves_flags) + ) + }; + addr += line; + } +} + +/// Completes earlier point-of-coherency maintenance for the outer shareable domain (`dsb osh`). +pub(crate) fn poc_barrier() { + // SAFETY: Barriers are always safe. + unsafe { core::arch::asm!("dsb osh", options(nostack, preserves_flags)) }; +} + /// Runs `f` on the first `count` top-level entries of the root page table at `ttb`. /// /// # Safety @@ -184,6 +253,11 @@ impl Prot { Self { memattr, ..*self } } + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) const fn leaf_bits(&self) -> u64 { + self.as_pte() + } + const fn as_pte(&self) -> u64 { (self.ap as u64) << UAT_AP_SHIFT | (self.high_bits as u64) << UAT_HIGH_BITS_SHIFT @@ -229,6 +303,10 @@ pub(crate) struct UatPageTable { va_range: Range, ias_mask: u64, oas_mask: u64, + /// Clean new tables and changed entries to the point of coherency, for a UAT walker that does + /// not snoop the CPU caches ([`Self::enable_clean_poc`]). + #[cfg(CONFIG_DRM_ASAHI_G15)] + clean_poc: bool, } impl UatPageTable { @@ -243,9 +321,43 @@ impl UatPageTable { va_range: 0..(1u64 << ias), ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, + #[cfg(CONFIG_DRM_ASAHI_G15)] + clean_poc: false, }) } + /// Makes every later change of this page table visible to a UAT walker that does not snoop the + /// CPU caches (G15), and cleans the root, which was zeroed through the cacheable linear map. + /// + /// From now on, a new table page is cleaned before its parent descriptor names it, a changed + /// table or leaf descriptor is cleaned after the store, and every mutating walk ends with a + /// `dsb osh`. Call it before anything is mapped and before any context names the root. + #[cfg(CONFIG_DRM_ASAHI_G15)] + pub(crate) fn enable_clean_poc(&mut self) -> Result { + self.clean_poc = true; + // SAFETY: The root is a RAM page owned or borrowed by this page table for its lifetime. + let root = unsafe { Page::borrow_phys(&self.ttb) }.ok_or(EIO)?; + root.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; + poc_barrier(); + Ok(()) + } + + /// Whether changes are cleaned to the point of coherency (always false without G15 support). + #[inline(always)] + fn cleans_poc(&self) -> bool { + #[cfg(CONFIG_DRM_ASAHI_G15)] + { + self.clean_poc + } + #[cfg(not(CONFIG_DRM_ASAHI_G15))] + { + false + } + } + /// Borrow a root; track individual child ownership only when other agents /// can install tables below it. Handoff roots have driver-owned children. pub(crate) fn new_with_ttb( @@ -284,6 +396,8 @@ impl UatPageTable { va_range, ias_mask: (1u64 << ias) - 1, oas_mask: (1u64 << oas) - 1, + #[cfg(CONFIG_DRM_ASAHI_G15)] + clean_poc: false, }) } @@ -291,16 +405,48 @@ impl UatPageTable { self.ttb } - fn with_pages( + /// Walks the leaves of `iova_range` without changing them. + fn with_pages(&mut self, iova_range: Range, alloc: bool, free: bool, cb: F) -> Result + where + F: FnMut(u64, &[Pte]) -> Result, + { + self.walk(iova_range, alloc, free, false, cb) + } + + /// Walks the leaves of `iova_range`, which `cb` may change (`mutate`). + /// + /// With point-of-coherency cleaning enabled, the changed leaves are cleaned after `cb`, and a + /// walk that may have changed any descriptor ends with a barrier, also on failure. + fn walk( + &mut self, + iova_range: Range, + alloc: bool, + free: bool, + mutate: bool, + cb: F, + ) -> Result + where + F: FnMut(u64, &[Pte]) -> Result, + { + let result = self.walk_inner(iova_range, alloc, free, mutate, cb); + if self.cleans_poc() && (alloc || free || mutate) { + poc_barrier(); + } + result + } + + fn walk_inner( &mut self, iova_range: Range, alloc: bool, free: bool, + mutate: bool, mut cb: F, ) -> Result where F: FnMut(u64, &[Pte]) -> Result, { + let clean = self.cleans_poc(); mod_pr_debug!( "UATPageTable::with_pages: {:#x?} alloc={} free={}\n", iova_range, @@ -382,6 +528,14 @@ impl UatPageTable { if upte_val == 0 && alloc { let pt_page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; mod_pr_debug!("UATPageTable::with_pages: alloc PT at {:#x}\n", pt_page.phys()); + if clean { + // A non-snooping walker must never follow a parent to a + // table whose zeroes are still in the CPU caches. + pt_page.with_pointer_into_page(0, UAT_PGSZ, |p| { + clean_poc(p, UAT_PGSZ); + Ok(()) + })?; + } if let Some(tables) = self.driver_tables.as_mut() { tables.push(pt_page.phys(), GFP_KERNEL)?; } @@ -393,6 +547,9 @@ impl UatPageTable { // already-visible parent against earlier clears. crate::mem::sync(); upte.store(upte_val, Ordering::Relaxed); + if clean { + clean_poc(uptep.cast(), PTE_SIZE); + } } if upte_val & PTE_TYPE_BITS == PTE_TYPE_LEAF_TABLE { Ok(Some(upte_val & self.oas_mask & (!UAT_PGMSK as u64))) @@ -448,8 +605,12 @@ impl UatPageTable { // SAFETY: We know this is a valid pointer to PTEs and the range is valid and // checked by with_pointer_into_page(). let ptes = unsafe { core::slice::from_raw_parts(ptep, count) }; - cb(iova, ptes)?; - Ok(()) + let result = cb(iova, ptes); + if clean && mutate { + // Also after a partial change: the callback may have stored some leaves. + clean_poc(p, count * PTE_SIZE); + } + result })?; let block = 1 << (UAT_PGBIT + UAT_LVBIT); @@ -505,6 +666,9 @@ impl UatPageTable { return Err(EIO); } pte.store(0, Ordering::Release); + if self.cleans_poc() { + clean_poc(p, PTE_SIZE); + } Ok(()) })?; } @@ -625,7 +789,7 @@ impl UatPageTable { let pte_bits = self.pte_bits(); - self.with_pages(iova_range, true, false, |iova, ptes| { + self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); if ptev != 0 { @@ -650,7 +814,7 @@ impl UatPageTable { iova_range, prot ); - self.with_pages(iova_range, true, false, |iova, ptes| { + self.walk(iova_range, true, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { let ptev = pte.load(Ordering::Relaxed); if ptev & PTE_TYPE_BITS != PTE_TYPE_LEAF_TABLE { @@ -669,7 +833,7 @@ impl UatPageTable { pub(crate) fn unmap_pages(&mut self, iova_range: Range) -> Result { mod_pr_debug!("UATPageTable::unmap_pages: {:#x?}\n", iova_range); - self.with_pages(iova_range, false, false, |iova, ptes| { + self.walk(iova_range, false, false, true, |iova, ptes| { for (idx, pte) in ptes.iter().enumerate() { if pte.load(Ordering::Relaxed) & PTE_TYPE_LEAF_TABLE == 0 { pr_err!( @@ -686,7 +850,7 @@ impl UatPageTable { /// Discards an aligned map operation that failed before acquiring a GPUVA owner. /// Missing tables and leaves are expected; existing table storage stays allocated. pub(crate) fn discard_partial_map(&mut self, iova_range: Range) -> Result { - self.with_pages(iova_range, false, false, |_, ptes| { + self.walk(iova_range, false, false, true, |_, ptes| { for pte in ptes { pte.store(0, Ordering::Relaxed); } diff --git a/drivers/gpu/drm/asahi/regs.rs b/drivers/gpu/drm/asahi/regs.rs index fb22569ea5fdf..aa7948b9792c8 100644 --- a/drivers/gpu/drm/asahi/regs.rs +++ b/drivers/gpu/drm/asahi/regs.rs @@ -39,10 +39,6 @@ pub(crate) const SGX_SIZE: usize = 0x1000000; const FAULT_INFO: usize = 0x17030; -/// A nonzero low nibble permits writes through the G17 queue configuration port. -const G17_HOST_IRQ_SUMMARY: usize = 0xe01000; -const G17_HOST_POWERED: u32 = 0xf; - const ID_VERSION: usize = 0xd04000; const ID_UNK08: usize = 0xd04008; const ID_COUNTS_1: usize = 0xd04010; @@ -64,6 +60,8 @@ const G17_BRIDGE_ENABLE: u32 = 0x1; // BIT(0) /// G17 dynamic power gating control. Clearing these bits enables gating of idle GPU cores. const G17_GATING_CONTROL: usize = 0xd06030; const G17_GATING_DISABLE: u32 = 0x6; // BIT(1) | BIT(2) +/// Held while the host writes the KSM ports. +const G17_GATING_HOLD: u32 = 0x4; // BIT(2) /// These performance selectors must be sampled before firmware can gate the core registers. const G17_PERF_CONTROL: usize = 0xe0141c; @@ -568,21 +566,26 @@ impl G17Resources { }) } - /// Installs both render queues while the caller holds the device mutex. - /// No other port publisher may interleave between this power witness and - /// the final validity strobe. False means no write was attempted. - pub(crate) fn install_render_pair( - &self, - registration: &crate::g17::fw::kick::RenderRegistration, - ) -> Result { + /// Installs kick queues ([`crate::g17::fw::kick::KickRegistration`] writes) while the + /// caller holds the device mutex. No other port publisher may interleave between the + /// gating hold and the final validity strobe. AGXG17X configureHardware writes without + /// a power witness (G17C's reads 0 before and after a kick), so this always writes and + /// returns true. + pub(crate) fn install_kick_queues(&self, writes: &[(usize, u64)]) -> Result { let sgx = self.sgx.try_access().ok_or(ENODEV)?; let sgx = (&*sgx).as_view(); - if sgx.relaxed().read32(G17_HOST_IRQ_SUMMARY) & G17_HOST_POWERED == 0 { - return Ok(false); - } - for &(offset, value) in registration.writes() { + // AGXG17X brackets every KSM port write with AGXArmFenderChinook's + // disableDynamicFenderGating (BIT(2) set) and enableDynamicFenderGating + // (BIT(1) | BIT(2) cleared), each followed by a read-back. + let gating = sgx.try_read32(G17_GATING_CONTROL)?; + sgx.try_write32(gating | G17_GATING_HOLD, G17_GATING_CONTROL)?; + sgx.try_read32(G17_GATING_CONTROL)?; + for &(offset, value) in writes { sgx.try_write64(value, offset)?; } + let released = sgx.try_read32(G17_GATING_CONTROL)? & !G17_GATING_DISABLE; + sgx.try_write32(released, G17_GATING_CONTROL)?; + sgx.try_read32(G17_GATING_CONTROL)?; Ok(true) } diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c index d4577663a1f0e..aa36b79d0f6c4 100644 --- a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c +++ b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c @@ -189,13 +189,12 @@ static int aspeed_gfx_load(struct drm_device *drm) } reset_control_deassert(priv->rst); - priv->clk = devm_clk_get(drm->dev, NULL); + priv->clk = devm_clk_get_enabled(drm->dev, NULL); if (IS_ERR(priv->clk)) { dev_err(&pdev->dev, - "missing or invalid clk device tree entry"); + "failed to get and enable clock\n"); return PTR_ERR(priv->clk); } - clk_prepare_enable(priv->clk); /* Sanitize control registers */ writel(0, priv->base + CRT_CTRL1); diff --git a/drivers/gpu/drm/bridge/th1520-dw-hdmi.c b/drivers/gpu/drm/bridge/th1520-dw-hdmi.c index cbea8b14cd4b0..2fa7f2d46b3dc 100644 --- a/drivers/gpu/drm/bridge/th1520-dw-hdmi.c +++ b/drivers/gpu/drm/bridge/th1520-dw-hdmi.c @@ -135,8 +135,8 @@ static int th1520_dw_hdmi_probe(struct platform_device *pdev) plat_data->priv_data = hdmi; hdmi->dw_hdmi = dw_hdmi_probe(pdev, plat_data); - if (IS_ERR(hdmi)) - return PTR_ERR(hdmi); + if (IS_ERR(hdmi->dw_hdmi)) + return PTR_ERR(hdmi->dw_hdmi); platform_set_drvdata(pdev, hdmi); @@ -145,9 +145,9 @@ static int th1520_dw_hdmi_probe(struct platform_device *pdev) static void th1520_dw_hdmi_remove(struct platform_device *pdev) { - struct dw_hdmi *hdmi = platform_get_drvdata(pdev); + struct th1520_hdmi *hdmi = platform_get_drvdata(pdev); - dw_hdmi_remove(hdmi); + dw_hdmi_remove(hdmi->dw_hdmi); } static const struct of_device_id th1520_dw_hdmi_of_table[] = { diff --git a/drivers/gpu/drm/display/drm_dp_helper.c b/drivers/gpu/drm/display/drm_dp_helper.c index e623ccb4c1d8a..488fd123a01ad 100644 --- a/drivers/gpu/drm/display/drm_dp_helper.c +++ b/drivers/gpu/drm/display/drm_dp_helper.c @@ -4634,7 +4634,9 @@ static int dp_aux_backlight_update_status(struct backlight_device *bd) if (!backlight_is_blank(bd)) { if (!bl->enabled) { - drm_edp_backlight_enable(bl->aux, &bl->info, brightness); + ret = drm_edp_backlight_enable(bl->aux, &bl->info, brightness); + if (ret) + return ret; bl->enabled = true; return 0; } diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c index 687b36eea0c70..9124cbe1ff889 100644 --- a/drivers/gpu/drm/drm_bridge.c +++ b/drivers/gpu/drm/drm_bridge.c @@ -414,8 +414,10 @@ void drm_bridge_add(struct drm_bridge *bridge) * in bridge_lingering_list. Remove it or bridge_lingering_list will be * corrupted when adding this bridge to bridge_list below. */ + mutex_lock(&bridge_lock); if (!list_empty(&bridge->list)) list_del_init(&bridge->list); + mutex_unlock(&bridge_lock); mutex_init(&bridge->hpd_mutex); @@ -556,8 +558,10 @@ int drm_bridge_attach(struct drm_encoder *encoder, struct drm_bridge *bridge, if (!bridge->container) DRM_WARN("DRM bridge corrupted or not allocated by devm_drm_bridge_alloc()\n"); - if (list_empty(&bridge->list)) - DRM_WARN("Missing drm_bridge_add() before attach\n"); + scoped_guard(mutex, &bridge_lock) { + if (list_empty(&bridge->list)) + DRM_WARN("Missing drm_bridge_add() before attach\n"); + } drm_bridge_get(bridge); diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 80ca785bdb262..3dfa64dc31096 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -1724,10 +1724,10 @@ __drm_fb_helper_initial_config_and_unlock(struct drm_fb_helper *fb_helper) ret = drm_fb_helper_single_fb_probe(fb_helper); if (ret < 0) { - if (ret == -EAGAIN) { - fb_helper->deferred_setup = true; + /* modesets are probed but fb_helper->fb isn't; defer restore too */ + fb_helper->deferred_setup = true; + if (ret == -EAGAIN) ret = 0; - } mutex_unlock(&fb_helper->lock); goto err_drm_fb_helper_release_info; diff --git a/drivers/gpu/drm/gud/gud_pipe.c b/drivers/gpu/drm/gud/gud_pipe.c index aa7792966287b..5aa5ac2f37274 100644 --- a/drivers/gpu/drm/gud/gud_pipe.c +++ b/drivers/gpu/drm/gud/gud_pipe.c @@ -507,6 +507,7 @@ int gud_plane_atomic_check(struct drm_plane *plane, for_each_new_connector_in_state(state, connector, connector_state, i) { if (connector_state->crtc) break; + connector_state = NULL; } /* diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 1f303d4eaa4d4..d5aadced7da0c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -1766,6 +1766,7 @@ static int eb_reinit_userptr(struct i915_execbuffer *eb) static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb) { + const unsigned int count = eb->buffer_count; bool have_copy = false; struct eb_vma *ev; int err = 0; @@ -1868,7 +1869,6 @@ static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb) out: if (have_copy) { - const unsigned int count = eb->buffer_count; unsigned int i; for (i = 0; i < count; i++) { diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index a33b19c04737a..0f9b4f1dd3cb6 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -1904,7 +1904,8 @@ static void rps_work(struct work_struct *work) out: spin_lock_irq(gt->irq_lock); - gen6_gt_pm_unmask_irq(gt, rps->pm_events); + if (intel_rps_is_active(rps)) + gen6_gt_pm_unmask_irq(gt, rps->pm_events); spin_unlock_irq(gt->irq_lock); } diff --git a/drivers/gpu/drm/imagination/pvr_mmu.c b/drivers/gpu/drm/imagination/pvr_mmu.c index 62eae7fcd5a26..67e73d7d5d0e9 100644 --- a/drivers/gpu/drm/imagination/pvr_mmu.c +++ b/drivers/gpu/drm/imagination/pvr_mmu.c @@ -2337,7 +2337,7 @@ void pvr_mmu_op_context_destroy(struct pvr_mmu_op_context *op_ctx) * @ctx: MMU context associated with owning VM context. * @sgt: Scatter gather table containing pages pinned for use by this context. * @device_addr: Virtual device address at the start of the requested mapping. - * @sgt_offset: Start offset of the requested device-virtual memory mapping. + * @sgt_offset: Offset into @sgt of the start of the requested mapping. * @size: Size in bytes of the requested device-virtual memory mapping. For an * unmapping, this should be zero so that no page tables are allocated. * @@ -2350,7 +2350,6 @@ struct pvr_mmu_op_context * pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt, u64 device_addr, u64 sgt_offset, u64 size) { - u64 start_addr = device_addr + sgt_offset; int err; struct pvr_mmu_op_context *op_ctx = kzalloc_obj(*op_ctx); @@ -2365,18 +2364,18 @@ pvr_mmu_op_context_create(struct pvr_mmu_context *ctx, struct sg_table *sgt, if (size) { /* - * The number of page table objects we need to prealloc is - * indicated by the mapping size, start address and the sizes - * of the areas mapped per PT or PD. The range calculation is - * identical to that for the index into a table for a device - * address, so we reuse those functions here. + * The page tables needed are set by the device-virtual range + * being mapped: one level 1 table per 1GiB region and one + * level 0 table per 2MiB region the range touches. Tables that + * already exist are not consumed, so this is an upper bound. */ - const u32 l1_start_idx = pvr_page_table_l2_idx(start_addr); - const u32 l1_end_idx = pvr_page_table_l2_idx(start_addr + size); - const u32 l1_count = l1_end_idx - l1_start_idx + 1; - const u32 l0_start_idx = pvr_page_table_l1_idx(start_addr); - const u32 l0_end_idx = pvr_page_table_l1_idx(start_addr + size); - const u32 l0_count = l0_end_idx - l0_start_idx + 1; + const u64 last_addr = device_addr + size - 1; + const u64 l1_count = + (last_addr >> ROGUE_MMUCTRL_VADDR_PC_INDEX_SHIFT) - + (device_addr >> ROGUE_MMUCTRL_VADDR_PC_INDEX_SHIFT) + 1; + const u64 l0_count = + (last_addr >> ROGUE_MMUCTRL_VADDR_PD_INDEX_SHIFT) - + (device_addr >> ROGUE_MMUCTRL_VADDR_PD_INDEX_SHIFT) + 1; /* * Alloc and push page table entries until we have enough of diff --git a/drivers/gpu/drm/imagination/pvr_vm.c b/drivers/gpu/drm/imagination/pvr_vm.c index 55cc999f3708b..cbdd15ed74fdb 100644 --- a/drivers/gpu/drm/imagination/pvr_vm.c +++ b/drivers/gpu/drm/imagination/pvr_vm.c @@ -418,6 +418,8 @@ pvr_vm_gpuva_unmap(struct drm_gpuva_op *op, void *op_ctx) static int pvr_vm_gpuva_remap(struct drm_gpuva_op *op, void *op_ctx) { + /* The split parts belong to the object of the mapping being split. */ + struct drm_gpuvm_bo *vm_bo = op->remap.unmap->va->vm_bo; struct pvr_vm_bind_op *ctx = op_ctx; u64 va_start = 0, va_range = 0; int err; @@ -433,14 +435,12 @@ pvr_vm_gpuva_remap(struct drm_gpuva_op *op, void *op_ctx) drm_gpuva_remap(&ctx->prev_va->base, &ctx->next_va->base, &op->remap); if (op->remap.prev) { - pvr_gem_object_get(gem_to_pvr_gem(ctx->prev_va->base.gem.obj)); - drm_gpuva_link(&ctx->prev_va->base, ctx->gpuvm_bo); + drm_gpuva_link(&ctx->prev_va->base, vm_bo); ctx->prev_va = NULL; } if (op->remap.next) { - pvr_gem_object_get(gem_to_pvr_gem(ctx->next_va->base.gem.obj)); - drm_gpuva_link(&ctx->next_va->base, ctx->gpuvm_bo); + drm_gpuva_link(&ctx->next_va->base, vm_bo); ctx->next_va = NULL; } diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c index 369ee06434bbd..765b895619e8b 100644 --- a/drivers/gpu/drm/panthor/panthor_sched.c +++ b/drivers/gpu/drm/panthor/panthor_sched.c @@ -3675,6 +3675,9 @@ int panthor_group_create(struct panthor_file *pfile, hweight64(group_args->tiler_core_mask) < group_args->max_tiler_cores) return -EINVAL; + if (group_args->queues.count > sched->cs_slot_count) + return -EINVAL; + group = kzalloc_obj(*group); if (!group) return -ENOMEM; diff --git a/drivers/gpu/drm/sysfb/simpledrm.c b/drivers/gpu/drm/sysfb/simpledrm.c index bb32b258a9b5f..27208f700de45 100644 --- a/drivers/gpu/drm/sysfb/simpledrm.c +++ b/drivers/gpu/drm/sysfb/simpledrm.c @@ -839,8 +839,10 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv, /* Connector */ connector = &sdev->connector; + /* A built-in laptop panel the loader hands over (Apple, before DCP). */ ret = drm_connector_init(dev, connector, &simpledrm_connector_funcs, - DRM_MODE_CONNECTOR_Unknown); + of_property_read_bool(dev->dev->of_node, "apple,internal-panel") ? + DRM_MODE_CONNECTOR_eDP : DRM_MODE_CONNECTOR_Unknown); if (ret) return ERR_PTR(ret); drm_connector_helper_add(connector, &simpledrm_connector_helper_funcs); diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c index 49ea2ed0996b4..7b8b3364000fc 100644 --- a/drivers/gpu/drm/vc4/vc4_bo.c +++ b/drivers/gpu/drm/vc4/vc4_bo.c @@ -204,7 +204,7 @@ static struct list_head *vc4_get_cache_list_for_size(struct drm_device *dev, struct list_head *new_list; uint32_t i; - new_list = kmalloc_objs(struct list_head, new_size); + new_list = kvmalloc_objs(struct list_head, new_size); if (!new_list) return NULL; @@ -224,7 +224,7 @@ static struct list_head *vc4_get_cache_list_for_size(struct drm_device *dev, for (i = vc4->bo_cache.size_list_size; i < new_size; i++) INIT_LIST_HEAD(&new_list[i]); - kfree(vc4->bo_cache.size_list); + kvfree(vc4->bo_cache.size_list); vc4->bo_cache.size_list = new_list; vc4->bo_cache.size_list_size = new_size; } @@ -1052,6 +1052,7 @@ static void vc4_bo_cache_destroy(struct drm_device *dev, void *unused) cancel_work_sync(&vc4->bo_cache.time_work); vc4_bo_cache_purge(dev); + kvfree(vc4->bo_cache.size_list); for (i = 0; i < vc4->num_labels; i++) { if (vc4->bo_labels[i].num_allocated) { diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c index 8e4c665ca159d..284a971e28b31 100644 --- a/drivers/gpu/drm/vc4/vc4_drv.c +++ b/drivers/gpu/drm/vc4/vc4_drv.c @@ -360,8 +360,11 @@ static int vc4_drm_bind(struct device *dev) } ret = aperture_remove_all_conflicting_devices(driver->name); - if (ret) + if (ret) { + if (firmware) + rpi_firmware_put(firmware); goto err; + } if (firmware) { ret = rpi_firmware_property(firmware, diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h index 0f5958c1e6b60..b0e82c2dc1bae 100644 --- a/drivers/gpu/drm/vc4/vc4_drv.h +++ b/drivers/gpu/drm/vc4/vc4_drv.h @@ -1051,7 +1051,7 @@ void vc4_plane_async_set_fb(struct drm_plane *plane, /* vc4_v3d.c */ extern struct platform_driver vc4_v3d_driver; extern const struct of_device_id vc4_v3d_dt_match[]; -int vc4_v3d_get_bin_slot(struct vc4_dev *vc4); +int vc4_v3d_get_bin_slot(struct vc4_dev *vc4, long timeout); int vc4_v3d_bin_bo_get(struct vc4_dev *vc4, bool *used); void vc4_v3d_bin_bo_put(struct vc4_dev *vc4); int vc4_v3d_pm_get(struct vc4_dev *vc4); diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c index e231c906709c1..6ee4bd77d4331 100644 --- a/drivers/gpu/drm/vc4/vc4_gem.c +++ b/drivers/gpu/drm/vc4/vc4_gem.c @@ -877,11 +877,17 @@ vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec) drm_gem_object_put(&bo->base.base); } - /* Free up the allocation of any bin slots we used. */ + /* Free up the allocation of any bin slots we used. Jobs that ran to + * completion had their slots released in vc4_irq_finish_render_job(). + * Only jobs that never completed still have slots to be released here. + */ spin_lock_irqsave(&vc4->job_lock, irqflags); vc4->bin_alloc_used &= ~exec->bin_slots; spin_unlock_irqrestore(&vc4->job_lock, irqflags); + /* Let anyone waiting on the binner pool retry. */ + wake_up_all(&vc4->job_wait_queue); + /* Release the reference on the binner BO if needed. */ if (exec->bin_bo_used) vc4_v3d_bin_bo_put(vc4); diff --git a/drivers/gpu/drm/vc4/vc4_irq.c b/drivers/gpu/drm/vc4/vc4_irq.c index 7877d493d80e6..dc10d3a4d90f2 100644 --- a/drivers/gpu/drm/vc4/vc4_irq.c +++ b/drivers/gpu/drm/vc4/vc4_irq.c @@ -74,7 +74,7 @@ vc4_overflow_mem_work(struct work_struct *work) bo = vc4->bin_bo; - bin_bo_slot = vc4_v3d_get_bin_slot(vc4); + bin_bo_slot = vc4_v3d_get_bin_slot(vc4, msecs_to_jiffies(500)); if (bin_bo_slot < 0) { drm_err(&vc4->base, "Couldn't allocate binner overflow mem\n"); goto complete; @@ -165,6 +165,10 @@ vc4_irq_finish_render_job(struct drm_device *dev) trace_vc4_rcl_end_irq(dev, exec->seqno); vc4->finished_seqno++; + + vc4->bin_alloc_used &= ~exec->bin_slots; + exec->bin_slots = 0; + list_move_tail(&exec->head, &vc4->job_done_list); nextbin = vc4_first_bin_job(vc4); @@ -276,7 +280,7 @@ vc4_irq_disable(struct drm_device *dev) V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS); /* Finish any interrupt handler still in flight. */ - synchronize_irq(vc4->irq); + disable_irq(vc4->irq); cancel_work_sync(&vc4->overflow_mem_work); } @@ -284,7 +288,6 @@ vc4_irq_disable(struct drm_device *dev) int vc4_irq_install(struct drm_device *dev, int irq) { struct vc4_dev *vc4 = to_vc4_dev(dev); - int ret; if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4)) return -ENODEV; @@ -298,19 +301,8 @@ int vc4_irq_install(struct drm_device *dev, int irq) init_waitqueue_head(&vc4->job_wait_queue); INIT_WORK(&vc4->overflow_mem_work, vc4_overflow_mem_work); - /* Clear any pending interrupts someone might have left around - * for us. - */ - V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS); - - ret = devm_request_irq(dev->dev, irq, vc4_irq, 0, - dev_name(dev->dev), dev); - if (ret) - return ret; - - vc4_irq_enable(dev); - - return 0; + return devm_request_irq(dev->dev, irq, vc4_irq, IRQF_NO_AUTOEN, + dev_name(dev->dev), dev); } void vc4_irq_uninstall(struct drm_device *dev) diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c index f32410420d3e4..6844e8f3948cf 100644 --- a/drivers/gpu/drm/vc4/vc4_v3d.c +++ b/drivers/gpu/drm/vc4/vc4_v3d.c @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -172,46 +173,42 @@ static void vc4_v3d_init_hw(struct drm_device *dev) V3D_WRITE(V3D_VPMBASE, 0); } -int vc4_v3d_get_bin_slot(struct vc4_dev *vc4) +static int bin_slot_try_alloc(struct vc4_dev *vc4) { - struct drm_device *dev = &vc4->base; - unsigned long irqflags; int slot; - uint64_t seqno = 0; - struct vc4_exec_info *exec; - if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4)) - return -ENODEV; + guard(spinlock_irqsave)(&vc4->job_lock); -try_again: - spin_lock_irqsave(&vc4->job_lock, irqflags); slot = ffs(~vc4->bin_alloc_used); if (slot != 0) { /* Switch from ffs() bit index to a 0-based index. */ slot--; vc4->bin_alloc_used |= BIT(slot); - spin_unlock_irqrestore(&vc4->job_lock, irqflags); - return slot; + } else { + slot = -ENOMEM; } - /* Couldn't find an open slot. Wait for render to complete - * and try again. - */ - exec = vc4_last_render_job(vc4); - if (exec) - seqno = exec->seqno; - spin_unlock_irqrestore(&vc4->job_lock, irqflags); + return slot; +} - if (seqno) { - int ret = vc4_wait_for_seqno(dev, seqno, ~0ull, true); +int vc4_v3d_get_bin_slot(struct vc4_dev *vc4, long timeout) +{ + int slot; + long ret; - if (ret == 0) - goto try_again; + if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4)) + return -ENODEV; + /* If the pool is full, wait for a job to release its slots. */ + ret = wait_event_interruptible_timeout(vc4->job_wait_queue, + (slot = bin_slot_try_alloc(vc4)) >= 0, + timeout); + if (ret < 0) return ret; - } + if (ret == 0) + return -ENOMEM; - return -ENOMEM; + return slot; } /* @@ -396,6 +393,7 @@ static int vc4_v3d_runtime_resume(struct device *dev) vc4_v3d_init_hw(&vc4->base); vc4_irq_enable(&vc4->base); + enable_irq(vc4->irq); return 0; } @@ -452,6 +450,12 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data) return ret; vc4->irq = ret; + ret = vc4_irq_install(drm, vc4->irq); + if (ret) { + drm_err(drm, "Failed to install IRQ handler\n"); + return ret; + } + ret = devm_pm_runtime_enable(dev); if (ret) return ret; @@ -473,12 +477,6 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data) V3D_WRITE(V3D_BPOA, 0); V3D_WRITE(V3D_BPOS, 0); - ret = vc4_irq_install(drm, vc4->irq); - if (ret) { - drm_err(drm, "Failed to install IRQ handler\n"); - goto err_put_runtime_pm; - } - pm_runtime_use_autosuspend(dev); pm_runtime_set_autosuspend_delay(dev, 40); /* a little over 2 frames. */ pm_runtime_put_autosuspend(dev); @@ -497,8 +495,15 @@ static void vc4_v3d_unbind(struct device *dev, struct device *master, struct drm_device *drm = data; struct vc4_dev *vc4 = to_vc4_dev(drm); + /* A straggler vc4_reset() re-enables the interrupt. */ + timer_shutdown_sync(&vc4->hangcheck.timer); + cancel_work_sync(&vc4->hangcheck.reset_work); + vc4_irq_uninstall(drm); + /* Flush rather than cancel, so queued completions release their jobs. */ + flush_work(&vc4->job_done_work); + /* Disable the binner's overflow memory address, so the next * driver probe (if any) doesn't try to reuse our old * allocation. diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c index d2a65c968b1f9..565227657b43b 100644 --- a/drivers/gpu/drm/vc4/vc4_validate.c +++ b/drivers/gpu/drm/vc4/vc4_validate.c @@ -402,7 +402,7 @@ validate_tile_binning_config(VALIDATE_ARGS) return -EINVAL; } - bin_slot = vc4_v3d_get_bin_slot(vc4); + bin_slot = vc4_v3d_get_bin_slot(vc4, MAX_SCHEDULE_TIMEOUT); if (bin_slot < 0) { if (bin_slot != -EINTR && bin_slot != -ERESTARTSYS) { drm_err(dev, "Failed to allocate binner memory: %d\n", diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c index d1e7df5001903..5a69606a0593b 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c @@ -324,6 +324,7 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf, unsigned long kmap_num; SVGA3dCopyBox *box; u32 box_count; + u64 src_extent; void *virtual; bool is_iomem; struct vmw_dma_cmd { @@ -335,6 +336,8 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf, vmw_surface_get_desc(VMW_CURSOR_SNOOP_FORMAT); const u32 image_pitch = VMW_CURSOR_SNOOP_WIDTH * desc->pitchBytesPerBlock; + BUILD_BUG_ON(sizeof(*srf->snooper.image) != desc->pitchBytesPerBlock); + cmd = container_of(header, struct vmw_dma_cmd, header); /* No snooper installed, nothing to copy */ @@ -372,8 +375,19 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf, return; } + if (box->w == 0 || box->h == 0) + return; + + src_extent = (u64)(box->h - 1) * cmd->dma.guest.pitch + + (u64)box->w * desc->pitchBytesPerBlock; + if (src_extent > bo->base.size) { + DRM_ERROR("Cursor snoop source of %llu bytes exceeds the %zu byte buffer\n", + src_extent, bo->base.size); + return; + } + kmap_offset = cmd->dma.guest.ptr.offset >> PAGE_SHIFT; - kmap_num = (VMW_CURSOR_SNOOP_HEIGHT * image_pitch) >> PAGE_SHIFT; + kmap_num = PFN_UP(src_extent); ret = ttm_bo_reserve(bo, true, false, NULL); if (unlikely(ret != 0)) { @@ -387,14 +401,16 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf, virtual = ttm_kmap_obj_virtual(&map, &is_iomem); - if (box->w == VMW_CURSOR_SNOOP_WIDTH && cmd->dma.guest.pitch == image_pitch) { + if (box->w == VMW_CURSOR_SNOOP_WIDTH && + box->h == VMW_CURSOR_SNOOP_HEIGHT && + cmd->dma.guest.pitch == image_pitch) { memcpy(srf->snooper.image, virtual, VMW_CURSOR_SNOOP_HEIGHT * image_pitch); } else { - /* Image is unsigned pointer. */ + /* The destination stride is in pixels. */ for (i = 0; i < box->h; i++) - memcpy(srf->snooper.image + i * image_pitch, - virtual + i * cmd->dma.guest.pitch, + memcpy(srf->snooper.image + i * VMW_CURSOR_SNOOP_WIDTH, + virtual + (size_t)i * cmd->dma.guest.pitch, box->w * desc->pitchBytesPerBlock); } srf->snooper.id++; diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index 73469ea5f333d..f1dbdc802df81 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -356,7 +356,7 @@ static struct i915_vma *__xe_pin_fb_vma(struct drm_gem_object *obj, bool is_dpt, drm_exec_retry_on_contention(&exec); xe_validation_retry_on_oom(&ctx, &ret); if (!ret) - ttm_bo_pin(&bo->ttm); + xe_bo_pin_account(bo); } if (ret) goto err; @@ -373,7 +373,7 @@ static struct i915_vma *__xe_pin_fb_vma(struct drm_gem_object *obj, bool is_dpt, err_unpin: ttm_bo_reserve(&bo->ttm, false, false, NULL); - ttm_bo_unpin(&bo->ttm); + xe_bo_unpin_account(bo); ttm_bo_unreserve(&bo->ttm); err: kfree(vma); @@ -393,7 +393,7 @@ static void __xe_unpin_fb_vma(struct i915_vma *vma) xe_ggtt_node_remove(vma->node, false); ttm_bo_reserve(&vma->bo->ttm, false, false, NULL); - ttm_bo_unpin(&vma->bo->ttm); + xe_bo_unpin_account(vma->bo); ttm_bo_unreserve(&vma->bo->ttm); kfree(vma); } diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 2c60e99249f93..f4c5151d61c7c 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -443,6 +443,100 @@ static void xe_ttm_tt_account_subtract(struct xe_device *xe, struct ttm_tt *tt) xe_shrinker_mod_pages(xe->mem.shrinker, -(long)tt->num_pages, 0); } +static bool xe_ttm_bo_is_imported(struct ttm_buffer_object *tbo) +{ + dma_resv_assert_held(tbo->base.resv); + + return tbo->ttm && + (tbo->ttm->page_flags & (TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_EXTERNAL_MAPPABLE)) == + TTM_TT_FLAG_EXTERNAL; +} + +/* + * Account @bo's pages as pinned for the shrinker. Removes @bo's pages + * from the shrinker's shrinkable / purgeable counts on the transition + * from unpinned to pinned. Must be called with @bo's dma-resv held, + * after &ttm_buffer_object.pin_count has been incremented by + * ttm_bo_pin(). Safe to call unconditionally regardless of which pin + * path (kernel, external, framebuffer, backup object, ...) is pinning + * @bo, since it only acts on the true 0->1 pin_count transition. + * + * Imported bos (xe_ttm_bo_is_imported()) are excluded: for those, + * xe_ttm_tt_populate() never actually populates the tt or calls + * xe_ttm_tt_account_add(), but TTM core's ttm_tt_populate() still + * unconditionally marks the tt as populated on driver-hook success. + * Relying on ttm_tt_is_populated() for these would subtract pages that + * were never added, underflowing the shrinker's counts. + */ +static void xe_bo_account_pin(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + + if (bo->ttm.pin_count == 1 && bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm) && + !xe_ttm_bo_is_imported(&bo->ttm)) + xe_ttm_tt_account_subtract(xe, bo->ttm.ttm); +} + +/* + * Counterpart to xe_bo_account_pin(). Must be called with @bo's dma-resv + * held, before &ttm_buffer_object.pin_count is decremented by + * ttm_bo_unpin(), so that the check against the true 1->0 transition sees + * the pin count that is about to be released. See xe_bo_account_pin() for + * why imported bos are excluded. + * + * On the true last unpin, also removes @bo from whichever pinned-bo list + * (external or kernel_bo_present) it may currently be linked on, since a + * bo's final unpin can happen through a pin path (e.g. framebuffer, + * backup object) that has no notion of, or ownership over, that list. + * This is safe and list-agnostic: list_del_init() only needs the node + * itself, not knowledge of which list it is threaded through, and is a + * no-op if @bo is not linked. + */ +static void xe_bo_account_unpin(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + bool last_unpin = bo->ttm.pin_count == 1; + + if (last_unpin && !list_empty(&bo->pinned_link)) { + spin_lock(&xe->pinned.lock); + list_del_init(&bo->pinned_link); + spin_unlock(&xe->pinned.lock); + } + + if (last_unpin && bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm) && + !xe_ttm_bo_is_imported(&bo->ttm)) + xe_ttm_tt_account_add(xe, bo->ttm.ttm); +} + +/** + * xe_bo_pin_account() - Pin a bo and account its pages for the shrinker + * @bo: The buffer object to pin. + * + * Pins @bo via ttm_bo_pin() and updates the shrinker's shrinkable / + * purgeable page accounting to match. Must be called with @bo's dma-resv + * held. Safe to call regardless of which pin path (kernel, external, + * framebuffer, backup object, ...) is pinning @bo. + */ +void xe_bo_pin_account(struct xe_bo *bo) +{ + ttm_bo_pin(&bo->ttm); + xe_bo_account_pin(bo); +} + +/** + * xe_bo_unpin_account() - Unpin a bo and account its pages for the shrinker + * @bo: The buffer object to unpin. + * + * Counterpart to xe_bo_pin_account(). Updates the shrinker's shrinkable / + * purgeable page accounting to match, then unpins @bo via ttm_bo_unpin(). + * Must be called with @bo's dma-resv held. + */ +void xe_bo_unpin_account(struct xe_bo *bo) +{ + xe_bo_account_unpin(bo); + ttm_bo_unpin(&bo->ttm); +} + static void update_global_total_pages(struct ttm_device *ttm_dev, long num_pages) { @@ -1377,7 +1471,8 @@ int xe_bo_notifier_prepare_pinned(struct xe_bo *bo) } backup->parent_obj = xe_bo_get(bo); /* Released by bo_destroy */ - ttm_bo_pin(&backup->ttm); + /* Note: XE_BO_FLAG_SYSTEM resolves to XE_PL_TT so already populated. */ + xe_bo_pin_account(backup); bo->backup_obj = backup; } @@ -1397,7 +1492,7 @@ int xe_bo_notifier_unprepare_pinned(struct xe_bo *bo) { xe_bo_lock(bo, false); if (bo->backup_obj) { - ttm_bo_unpin(&bo->backup_obj->ttm); + xe_bo_unpin_account(bo->backup_obj); xe_bo_put(bo->backup_obj); bo->backup_obj = NULL; } @@ -1608,7 +1703,7 @@ int xe_bo_restore_pinned(struct xe_bo *bo) xe_bo_vunmap(backup); if (!bo->backup_obj) { if (xe_bo_is_pinned(backup)) - ttm_bo_unpin(&backup->ttm); + xe_bo_unpin_account(backup); xe_bo_put(backup); } out_unlock_bo: @@ -2000,15 +2095,6 @@ static vm_fault_t xe_err_to_fault_t(int err) return VM_FAULT_SIGBUS; } -static bool xe_ttm_bo_is_imported(struct ttm_buffer_object *tbo) -{ - dma_resv_assert_held(tbo->base.resv); - - return tbo->ttm && - (tbo->ttm->page_flags & (TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_EXTERNAL_MAPPABLE)) == - TTM_TT_FLAG_EXTERNAL; -} - static vm_fault_t xe_bo_cpu_fault_fastpath(struct vm_fault *vmf, struct xe_device *xe, struct xe_bo *bo, bool needs_rpm) { @@ -2975,12 +3061,13 @@ uint64_t vram_region_gpu_offset(struct ttm_resource *res) int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec) { struct xe_device *xe = xe_bo_device(bo); + bool first_pin = !xe_bo_is_pinned(bo); int err; xe_assert(xe, !bo->vm); xe_assert(xe, xe_bo_is_user(bo)); - if (!xe_bo_is_pinned(bo)) { + if (first_pin) { if (!in_place) { err = xe_bo_validate(bo, NULL, false, exec); if (err) @@ -2992,9 +3079,7 @@ int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec) spin_unlock(&xe->pinned.lock); } - ttm_bo_pin(&bo->ttm); - if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) - xe_ttm_tt_account_subtract(xe, bo->ttm.ttm); + xe_bo_pin_account(bo); /* * FIXME: If we always use the reserve / unreserve functions for locking @@ -3050,9 +3135,7 @@ int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec) spin_unlock(&xe->pinned.lock); } - ttm_bo_pin(&bo->ttm); - if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) - xe_ttm_tt_account_subtract(xe, bo->ttm.ttm); + xe_bo_pin_account(bo); /* * FIXME: If we always use the reserve / unreserve functions for locking @@ -3081,14 +3164,7 @@ void xe_bo_unpin_external(struct xe_bo *bo) xe_assert(xe, xe_bo_is_pinned(bo)); xe_assert(xe, xe_bo_is_user(bo)); - spin_lock(&xe->pinned.lock); - if (bo->ttm.pin_count == 1 && !list_empty(&bo->pinned_link)) - list_del_init(&bo->pinned_link); - spin_unlock(&xe->pinned.lock); - - ttm_bo_unpin(&bo->ttm); - if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) - xe_ttm_tt_account_add(xe, bo->ttm.ttm); + xe_bo_unpin_account(bo); /* * FIXME: If we always use the reserve / unreserve functions for locking @@ -3106,21 +3182,16 @@ void xe_bo_unpin(struct xe_bo *bo) xe_assert(xe, xe_bo_is_pinned(bo)); if (mem_type_is_vram(place->mem_type) || bo->flags & XE_BO_FLAG_GGTT) { - spin_lock(&xe->pinned.lock); xe_assert(xe, !list_empty(&bo->pinned_link)); - list_del_init(&bo->pinned_link); - spin_unlock(&xe->pinned.lock); if (bo->backup_obj) { if (xe_bo_is_pinned(bo->backup_obj)) - ttm_bo_unpin(&bo->backup_obj->ttm); + xe_bo_unpin_account(bo->backup_obj); xe_bo_put(bo->backup_obj); bo->backup_obj = NULL; } } - ttm_bo_unpin(&bo->ttm); - if (bo->ttm.ttm && ttm_tt_is_populated(bo->ttm.ttm)) - xe_ttm_tt_account_add(xe, bo->ttm.ttm); + xe_bo_unpin_account(bo); } /** diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index a9e5e1bb8e4f7..f9c8c73a1e69c 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -228,6 +228,8 @@ int xe_bo_pin_external(struct xe_bo *bo, bool in_place, struct drm_exec *exec); int xe_bo_pin(struct xe_bo *bo, struct drm_exec *exec); void xe_bo_unpin_external(struct xe_bo *bo); void xe_bo_unpin(struct xe_bo *bo); +void xe_bo_pin_account(struct xe_bo *bo); +void xe_bo_unpin_account(struct xe_bo *bo); int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict, struct drm_exec *exec); diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index b376a9167cc1c..3127fb89d965f 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -330,12 +330,15 @@ static void xe_i2c_remove(void *data) xe_i2c_irq_reset(xe); xe_amc_exit(i2c); + /* Stop the notifier from arming the client work before teardown. */ + bus_unregister_notifier(&i2c_bus_type, &i2c->bus_notifier); + cancel_work_sync(&i2c->work); + for (i = 0; i < XE_I2C_MAX_CLIENTS; i++) { i2c_unregister_device(i2c->client[i]); i2c->client[i] = NULL; } - bus_unregister_notifier(&i2c_bus_type, &i2c->bus_notifier); xe_i2c_unregister_adapter(i2c); xe->i2c = NULL; } diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 9f8217ff1904c..224e64bccaebc 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -1099,8 +1099,7 @@ static void xe_lrc_finish(struct xe_lrc *lrc) #define CONTEXT_ACTIVE XE_LRC_CTX_TIMESTAMP_ACTIVE static ssize_t setup_utilization_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - u32 *batch, - size_t max_len) + u32 *batch, size_t max_len, bool indirect) { u32 *cmd = batch; @@ -1131,19 +1130,25 @@ static ssize_t setup_utilization_wa(struct xe_lrc *lrc, } static ssize_t setup_timestamp_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - u32 *batch, size_t max_len) + u32 *batch, size_t max_len, bool indirect) { const u32 ts_addr = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); u32 *cmd = batch; - if (!XE_GT_WA(lrc->gt, 16010904313) || - !(hwe->class == XE_ENGINE_CLASS_RENDER || - hwe->class == XE_ENGINE_CLASS_COMPUTE || - hwe->class == XE_ENGINE_CLASS_COPY || + if (!XE_GT_WA(lrc->gt, 16010904313)) + return 0; + + if (!indirect && + !(hwe->class == XE_ENGINE_CLASS_COPY || hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE || hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE)) return 0; + if (indirect && + !(hwe->class == XE_ENGINE_CLASS_RENDER || + hwe->class == XE_ENGINE_CLASS_COMPUTE)) + return 0; + if (xe_gt_WARN_ON(lrc->gt, max_len < 12)) return -ENOSPC; @@ -1169,7 +1174,8 @@ static ssize_t setup_timestamp_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, static ssize_t setup_configfs_post_ctx_restore_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - u32 *batch, size_t max_len) + u32 *batch, size_t max_len, + bool indirect) { struct xe_device *xe = gt_to_xe(lrc->gt); const u32 *user_batch; @@ -1198,7 +1204,8 @@ static ssize_t setup_configfs_post_ctx_restore_bb(struct xe_lrc *lrc, static ssize_t setup_configfs_mid_ctx_restore_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - u32 *batch, size_t max_len) + u32 *batch, size_t max_len, + bool indirect) { struct xe_device *xe = gt_to_xe(lrc->gt); const u32 *user_batch; @@ -1227,7 +1234,8 @@ static ssize_t setup_configfs_mid_ctx_restore_bb(struct xe_lrc *lrc, static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - u32 *batch, size_t max_len) + u32 *batch, size_t max_len, + bool indirect) { u32 *cmd = batch; @@ -1247,7 +1255,8 @@ static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, static ssize_t setup_invalidate_auxccs_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - u32 *batch, size_t max_len) + u32 *batch, size_t max_len, + bool indirect) { struct xe_gt *gt = lrc->gt; u32 *(*emit)(struct xe_gt *gt, u32 *cmd) = @@ -1264,7 +1273,7 @@ static ssize_t setup_invalidate_auxccs_wa(struct xe_lrc *lrc, struct bo_setup { ssize_t (*setup)(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - u32 *batch, size_t max_size); + u32 *batch, size_t max_size, bool indirect); }; struct bo_setup_state { @@ -1273,6 +1282,7 @@ struct bo_setup_state { struct xe_hw_engine *hwe; size_t max_size; size_t reserve_dw; + bool indirect; unsigned int offset; const struct bo_setup *funcs; unsigned int num_funcs; @@ -1298,7 +1308,8 @@ static int setup_bo(struct bo_setup_state *state) for (size_t i = 0; i < state->num_funcs; i++) { ssize_t len = state->funcs[i].setup(state->lrc, state->hwe, - state->ptr, remain); + state->ptr, remain, + state->indirect); remain -= len; @@ -1404,6 +1415,7 @@ setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) struct bo_setup_state state = { .lrc = lrc, .hwe = hwe, + .indirect = true, .max_size = (63 * 64) /* max 63 cachelines */, .buffer = NULL, .offset = __xe_lrc_indirect_ctx_offset(lrc), diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c index 78adb303b6630..6907c1127d326 100644 --- a/drivers/gpu/drm/xe/xe_mmio.c +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -288,6 +288,7 @@ u64 xe_mmio_read64_2x32(struct xe_mmio *mmio, struct xe_reg reg) return (u64)udw << 32 | ldw; } +#define __XE_MMIO_WAIT_MAX_BACKOFF_100MS (100 * USEC_PER_MSEC) static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u32 val, u32 timeout_us, u32 *out_val, bool atomic, bool expect_match) { @@ -318,10 +319,10 @@ static int __xe_mmio_wait32(struct xe_mmio *mmio, struct xe_reg reg, u32 mask, u wait = ktime_us_delta(end, cur); if (atomic) - udelay(wait); + udelay(min_t(s64, wait, (MAX_UDELAY_MS * 1000))); else usleep_range(wait, wait << 1); - wait <<= 1; + wait = min_t(s64, wait << 1, __XE_MMIO_WAIT_MAX_BACKOFF_100MS); } if (ret != 0) { diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c index 72baf1aa4b3ad..a0179e62abaaf 100644 --- a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c +++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c @@ -19,20 +19,20 @@ #include "xe_sysctrl_mailbox_types.h" struct xe_sysctrl_mailbox_msg_hdr { - __le32 data; -} __packed; + u32 data; +}; #define XE_SYSCTRL_HDR_GROUP_ID(hdr) \ - FIELD_GET(SYSCTRL_HDR_GROUP_ID_MASK, le32_to_cpu((hdr)->data)) + FIELD_GET(SYSCTRL_HDR_GROUP_ID_MASK, (hdr)->data) #define XE_SYSCTRL_HDR_COMMAND(hdr) \ - FIELD_GET(SYSCTRL_HDR_COMMAND_MASK, le32_to_cpu((hdr)->data)) + FIELD_GET(SYSCTRL_HDR_COMMAND_MASK, (hdr)->data) #define XE_SYSCTRL_HDR_IS_RESPONSE(hdr) \ - FIELD_GET(SYSCTRL_HDR_IS_RESPONSE, le32_to_cpu((hdr)->data)) + FIELD_GET(SYSCTRL_HDR_IS_RESPONSE, (hdr)->data) #define XE_SYSCTRL_HDR_RESULT(hdr) \ - FIELD_GET(SYSCTRL_HDR_RESULT_MASK, le32_to_cpu((hdr)->data)) + FIELD_GET(SYSCTRL_HDR_RESULT_MASK, (hdr)->data) static bool sysctrl_wait_bit_clear(struct xe_sysctrl *sc, u32 bit_mask, unsigned int timeout_ms) @@ -126,8 +126,8 @@ static int sysctrl_prepare_command(struct xe_device *xe, return -ENOMEM; hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer; - hdr->data = cpu_to_le32(FIELD_PREP(SYSCTRL_HDR_GROUP_ID_MASK, group_id) | - FIELD_PREP(SYSCTRL_HDR_COMMAND_MASK, command)); + hdr->data = FIELD_PREP(SYSCTRL_HDR_GROUP_ID_MASK, group_id) | + FIELD_PREP(SYSCTRL_HDR_COMMAND_MASK, command); if (data_in && data_in_len) memcpy(buffer + sizeof(*hdr), data_in, data_in_len); diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig index 9fee16e37fef0..d8ca428276efc 100644 --- a/drivers/hid/Kconfig +++ b/drivers/hid/Kconfig @@ -146,6 +146,16 @@ config HID_APPLE Say Y here if you want support for keyboards of Apple iBooks, PowerBooks, MacBooks, MacBook Pros and Apple Aluminum. +config HID_APPLE_MTP_KIWI + tristate "Apple MTP touch-screen (kiwi) interface" + depends on HID + help + Exposes the raw touch-screen report pipe of the kiwi interface of + the Apple dockchannel MTP on hidraw. + + This driver can also be built as a module. If so, it will be called + hid-apple-mtp-kiwi. + config HID_APPLE_MTP_HAPTIC tristate "Apple MTP Haptic Actuator support" depends on HID diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile index 4750e03a98b0e..1f0c5ecbb8a28 100644 --- a/drivers/hid/Makefile +++ b/drivers/hid/Makefile @@ -30,6 +30,7 @@ obj-$(CONFIG_HID_ALPS) += hid-alps.o obj-$(CONFIG_HID_ACRUX) += hid-axff.o obj-$(CONFIG_HID_APPLE) += hid-apple.o obj-$(CONFIG_HID_APPLE_MTP_HAPTIC) += hid-apple-mtp-haptic.o +obj-$(CONFIG_HID_APPLE_MTP_KIWI) += hid-apple-mtp-kiwi.o obj-$(CONFIG_HID_APPLEIR) += hid-appleir.o obj-$(CONFIG_HID_APPLETB_BL) += hid-appletb-bl.o obj-$(CONFIG_HID_APPLETB_KBD) += hid-appletb-kbd.o diff --git a/drivers/hid/dockchannel-hid/dockchannel-hid.c b/drivers/hid/dockchannel-hid/dockchannel-hid.c index bb19561562ab7..bf037dfbfc328 100644 --- a/drivers/hid/dockchannel-hid/dockchannel-hid.c +++ b/drivers/hid/dockchannel-hid/dockchannel-hid.c @@ -23,6 +23,7 @@ #define COMMAND_TIMEOUT_MS 1000 #define START_TIMEOUT_MS 2000 +#define START_ATTEMPTS 3 #define MAX_INTERFACES 16 @@ -163,8 +164,15 @@ struct dchid_iface { size_t hid_desc_len; struct gpio_desc *gpio; - char gpio_name[MAX_GPIO_NAME]; - int gpio_id; + /* The k114c MTP announces several GPIOs per interface (kiwi: + * clock_24MHz_enable, display_sync, power, reset). Track them all; + * gpio keeps the legacy single-GPIO semantics for existing paths. */ +#define MAX_IFACE_GPIOS 4 + struct gpio_desc *gpios[MAX_IFACE_GPIOS]; + char gpio_names[MAX_IFACE_GPIOS][MAX_GPIO_NAME]; + int gpio_ids[MAX_IFACE_GPIOS]; + int gpio_count; + bool gpios_requested; bool gpio_announced; struct mutex out_mutex; @@ -178,6 +186,11 @@ struct dchid_iface { u32 keyboard_layout_id; u32 power_method; bool firmware_managed_reset; + + /* Power method 2: the line that resets this interface, and its optional supply. */ + struct gpio_desc *reset; + struct gpio_desc *power; + struct gpio_desc *clock; }; struct dockchannel_hid { @@ -220,6 +233,7 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) u32 power_method = DCHID_POWER_METHOD_1; bool firmware_managed_reset = false; struct device_node *of_node = NULL; + struct gpio_desc *reset = NULL, *power = NULL, *clock = NULL; struct dchid_iface *iface; if (READ_ONCE(dchid->stopping)) @@ -256,11 +270,44 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) power_method, name); goto err_put_node; } - if (power_method == DCHID_POWER_METHOD_2 && - !firmware_managed_reset && !dchid->afe_reset) { - dev_err(dchid->dev, "Power method 2 for %s needs the AFE reset line\n", - name); - goto err_put_node; + if (power_method == DCHID_POWER_METHOD_2 && !firmware_managed_reset) { + struct fwnode_handle *fwnode = of_fwnode_handle(of_node); + + /* An interface with its own reset line (kiwi) does not share the AFE one. */ + reset = devm_fwnode_gpiod_get(dchid->dev, fwnode, "apple,reset", + GPIOD_OUT_LOW, "dchid-reset"); + if (IS_ERR(reset)) { + if (PTR_ERR(reset) != -ENOENT) { + dev_err(dchid->dev, "Reset line of %s: %pe\n", name, reset); + goto err_put_node; + } + reset = dchid->afe_reset; + } + if (!reset) { + dev_err(dchid->dev, "Power method 2 for %s needs a reset line\n", + name); + goto err_put_node; + } + + power = devm_fwnode_gpiod_get(dchid->dev, fwnode, "apple,power", + GPIOD_OUT_HIGH, "dchid-power"); + if (IS_ERR(power)) { + if (PTR_ERR(power) != -ENOENT) { + dev_err(dchid->dev, "Power line of %s: %pe\n", name, power); + goto err_put_node; + } + power = NULL; + } + + clock = devm_fwnode_gpiod_get(dchid->dev, fwnode, "apple,clock", + GPIOD_OUT_HIGH, "dchid-clock"); + if (IS_ERR(clock)) { + if (PTR_ERR(clock) != -ENOENT) { + dev_err(dchid->dev, "Clock line of %s: %pe\n", name, clock); + goto err_put_node; + } + clock = NULL; + } } } @@ -276,6 +323,9 @@ dchid_get_interface(struct dockchannel_hid *dchid, int index, const char *name) iface->of_node = of_node; iface->power_method = power_method; iface->firmware_managed_reset = firmware_managed_reset; + iface->reset = reset; + iface->power = power; + iface->clock = clock; iface->out_report = -1; init_completion(&iface->out_complete); init_completion(&iface->ready); @@ -454,7 +504,12 @@ static int dchid_pm2_set_power(struct dchid_iface *iface, u8 state) if (state == DCHID_POWER_STATE_ON) fsleep(50 * USEC_PER_MSEC); } else { - ret = gpiod_set_value_cansleep(dchid->afe_reset, + if (state == DCHID_POWER_STATE_ON && iface->power) + gpiod_set_value_cansleep(iface->power, 1); + if (iface->clock) + gpiod_set_value_cansleep(iface->clock, + state == DCHID_POWER_STATE_ON); + ret = gpiod_set_value_cansleep(iface->reset, state == DCHID_POWER_STATE_OFF); if (ret < 0) return ret; @@ -560,30 +615,44 @@ static int dchid_request_gpio(struct dchid_iface *iface) if (iface->firmware_managed_reset) return -EINVAL; - if (iface->gpio) - return 0; + int g; - dev_dbg(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", - iface->name, iface->gpio_id, iface->gpio_name); - - /* A power method 2 interface holds its reset line from probe on. */ - if (iface->power_method == DCHID_POWER_METHOD_2 && - !strcmp(iface->gpio_name, "afe-reset")) { - iface->gpio = iface->dchid->afe_reset; + if (iface->gpio_count && iface->gpios_requested) + return 0; + if (!iface->gpio_count && iface->gpio) return 0; - } - snprintf(prop_name, sizeof(prop_name), "apple,%s", iface->gpio_name); + for (g = 0; g < iface->gpio_count; g++) { + const char *name = iface->gpio_names[g]; - iface->gpio = devm_gpiod_get_index(iface->dchid->dev, prop_name, 0, GPIOD_OUT_LOW); + dev_dbg(iface->dchid->dev, "Requesting GPIO %s#%d: %s\n", + iface->name, iface->gpio_ids[g], name); - if (IS_ERR(iface->gpio)) { - ret = PTR_ERR(iface->gpio); - iface->gpio = NULL; - dev_err(iface->dchid->dev, "Failed to request GPIO %s-gpios: %d\n", - prop_name, ret); - return ret; + /* A power method 2 multi-touch holds its AFE reset line from + * probe on; the pm2 transitions drive it, not this path. */ + if (iface->power_method == DCHID_POWER_METHOD_2 && + !strcmp(name, "afe-reset")) { + if (g == 0) + iface->gpio = iface->dchid->afe_reset; + iface->gpios[g] = iface->dchid->afe_reset; + continue; + } + + snprintf(prop_name, sizeof(prop_name), "apple,%s", name); + iface->gpios[g] = devm_gpiod_get_index(iface->dchid->dev, prop_name, + 0, GPIOD_OUT_LOW); + if (IS_ERR(iface->gpios[g])) { + ret = PTR_ERR(iface->gpios[g]); + iface->gpios[g] = NULL; + dev_err(iface->dchid->dev, + "Failed to request GPIO %s-gpios: %d\n", + prop_name, ret); + return ret; + } + if (g == 0) + iface->gpio = iface->gpios[g]; } + iface->gpios_requested = 1; return 0; } @@ -674,20 +743,27 @@ static void dchid_stop(struct hid_device *hdev) static int dchid_open(struct hid_device *hdev) { struct dchid_iface *iface = hdev->driver_data; - int ret; + int attempt, ret; if (READ_ONCE(iface->dchid->stopping)) return -ESHUTDOWN; - if (!completion_done(&iface->ready)) { + for (attempt = 1; !completion_done(&iface->ready); attempt++) { ret = dchid_start_interface(iface); if (ret < 0) return ret; - if (!wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) { + if (wait_for_completion_timeout(&iface->ready, msecs_to_jiffies(START_TIMEOUT_MS))) + break; + + /* No ready message: the next attempt, or the next open, starts it again. */ + iface->starting = false; + if (attempt == START_ATTEMPTS) { dev_err(iface->dchid->dev, "iface %s start timed out\n", iface->name); return -ETIMEDOUT; } + dev_warn(iface->dchid->dev, "iface %s start timed out, starting it again\n", + iface->name); } /* Pairs with the acquire in dchid_handle_report(); publishes iface->hid. */ @@ -833,6 +909,51 @@ static int dchid_create_interface(struct dchid_iface *iface) return queue_work(iface->dchid->new_iface_wq, &iface->create_work); } +/* hid-core rejects any Report Size above this (see hid_parser_global()). */ +#define DCHID_MAX_REPORT_SIZE 16384 + +/* + * The T6050 multi-touch firmware declares its raw touch report as one item of + * 0x80000 bits. The magicmouse driver reads those reports in raw_event() and + * never looks at the field layout, so clamp every Report Size item to what + * hid-core accepts instead of failing the whole interface. + */ +static void dchid_clamp_report_sizes(struct dchid_iface *iface, u8 *desc, size_t len) +{ + size_t i = 0; + + while (i < len) { + u8 prefix = desc[i]; + size_t n = prefix & 3; + u32 val = 0; + size_t j; + + if (prefix == 0xfe) { + if (i + 1 >= len) + return; + i += 3 + desc[i + 1]; + continue; + } + if (n == 3) + n = 4; + if (i + 1 + n > len) + return; + + if ((prefix & 0xfc) == 0x74 && n >= 2) { + for (j = 0; j < n; j++) + val |= (u32)desc[i + 1 + j] << (8 * j); + if (val > DCHID_MAX_REPORT_SIZE) { + dev_info(iface->dchid->dev, + "%s: clamping Report Size %u to %u\n", + iface->name, val, DCHID_MAX_REPORT_SIZE); + for (j = 0; j < n; j++) + desc[i + 1 + j] = (DCHID_MAX_REPORT_SIZE >> (8 * j)) & 0xff; + } + } + i += 1 + n; + } +} + static void dchid_handle_descriptor(struct dchid_iface *iface, void *hid_desc, size_t desc_len) { if (iface->hid) { @@ -846,6 +967,7 @@ static void dchid_handle_descriptor(struct dchid_iface *iface, void *hid_desc, s return; iface->hid_desc_len = desc_len; + dchid_clamp_report_sizes(iface, iface->hid_desc, desc_len); } static void dchid_handle_ready(struct dockchannel_hid *dchid, void *data, size_t length) @@ -944,14 +1066,16 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t break; } - if (iface->gpio_announced) { + if (iface->gpio_count >= MAX_IFACE_GPIOS) { dev_err(dchid->dev, - "Cannot request more than one GPIO per interface!\n"); + "Too many GPIO requests for %s\n", iface->name); break; } - strscpy(iface->gpio_name, req->name, MAX_GPIO_NAME); - iface->gpio_id = req->id; + strscpy(iface->gpio_names[iface->gpio_count], req->name, + MAX_GPIO_NAME); + iface->gpio_ids[iface->gpio_count] = req->id; + iface->gpio_count++; iface->gpio_announced = true; break; } @@ -998,6 +1122,9 @@ static void dchid_handle_init(struct dockchannel_hid *dchid, void *data, size_t static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t length) { + struct gpio_desc *cmd_gpio = NULL; + unsigned int g; + struct dchid_gpio_cmd *cmd = data; struct dchid_iface *iface; u32 retcode = 0xe000f00d; /* Give it a random Apple-style error code */ @@ -1020,7 +1147,14 @@ static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t if (dchid_request_gpio(iface) < 0) goto err; - if (!iface->gpio || cmd->gpio != iface->gpio_id) { + cmd_gpio = NULL; + for (g = 0; g < iface->gpio_count; g++) { + if (iface->gpio_ids[g] == cmd->gpio) { + cmd_gpio = iface->gpios[g]; + break; + } + } + if (!cmd_gpio) { dev_err(dchid->dev, "Got GPIO command for bad GPIO %s#%d\n", iface->name, cmd->gpio); goto err; @@ -1031,9 +1165,9 @@ static void dchid_handle_gpio(struct dockchannel_hid *dchid, void *data, size_t switch (cmd->cmd) { case 3: /* Pulse. */ - gpiod_set_value_cansleep(iface->gpio, 1); + gpiod_set_value_cansleep(cmd_gpio, 1); msleep(10); /* Random guess... */ - gpiod_set_value_cansleep(iface->gpio, 0); + gpiod_set_value_cansleep(cmd_gpio, 0); retcode = 0; break; default: @@ -1267,6 +1401,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct dockchannel_hid *dchid; struct device_node *child, *helper; + struct fwnode_handle *stm; struct platform_device *helper_pdev; struct property *prop; int ret; @@ -1281,6 +1416,9 @@ static int dockchannel_hid_probe(struct platform_device *pdev) } dchid->dev = dev; + stm = device_get_named_child_node(dev, "stm"); + dchid->id_ready = !stm || !fwnode_device_is_available(stm); + fwnode_handle_put(stm); /* * The STM interface reports the machine's vendor and product IDs, @@ -1394,8 +1532,7 @@ static int dockchannel_hid_probe(struct platform_device *pdev) if (ret) dev_warn(dev, "Failed to set up wakeup: %d\n", ret); - dchid->new_iface_wq = alloc_workqueue("dchid-new", - WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + dchid->new_iface_wq = alloc_ordered_workqueue("dchid-new", 0); if (!dchid->new_iface_wq) return -ENOMEM; diff --git a/drivers/hid/hid-apple-mtp-kiwi.c b/drivers/hid/hid-apple-mtp-kiwi.c new file mode 100644 index 0000000000000..b0d686dc0ba13 --- /dev/null +++ b/drivers/hid/hid-apple-mtp-kiwi.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Apple MTP touch-screen (kiwi) interface. + * + * The kiwi interface of the dockchannel MTP carries its touches as a raw + * report pipe. No other HID driver may claim it: the Apple wildcard entries of + * hid-apple and hid-magicmouse make hid-generic stand off from every + * BUS_HOST Apple device, and both reject the interface in probe. This driver + * claims it by name and exposes the pipe on hidraw. The interface is started + * (firmware, reset) when hidraw is first opened. + */ + +#include +#include +#include + +#include "hid-ids.h" + +static bool kiwi_match(struct hid_device *hdev, bool ignore_special_drivers) +{ + return hdev->bus == BUS_HOST && !strcmp(hdev->name, "Apple MTP kiwi"); +} + +static int kiwi_probe(struct hid_device *hdev, const struct hid_device_id *id) +{ + int ret; + + ret = hid_parse(hdev); + if (ret) + return ret; + + return hid_hw_start(hdev, HID_CONNECT_HIDRAW); +} + +static const struct hid_device_id kiwi_devices[] = { + { HID_DEVICE(BUS_HOST, HID_GROUP_ANY, HOST_VENDOR_ID_APPLE, HID_ANY_ID) }, + { } +}; +MODULE_DEVICE_TABLE(hid, kiwi_devices); + +static struct hid_driver kiwi_driver = { + .name = "hid-apple-mtp-kiwi", + .id_table = kiwi_devices, + .match = kiwi_match, + .probe = kiwi_probe, +}; +module_hid_driver(kiwi_driver); + +MODULE_DESCRIPTION("Apple MTP touch-screen (kiwi) interface"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 970a10d2b9509..de661e8ebbdbd 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -171,6 +171,7 @@ MODULE_PARM_DESC(haptic_pulse_gap_ms, "milliseconds between pulses when more tha #define SPI_REPORT_ID 0x02 #define SPI_RESET_REPORT_ID 0x60 #define MTP_REPORT_ID 0x75 +#define MTP_V2_REPORT_ID 0x5a #define SENSOR_DIMENSIONS_REPORT_ID 0xd9 #define USB_BATTERY_TIMEOUT_SEC 60 @@ -1042,6 +1043,114 @@ static int magicmouse_raw_event_mtp(struct hid_device *hdev, return 1; } +/* + * The K114c MTP firmware (personality K2C1) sends report 0x5a. Its contact + * section sits at a fixed offset and holds 48-byte records of IEEE 754 + * floats, in micrometres. All offsets are bytes from the report id. + * + * 48 section tag, 0x02 + * 49 section header size (8) + * 50 contact record size (48) + * 51 section payload size, header + count * record size + * 53 number of contact records + * 59 bit 4: physical button + * 52 section header, 8 bytes, then count records of 48 bytes: + * record n: + * +1 tracking id, rolling + * +2 phase: 1 start, 2-3 approach, 4 touching, 5-7 lifting, 0 gone + * +5 float x, y; velocity x, y; major, minor axis; three unnamed + * floats; force (about 0 to 140 at a hard click) + */ +#define MTP_V2_SECTION 48 +#define MTP_V2_SECTION_TAG 0x02 +#define MTP_V2_COUNT 53 +#define MTP_V2_BUTTONS 59 +#define MTP_V2_BUTTON_MASK 0x10 +#define MTP_V2_PAYLOAD 52 +#define MTP_V2_RECORD_SIZE 48 +#define MTP_V2_PHASE_TOUCH 3 +#define MTP_V2_PHASE_LEAVE 5 + +enum { MTP_V2_X = 5, MTP_V2_Y = 9, MTP_V2_MAJOR = 21, MTP_V2_MINOR = 25, MTP_V2_FORCE = 41 }; + +/* Integer part of an IEEE 754 single, without touching the FPU. */ +static int magicmouse_f32_to_int(const u8 *p) +{ + u32 bits = get_unaligned_le32(p); + int exp = (int)((bits >> 23) & 0xff) - 127; + u32 mant = (bits & 0x7fffff) | 0x800000; + u32 v; + + if (exp < 0 || exp > 30) + return 0; + v = exp >= 23 ? mant << (exp - 23) : mant >> (23 - exp); + return (bits >> 31) ? -(int)v : (int)v; +} + +static int magicmouse_raw_event_mtp_v2(struct hid_device *hdev, u8 *data, int size) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + struct input_dev *input = msc->input; + int count, i, n = 0; + int major[MAX_CONTACTS], minor[MAX_CONTACTS], force[MAX_CONTACTS]; + u32 pressure = 0; + + if (size <= MTP_V2_COUNT || data[MTP_V2_SECTION] != MTP_V2_SECTION_TAG || + data[MTP_V2_SECTION + 2] < MTP_V2_FORCE + 4) + return 0; + + count = data[MTP_V2_COUNT]; + if (count > MAX_CONTACTS || + data[MTP_V2_SECTION + 3] != data[MTP_V2_SECTION + 1] + count * data[MTP_V2_SECTION + 2] || + size < MTP_V2_SECTION + 4 + data[MTP_V2_SECTION + 3]) + return 0; + + for (i = 0; i < count; i++) { + const u8 *rec = data + MTP_V2_PAYLOAD + data[MTP_V2_SECTION + 1] + + i * data[MTP_V2_SECTION + 2]; + + if (rec[2] < MTP_V2_PHASE_TOUCH || rec[2] >= MTP_V2_PHASE_LEAVE) + continue; + + /* Micrometres to the hundredths of a millimetre of the J314 ranges. */ + msc->pos[n].x = magicmouse_f32_to_int(rec + MTP_V2_X) / 10; + msc->pos[n].y = -magicmouse_f32_to_int(rec + MTP_V2_Y) / 10; + major[n] = magicmouse_f32_to_int(rec + MTP_V2_MAJOR) / 10; + minor[n] = magicmouse_f32_to_int(rec + MTP_V2_MINOR) / 10; + force[n] = magicmouse_f32_to_int(rec + MTP_V2_FORCE); + n++; + } + + input_mt_assign_slots(input, msc->tracking_ids, msc->pos, n, 0); + + for (i = 0; i < n; i++) { + input_mt_slot(input, msc->tracking_ids[i]); + input_mt_report_slot_state(input, MT_TOOL_FINGER, true); + input_report_abs(input, ABS_MT_TOUCH_MAJOR, major[i]); + input_report_abs(input, ABS_MT_TOUCH_MINOR, minor[i]); + input_report_abs(input, ABS_MT_PRESSURE, force[i]); + input_report_abs(input, ABS_MT_POSITION_X, msc->pos[i].x); + input_report_abs(input, ABS_MT_POSITION_Y, msc->pos[i].y); + pressure += force[i]; + } + + input_mt_sync_frame(input); + input_report_key(input, BTN_MOUSE, + magicmouse_haptic_click(msc, pressure, + data[MTP_V2_BUTTONS] & MTP_V2_BUTTON_MASK)); + input_sync(input); + return 1; +} + +static int magicmouse_raw_event_dockchannel(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + if (size && data[0] == MTP_V2_REPORT_ID) + return magicmouse_raw_event_mtp_v2(hdev, data, size); + + return magicmouse_raw_event_mtp(hdev, report, data, size); +} + static int magicmouse_raw_event_spi(struct hid_device *hdev, struct hid_report *report, u8 *data, int size) { @@ -1274,6 +1383,12 @@ static int magicmouse_setup_input_mtp(struct input_dev *input, int mt_flags = 0; struct magicmouse_sc *msc = hid_get_drvdata(hdev); + /* + * The K114c descriptor puts a Digitizer-page field under a Mouse + * application. hid-input takes that for a pen and sets DIRECT, which + * makes udev call the pad a touchscreen. This is an indirect touchpad. + */ + __clear_bit(INPUT_PROP_DIRECT, input->propbit); __set_bit(INPUT_PROP_BUTTONPAD, input->propbit); /* The C1FE controller reports no force: it is not a pressure pad. */ if (!msc->mtp_c1fe) @@ -1396,6 +1511,15 @@ static int magicmouse_input_mapping(struct hid_device *hdev, if (!msc->input) msc->input = hi->input; + /* The dockchannel MTP descriptors (K114c and later) open with a GD + * Mouse collection by design; macOS consumes both halves. Mouse + * usages must not reach the input: libinput refuses a mouse+MT + * hybrid. The keyboard interface (GD Keyboard) is unaffected, and + * kiwi never probes here (type guard in magicmouse_probe). */ + if (hdev->bus == BUS_HOST && !strncmp(hdev->name, "Apple MTP", 9) && + field->application == HID_GD_MOUSE) + return -1; + /* Magic Trackpad does not give relative data after switching to MT */ if ((hi->input->id.product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD || hi->input->id.product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2 || @@ -2304,7 +2428,7 @@ static int magicmouse_probe(struct hid_device *hdev, msc->mtp_c1fe = of_device_is_compatible(np, "apple,j700-multitouch"); of_node_put(np); - msc->input_ops.raw_event = magicmouse_raw_event_mtp; + msc->input_ops.raw_event = magicmouse_raw_event_dockchannel; msc->input_ops.setup_input = magicmouse_setup_input_mtp; } else if (id->bus == BUS_SPI) { msc->input_ops.raw_event = magicmouse_raw_event_spi; @@ -2391,6 +2515,10 @@ static int magicmouse_probe(struct hid_device *hdev, switch (id->bus) { case BUS_HOST: report = hid_register_report(hdev, HID_INPUT_REPORT, MTP_REPORT_ID, 0); + if (report) { + report->size = 6; + report = hid_register_report(hdev, HID_INPUT_REPORT, MTP_V2_REPORT_ID, 0); + } break; case BUS_SPI: report = hid_register_report(hdev, HID_INPUT_REPORT, SPI_REPORT_ID, 0); @@ -2503,6 +2631,46 @@ static const __u8 *magicmouse_report_fixup(struct hid_device *hdev, __u8 *rdesc, rdesc[3] = 0x02; } + /* The K114c dockchannel MTP declares its input as a raw buffer pipe: + * REPORT_SIZE 0x80000 bits (64 KiB), one field, no report id. macOS + * reads variable-length reports against it. hid core refuses any + * REPORT_SIZE above 16384 bits and aborts the whole parse, so no + * report ever registers and every finger report is dropped before + * hidraw. Clamp the item to the hid core maximum; real reports are + * far shorter and hid core accepts reports shorter than declared. */ + if (hdev->bus == BUS_HOST && !strncmp(hdev->name, "Apple MTP", 9)) { + unsigned int i = 0; + + while (i < *rsize) { + __u8 b = rdesc[i]; + unsigned int len; + static const unsigned int lens[4] = { 0, 1, 2, 4 }; + + if ((b >> 2 & 3) == 3) { /* long item */ + i += 3 + rdesc[i + 1]; + continue; + } + len = lens[b & 3]; + if ((b >> 2 & 3) == 1 && (b >> 4) == 0x7 && len) { + __u32 v = 0; + unsigned int k; + + for (k = 0; k < len; k++) + v |= (__u32)rdesc[i + 1 + k] << (8 * k); + if (v > 16384) { + rdesc[i + 1] = 0x00; + rdesc[i + 2] = 0x40; + rdesc[i + 3] = 0x00; + rdesc[i + 4] = 0x00; + hid_info(hdev, + "clamped MTP REPORT_SIZE %u bits to 16384\n", + v); + } + } + i += 1 + len; + } + } + return rdesc; } diff --git a/drivers/hid/hid-steam.c b/drivers/hid/hid-steam.c index 80d2eabc930ec..84c90091d45fc 100644 --- a/drivers/hid/hid-steam.c +++ b/drivers/hid/hid-steam.c @@ -327,12 +327,14 @@ struct steam_device { spinlock_t lock; struct hid_device *hdev, *client_hdev; struct mutex report_mutex; + struct mutex registration_mutex; unsigned long client_opened; struct input_dev __rcu *input; struct input_dev __rcu *sensors; unsigned long quirks; struct work_struct work_connect; bool connected; + bool registered; char serial_no[STEAM_SERIAL_LEN + 1]; struct power_supply_desc battery_desc; struct power_supply __rcu *battery; @@ -346,7 +348,6 @@ struct steam_device { u16 rumble_right; unsigned int sensor_timestamp_us; unsigned int sensor_update_rate_us; - struct work_struct unregister_work; }; static int steam_recv_report(struct steam_device *steam, @@ -510,7 +511,6 @@ static int steam_get_serial(struct steam_device *steam) u8 cmd[] = {ID_GET_STRING_ATTRIBUTE, sizeof(steam->serial_no), ATTRIB_STR_UNIT_SERIAL}; u8 reply[3 + STEAM_SERIAL_LEN + 1]; - guard(mutex)(&steam->report_mutex); ret = steam_send_report(steam, cmd, sizeof(cmd)); if (ret < 0) return ret; @@ -537,7 +537,6 @@ static int steam_get_attributes(struct steam_device *steam) int i; struct steam_controller_attribute *attr; - guard(mutex)(&steam->report_mutex); ret = steam_send_report(steam, cmd, sizeof(cmd)); if (ret < 0) return ret; @@ -599,9 +598,7 @@ static inline int steam_haptic_pulse(struct steam_device *steam, u8 pad, report[8] = count >> 8; report[9] = gain; - mutex_lock(&steam->report_mutex); ret = steam_send_report(steam, report, sizeof(report)); - mutex_unlock(&steam->report_mutex); return ret; } @@ -621,9 +618,7 @@ static inline int steam_haptic_rumble(struct steam_device *steam, report[9] = left_gain; report[10] = right_gain; - mutex_lock(&steam->report_mutex); ret = steam_send_report(steam, report, sizeof(report)); - mutex_unlock(&steam->report_mutex); return ret; } @@ -631,8 +626,11 @@ static void steam_haptic_rumble_cb(struct work_struct *work) { struct steam_device *steam = container_of(work, struct steam_device, rumble_work); + + mutex_lock(&steam->report_mutex); steam_haptic_rumble(steam, 0, steam->rumble_left, steam->rumble_right, 2, 0); + mutex_unlock(&steam->report_mutex); } #ifdef CONFIG_STEAM_FF @@ -653,7 +651,6 @@ static void steam_set_lizard_mode(struct steam_device *steam, bool enable) if (steam->gamepad_mode) enable = false; - mutex_lock(&steam->report_mutex); if (enable) { /* enable esc, enter, cursors */ steam_send_report_byte(steam, ID_SET_DEFAULT_DIGITAL_MAPPINGS); @@ -678,14 +675,13 @@ static void steam_set_lizard_mode(struct steam_device *steam, bool enable) 0); } } - mutex_unlock(&steam->report_mutex); } static int steam_input_open(struct input_dev *dev) { struct steam_device *steam = input_get_drvdata(dev); unsigned long flags; - bool set_lizard_mode; + bool client_opened; /* * Disabling lizard mode automatically is only done on the Steam @@ -694,9 +690,10 @@ static int steam_input_open(struct input_dev *dev) */ if (!(steam->quirks & STEAM_QUIRK_DECK)) { spin_lock_irqsave(&steam->lock, flags); - set_lizard_mode = !steam->client_opened && lizard_mode; + client_opened = steam->client_opened; spin_unlock_irqrestore(&steam->lock, flags); - if (set_lizard_mode) + guard(mutex)(&steam->report_mutex); + if (!client_opened && lizard_mode) steam_set_lizard_mode(steam, false); } @@ -707,13 +704,14 @@ static void steam_input_close(struct input_dev *dev) { struct steam_device *steam = input_get_drvdata(dev); unsigned long flags; - bool set_lizard_mode; + bool client_opened; if (!(steam->quirks & STEAM_QUIRK_DECK)) { spin_lock_irqsave(&steam->lock, flags); - set_lizard_mode = !steam->client_opened && lizard_mode; + client_opened = steam->client_opened; spin_unlock_irqrestore(&steam->lock, flags); - if (set_lizard_mode) + guard(mutex)(&steam->report_mutex); + if (!client_opened && lizard_mode) steam_set_lizard_mode(steam, true); } } @@ -721,14 +719,11 @@ static void steam_input_close(struct input_dev *dev) static int steam_sensor_open(struct input_dev *dev) { struct steam_device *steam = input_get_drvdata(dev); - unsigned long flags; - bool client_opened; - spin_lock_irqsave(&steam->lock, flags); - client_opened = steam->client_opened; - spin_unlock_irqrestore(&steam->lock, flags); - if (client_opened) - return 0; + scoped_guard(spinlock_irqsave, &steam->lock) { + if (steam->client_opened) + return 0; + } guard(mutex)(&steam->report_mutex); steam_write_settings(steam, SETTING_IMU_MODE, @@ -741,14 +736,11 @@ static int steam_sensor_open(struct input_dev *dev) static void steam_sensor_close(struct input_dev *dev) { struct steam_device *steam = input_get_drvdata(dev); - unsigned long flags; - bool client_opened; - spin_lock_irqsave(&steam->lock, flags); - client_opened = steam->client_opened; - spin_unlock_irqrestore(&steam->lock, flags); - if (client_opened) - return; + scoped_guard(spinlock_irqsave, &steam->lock) { + if (steam->client_opened) + return; + } guard(mutex)(&steam->report_mutex); steam_write_settings(steam, SETTING_IMU_MODE, 0, 0); @@ -823,6 +815,7 @@ static int steam_battery_register(struct steam_device *steam) &steam->battery_desc, &battery_cfg); if (IS_ERR(battery)) { ret = PTR_ERR(battery); + devm_kfree(&steam->hdev->dev, steam->battery_desc.name); hid_err(steam->hdev, "%s:power_supply_register failed with error %d\n", __func__, ret); @@ -1082,103 +1075,117 @@ static void steam_battery_unregister(struct steam_device *steam) RCU_INIT_POINTER(steam->battery, NULL); synchronize_rcu(); power_supply_unregister(battery); + devm_kfree(&steam->hdev->dev, steam->battery_desc.name); } static int steam_register(struct steam_device *steam) { int ret; - unsigned long client_opened; - unsigned long flags; + mutex_lock(&steam->registration_mutex); /* * This function can be called several times in a row with the * wireless adaptor, without steam_unregister() between them, because * another client send a get_connection_status command, for example. * The battery and serial number are set just once per device. */ - if (!steam->serial_no[0]) { - /* - * Unlikely, but getting the serial could fail, and it is not so - * important, so make up a serial number and go on. - */ - if (steam_get_serial(steam) < 0) - strscpy(steam->serial_no, "XXXXXXXXXX", - sizeof(steam->serial_no)); + if (steam->registered) { + mutex_unlock(&steam->registration_mutex); + return 0; + } - ret = steam_get_attributes(steam); - if (ret < 0) - hid_err(steam->hdev, - "%s:steam_get_attributes failed with error %d\n", - __func__, ret); + mutex_lock(&steam->report_mutex); + /* + * Unlikely, but getting the serial could fail, and it is not so + * important, so make up a serial number and go on. + */ + if (steam_get_serial(steam) < 0 || !steam->serial_no[0]) + strscpy(steam->serial_no, "XXXXXXXXXX", + sizeof(steam->serial_no)); - hid_info(steam->hdev, "Steam Controller '%s' connected", - steam->serial_no); + ret = steam_get_attributes(steam); + if (ret < 0) + hid_err(steam->hdev, + "%s:steam_get_attributes failed with error %d\n", + __func__, ret); - /* ignore battery errors, we can live without it */ - if (steam->quirks & STEAM_QUIRK_WIRELESS) - steam_battery_register(steam); + hid_info(steam->hdev, "Steam Controller '%s' connected", + steam->serial_no); - mutex_lock(&steam_devices_lock); - if (list_empty(&steam->list)) - list_add(&steam->list, &steam_devices); - mutex_unlock(&steam_devices_lock); - } + steam_set_lizard_mode(steam, lizard_mode); + mutex_unlock(&steam->report_mutex); - spin_lock_irqsave(&steam->lock, flags); - client_opened = steam->client_opened; - spin_unlock_irqrestore(&steam->lock, flags); + /* ignore battery errors, we can live without it */ + if (steam->quirks & STEAM_QUIRK_WIRELESS) + steam_battery_register(steam); - if (!client_opened) { - steam_set_lizard_mode(steam, lizard_mode); - ret = steam_input_register(steam); - if (ret != 0) - goto steam_register_input_fail; - ret = steam_sensors_register(steam); - if (ret != 0) - goto steam_register_sensors_fail; - } + ret = steam_input_register(steam); + if (ret != 0) + goto steam_register_input_fail; + ret = steam_sensors_register(steam); + if (ret != 0) + goto steam_register_sensors_fail; + + steam->registered = true; + mutex_unlock(&steam->registration_mutex); + mutex_lock(&steam_devices_lock); + if (list_empty(&steam->list)) + list_add(&steam->list, &steam_devices); + mutex_unlock(&steam_devices_lock); return 0; steam_register_sensors_fail: steam_input_unregister(steam); steam_register_input_fail: + steam_battery_unregister(steam); + mutex_unlock(&steam->registration_mutex); + cancel_work_sync(&steam->rumble_work); + cancel_delayed_work_sync(&steam->mode_switch); return ret; } static void steam_unregister(struct steam_device *steam) { + if (!steam->registered) + return; + + hid_info(steam->hdev, "Steam Controller '%s' disconnected", + steam->serial_no); + mutex_lock(&steam->registration_mutex); + steam->registered = false; steam_battery_unregister(steam); steam_sensors_unregister(steam); steam_input_unregister(steam); - if (steam->serial_no[0]) { - hid_info(steam->hdev, "Steam Controller '%s' disconnected", - steam->serial_no); - mutex_lock(&steam_devices_lock); - list_del_init(&steam->list); - mutex_unlock(&steam_devices_lock); - steam->serial_no[0] = 0; - } + mutex_unlock(&steam->registration_mutex); + cancel_work_sync(&steam->rumble_work); + cancel_delayed_work_sync(&steam->mode_switch); + mutex_lock(&steam_devices_lock); + list_del_init(&steam->list); + mutex_unlock(&steam_devices_lock); + steam->serial_no[0] = 0; } static void steam_work_connect_cb(struct work_struct *work) { struct steam_device *steam = container_of(work, struct steam_device, work_connect); + unsigned long flags; bool connected; + bool opened; int ret; spin_lock_irqsave(&steam->lock, flags); + opened = steam->client_opened; connected = steam->connected; spin_unlock_irqrestore(&steam->lock, flags); - if (connected) { + if (connected && !opened) { ret = steam_register(steam); - if (ret) { + if (ret) hid_err(steam->hdev, "%s:steam_register failed with error %d\n", __func__, ret); - } } else { steam_unregister(steam); } @@ -1190,54 +1197,32 @@ static void steam_mode_switch_cb(struct work_struct *work) struct steam_device, mode_switch); unsigned long flags; bool client_opened; + bool gamepad_mode; + if (!lizard_mode) return; + spin_lock_irqsave(&steam->lock, flags); steam->gamepad_mode = !steam->gamepad_mode; - hid_dbg(steam->hdev, "%s: switching gamepad mode to %i\n", __func__, steam->gamepad_mode); - if (steam->gamepad_mode) + gamepad_mode = steam->gamepad_mode; + client_opened = steam->client_opened; + spin_unlock_irqrestore(&steam->lock, flags); + + guard(mutex)(&steam->report_mutex); + hid_dbg(steam->hdev, "%s: switching gamepad mode to %i\n", __func__, gamepad_mode); + if (gamepad_mode) steam_set_lizard_mode(steam, false); - else { - spin_lock_irqsave(&steam->lock, flags); - client_opened = steam->client_opened; - spin_unlock_irqrestore(&steam->lock, flags); - if (!client_opened) - steam_set_lizard_mode(steam, lizard_mode); - } + else if (!client_opened) + steam_set_lizard_mode(steam, lizard_mode); steam_haptic_pulse(steam, STEAM_PAD_RIGHT, 0x190, 0, 1, 0); - if (steam->gamepad_mode) { + if (gamepad_mode) { steam_haptic_pulse(steam, STEAM_PAD_LEFT, 0x14D, 0x14D, 0x2D, 0); } else { steam_haptic_pulse(steam, STEAM_PAD_LEFT, 0x1F4, 0x1F4, 0x1E, 0); } } -static void steam_work_unregister_cb(struct work_struct *work) -{ - struct steam_device *steam = container_of(work, struct steam_device, - unregister_work); - unsigned long flags; - bool connected; - bool opened; - - spin_lock_irqsave(&steam->lock, flags); - opened = steam->client_opened; - connected = steam->connected; - spin_unlock_irqrestore(&steam->lock, flags); - - if (connected) { - if (opened) { - steam_sensors_unregister(steam); - steam_input_unregister(steam); - } else { - steam_set_lizard_mode(steam, lizard_mode); - steam_input_register(steam); - steam_sensors_register(steam); - } - } -} - static bool steam_is_valve_interface(struct hid_device *hdev) { struct hid_report_enum *rep_enum; @@ -1283,7 +1268,7 @@ static int steam_client_ll_open(struct hid_device *hdev) steam->client_opened++; spin_unlock_irqrestore(&steam->lock, flags); - schedule_work(&steam->unregister_work); + schedule_work(&steam->work_connect); return 0; } @@ -1298,7 +1283,7 @@ static void steam_client_ll_close(struct hid_device *hdev) steam->client_opened--; spin_unlock_irqrestore(&steam->lock, flags); - schedule_work(&steam->unregister_work); + schedule_work(&steam->work_connect); } static int steam_client_ll_raw_request(struct hid_device *hdev, @@ -1384,6 +1369,7 @@ static int steam_probe(struct hid_device *hdev, hid_set_drvdata(hdev, steam); spin_lock_init(&steam->lock); mutex_init(&steam->report_mutex); + mutex_init(&steam->registration_mutex); steam->quirks = id->driver_data; INIT_WORK(&steam->work_connect, steam_work_connect_cb); INIT_DELAYED_WORK(&steam->mode_switch, steam_mode_switch_cb); @@ -1394,7 +1380,6 @@ static int steam_probe(struct hid_device *hdev, steam->sensor_update_rate_us = 4000; else steam->sensor_update_rate_us = 9000; - INIT_WORK(&steam->unregister_work, steam_work_unregister_cb); /* * With the real steam controller interface, do not connect hidraw. @@ -1455,7 +1440,6 @@ static int steam_probe(struct hid_device *hdev, cancel_work_sync(&steam->work_connect); cancel_delayed_work_sync(&steam->mode_switch); cancel_work_sync(&steam->rumble_work); - cancel_work_sync(&steam->unregister_work); return ret; } @@ -1463,36 +1447,32 @@ static int steam_probe(struct hid_device *hdev, static void steam_remove(struct hid_device *hdev) { struct steam_device *steam = hid_get_drvdata(hdev); + unsigned long flags; if (!steam || hdev->group == HID_GROUP_STEAM) { hid_hw_stop(hdev); return; } + hid_hw_close(hdev); hid_destroy_device(steam->client_hdev); - cancel_delayed_work_sync(&steam->mode_switch); - cancel_work_sync(&steam->work_connect); - cancel_work_sync(&steam->rumble_work); - cancel_work_sync(&steam->unregister_work); - steam->client_hdev = NULL; + spin_lock_irqsave(&steam->lock, flags); steam->client_opened = 0; + spin_unlock_irqrestore(&steam->lock, flags); + cancel_work_sync(&steam->work_connect); if (steam->quirks & STEAM_QUIRK_WIRELESS) { hid_info(hdev, "Steam wireless receiver disconnected"); } - hid_hw_close(hdev); - hid_hw_stop(hdev); steam_unregister(steam); + hid_hw_stop(hdev); } static void steam_do_connect_event(struct steam_device *steam, bool connected) { - unsigned long flags; bool changed; - spin_lock_irqsave(&steam->lock, flags); changed = steam->connected != connected; steam->connected = connected; - spin_unlock_irqrestore(&steam->lock, flags); if (changed && schedule_work(&steam->work_connect) == 0) dbg_hid("%s: connected=%d event already queued\n", @@ -1921,8 +1901,6 @@ static void steam_do_deck_sensors_event(struct steam_device *steam, static void steam_do_battery_event(struct steam_device *steam, struct power_supply *battery, u8 *data) { - unsigned long flags; - s16 volts = steam_le16(data + 12); u8 batt = data[14]; @@ -1930,10 +1908,8 @@ static void steam_do_battery_event(struct steam_device *steam, rcu_read_lock(); battery = rcu_dereference(steam->battery); if (likely(battery)) { - spin_lock_irqsave(&steam->lock, flags); steam->voltage = volts; steam->battery_charge = batt; - spin_unlock_irqrestore(&steam->lock, flags); power_supply_changed(battery); } rcu_read_unlock(); @@ -1951,6 +1927,7 @@ static int steam_raw_event(struct hid_device *hdev, if (!steam) return 0; + guard(spinlock_irqsave)(&steam->lock); if (steam->client_opened) hid_input_report(steam->client_hdev, HID_FEATURE_REPORT, data, size, 0); @@ -2038,6 +2015,8 @@ static int steam_param_set_lizard_mode(const char *val, { struct steam_device *steam; int ret; + bool client_opened; + unsigned long flags; ret = param_set_bool(val, kp); if (ret) @@ -2045,8 +2024,13 @@ static int steam_param_set_lizard_mode(const char *val, mutex_lock(&steam_devices_lock); list_for_each_entry(steam, &steam_devices, list) { - if (!steam->client_opened) + spin_lock_irqsave(&steam->lock, flags); + client_opened = steam->client_opened; + spin_unlock_irqrestore(&steam->lock, flags); + if (!client_opened) { + guard(mutex)(&steam->report_mutex); steam_set_lizard_mode(steam, lizard_mode); + } } mutex_unlock(&steam_devices_lock); return 0; diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index 6406ceab988ad..810e52782ae89 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -302,6 +302,8 @@ static const struct xpad_device { { 0x17ef, 0x6182, "Lenovo Legion Controller for Windows", 0, XTYPE_XBOX360 }, { 0x1949, 0x041a, "Amazon Game Controller", 0, XTYPE_XBOX360 }, { 0x1a86, 0xe310, "Legion Go S", 0, XTYPE_XBOX360 }, + { 0x1b1c, 0x2b1f, "Corsair Novablade Pro (Wired)", 0, XTYPE_XBOX360 }, + { 0x1b1c, 0x2b2b, "Corsair Novablade Pro (2.4 GHz Receiver)", 0, XTYPE_XBOX360 }, { 0x1bad, 0x0002, "Harmonix Rock Band Guitar", 0, XTYPE_XBOX360 }, { 0x1bad, 0x0003, "Harmonix Rock Band Drumkit", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360 }, { 0x1bad, 0x0130, "Ion Drum Rocker", MAP_DPAD_TO_BUTTONS, XTYPE_XBOX360 }, @@ -379,6 +381,7 @@ static const struct xpad_device { { 0x2993, 0x2001, "TECNO Pocket Go", 0, XTYPE_XBOX360 }, { 0x2dc8, 0x2000, "8BitDo Pro 2 Wired Controller fox Xbox", 0, XTYPE_XBOXONE }, { 0x2dc8, 0x200f, "8BitDo Ultimate 3-mode Controller for Xbox", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, + { 0x2dc8, 0x201e, "8BitDo Pro 2 Wired Controller for Xbox", MAP_SHARE_BUTTON, XTYPE_XBOXONE }, { 0x2dc8, 0x3106, "8BitDo Ultimate Wireless / Pro 2 Wired Controller", 0, XTYPE_XBOX360 }, { 0x2dc8, 0x3109, "8BitDo Ultimate Wireless Bluetooth", 0, XTYPE_XBOX360 }, { 0x2dc8, 0x310a, "8BitDo Ultimate 2C Wireless Controller", 0, XTYPE_XBOX360 }, @@ -545,6 +548,7 @@ static const struct usb_device_id xpad_table[] = { XPAD_XBOX360_VENDOR(0x17ef), /* Lenovo */ XPAD_XBOX360_VENDOR(0x1949), /* Amazon controllers */ XPAD_XBOX360_VENDOR(0x1a86), /* Nanjing Qinheng Microelectronics (WCH) */ + XPAD_XBOX360_VENDOR(0x1b1c), /* Corsair controllers */ XPAD_XBOX360_VENDOR(0x1bad), /* Harmonix Rock Band guitar and drums */ XPAD_XBOX360_VENDOR(0x1ee9), /* ZOTAC Technology Limited */ XPAD_XBOX360_VENDOR(0x20bc), /* BETOP wireless dongles */ diff --git a/drivers/input/misc/ati_remote2.c b/drivers/input/misc/ati_remote2.c index 8db2dca84975c..3e0f874e9b16f 100644 --- a/drivers/input/misc/ati_remote2.c +++ b/drivers/input/misc/ati_remote2.c @@ -672,8 +672,7 @@ static ssize_t ati_remote2_show_channel_mask(struct device *dev, struct device_attribute *attr, char *buf) { - struct usb_device *udev = to_usb_device(dev); - struct usb_interface *intf = usb_ifnum_to_if(udev, 0); + struct usb_interface *intf = to_usb_interface(dev); struct ati_remote2 *ar2 = usb_get_intfdata(intf); return sprintf(buf, "0x%04x\n", ar2->channel_mask); @@ -683,8 +682,7 @@ static ssize_t ati_remote2_store_channel_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct usb_device *udev = to_usb_device(dev); - struct usb_interface *intf = usb_ifnum_to_if(udev, 0); + struct usb_interface *intf = to_usb_interface(dev); struct ati_remote2 *ar2 = usb_get_intfdata(intf); unsigned int mask; int r; @@ -720,8 +718,7 @@ static ssize_t ati_remote2_show_mode_mask(struct device *dev, struct device_attribute *attr, char *buf) { - struct usb_device *udev = to_usb_device(dev); - struct usb_interface *intf = usb_ifnum_to_if(udev, 0); + struct usb_interface *intf = to_usb_interface(dev); struct ati_remote2 *ar2 = usb_get_intfdata(intf); return sprintf(buf, "0x%02x\n", ar2->mode_mask); @@ -731,8 +728,7 @@ static ssize_t ati_remote2_store_mode_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { - struct usb_device *udev = to_usb_device(dev); - struct usb_interface *intf = usb_ifnum_to_if(udev, 0); + struct usb_interface *intf = to_usb_interface(dev); struct ati_remote2 *ar2 = usb_get_intfdata(intf); unsigned int mask; int err; diff --git a/drivers/iommu/apple-dart-dcp-format.h b/drivers/iommu/apple-dart-dcp-format.h new file mode 100644 index 0000000000000..124f20977afd7 --- /dev/null +++ b/drivers/iommu/apple-dart-dcp-format.h @@ -0,0 +1,407 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef APPLE_DART_DCP_FORMAT_H +#define APPLE_DART_DCP_FORMAT_H + +#include +#include +#include +#include + +#define DCP_DART_PAGE 0x4000ULL +#define DCP_DART_SLICE 0x2000000ULL +#define DCP_DART_ALIAS 0x10000000000ULL +#define DCP_DART_APERTURE 0x1000000000ULL +#define DCP_DART_ENTRIES 2048 +#define DCP_DART_PADDR 0x3ffffffc00ULL +#define DCP_DART_LEAF_FIELDS 0xffffff3ffffffc0fULL +#define DCP_ADT_MAX_SEGMENTS 16 + +struct dcp_adt_segment { + u64 phys; + u64 iova; + u64 size; + u32 kind; +}; + +struct dcp_adt_tables { + u64 start; + u64 end; + u64 dram_start; + u64 dram_size; + u32 seen; + u32 num_segments; + struct dcp_adt_segment segment[DCP_ADT_MAX_SEGMENTS]; +}; + +/* Internal display frontend: separate inherited storage for SID0 and SID16. */ +struct dcp_adt_frontend { + u64 start[2]; + u64 end[2]; + u32 seen; + u32 nodes; +}; + +/* One J713 display coprocessor: its ADT nodes and its firmware segments. */ +struct dcp_adt_instance { + const char *dart; /* firmware DART, SID23 */ + const char *dcp; + const char *nub; + const char *disp; /* display DART; only the internal frontend is admitted */ + const char *segment_names; + const u32 *kinds; + u32 num_segments; +}; + +static const u32 dcp_adt_kinds_internal[] = {1, 0, 10, 2, 2, 2, 2}; +static const u32 dcp_adt_kinds_external[] = {1, 0, 10, 2}; + +static const struct dcp_adt_instance dcp_adt_internal = { + .dart = "dart-dcp", + .dcp = "dcp", + .nub = "iop-dcp-nub", + .disp = "dart-disp0", + .segment_names = "__TEXT;__DATA;__OS_LOG;unknown;unknown;unknown;unknown", + .kinds = dcp_adt_kinds_internal, + .num_segments = ARRAY_SIZE(dcp_adt_kinds_internal), +}; + +/* The external DCPs run the same firmware with a single extra segment. */ +static const struct dcp_adt_instance dcp_adt_external[] = { + { + .dart = "dart-dcpext0", + .dcp = "dcpext0", + .nub = "iop-dcpext0-nub", + .disp = "dart-dispext0", + .segment_names = "__TEXT;__DATA;__OS_LOG;unknown", + .kinds = dcp_adt_kinds_external, + .num_segments = ARRAY_SIZE(dcp_adt_kinds_external), + }, + { + .dart = "dart-dcpext1", + .dcp = "dcpext1", + .nub = "iop-dcpext1-nub", + .disp = "dart-dispext1", + .segment_names = "__TEXT;__DATA;__OS_LOG;unknown", + .kinds = dcp_adt_kinds_external, + .num_segments = ARRAY_SIZE(dcp_adt_kinds_external), + }, +}; + +static inline int dcp_dart_pte_check(u64 pte, bool leaf) +{ + u64 fields = leaf ? DCP_DART_LEAF_FIELDS : DCP_DART_PADDR | 1; + + if (!pte) + return 0; + if (!(pte & 1) || (pte & ~fields)) + return -EINVAL; + if (leaf && ((pte >> 52) > ((pte >> 40) & 0xfff))) + return -EINVAL; + return 0; +} + +static inline u64 dcp_dart_pte_phys(u64 pte) +{ + return (pte & DCP_DART_PADDR) << 4; +} + +static inline bool dcp_dart_pte_full_rw(u64 pte, bool coherent) +{ + return pte && !dcp_dart_pte_check(pte, true) && !(pte & 0xc) && + (!coherent || !(pte & 2)) && !(pte >> 52) && + ((pte >> 40) & 0xfff) == 0xfff; +} + +/* Observed J713 profile classes, not generic ADT flag bits. */ +static inline bool dcp_dart_pte_firmware(u64 pte, u32 kind) +{ + if (kind != 0 && kind != 1 && kind != 2) + return false; + return !!(pte & 4) == (kind == 1) && dcp_dart_pte_full_rw(pte & ~4ULL, true); +} + +static inline bool dcp_dart_pte_frontend(u64 pte) +{ + /* Frontend requires no-cache, unlike the coherent SID23 firmware leaves. */ + return (pte & 2) && dcp_dart_pte_full_rw(pte, false); +} + +static inline const struct dcp_adt_segment * +dcp_adt_segment_match(const struct dcp_adt_tables *tables, u64 phys, u64 iova, u64 size) +{ + const struct dcp_adt_segment *found = NULL; + + for (u32 i = 0; i < tables->num_segments; i++) { + const struct dcp_adt_segment *seg = &tables->segment[i]; + + /* OS log is relocated/published separately as a physical carveout. */ + if (seg->kind == 10 || seg->phys != phys || seg->iova != iova || seg->size != size) + continue; + if (found) + return NULL; + found = seg; + } + return found; +} + +static inline bool dcp_dart_contains(u64 base, u64 end, u64 start, u64 size) +{ + return base < end && size && start >= base && start < end && size <= end - start; +} + +static inline int dcp_adt_value(struct dcp_adt_tables *out, + const struct dcp_adt_instance *inst, unsigned int kind, + const char *name, const u8 *value, u32 length) +{ + u32 bit = 0; + + if (kind == 1 && !strcmp(name, "target-type")) { + bit = 1; + if (length != 5 || memcmp(value, "J713", 5)) + return -ENODEV; + } else if (kind == 3 && !strcmp(name, "compatible")) { + bit = 2; + if (length != 11 || memcmp(value, "dart,t8110", 11)) + return -ENODEV; + } else if (kind == 3 && !strcmp(name, "pt-region-23")) { + bit = 4; + if (length != 16) + return -EINVAL; + out->start = get_unaligned_le64(value); + out->end = get_unaligned_le64(value + 8); + } else if (kind == 4 && !strcmp(name, "dram-base")) { + bit = 8; + if (length != 8) + return -EINVAL; + out->dram_start = get_unaligned_le64(value); + } else if (kind == 4 && !strcmp(name, "dram-size")) { + bit = 16; + if (length != 8) + return -EINVAL; + out->dram_size = get_unaligned_le64(value); + } else if (kind == 6 && !strcmp(name, "segment-ranges")) { + bit = 32; + if (!length || length % 32 || length / 32 > DCP_ADT_MAX_SEGMENTS) + return -EINVAL; + out->num_segments = length / 32; + for (u32 i = 0; i < out->num_segments; i++) { + struct dcp_adt_segment *seg = &out->segment[i]; + const u8 *record = value + i * 32; + + seg->phys = get_unaligned_le64(record); + seg->iova = get_unaligned_le64(record + 16); /* remap, not original IOVA */ + seg->size = (get_unaligned_le32(record + 24) + 0x3fffULL) & ~0x3fffULL; + seg->kind = get_unaligned_le32(record + 28); + } + } else if (kind == 6 && !strcmp(name, "uuid")) { + static const char uuid[] = "81BAF476-679D-3CC0-8C6A-E6B6FA659D5E"; + + bit = 64; + if (length != sizeof(uuid) || memcmp(value, uuid, sizeof(uuid))) + return -ENODEV; + } else if (kind == 6 && !strcmp(name, "segment-names")) { + size_t names = strlen(inst->segment_names) + 1; + + bit = 128; + if (length != names || memcmp(value, inst->segment_names, names)) + return -ENODEV; + } + if (out->seen & bit) + return -EINVAL; + out->seen |= bit; + return 0; +} + +static inline int dcp_adt_frontend_value(struct dcp_adt_frontend *out, + const char *name, const u8 *value, u32 length) +{ + u32 bit; + unsigned int index; + + if (!strcmp(name, "compatible")) { + bit = 1; + if (length != 11 || memcmp(value, "dart,t8110", 11)) + return -ENODEV; + } else if (!strcmp(name, "pt-region-0") || !strcmp(name, "pt-region-16")) { + index = !strcmp(name, "pt-region-16"); + bit = 2U << index; + if (length != 16) + return -EINVAL; + out->start[index] = get_unaligned_le64(value); + out->end[index] = get_unaligned_le64(value + 8); + } else { + return 0; + } + if (out->seen & bit) + return -EINVAL; + out->seen |= bit; + return 0; +} + +/* Bounded, read-only ADT subset: one coprocessor's nodes under ROOT / arm-io, and ROOT / chosen. */ +static inline int dcp_adt_node(const u8 *data, size_t size, size_t *cursor, + unsigned int parent, unsigned int depth, + const struct dcp_adt_instance *inst, + struct dcp_adt_tables *out, struct dcp_adt_frontend *frontend) +{ + size_t first, pos; + u32 props, children; + const char *node_name = NULL; + unsigned int kind = 0; + int ret; + + if (depth > 32 || *cursor > size || size - *cursor < 8) + return -EINVAL; + props = get_unaligned_le32(data + *cursor); + children = get_unaligned_le32(data + *cursor + 4); + if (props > 4096 || children > 4096) + return -EINVAL; + first = *cursor + 8; + pos = first; + for (u32 i = 0; i < props; i++) { + u32 length; + + if (pos > size || size - pos < 36 || !memchr(data + pos, 0, 32)) + return -EINVAL; + length = get_unaligned_le32(data + pos + 32) & 0x7fffffff; + if (length > size - pos - 36 || ((length + 3ULL) & ~3ULL) > size - pos - 36) + return -EINVAL; + if (!strcmp((const char *)data + pos, "name")) { + if (node_name || !length || !memchr(data + pos + 36, 0, length)) + return -EINVAL; + node_name = (const char *)data + pos + 36; + } + pos += 36 + ((length + 3ULL) & ~3ULL); + } + if (!node_name) + return -EINVAL; + if (!depth) + kind = 1; + else if (parent == 1 && !strcmp(node_name, "arm-io")) + kind = 2; + else if (parent == 2 && !strcmp(node_name, inst->dart)) + kind = 3; + else if (parent == 1 && !strcmp(node_name, "chosen")) + kind = 4; + else if (parent == 2 && !strcmp(node_name, inst->dcp)) + kind = 5; + else if (parent == 5 && !strcmp(node_name, inst->nub)) + kind = 6; + else if (parent == 2 && !strcmp(node_name, inst->disp)) + kind = 7; + if (kind == 7 && frontend && frontend->nodes++) + return -EINVAL; + *cursor = pos; + pos = first; + for (u32 i = 0; i < props; i++) { + u32 length = get_unaligned_le32(data + pos + 32) & 0x7fffffff; + + ret = dcp_adt_value(out, inst, kind, (const char *)data + pos, data + pos + 36, + length); + if (ret) + return ret; + if (kind == 7 && frontend) { + ret = dcp_adt_frontend_value(frontend, (const char *)data + pos, + data + pos + 36, length); + if (ret) + return ret; + } + pos += 36 + ((length + 3ULL) & ~3ULL); + } + for (u32 i = 0; i < children; i++) { + ret = dcp_adt_node(data, size, cursor, kind, depth + 1, inst, out, frontend); + if (ret) + return ret; + } + return 0; +} + +static inline int dcp_adt_parse_common(const void *data, size_t size, + const struct dcp_adt_instance *inst, + struct dcp_adt_tables *out, + struct dcp_adt_frontend *frontend) +{ + size_t cursor = 0; + int ret; + + *out = (struct dcp_adt_tables){}; + if (frontend) + *frontend = (struct dcp_adt_frontend){}; + if (size > 0x200000) + return -E2BIG; + ret = dcp_adt_node(data, size, &cursor, 0, 0, inst, out, frontend); + if (ret) + return ret; + if (out->seen != 255 || out->num_segments != inst->num_segments || !out->dram_size || + out->dram_start + out->dram_size < out->dram_start || + out->start >= out->end || out->end - out->start > 0x100000 || + ((out->start | out->end) & (DCP_DART_PAGE - 1)) || + !dcp_dart_contains(out->dram_start, out->dram_start + out->dram_size, + out->start, out->end - out->start)) + return -EINVAL; + for (u32 i = 0; i < out->num_segments; i++) { + const struct dcp_adt_segment *seg = &out->segment[i]; + + if (seg->kind != inst->kinds[i] || (seg->phys & (DCP_DART_PAGE - 1)) || + !dcp_dart_contains(out->dram_start, out->dram_start + out->dram_size, + seg->phys, seg->size)) + return -EINVAL; + if (seg->kind == 10) + continue; /* DT's explicitly physical OS log can be relocated. */ + if ((seg->kind != 0 && seg->kind != 1 && seg->kind != 2) || + (seg->iova & (DCP_DART_PAGE - 1)) || + !dcp_dart_contains(DCP_DART_ALIAS, DCP_DART_ALIAS + DCP_DART_APERTURE, + seg->iova, seg->size)) + return -EINVAL; + for (u32 j = 0; j < i; j++) { + const struct dcp_adt_segment *other = &out->segment[j]; + + if (other->kind != 10 && seg->iova < other->iova + other->size && + other->iova < seg->iova + seg->size) + return -EINVAL; + } + } + return 0; +} + +static inline int dcp_adt_parse(const void *data, size_t size, struct dcp_adt_tables *out) +{ + return dcp_adt_parse_common(data, size, &dcp_adt_internal, out, NULL); +} + +static inline int dcp_adt_parse_instance(const void *data, size_t size, + const struct dcp_adt_instance *inst, + struct dcp_adt_tables *out) +{ + return dcp_adt_parse_common(data, size, inst, out, NULL); +} + +/* Parse metadata only: live MMIO, RAM exclusion and page tables remain unproven. */ +static inline int dcp_adt_parse_frontend(const void *data, size_t size, + struct dcp_adt_tables *firmware, + struct dcp_adt_frontend *frontend) +{ + int ret = dcp_adt_parse_common(data, size, &dcp_adt_internal, firmware, frontend); + + if (ret) + return ret; + if (frontend->nodes != 1 || frontend->seen != 7) + return -ENODEV; + for (unsigned int i = 0; i < 2; i++) { + u64 start = frontend->start[i], end = frontend->end[i]; + + if (end <= start || end - start != (i ? 0x10000 : 0x20000) || + ((start | end) & (DCP_DART_PAGE - 1)) || + !dcp_dart_contains(firmware->dram_start, + firmware->dram_start + firmware->dram_size, + start, end - start) || + (start < firmware->end && firmware->start < end)) + return -EINVAL; + } + if (frontend->start[0] < frontend->end[1] && + frontend->start[1] < frontend->end[0]) + return -EINVAL; + return 0; +} + +#endif diff --git a/drivers/iommu/apple-dart-dcp-scanout.h b/drivers/iommu/apple-dart-dcp-scanout.h new file mode 100644 index 0000000000000..770cbcf7518bc --- /dev/null +++ b/drivers/iommu/apple-dart-dcp-scanout.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Retained SID0 copy surfaces, confined to the arena fixed at admission. */ +void *apple_dart_dcp_scanout_alloc(struct device *dev, size_t size, dma_addr_t *dva) +{ + struct apple_dart_dcp_allocation *allocation; + struct apple_dart_dcp *dcp; + struct device_node *node; + unsigned int root, count, allocated = 0; + unsigned long flags; + int ret; + void *result; + bool match; + + if (!dev || !dva || !size || size > SZ_32M) + return ERR_PTR(-EINVAL); + node = of_find_node_by_path("disp0"); + match = node && node == dev->of_node && + apple_dart_j713_scanout_node(node); + of_node_put(node); + if (!match || dev_is_dma_coherent(dev)) + return ERR_PTR(-ENODEV); + ret = apple_dart_dcp_check(dev); + if (ret) + return ERR_PTR(ret); + dcp = to_dart_domain(iommu_get_domain_for_dev(dev))->dcp; + if (dcp->sid != 0) + return ERR_PTR(-ENODEV); + size = PAGE_ALIGN(size); + count = size / PAGE_SIZE; + mutex_lock(&dcp->alloc_lock); + result = ERR_PTR(-ENOSPC); + if (READ_ONCE(dcp->failed) || dcp->allocated > SZ_64M - size) + goto unlock; + /* Claim under the PTE lock too: generic callers cannot race publication. */ + spin_lock_irqsave(&dcp->lock, flags); + for (root = 0; root < DCP_DART_ENTRIES; root++) + if (test_bit(root, dcp->arena) && !test_bit(root, dcp->claimed) && + !test_bit(root, dcp->foreign) && !dcp->entry[root] && !READ_ONCE(dcp->root[root])) + break; + if (root != DCP_DART_ENTRIES) { + set_bit(root, dcp->claimed); + set_bit(root, dcp->shared_roots); /* Private map only, never generic unmap. */ + } + spin_unlock_irqrestore(&dcp->lock, flags); + if (root == DCP_DART_ENTRIES) + goto unlock; + result = ERR_PTR(-ENOMEM); + allocation = kzalloc(struct_size(allocation, pages, count), GFP_KERNEL); + if (!allocation) + goto unlock; + allocation->size = size; + allocation->dva = DCP_DART_ALIAS + root * DCP_DART_SLICE; + for (allocated = 0; allocated < count; allocated++) { + allocation->pages[allocated] = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!allocation->pages[allocated]) + goto free_unpublished; + arch_dma_prep_coherent(allocation->pages[allocated], PAGE_SIZE); + } + /* Only this Normal-NC mapping may access payload bytes after cache cleaning. */ + allocation->cpu = dma_common_pages_remap(allocation->pages, size, + dma_pgprot(dev, PAGE_KERNEL, 0), + __builtin_return_address(0)); + if (!allocation->cpu) + goto free_unpublished; + list_add_tail(&allocation->link, &dcp->allocations); + dcp->allocated += size; + /* Every outcome after the first possible PTE retains all backing and claims. */ + for (unsigned int i = 0; i < count; i++) { + size_t mapped = 0; + + ret = apple_dart_dcp_map_owned(dcp, allocation->dva + i * PAGE_SIZE, + page_to_phys(allocation->pages[i]), PAGE_SIZE, 1, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL, &mapped, true); + if (!ret && mapped != PAGE_SIZE) + ret = -EIO; + if (ret) + goto pin_failure; + } + ret = apple_dart_dcp_flush(dcp); + if (!ret) + ret = apple_dart_dcp_check(dev); + if (ret) + goto pin_failure; + for (unsigned int i = 0; i < count; i++) { + if (apple_dart_dcp_translate(dcp, allocation->dva + i * PAGE_SIZE) != + page_to_phys(allocation->pages[i]) || + !dcp_dart_pte_frontend(READ_ONCE(dcp->table[root][i]))) { + ret = -EIO; + goto pin_failure; + } + } + *dva = allocation->dva; + result = allocation->cpu; + goto unlock; +pin_failure: + apple_dart_dcp_fail(dcp, "scanout mapping failed: %d", ret); + dev_err(dev, "Retaining %zu scanout bytes after SID0 mapping failure: %d\n", size, ret); + result = ERR_PTR(ret); + goto unlock; +free_unpublished: + while (allocated) + __free_page(allocation->pages[--allocated]); + kfree(allocation); +unlock: + mutex_unlock(&dcp->alloc_lock); + return result; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_scanout_alloc); diff --git a/drivers/iommu/apple-dart-dcp-shared.h b/drivers/iommu/apple-dart-dcp-shared.h new file mode 100644 index 0000000000000..f597edd6b6128 --- /dev/null +++ b/drivers/iommu/apple-dart-dcp-shared.h @@ -0,0 +1,899 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* Included after the retained DART and allocation helpers in apple-dart.c. */ + +static int apple_dart_dcp_shared_pair(struct device *dev, struct device *piodma, + struct apple_dart_dcp **firmware, + struct apple_dart_dcp **frontend) +{ + struct apple_dart_domain *a, *b; + int ret; + + if (!dev || !piodma || dev == piodma || piodma->parent != dev || + dev_is_dma_coherent(dev) || dev_is_dma_coherent(piodma)) + return -ENODEV; + ret = __apple_dart_dcp_check(dev); + if (ret) + return ret; + ret = __apple_dart_dcp_check(piodma); + if (ret) + return ret; + a = to_dart_domain(iommu_get_domain_for_dev(dev)); + b = to_dart_domain(iommu_get_domain_for_dev(piodma)); + if (a->dcp->sid != 23 || b->dcp->sid != 16 || a->dcp->dart == b->dcp->dart) + return -ENODEV; + /* No pair on an external display DART either of whose streams is retired. */ + if (b->dcp->dart->dispext && apple_dart_dispext_retired(b->dcp->dart)) + return -EIO; + *firmware = a->dcp; + *frontend = b->dcp; + return 0; +} + +static int apple_dart_dcp_shared_verify(struct apple_dart_dcp *dcp, + struct apple_dart_dcp_shared *object) +{ + for (size_t offset = 0; offset < object->size; offset += PAGE_SIZE) { + u64 dva = object->dva + offset; + unsigned int root = (dva >> 25) & 2047, leaf = (dva >> 14) & 2047; + + if (apple_dart_dcp_translate(dcp, dva) != object->phys + offset || + test_bit(root, dcp->foreign) || !test_bit(root, dcp->claimed) || + !dcp_dart_pte_frontend(READ_ONCE(dcp->table[root][leaf]))) + return -EIO; + } + return 0; +} + +static int __apple_dart_dcp_shared_alloc(struct device *dev, struct device *piodma, + size_t size, u64 *phys, u64 *dva) +{ + struct apple_dart_dcp *a, *b; + struct apple_dart_dcp_shared *object; + unsigned int root; + unsigned long flags; + size_t mapped = 0; + int ret; + + if (!phys || !dva || !size || size > SZ_16M) + return -EINVAL; + ret = apple_dart_dcp_shared_pair(dev, piodma, &a, &b); + if (ret) + return ret; + size = PAGE_ALIGN(size); + /* Always SID23 then SID16; ordinary RTKit allocation holds SID23 only. */ + mutex_lock(&a->alloc_lock); + mutex_lock_nested(&b->alloc_lock, SINGLE_DEPTH_NESTING); + ret = -EIO; + if (READ_ONCE(a->failed) || READ_ONCE(b->failed)) + goto unlock; + ret = -ENOSPC; + if (a->shared_count >= 64 || a->allocated > SZ_64M - size || b->allocated > SZ_64M - size) + goto unlock; + for (root = 0; root < DCP_DART_ENTRIES; root++) + if (!test_bit(root, a->claimed) && !test_bit(root, b->claimed) && + !a->entry[root] && !b->entry[root] && + !READ_ONCE(a->root[root]) && !READ_ONCE(b->root[root])) + break; + if (root == DCP_DART_ENTRIES) + goto unlock; + /* Serialize the claim against explicit generic map/unmap callers too. */ + spin_lock_irqsave(&a->lock, flags); + spin_lock_nested(&b->lock, SINGLE_DEPTH_NESTING); + if (a->entry[root] || b->entry[root] || READ_ONCE(a->root[root]) || READ_ONCE(b->root[root])) { + spin_unlock(&b->lock); + spin_unlock_irqrestore(&a->lock, flags); + ret = -EBUSY; + goto unlock; + } + /* Failed attempts retain both claims, even before any page is published. */ + set_bit(root, a->claimed); + set_bit(root, b->claimed); + set_bit(root, a->shared_roots); + set_bit(root, b->shared_roots); + spin_unlock(&b->lock); + spin_unlock_irqrestore(&a->lock, flags); + ret = -ENOMEM; + object = kzalloc(sizeof(*object), GFP_KERNEL); + if (!object) + goto unlock; + object->backing = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN); + if (!object->backing) { + kfree(object); + goto unlock; + } + object->phys = virt_to_phys(object->backing); + object->dva = DCP_DART_ALIAS + root * DCP_DART_SLICE; + object->size = size; + object->frontend = b; + if ((object->phys & (PAGE_SIZE - 1)) || object->phys >= BIT_ULL(42) || + size > BIT_ULL(42) - object->phys) { + free_pages_exact(object->backing, size); + kfree(object); + ret = -ERANGE; + goto unlock; + } + /* The cached direct alias is never touched again. Firmware owns the bytes. */ + arch_dma_prep_coherent(virt_to_page(object->backing), size); + list_add_tail(&object->link, &a->shared); + a->shared_count++; + a->allocated += size; + b->allocated += size; + /* Backing and paired claims are permanently owned before the first PTE. */ + ret = apple_dart_dcp_map_owned(a, object->dva, object->phys, PAGE_SIZE, + size / PAGE_SIZE, IOMMU_READ | IOMMU_WRITE, + GFP_KERNEL, &mapped, true); + if (!ret && mapped != size) + ret = -EIO; + if (!ret) + ret = apple_dart_domain_flush_tlb_range(a->domain, false, 0, 0); + if (!ret) + ret = apple_dart_dcp_shared_verify(a, object); + if (!ret) + ret = __apple_dart_dcp_check(dev); + if (!ret) + ret = __apple_dart_dcp_check(piodma); + if (ret) { + apple_dart_dcp_fail(a, "shared buffer publication failed: %d", ret); + apple_dart_dcp_fail(b, "shared buffer publication failed: %d", ret); + dev_err(dev, "Retaining shared DCP backing after SID23 publication failure: %d\n", ret); + goto unlock; + } + *phys = object->phys; + *dva = object->dva; +unlock: + mutex_unlock(&b->alloc_lock); + mutex_unlock(&a->alloc_lock); + return ret; +} + +#define apple_dart_dcp_dispext_fail(dart, fmt, ...) \ + do { \ + struct apple_dart *__dd = (dart); \ + \ + for (unsigned int __i = 0; __i < 2; __i++) \ + if (__dd->frontend[__i]) \ + apple_dart_dcp_fail(__dd->frontend[__i], fmt, \ + ##__VA_ARGS__); \ + } while (0) + +/* + * J713 external DCP: a shared buffer on the coprocessor's own DART + * only, SID23, as the firmware's D451 asks: the display DART comes in only + * if the firmware maps the buffer there (D201). Its root slot is claimed in + * SID23 alone, so apple_dart_dcp_shared_map() refuses it (frontend NULL). + * Up to 16 MiB, as the pair; backing and translation are retained until reboot. + */ +int apple_dart_dcp_shared_alloc_link(struct device *dev, size_t size, u64 *phys, u64 *dva) +{ + struct apple_dart_dcp_shared *object; + struct apple_dart_dcp *a; + struct apple_dart_domain *dd; + unsigned int root; + unsigned long flags; + size_t mapped = 0; + int ret; + + if (!dev || dev_is_dma_coherent(dev)) + return -ENODEV; + if (!phys || !dva || !size || size > SZ_16M) + return -EINVAL; + ret = apple_dart_dcp_check(dev); + if (ret) + return ret; + dd = to_dart_domain(iommu_get_domain_for_dev(dev)); + a = dd->dcp; + if (a->sid != 23) + return -ENODEV; + size = PAGE_ALIGN(size); + mutex_lock(&a->alloc_lock); + ret = -EIO; + if (READ_ONCE(a->failed)) + goto unlock; + ret = -ENOSPC; + if (a->shared_count >= 64 || a->allocated > SZ_64M - size) + goto unlock; + for (root = 0; root < DCP_DART_ENTRIES; root++) + if (!test_bit(root, a->claimed) && !a->entry[root] && !READ_ONCE(a->root[root])) + break; + if (root == DCP_DART_ENTRIES) + goto unlock; + spin_lock_irqsave(&a->lock, flags); + if (a->entry[root] || READ_ONCE(a->root[root])) { + spin_unlock_irqrestore(&a->lock, flags); + ret = -EBUSY; + goto unlock; + } + /* A failed attempt retains its claim, even before any page is published. */ + set_bit(root, a->claimed); + set_bit(root, a->shared_roots); + spin_unlock_irqrestore(&a->lock, flags); + ret = -ENOMEM; + object = kzalloc_obj(*object); + if (!object) + goto unlock; + object->backing = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN); + if (!object->backing) { + kfree(object); + goto unlock; + } + object->phys = virt_to_phys(object->backing); + object->dva = DCP_DART_ALIAS + root * DCP_DART_SLICE; + object->size = size; + object->frontend = NULL; + if ((object->phys & (PAGE_SIZE - 1)) || object->phys >= BIT_ULL(42) || + size > BIT_ULL(42) - object->phys) { + free_pages_exact(object->backing, size); + kfree(object); + ret = -ERANGE; + goto unlock; + } + /* The cached direct alias is never touched again. Firmware owns the bytes. */ + arch_dma_prep_coherent(virt_to_page(object->backing), size); + list_add_tail(&object->link, &a->shared); + a->shared_count++; + a->allocated += size; + ret = apple_dart_dcp_map_owned(a, object->dva, object->phys, PAGE_SIZE, + size / PAGE_SIZE, IOMMU_READ | IOMMU_WRITE, + GFP_KERNEL, &mapped, true); + if (!ret && mapped != size) + ret = -EIO; + if (!ret) + ret = apple_dart_domain_flush_tlb_range(a->domain, false, 0, 0); + if (!ret) + ret = apple_dart_dcp_shared_verify(a, object); + if (!ret) + ret = apple_dart_dcp_check(dev); + if (ret) { + apple_dart_dcp_fail(a, "SID23 shared buffer publication failed: %d", ret); + dev_err(dev, "Retaining SID23 shared backing after publication failure: %d\n", ret); + goto unlock; + } + *phys = object->phys; + *dva = object->dva; +unlock: + mutex_unlock(&a->alloc_lock); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_shared_alloc_link); + +/* + * J713 external DCP: map a D451 buffer's backing (@phys, @size) in + * stream @sid (0 scanout, 16 PIODMA) of the display DART @piodma is on, for + * the firmware's D201, at a DVA of its own (*@dva). Mappings fill a claimed + * 32-MiB slice one after the other (SID0: a slice of the arena fixed at + * admission; SID16: any free slice, so the pair allocator never takes it); + * a new slice is claimed only when one is full. Retained until reboot. + */ +static int __apple_dart_dcp_dispext_map(struct device *piodma, unsigned int sid, u64 phys, + size_t size, u64 *dva) +{ + struct apple_dart_dcp *pio, *dcp; + unsigned int root, count; + unsigned long flags; + u64 base; + int ret; + + if (!piodma || !dva || !size || size > DCP_DART_SLICE || (sid != 0 && sid != 16) || + ((phys | size) & (PAGE_SIZE - 1)) || phys >= BIT_ULL(42) || size > BIT_ULL(42) - phys) + return -EINVAL; + ret = __apple_dart_dcp_check(piodma); + if (ret) + return ret; + pio = to_dart_domain(iommu_get_domain_for_dev(piodma))->dcp; + if (pio->sid != 16 || !pio->dart->dispext) + return -ENODEV; + dcp = sid ? pio : pio->dart->frontend[0]; + if (!dcp) + return -ENODEV; + count = size / PAGE_SIZE; + mutex_lock(&dcp->alloc_lock); + ret = -EIO; + if (apple_dart_dispext_retired(dcp->dart)) + goto unlock; + if (!apple_dart_dcp_registers(dcp)) + goto unlock; + if (apple_dart_fault_pending(dcp->dart)) { + apple_dart_dcp_dispext_fail(dcp->dart, "a DART fault is pending (mapping)"); + goto unlock; + } + ret = -ENOSPC; + if (dcp->allocated > SZ_64M - size) + goto unlock; + if (dcp->sub_root < 0 || dcp->sub_used > DCP_DART_SLICE - size) { + spin_lock_irqsave(&dcp->lock, flags); + for (root = 0; root < DCP_DART_ENTRIES; root++) + if ((sid || test_bit(root, dcp->arena)) && !test_bit(root, dcp->claimed) && + !test_bit(root, dcp->foreign) && !dcp->entry[root] && + !READ_ONCE(dcp->root[root])) + break; + if (root != DCP_DART_ENTRIES) { + set_bit(root, dcp->claimed); + /* Private map only, never generic unmap. */ + set_bit(root, dcp->shared_roots); + } + spin_unlock_irqrestore(&dcp->lock, flags); + if (root == DCP_DART_ENTRIES) + goto unlock; + dcp->sub_root = root; + dcp->sub_used = 0; + } + root = dcp->sub_root; + base = DCP_DART_ALIAS + root * DCP_DART_SLICE + dcp->sub_used; + /* Used before the first possible PTE: a failure never reuses the range. */ + dcp->sub_used += size; + dcp->allocated += size; + for (unsigned int i = 0; i < count; i++) { + size_t mapped = 0; + + ret = apple_dart_dcp_map_owned(dcp, base + i * PAGE_SIZE, phys + i * PAGE_SIZE, + PAGE_SIZE, 1, IOMMU_READ | IOMMU_WRITE, GFP_KERNEL, + &mapped, true); + if (!ret && mapped != PAGE_SIZE) + ret = -EIO; + if (ret) + goto pin_failure; + } + ret = sid ? apple_dart_dcp_shared_flush(dcp) : apple_dart_dcp_flush(dcp); + if (!ret) + ret = __apple_dart_dcp_check(piodma); + for (unsigned int i = 0; !ret && i < count; i++) { + unsigned int leaf = ((base + i * PAGE_SIZE) >> 14) & 2047; + u64 pte = READ_ONCE(dcp->table[root][leaf]); + + if (apple_dart_dcp_translate(dcp, base + i * PAGE_SIZE) != phys + i * PAGE_SIZE || + pte != dcp->expected[root][leaf] || !dcp_dart_pte_frontend(pte)) + ret = -EIO; + } + if (ret) + goto pin_failure; + *dva = base; + goto unlock; +pin_failure: + apple_dart_dcp_dispext_fail(dcp->dart, "external display mapping failed: %d", ret); + dev_err(piodma, "retaining a SID%u mapping after its failure: %d\n", sid, ret); +unlock: + mutex_unlock(&dcp->alloc_lock); + return ret; +} + +/* + * J713 external DCP: whether stream @sid (0 or 16) of the display DART + * @piodma is on is adopted and not retired (a failed mapping, a lost state, + * a sleep). Its flags only: no register is read. + */ +int apple_dart_dcp_dispext_stream_ok(struct device *piodma, unsigned int sid) +{ + struct iommu_domain *domain = piodma ? iommu_get_domain_for_dev(piodma) : NULL; + struct apple_dart_dcp *pio, *dcp; + + if (!domain || domain->owner != &apple_dart_iommu_ops || (sid != 0 && sid != 16)) + return -ENODEV; + pio = to_dart_domain(domain)->dcp; + if (!pio || pio->sid != 16 || !pio->dart->dispext) + return -ENODEV; + dcp = sid ? pio : pio->dart->frontend[0]; + if (!dcp) + return -ENODEV; + /* Either frontend failed retires both streams (apple_dart_dcp_dispext_fail()). */ + if (apple_dart_dispext_retired(pio->dart)) + return -EIO; + /* Its power not confirmed since sleep: usable later, not now. */ + return apple_dart_dispext_unconfirmed(pio->dart) ? -EAGAIN : 0; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_stream_ok); + +/* + * J713 external DCP: a host-written scanout surface in SID0 of the + * display DART @piodma is on: zeroed pages behind a Normal-NC CPU mapping + * (the return value), in one whole 32-MiB slice of the SID0 arena at *@dva. + * Retained until reboot, like every display-visible allocation. + */ +static void *__apple_dart_dcp_dispext_scanout_alloc(struct device *piodma, size_t size, u64 *dva) +{ + struct apple_dart_dcp_allocation *allocation; + struct apple_dart_dcp *pio, *dcp; + unsigned int root, count, allocated = 0; + unsigned long flags; + void *result; + int ret; + + if (!piodma || !dva || !size || size > DCP_DART_SLICE || dev_is_dma_coherent(piodma)) + return ERR_PTR(-EINVAL); + ret = __apple_dart_dcp_check(piodma); + if (ret) + return ERR_PTR(ret); + pio = to_dart_domain(iommu_get_domain_for_dev(piodma))->dcp; + if (pio->sid != 16 || !pio->dart->dispext) + return ERR_PTR(-ENODEV); + dcp = pio->dart->frontend[0]; + if (!dcp) + return ERR_PTR(-ENODEV); + size = PAGE_ALIGN(size); + count = size / PAGE_SIZE; + mutex_lock(&dcp->alloc_lock); + result = ERR_PTR(-EIO); + if (apple_dart_dispext_retired(dcp->dart)) + goto unlock; + if (!apple_dart_dcp_registers(dcp)) + goto unlock; + if (apple_dart_fault_pending(dcp->dart)) { + apple_dart_dcp_dispext_fail(dcp->dart, "a DART fault is pending (scanout surface)"); + goto unlock; + } + /* Bounded by the arena: two 4K surfaces are more than 64 MiB. */ + result = ERR_PTR(-ENOSPC); + if (dcp->allocated > DCP_DART_SCANOUT_SLOTS * DCP_DART_SLICE - size) + goto unlock; + spin_lock_irqsave(&dcp->lock, flags); + for (root = 0; root < DCP_DART_ENTRIES; root++) + if (test_bit(root, dcp->arena) && !test_bit(root, dcp->claimed) && + !test_bit(root, dcp->foreign) && !dcp->entry[root] && + !READ_ONCE(dcp->root[root])) + break; + if (root != DCP_DART_ENTRIES) { + set_bit(root, dcp->claimed); + set_bit(root, dcp->shared_roots); /* Private map only, never generic unmap. */ + } + spin_unlock_irqrestore(&dcp->lock, flags); + if (root == DCP_DART_ENTRIES) + goto unlock; + result = ERR_PTR(-ENOMEM); + allocation = kzalloc(struct_size(allocation, pages, count), GFP_KERNEL); + if (!allocation) + goto unlock; + allocation->size = size; + allocation->dva = DCP_DART_ALIAS + root * DCP_DART_SLICE; + for (allocated = 0; allocated < count; allocated++) { + allocation->pages[allocated] = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!allocation->pages[allocated]) + goto free_unpublished; + arch_dma_prep_coherent(allocation->pages[allocated], PAGE_SIZE); + } + /* Only this Normal-NC mapping may access payload bytes after cache cleaning. */ + allocation->cpu = dma_common_pages_remap(allocation->pages, size, + dma_pgprot(piodma, PAGE_KERNEL, 0), + __builtin_return_address(0)); + if (!allocation->cpu) + goto free_unpublished; + list_add_tail(&allocation->link, &dcp->allocations); + dcp->allocated += size; + /* Every outcome after the first possible PTE retains all backing and claims. */ + for (unsigned int i = 0; i < count; i++) { + size_t mapped = 0; + + ret = apple_dart_dcp_map_owned(dcp, allocation->dva + i * PAGE_SIZE, + page_to_phys(allocation->pages[i]), PAGE_SIZE, 1, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL, &mapped, true); + if (!ret && mapped != PAGE_SIZE) + ret = -EIO; + if (ret) + goto pin_failure; + } + ret = apple_dart_dcp_flush(dcp); + if (!ret) + ret = __apple_dart_dcp_check(piodma); + for (unsigned int i = 0; !ret && i < count; i++) { + u64 pte = READ_ONCE(dcp->table[root][i]); + + if (apple_dart_dcp_translate(dcp, allocation->dva + i * PAGE_SIZE) != + page_to_phys(allocation->pages[i]) || + pte != dcp->expected[root][i] || !dcp_dart_pte_frontend(pte)) + ret = -EIO; + } + if (ret) + goto pin_failure; + *dva = allocation->dva; + result = allocation->cpu; + goto unlock; +pin_failure: + apple_dart_dcp_dispext_fail(dcp->dart, "SID0 scanout surface mapping failed: %d", ret); + dev_err(piodma, "retaining %zu SID0 scanout bytes after their mapping failure: %d\n", + size, ret); + result = ERR_PTR(ret); + goto unlock; +free_unpublished: + while (allocated) + __free_page(allocation->pages[--allocated]); + kfree(allocation); +unlock: + mutex_unlock(&dcp->alloc_lock); + return result; +} + +/* + * A hardware qualification failure of an external display DART (a pending + * fault, registers not as adopted, a failed check) retires the whole DART: + * both frontends, SID0 and SID16 (PIODMA). + */ + +/* The SID16 (PIODMA) frontend of an external display DART, or NULL. */ +static struct apple_dart_dcp *apple_dart_dcp_dispext_pio(struct device *piodma) +{ + struct iommu_domain *domain = piodma ? iommu_get_domain_for_dev(piodma) : NULL; + struct apple_dart_dcp *pio; + + if (!domain || domain->owner != &apple_dart_iommu_ops) + return NULL; + pio = to_dart_domain(domain)->dcp; + if (!pio || pio->sid != 16 || !pio->dart->dispext) + return NULL; + return pio; +} + +/* + * The SID0 frontend of an external display DART, from its PIODMA + * child, once that child's check (registers, roots, ownership) passed; a + * failed check retires the whole DART. + */ +static struct apple_dart_dcp *apple_dart_dcp_dispext_sid0(struct device *piodma) +{ + struct apple_dart_dcp *pio = apple_dart_dcp_dispext_pio(piodma); + int ret; + + if (!pio) + return NULL; + /* Unconfirmed power since sleep (-EAGAIN) retires nothing: it waits. */ + ret = __apple_dart_dcp_check(piodma); + if (ret == -EAGAIN) + return NULL; + if (ret) { + apple_dart_dcp_dispext_fail(pio->dart, "the PIODMA check failed: %d", ret); + return NULL; + } + return pio->dart->frontend[0]; +} + +/* + * J713 external DCP: with the DART's power confirmed on by the caller + * (its firmware last powered it up, under the lock that serialises that), both + * frontends as adopted and no fault pending; else the whole DART is retired. + * Register reads only: PROTECT, both TTBRs and TCRs, ENABLE_STREAMS and the + * exception/error state. + */ +static int __apple_dart_dcp_dispext_qualify(struct device *piodma) +{ + struct apple_dart_dcp *pio = apple_dart_dcp_dispext_pio(piodma); + struct apple_dart_dcp *sid0 = pio ? pio->dart->frontend[0] : NULL; + + if (!sid0) + return -ENODEV; + if (apple_dart_dispext_retired(pio->dart)) + return -EIO; + /* A mismatch retires the whole DART, see apple_dart_dcp_registers(). */ + if (!apple_dart_dcp_registers(pio) || !apple_dart_dcp_registers(sid0)) + return -EIO; + if (apple_dart_fault_pending(pio->dart)) { + apple_dart_dcp_dispext_fail(pio->dart, "a DART fault is pending"); + return -EIO; + } + return 0; +} + +/* + * J713 external DCP: claim a free SID0 arena slice (one not yet used + * by a surface, firmware-owned or holding a table) as a remappable host + * window at *@dva, until reboot. -ENOSPC without one. + */ +static int __apple_dart_dcp_dispext_window_claim(struct device *piodma, u64 *dva) +{ + struct apple_dart_dcp *dcp = apple_dart_dcp_dispext_sid0(piodma); + unsigned int root; + unsigned long flags; + int ret = -ENOSPC; + + if (!dcp || !dva) + return -ENODEV; + mutex_lock(&dcp->alloc_lock); + /* A register mismatch retires the whole DART, see apple_dart_dcp_registers(). */ + if (apple_dart_dispext_retired(dcp->dart) || !apple_dart_dcp_registers(dcp)) { + ret = -EIO; + goto unlock; + } + spin_lock_irqsave(&dcp->lock, flags); + for (root = 0; root < DCP_DART_ENTRIES; root++) + if (test_bit(root, dcp->arena) && !test_bit(root, dcp->claimed) && + !test_bit(root, dcp->foreign) && !dcp->entry[root] && + !READ_ONCE(dcp->root[root])) + break; + if (root != DCP_DART_ENTRIES) { + set_bit(root, dcp->claimed); + set_bit(root, dcp->shared_roots); /* Private map only, never generic unmap. */ + set_bit(root, dcp->window); + *dva = DCP_DART_ALIAS + root * DCP_DART_SLICE; + ret = 0; + } + spin_unlock_irqrestore(&dcp->lock, flags); +unlock: + mutex_unlock(&dcp->alloc_lock); + return ret; +} + +/* The window slice at @dva, or -1 if it is none. */ +static int apple_dart_dcp_window_root(struct apple_dart_dcp *dcp, u64 dva) +{ + unsigned int root = (dva >> 25) & 2047; + + if (!apple_dart_dcp_iova(dva) || (dva & (DCP_DART_SLICE - 1)) || + dva != DCP_DART_ALIAS + root * DCP_DART_SLICE || !test_bit(root, dcp->window) || + !test_bit(root, dcp->claimed) || !test_bit(root, dcp->arena) || + test_bit(root, dcp->foreign)) + return -1; + return root; +} + +/* + * J713 external DCP: map @count caller-owned pages (@phys, 16 KiB each) + * at the start of the empty window @dva; TLB flush, then every PTE read back. + * On any error the window and the pages stay as they are: the caller keeps + * the pages until reboot. + */ +static int __apple_dart_dcp_dispext_window_map(struct device *piodma, u64 dva, + const phys_addr_t *phys, unsigned int count) +{ + struct apple_dart_dcp *dcp = apple_dart_dcp_dispext_sid0(piodma); + int root, ret; + + if (!dcp) + return -ENODEV; + if (!phys || !count || count > DCP_DART_ENTRIES) + return -EINVAL; + for (unsigned int i = 0; i < count; i++) + if ((phys[i] & (DCP_DART_PAGE - 1)) || phys[i] >> 42) + return -EINVAL; + mutex_lock(&dcp->alloc_lock); + root = apple_dart_dcp_window_root(dcp, dva); + ret = -EINVAL; + if (root < 0) + goto unlock; + ret = -EIO; + if (apple_dart_dispext_retired(dcp->dart)) + goto unlock; + if (!apple_dart_dcp_registers(dcp) || apple_dart_fault_pending(dcp->dart)) + goto failed; + ret = -EBUSY; + for (unsigned int i = 0; dcp->expected[root] && i < DCP_DART_ENTRIES; i++) + if (dcp->expected[root][i]) + goto unlock; + for (unsigned int i = 0; i < count; i++) { + size_t mapped = 0; + + ret = apple_dart_dcp_map_owned(dcp, dva + (u64)i * DCP_DART_PAGE, phys[i], + DCP_DART_PAGE, 1, IOMMU_READ | IOMMU_WRITE, + GFP_KERNEL, &mapped, true); + if (!ret && mapped != DCP_DART_PAGE) + ret = -EIO; + if (ret) + goto failed; + } + ret = apple_dart_dcp_flush(dcp); + if (!ret) + ret = __apple_dart_dcp_check(piodma); + for (unsigned int i = 0; !ret && i < count; i++) { + u64 pte = READ_ONCE(dcp->table[root][i]); + + if (apple_dart_dcp_translate(dcp, dva + (u64)i * DCP_DART_PAGE) != phys[i] || + pte != dcp->expected[root][i] || !dcp_dart_pte_frontend(pte)) + ret = -EIO; + } + if (READ_ONCE(dcp->root[root]) != dcp->entry[root]) + ret = -EIO; + if (!ret) + goto unlock; +failed: + apple_dart_dcp_dispext_fail(dcp->dart, "window map at %#llx failed: %d", dva, ret); + dev_err(piodma, "SID0 window %#llx: mapping %u pages failed: %d; pages retained\n", + dva, count, ret); +unlock: + mutex_unlock(&dcp->alloc_lock); + return ret; +} + +/* + * J713 external DCP: clear every PTE of the window @dva, TLB flush, + * then read every one back as invalid. The number of pages unmapped, or an + * error, after which the window's pages must be kept until reboot (a stale + * translation may survive). + */ +static int __apple_dart_dcp_dispext_window_unmap(struct device *piodma, u64 dva) +{ + struct apple_dart_dcp *dcp = apple_dart_dcp_dispext_sid0(piodma); + unsigned long flags; + int root, ret, cleared = 0; + + if (!dcp) + return -ENODEV; + mutex_lock(&dcp->alloc_lock); + root = apple_dart_dcp_window_root(dcp, dva); + ret = -EINVAL; + if (root < 0) + goto unlock; + ret = 0; + if (!dcp->table[root]) + goto unlock; + ret = -EIO; + if (apple_dart_dispext_retired(dcp->dart)) + goto unlock; + if (!apple_dart_dcp_registers(dcp) || apple_dart_fault_pending(dcp->dart)) + goto failed; + spin_lock_irqsave(&dcp->lock, flags); + if (READ_ONCE(dcp->root[root]) != dcp->entry[root]) { + spin_unlock_irqrestore(&dcp->lock, flags); + goto failed; + } + for (unsigned int i = 0; i < DCP_DART_ENTRIES; i++) + if (READ_ONCE(dcp->table[root][i]) != dcp->expected[root][i]) { + spin_unlock_irqrestore(&dcp->lock, flags); + goto failed; + } + for (unsigned int i = 0; i < DCP_DART_ENTRIES; i++) { + if (!dcp->expected[root][i]) + continue; + WRITE_ONCE(dcp->table[root][i], 0); + dcp->expected[root][i] = 0; + cleared++; + } + dma_wmb(); + spin_unlock_irqrestore(&dcp->lock, flags); + ret = apple_dart_dcp_flush(dcp); + if (!ret) + ret = __apple_dart_dcp_check(piodma); + for (unsigned int i = 0; !ret && i < DCP_DART_ENTRIES; i++) + if (READ_ONCE(dcp->table[root][i]) || + apple_dart_dcp_translate(dcp, dva + (u64)i * DCP_DART_PAGE)) + ret = -EIO; + if (!ret) { + ret = cleared; + goto unlock; + } +failed: + if (!ret) + ret = -EIO; + apple_dart_dcp_dispext_fail(dcp->dart, "window unmap at %#llx failed: %d", dva, ret); + dev_err(piodma, "SID0 window %#llx: unmap failed: %d; pages retained\n", dva, ret); +unlock: + mutex_unlock(&dcp->alloc_lock); + return ret; +} + +static int __apple_dart_dcp_shared_map(struct device *dev, struct device *piodma, + u64 phys, u64 dva, size_t size) +{ + struct apple_dart_dcp *a, *b; + struct apple_dart_dcp_shared *object, *found = NULL; + size_t mapped = 0; + int ret = apple_dart_dcp_shared_pair(dev, piodma, &a, &b); + + if (ret) + return ret; + mutex_lock(&a->alloc_lock); + mutex_lock_nested(&b->alloc_lock, SINGLE_DEPTH_NESTING); + list_for_each_entry(object, &a->shared, link) + if (object->dva == dva) { + found = object; + break; + } + ret = -EINVAL; + if (!found || found->frontend != b || found->phys != phys || found->size != size || found->mapped) + goto unlock; + ret = apple_dart_dcp_shared_verify(a, found); + if (ret) + goto failed; + ret = apple_dart_dcp_map_owned(b, dva, phys, PAGE_SIZE, size / PAGE_SIZE, + IOMMU_READ | IOMMU_WRITE, GFP_KERNEL, &mapped, true); + if (!ret && mapped != size) + ret = -EIO; + if (!ret) + ret = apple_dart_dcp_shared_flush(b); + if (!ret) + ret = apple_dart_dcp_shared_verify(b, found); + if (!ret) + ret = __apple_dart_dcp_check(dev); + if (!ret) + ret = __apple_dart_dcp_check(piodma); + if (ret) + goto failed; + found->mapped = true; + WRITE_ONCE(b->shared_mapping, true); + goto unlock; +failed: + apple_dart_dcp_fail(a, "SID16 shared mapping failed: %d", ret); + apple_dart_dcp_fail(b, "SID16 shared mapping failed: %d", ret); + dev_err(dev, "Retaining shared DCP backing after SID16 publication failure: %d\n", ret); +unlock: + mutex_unlock(&b->alloc_lock); + mutex_unlock(&a->alloc_lock); + return ret; +} + +/* + * J713 external DCP: the external display DART operations above, each + * inside one use of the DART's registers (apple_dart_dispext_get()): with + * its power not confirmed since sleep, -EAGAIN before any step, nothing + * published, nothing retired. A sleep waits for a use under way to end, so + * no step of it (its flushes included) runs unconfirmed. + */ +static struct apple_dart *apple_dart_dcp_dispext_gate(struct device *piodma) +{ + struct apple_dart_dcp *pio = apple_dart_dcp_dispext_pio(piodma); + + return pio ? pio->dart : NULL; +} + +#define APPLE_DART_DISPEXT_GATED(piodma, call) \ + ({ \ + struct apple_dart *__gd = apple_dart_dcp_dispext_gate(piodma); \ + typeof(call) __ret; \ + \ + if (__gd && !apple_dart_dispext_get(__gd)) { \ + __ret = (typeof(call))-EAGAIN; \ + } else { \ + __ret = (call); \ + if (__gd) \ + apple_dart_dispext_put(__gd); \ + } \ + __ret; \ + }) + +int apple_dart_dcp_shared_alloc(struct device *dev, struct device *piodma, + size_t size, u64 *phys, u64 *dva) +{ + return APPLE_DART_DISPEXT_GATED(piodma, + __apple_dart_dcp_shared_alloc(dev, piodma, size, phys, dva)); +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_shared_alloc); + +int apple_dart_dcp_shared_map(struct device *dev, struct device *piodma, + u64 phys, u64 dva, size_t size) +{ + return APPLE_DART_DISPEXT_GATED(piodma, + __apple_dart_dcp_shared_map(dev, piodma, phys, dva, size)); +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_shared_map); + +int apple_dart_dcp_dispext_map(struct device *piodma, unsigned int sid, u64 phys, size_t size, + u64 *dva) +{ + return APPLE_DART_DISPEXT_GATED(piodma, + __apple_dart_dcp_dispext_map(piodma, sid, phys, size, dva)); +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_map); + +void *apple_dart_dcp_dispext_scanout_alloc(struct device *piodma, size_t size, u64 *dva) +{ + struct apple_dart *dart = apple_dart_dcp_dispext_gate(piodma); + void *ret; + + if (dart && !apple_dart_dispext_get(dart)) + return ERR_PTR(-EAGAIN); + ret = __apple_dart_dcp_dispext_scanout_alloc(piodma, size, dva); + if (dart) + apple_dart_dispext_put(dart); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_scanout_alloc); + +int apple_dart_dcp_dispext_qualify(struct device *piodma) +{ + return APPLE_DART_DISPEXT_GATED(piodma, __apple_dart_dcp_dispext_qualify(piodma)); +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_qualify); + +int apple_dart_dcp_dispext_window_claim(struct device *piodma, u64 *dva) +{ + return APPLE_DART_DISPEXT_GATED(piodma, __apple_dart_dcp_dispext_window_claim(piodma, dva)); +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_window_claim); + +int apple_dart_dcp_dispext_window_map(struct device *piodma, u64 dva, const phys_addr_t *phys, + unsigned int count) +{ + return APPLE_DART_DISPEXT_GATED(piodma, + __apple_dart_dcp_dispext_window_map(piodma, dva, phys, count)); +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_window_map); + +int apple_dart_dcp_dispext_window_unmap(struct device *piodma, u64 dva) +{ + return APPLE_DART_DISPEXT_GATED(piodma, __apple_dart_dcp_dispext_window_unmap(piodma, dva)); +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_window_unmap); diff --git a/drivers/iommu/apple-dart-dcp.h b/drivers/iommu/apple-dart-dcp.h new file mode 100644 index 0000000000000..3fbd919a93400 --- /dev/null +++ b/drivers/iommu/apple-dart-dcp.h @@ -0,0 +1,1444 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +/* + * Included by apple-dart.c. J713 display coprocessors: the internal DCP's + * firmware and frontend DARTs, and the external DCPs' firmware and display + * DARTs. Foreign tables stay live. + */ +#include +#include +#include "apple-dart-dcp-format.h" + +struct apple_dart_dcp { + struct apple_dart *dart; + struct apple_dart_domain *domain; + struct device *owner; + spinlock_t lock; + struct mutex alloc_lock; + struct list_head allocations; + struct list_head shared; + size_t allocated; + unsigned int shared_count; + struct dcp_adt_tables firmware; + u64 *root; + u64 *table[DCP_DART_ENTRIES]; + u64 *expected[DCP_DART_ENTRIES]; + u64 entry[DCP_DART_ENTRIES]; + struct resource leaf_regions[64]; + unsigned int nr_leaf_regions; + struct resource excluded_regions[128]; + unsigned int nr_excluded_regions; + DECLARE_BITMAP(foreign, DCP_DART_ENTRIES); + DECLARE_BITMAP(claimed, DCP_DART_ENTRIES); + DECLARE_BITMAP(shared_roots, DCP_DART_ENTRIES); + DECLARE_BITMAP(arena, DCP_DART_ENTRIES); /* SID0 retained surfaces only. */ + /* SID0 arena slices claimed as remappable host windows. */ + DECLARE_BITMAP(window, DCP_DART_ENTRIES); + unsigned long retained; /* SID0 pages pinned after an uncertain unmap. */ + u32 ttbr; + unsigned int sid; + bool failed; + /* the reason it failed was logged (apple_dart_dcp_fail()) */ + bool failed_logged; + bool shared_mapping; + bool shared_flushing; + /* + * J713 external display DART: the slice being filled by + * apple_dart_dcp_dispext_map() (-1: none), and how much of it is used. + */ + int sub_root; + u64 sub_used; +}; + +/* + * Latch @d failed (sticky), then say why, the first time: every + * failure of a borrowed DCP stream goes through here, so none is silent. + * The log arguments are evaluated after the latch, and must not dereference + * anything the failure may have left invalid. + */ +#define apple_dart_dcp_fail(d, fmt, ...) \ + do { \ + struct apple_dart_dcp *__dcp = (d); \ + \ + WRITE_ONCE(__dcp->failed, true); \ + if (!READ_ONCE(__dcp->failed_logged)) { \ + WRITE_ONCE(__dcp->failed_logged, true); \ + dev_err(__dcp->dart->dev, "DCP SID%u retired: " fmt "\n", \ + __dcp->sid, ##__VA_ARGS__); \ + } \ + } while (0) + +/* + * J713 external display DART: its power not confirmed since sleep (from + * suspend until its firmware powers it up again): no register of it may be + * read or written, and nothing is retired for that. + */ +static inline bool apple_dart_dispext_unconfirmed(const struct apple_dart *dart) +{ + return dart->dispext && READ_ONCE(dart->dispext_unverified); +} + +/* + * A use of an external display DART's registers, taken at the start of an + * operation and dropped at its end, and only with its power confirmed: + * false (the caller returns -EAGAIN, nothing retired) from sleep until its + * next confirmed power-up. Inside it the operation's steps, flushes included, + * run unchecked: an invalidation (sleep) waits for it to end. Any other + * DART: always true, no count. + */ +static inline bool apple_dart_dispext_get(struct apple_dart *dart) +{ + unsigned long flags; + bool ok; + + if (!dart->dispext) + return true; + spin_lock_irqsave(&dart->dispext_pm_lock, flags); + ok = !dart->dispext_unverified; + if (ok) + dart->dispext_users++; + spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + return ok; +} + +static inline void apple_dart_dispext_put(struct apple_dart *dart) +{ + unsigned long flags; + + if (!dart->dispext) + return; + spin_lock_irqsave(&dart->dispext_pm_lock, flags); + if (!--dart->dispext_users) + wake_up_all(&dart->dispext_wq); + spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); +} + +/* + * Sleep: no power confirmed from now on, a new generation (a check + * still under way confirms nothing), and every use in flight waited for. + * @suspend: asleep until the matching resume (no check runs then). + */ +static inline void apple_dart_dispext_invalidate(struct apple_dart *dart, bool suspend) +{ + unsigned long flags; + + spin_lock_irqsave(&dart->dispext_pm_lock, flags); + dart->dispext_unverified = true; + dart->dispext_sleeping = suspend; + dart->dispext_gen++; + spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + wait_event(dart->dispext_wq, !READ_ONCE(dart->dispext_users)); +} + +/* Every borrowed stream of @dart, for one reason. */ +#define apple_dart_fail_borrowed(dart, fmt, ...) \ + do { \ + struct apple_dart *__dart = (dart); \ + \ + if (__dart->dcp) \ + apple_dart_dcp_fail(__dart->dcp, fmt, ##__VA_ARGS__); \ + for (unsigned int __i = 0; __i < ARRAY_SIZE(__dart->frontend); __i++) \ + if (__dart->frontend[__i]) \ + apple_dart_dcp_fail(__dart->frontend[__i], fmt, \ + ##__VA_ARGS__); \ + } while (0) + +struct apple_dart_dcp_shared { + struct list_head link; + struct apple_dart_dcp *frontend; + void *backing; + u64 phys, dva; + size_t size; + bool mapped; +}; + +struct apple_dart_dcp_allocation { + struct list_head link; + void *cpu; + dma_addr_t dva; + size_t size; + struct page *pages[]; +}; + +/* SID0 slices for the retained copy surfaces; generic DMA gets the other free ones. */ +#define DCP_DART_SCANOUT_SLOTS 4 +/* An external display DART's SID0 arena: the surfaces and host windows. */ +#define DCP_DART_DISPEXT_SLOTS 10 + +static bool apple_dart_has_borrowed(struct apple_dart *dart) +{ + return dart->dcp || dart->frontend[0] || dart->frontend[1]; +} + +static struct apple_dart_dcp *apple_dart_borrowed_sid(struct apple_dart *dart, unsigned int sid) +{ + return sid == 23 ? dart->dcp : sid == 0 ? dart->frontend[0] : + sid == 16 ? dart->frontend[1] : NULL; +} + +static struct apple_dart_dcp *apple_dart_borrowed_cfg(struct apple_dart_master_cfg *cfg) +{ + struct apple_dart *dart = cfg->stream_maps[0].dart; + + if (cfg->stream_maps[1].dart || + bitmap_weight(cfg->stream_maps[0].sidmap, dart->num_streams) != 1) + return NULL; + return apple_dart_borrowed_sid(dart, + find_first_bit(cfg->stream_maps[0].sidmap, dart->num_streams)); +} + +/* + * Whether either stream of an external display DART was retired: a hardware + * failure of one retires the whole DART, so every mapping and publication + * refuses then. + */ +static inline bool apple_dart_dispext_retired(struct apple_dart *dart) +{ + for (unsigned int i = 0; i < ARRAY_SIZE(dart->frontend); i++) + if (dart->frontend[i] && READ_ONCE(dart->frontend[i]->failed)) + return true; + return false; +} + +#define DCP_DART_REGS_FMT \ + "registers not as adopted: PROTECT %08x, TTBR0 %08x (adopted %08x), TCR %08x, ENABLE_STREAMS %08x" + +static bool apple_dart_dcp_registers(struct apple_dart_dcp *dcp) +{ + struct apple_dart *dart = dcp->dart; + u32 protect, ttbr, tcr, streams; + + /* An external display DART's callers hold a use of it (apple_dart_dispext_get()). */ + if (READ_ONCE(dcp->failed)) + return false; + protect = readl(dart->regs + DART_T8110_PROTECT); + ttbr = readl(dart->regs + DART_TTBR(dart, dcp->sid, 0)); + tcr = readl(dart->regs + DART_TCR(dart, dcp->sid)); + streams = readl(dart->regs + DART_T8110_ENABLE_STREAMS); + if ((protect & DART_T8110_PROTECT_TTBR_TCR) && ttbr == dcp->ttbr && + tcr == DART_T8110_TCR_TRANSLATE_ENABLE && (streams & BIT(dcp->sid))) + return true; + /* On an external display DART a hardware failure retires the whole DART. */ + if (dart->dispext) + apple_dart_fail_borrowed(dart, "SID%u " DCP_DART_REGS_FMT, dcp->sid, protect, ttbr, + dcp->ttbr, tcr, streams); + else + apple_dart_dcp_fail(dcp, DCP_DART_REGS_FMT, protect, ttbr, dcp->ttbr, tcr, streams); + return false; +} + +static int apple_dart_dcp_read_adt(struct apple_dart *dart, struct dcp_adt_tables *tables, + struct dcp_adt_frontend *frontend) +{ + struct device_node *node, *found = NULL; + struct reserved_mem *rmem; + struct resource res; + void *data; + int ret; + + for_each_compatible_node(node, NULL, "phram") { + const char *label; + + if (of_property_read_string(node, "label", &label) || strcmp(label, "adt")) + continue; + if (found) { + of_node_put(found); + of_node_put(node); + return -EINVAL; + } + found = of_node_get(node); + } + if (!found) + return -ENODEV; + ret = of_address_to_resource(found, 0, &res); + rmem = of_reserved_mem_lookup(found); + if (ret || !rmem || rmem->base != res.start || rmem->size != resource_size(&res) || + !resource_size(&res) || resource_size(&res) > SZ_2M) { + ret = -EINVAL; + goto out_node; + } + data = memremap(res.start, resource_size(&res), MEMREMAP_WB); + if (!data) { + ret = -ENOMEM; + goto out_node; + } + /* An external display DART reads its instance's frontend. */ + if (frontend && dart->dispext) + ret = dcp_adt_parse_common(data, resource_size(&res), dart->dcp_instance, tables, + frontend); + else + ret = frontend ? dcp_adt_parse_frontend(data, resource_size(&res), tables, + frontend) : + dcp_adt_parse_instance(data, resource_size(&res), dart->dcp_instance, + tables); + memunmap(data); +out_node: + of_node_put(found); + return ret; +} + +static int apple_dart_dcp_excluded(struct apple_dart *dart, u64 start, u64 end) +{ + struct device_node *node, *reserved; + struct resource res; + + /* Inspect every final DT RAM tuple, not only Linux's currently free pages. */ + for_each_node_by_type(node, "memory") { + int cells = of_n_addr_cells(node) + of_n_size_cells(node); + int count = of_property_count_u32_elems(node, "reg"); + + if (cells <= 0 || count <= 0 || count % cells) { + of_node_put(node); + return -EINVAL; + } + for (int i = 0; i < count / cells; i++) { + if (of_address_to_resource(node, i, &res)) { + of_node_put(node); + return -EINVAL; + } + if (start <= res.end && res.start < end) { + of_node_put(node); + return -EBUSY; + } + } + } + if (region_intersects(start, end - start, IORESOURCE_SYSTEM_RAM, + IORES_DESC_NONE) != REGION_DISJOINT) + return -EBUSY; + for (u64 pa = start; pa < end; pa += PAGE_SIZE) { +#ifdef CONFIG_ARM64 + if (pfn_is_map_memory(PHYS_PFN(pa))) + return -EBUSY; +#else + if (pfn_valid(PHYS_PFN(pa))) + return -EBUSY; +#endif + } + /* Reserved DT regions outside System RAM need not appear in /proc/iomem. */ + reserved = of_find_node_by_path("/reserved-memory"); + for_each_child_of_node(reserved, node) { + int cells = of_n_addr_cells(node) + of_n_size_cells(node); + int count; + + if (!of_device_is_available(node) || !of_property_present(node, "reg")) + continue; + count = of_property_count_u32_elems(node, "reg"); + if (cells <= 0 || count <= 0 || count % cells) { + of_node_put(node); + of_node_put(reserved); + return -EINVAL; + } + for (int i = 0; i < count / cells; i++) { + if (of_address_to_resource(node, i, &res)) { + of_node_put(node); + of_node_put(reserved); + return -EINVAL; + } + if (start <= res.end && res.start < end) { + of_node_put(node); + of_node_put(reserved); + return -EBUSY; + } + } + } + of_node_put(reserved); + /* Includes conflicting non-RAM owners; kept until reboot. */ + return apple_dart_retained_request_region(dart, start, end - start, + "J713 DCP inherited tables"); +} + +/* + * Fixed at admission, so every reservation query reports the same SID0 slices. + * An external display DART takes up to DCP_DART_DISPEXT_SLOTS, for its host + * windows (direct scanout) besides the retained surfaces; it needs only + * DCP_DART_SCANOUT_SLOTS. + */ +static int apple_dart_dcp_arena(struct apple_dart_dcp *dcp) +{ + unsigned int want = dcp->dart->dispext ? DCP_DART_DISPEXT_SLOTS : + DCP_DART_SCANOUT_SLOTS; + unsigned int slots = 0; + + for (unsigned int i = 0; i < DCP_DART_ENTRIES && slots < want; i++) { + if (dcp->entry[i] || test_bit(i, dcp->foreign)) + continue; + set_bit(i, dcp->arena); + slots++; + } + if (dcp->dart->dispext) + dev_dbg(dcp->dart->dev, "SID0 arena: %u of %u slices (%u for surfaces)\n", + slots, want, DCP_DART_SCANOUT_SLOTS); + return slots >= DCP_DART_SCANOUT_SLOTS ? 0 : -ENOSPC; +} + +/* Snapshot only this consumer's real no-map reservations, before table admission. */ +static int apple_dart_dcp_leaf_regions(struct apple_dart_dcp *dcp) +{ + struct device_node *consumer, *node; + /* An external display DART's SID16 serves its dcpext's memory. */ + const char *alias = dcp->sid == 23 || dcp->dart->dispext ? + dcp->dart->dcp_instance->dcp : dcp->sid == 0 ? "disp0" : "disp0_piodma"; + int count, ret = 0; + + consumer = of_find_node_by_path(alias); + if (!consumer) + return -ENODEV; + count = of_property_count_u32_elems(consumer, "memory-region"); + if (count <= 0 || count > ARRAY_SIZE(dcp->leaf_regions)) { + ret = -EINVAL; + goto out; + } + for (int i = 0; i < count; i++) { + struct resource res; + struct reserved_mem *rmem; + + node = of_parse_phandle(consumer, "memory-region", i); + if (!node) { ret = -EINVAL; break; } + rmem = of_reserved_mem_lookup(node); + if (!of_property_read_bool(node, "no-map") || + of_address_to_resource(node, 0, &res) || !rmem || + rmem->base != res.start || rmem->size != resource_size(&res) || + !resource_size(&res)) + ret = -EINVAL; + of_node_put(node); + if (ret) + break; + dcp->leaf_regions[dcp->nr_leaf_regions++] = res; + } +out: + of_node_put(consumer); + return ret; +} + +/* Snapshot all reserved tuples at admission, never walk OF under the IRQ lock. */ +static int apple_dart_dcp_excluded_regions(struct apple_dart_dcp *dcp) +{ + struct device_node *reserved, *node; + int ret = 0; + + reserved = of_find_node_by_path("/reserved-memory"); + for_each_child_of_node(reserved, node) { + int cells = of_n_addr_cells(node) + of_n_size_cells(node); + int count; + + if (!of_device_is_available(node) || !of_property_present(node, "reg")) + continue; + count = of_property_count_u32_elems(node, "reg"); + if (cells <= 0 || count <= 0 || count % cells) { + ret = -EINVAL; + goto out_node; + } + for (int i = 0; i < count / cells; i++) { + if (dcp->nr_excluded_regions == ARRAY_SIZE(dcp->excluded_regions)) { + ret = -E2BIG; + goto out_node; + } + ret = of_address_to_resource(node, i, + &dcp->excluded_regions[dcp->nr_excluded_regions]); + if (ret) + goto out_node; + /* Empty loader placeholders wrap end to start - 1; they reserve + * no bytes and must never exclude the whole address space. + */ + if (!resource_size(&dcp->excluded_regions[dcp->nr_excluded_regions])) + continue; + dcp->nr_excluded_regions++; + } + } + goto out; +out_node: + of_node_put(node); +out: + of_node_put(reserved); + return ret; +} + +/* Foreign leaves must never DMA into ordinary Linux memory. Consumer no-map + * reservations may sit under a broad System RAM resource, but must really have + * been removed from the linear map. Other consumers' reserved carveouts are + * forbidden even when /proc/iomem does not label them System RAM. Host-owned + * later mappings use expected[]. + */ +static bool apple_dart_dcp_leaf_safe(const struct apple_dart_dcp *dcp, u64 leaf) +{ + u64 pa; + + if (!leaf) + return true; + pa = dcp_dart_pte_phys(leaf); + if (dcp_dart_pte_check(leaf, true) || pa > U64_MAX - DCP_DART_PAGE) + return false; +#ifdef CONFIG_ARM64 + if (pfn_is_map_memory(PHYS_PFN(pa))) + return false; +#else + if (pfn_valid(PHYS_PFN(pa))) + return false; +#endif + for (unsigned int i = 0; i < dcp->nr_leaf_regions; i++) + if (pa >= dcp->leaf_regions[i].start && + pa + DCP_DART_PAGE - 1 <= dcp->leaf_regions[i].end) + return true; + for (unsigned int i = 0; i < dcp->nr_excluded_regions; i++) + if (pa <= dcp->excluded_regions[i].end && + dcp->excluded_regions[i].start < pa + DCP_DART_PAGE) + return false; + return region_intersects(pa, DCP_DART_PAGE, IORESOURCE_SYSTEM_RAM, + IORES_DESC_NONE) == REGION_DISJOINT; +} + +static int apple_dart_dcp_import(struct apple_dart *dart, unsigned int sid, + struct dcp_adt_tables bounds, struct apple_dart_dcp **result) +{ + struct apple_dart_dcp *dcp; + u64 root_pa; + u8 *storage; + u32 tcr, enabled; + int ret; + + tcr = readl(dart->regs + DART_TCR(dart, sid)); + enabled = readl(dart->regs + DART_T8110_ENABLE_STREAMS); + if (!dart->locked || dart->hw->type != DART_T8110 || dart->num_streams != 32 || + PAGE_SIZE != SZ_16K || dart->pgsize != SZ_16K || dart->ias != 42 || dart->oas != 42 || + dart->dma_min != DCP_DART_ALIAS || + dart->dma_max != DCP_DART_ALIAS + DCP_DART_APERTURE - 1 || + tcr != DART_T8110_TCR_TRANSLATE_ENABLE || !(enabled & BIT(sid))) + return dev_err_probe(dart->dev, -ENODEV, + "DCP admission: lock=%d type=%u streams=%u page=%u IAS/OAS=%u/%u range=%pad..%pad TCR%u=%08x enabled=%08x\n", + dart->locked, dart->hw->type, dart->num_streams, dart->pgsize, + dart->ias, dart->oas, &dart->dma_min, &dart->dma_max, sid, tcr, enabled); + ret = apple_dart_dcp_excluded(dart, bounds.start, bounds.end); + if (ret) + return dev_err_probe(dart->dev, ret, "DCP table storage is not excluded memory\n"); + dcp = apple_dart_retained_alloc(dart, sizeof(*dcp)); + if (!dcp) + return -ENOMEM; + dcp->dart = dart; + dcp->sid = sid; + dcp->firmware = bounds; + ret = apple_dart_dcp_excluded_regions(dcp); + if (ret) + return dev_err_probe(dart->dev, ret, "invalid reserved-memory leaf exclusions\n"); + ret = apple_dart_dcp_leaf_regions(dcp); + if (ret) + return dev_err_probe(dart->dev, ret, "DCP consumer reservations are invalid\n"); + spin_lock_init(&dcp->lock); + mutex_init(&dcp->alloc_lock); + dcp->sub_root = -1; + INIT_LIST_HEAD(&dcp->allocations); + INIT_LIST_HEAD(&dcp->shared); + dcp->ttbr = readl(dart->regs + DART_TTBR(dart, sid, 0)); + root_pa = ((u64)dcp->ttbr >> 2) << 14; + if ((dcp->ttbr & 3) != 1 || (sid != 23 && root_pa != bounds.start) || + !dcp_dart_contains(bounds.start, bounds.end, root_pa, DCP_DART_PAGE)) + return dev_err_probe(dart->dev, -EINVAL, + "DCP admission: TTBR%u=%08x root=%llx outside ADT [%llx,%llx) or invalid flags\n", + sid, dcp->ttbr, root_pa, bounds.start, bounds.end); + storage = apple_dart_retained_memremap(dart, bounds.start, bounds.end - bounds.start); + if (IS_ERR(storage)) + return PTR_ERR(storage); + if (!storage) + return -ENOMEM; + dcp->root = (u64 *)(storage + root_pa - bounds.start); + for (unsigned int i = 0; i < DCP_DART_ENTRIES; i++) { + u64 pte = READ_ONCE(dcp->root[i]); + u64 pa = dcp_dart_pte_phys(pte); + + if (dcp_dart_pte_check(pte, false)) + return dev_err_probe(dart->dev, -EINVAL, + "DCP admission: root[%u]=%016llx has unsupported encoding\n", + i, pte); + dcp->entry[i] = pte; + if (!pte) + continue; + if (pa == root_pa || !dcp_dart_contains(bounds.start, bounds.end, pa, DCP_DART_PAGE)) + return dev_err_probe(dart->dev, -EINVAL, + "DCP admission: root[%u]=%016llx child=%llx outside ADT [%llx,%llx) or aliases root\n", + i, pte, pa, bounds.start, bounds.end); + for (unsigned int j = 0; j < i; j++) + if (dcp->entry[j] && dcp_dart_pte_phys(dcp->entry[j]) == pa) + return dev_err_probe(dart->dev, -EINVAL, + "DCP admission: roots %u and %u alias child=%llx\n", + j, i, pa); + dcp->table[i] = (u64 *)(storage + pa - bounds.start); + set_bit(i, dcp->foreign); + if (sid != 23) { + dcp->expected[i] = apple_dart_retained_alloc(dart, DCP_DART_PAGE); + if (!dcp->expected[i]) + return -ENOMEM; + } + for (unsigned int j = 0; j < DCP_DART_ENTRIES; j++) { + u64 leaf = READ_ONCE(dcp->table[i][j]); + + if (dcp_dart_pte_check(leaf, true)) + return dev_err_probe(dart->dev, -EINVAL, + "DCP admission: leaf[%u][%u]=%016llx has unsupported encoding\n", + i, j, leaf); + if (!apple_dart_dcp_leaf_safe(dcp, leaf)) + return dev_err_probe(dart->dev, -EBUSY, + "DCP inherited SID%u leaf[%u][%u] targets Linux RAM: %016llx\n", + sid, i, j, leaf); + if (sid != 23) + dcp->expected[i][j] = leaf; + } + } + /* Refuse an unstable root instead of trying to repair it. */ + for (unsigned int i = 0; i < DCP_DART_ENTRIES; i++) + if (READ_ONCE(dcp->root[i]) != dcp->entry[i]) + return dev_err_probe(dart->dev, -EBUSY, + "DCP admission: root[%u] changed during import\n", i); + if (sid != 23) + for (unsigned int i = 0; i < DCP_DART_ENTRIES; i++) + if (dcp->table[i]) + for (unsigned int j = 0; j < DCP_DART_ENTRIES; j++) + if (READ_ONCE(dcp->table[i][j]) != dcp->expected[i][j]) + return -EBUSY; + if (!apple_dart_dcp_registers(dcp)) + return -EBUSY; + if (sid == 0 && apple_dart_dcp_arena(dcp)) + return dev_err_probe(dart->dev, -ENOSPC, + "DCP admission: no free SID0 slices for retained surfaces\n"); + *result = dcp; + dev_info(dart->dev, "Preserving DCP SID%u root %pa and %u foreign 32-MiB slots; no writes\n", + sid, &root_pa, bitmap_weight(dcp->foreign, DCP_DART_ENTRIES)); + return 0; +} + +static int apple_dart_dcp_prepare(struct apple_dart *dart) +{ + struct dcp_adt_tables tables; + int ret = apple_dart_dcp_read_adt(dart, &tables, NULL); + + if (ret) + return dev_err_probe(dart->dev, ret, "cannot validate existing ADT table handoff\n"); + return apple_dart_dcp_import(dart, 23, tables, &dart->dcp); +} + +static bool apple_dart_dcp_iova(u64 iova) +{ + return iova >= DCP_DART_ALIAS && iova - DCP_DART_ALIAS < DCP_DART_APERTURE; +} + +static phys_addr_t apple_dart_dcp_translate(struct apple_dart_dcp *dcp, u64 iova) +{ + unsigned int root = (iova >> 25) & 2047; + unsigned int leaf = (iova >> 14) & 2047; + u64 pte; + phys_addr_t phys = 0; + unsigned long flags; + bool used = false; + + if (!apple_dart_dcp_iova(iova)) + return 0; + spin_lock_irqsave(&dcp->lock, flags); + /* DMA teardown still needs the host-owned physical address after quarantine. + * expected[] owns that provenance independently of failed hardware state. + */ + if (test_bit(root, dcp->claimed) && dcp->expected[root]) { + pte = dcp->expected[root][leaf]; + if (pte) + phys = dcp_dart_pte_phys(pte) | (iova & (DCP_DART_PAGE - 1)); + goto out; + } + /* + * An external display DART's registers only inside a use of them (its + * power confirmed, a sleep waits for it): none, no lookup. + */ + if (dcp->failed || !apple_dart_dispext_get(dcp->dart)) + goto out; + used = true; + if (!apple_dart_dcp_registers(dcp)) + goto out; + if (READ_ONCE(dcp->root[root]) != dcp->entry[root]) { + apple_dart_dcp_fail(dcp, "translate: root %u changed: %016llx, adopted %016llx", + root, READ_ONCE(dcp->root[root]), dcp->entry[root]); + goto out; + } + if (!dcp->table[root]) + goto out; + pte = READ_ONCE(dcp->table[root][leaf]); + if (dcp_dart_pte_check(pte, true) || + (test_bit(root, dcp->foreign) && !apple_dart_dcp_leaf_safe(dcp, pte)) || + ((dcp->sid != 23 || !test_bit(root, dcp->foreign)) && + pte != dcp->expected[root][leaf])) { + /* Inherited SID23 tables have no expected[]: their leaves on their own. */ + if (dcp->expected[root]) + apple_dart_dcp_fail(dcp, "translate: leaf %u/%u %016llx, expected %016llx", + root, leaf, pte, dcp->expected[root][leaf]); + else + apple_dart_dcp_fail(dcp, "translate: inherited leaf %u/%u %016llx invalid or not safe", + root, leaf, pte); + goto out; + } + if (!pte) + goto out; + phys = dcp_dart_pte_phys(pte) | (iova & (DCP_DART_PAGE - 1)); +out: + spin_unlock_irqrestore(&dcp->lock, flags); + if (used) + apple_dart_dispext_put(dcp->dart); + return phys; +} + +/* The old loader name selects the same common Apple DRM SID0 consumer. */ +static bool apple_dart_j713_scanout_node(struct device_node *node) +{ + return of_device_is_compatible(node, "apple,t8132-display-subsystem") || + of_device_is_compatible(node, "apple,t8132-display-diagnostics"); +} + +static int apple_dart_frontend_validate(struct apple_dart_dcp *dcp, + struct device_node *consumer) +{ + struct device_node *alias; + struct of_phandle_args spec; + u32 wanted = dcp->sid == 0 ? BIT(4) | BIT(6) : BIT(5); + u32 seen = 0; + unsigned long flags; + int nr, ret; + + if (!apple_dart_dcp_registers(dcp)) + return -EBUSY; + /* Read-only admission must not hide leaf drift behind stable root pointers. */ + ret = 0; + spin_lock_irqsave(&dcp->lock, flags); + for (unsigned int i = 0; i < DCP_DART_ENTRIES; i++) { + if (READ_ONCE(dcp->root[i]) != dcp->entry[i]) { + apple_dart_dcp_fail(dcp, "root %u changed: %016llx, adopted %016llx", i, + READ_ONCE(dcp->root[i]), dcp->entry[i]); + ret = -EBUSY; + break; + } + if (dcp->table[i]) + for (unsigned int j = 0; j < DCP_DART_ENTRIES; j++) + if (READ_ONCE(dcp->table[i][j]) != dcp->expected[i][j]) { + apple_dart_dcp_fail(dcp, "leaf %u/%u changed: %016llx, expected %016llx", + i, j, READ_ONCE(dcp->table[i][j]), + dcp->expected[i][j]); + ret = -EBUSY; + goto snapshot_done; + } + } +snapshot_done: + spin_unlock_irqrestore(&dcp->lock, flags); + if (ret) + return ret; + + alias = of_find_node_by_path(dcp->sid == 0 ? "disp0" : "disp0_piodma"); + ret = alias && consumer == alias && of_device_is_available(alias) && + of_property_read_bool(alias, "apple,j713-inherited-mappings") ? 0 : -ENODEV; + of_node_put(alias); + if (ret) + return ret; + if (dcp->sid == 0 && !apple_dart_j713_scanout_node(consumer)) + return -ENODEV; + if (of_count_phandle_with_args(consumer, "iommus", "#iommu-cells") != 1) + return -EINVAL; + ret = of_parse_phandle_with_args(consumer, "iommus", "#iommu-cells", 0, &spec); + if (ret) + return ret; + ret = spec.np == dcp->dart->dev->of_node && spec.args_count == 1 && + spec.args[0] == dcp->sid ? 0 : -EINVAL; + of_node_put(spec.np); + if (ret) + return ret; + nr = of_property_count_u32_elems(consumer, "memory-region"); + if (nr != (dcp->sid == 0 ? 2 : 1)) + return -EINVAL; + for (int i = 0; i < nr; i++) { + struct device_node *node = of_parse_phandle(consumer, "memory-region", i); + struct reserved_mem *rmem; + struct resource res; + const __be32 *maps; + unsigned int matches = 0; + int bytes; + + if (!node) + return -EINVAL; + rmem = of_reserved_mem_lookup(node); + ret = -EINVAL; + if (!of_device_is_compatible(node, "apple,asc-mem") || + !of_property_read_bool(node, "no-map") || + of_property_count_u32_elems(node, "reg") != 4 || + of_address_to_resource(node, 0, &res) || !rmem || + rmem->base != res.start || rmem->size != resource_size(&res)) + goto put_node; + maps = of_get_property(node, "iommu-addresses", &bytes); + if (!maps || bytes <= 0 || bytes % (5 * sizeof(*maps))) + goto put_node; + for (int j = 0; j < bytes / sizeof(*maps); j += 5) { + struct device_node *owner = of_find_node_by_phandle(be32_to_cpu(maps[j])); + const struct dcp_adt_segment *segment; + u64 iova = of_read_number(maps + j + 1, 2); + u64 size = of_read_number(maps + j + 3, 2); + bool ours = owner == consumer; + bool cells = owner && of_n_addr_cells(owner) == 2 && of_n_size_cells(owner) == 2; + u32 bit; + + of_node_put(owner); + /* Validate other owners' tuple extents too, before skipping them. */ + if (!cells || size != resource_size(&res) || + ((res.start | size | iova) & (DCP_DART_PAGE - 1)) || + !dcp_dart_contains(DCP_DART_ALIAS, DCP_DART_ALIAS + DCP_DART_APERTURE, + iova, size)) + goto put_node; + segment = dcp_adt_segment_match(&dcp->firmware, res.start, iova, size); + if (!segment || segment->kind != 2) + goto put_node; + if (!ours) + continue; + bit = BIT(segment - dcp->firmware.segment); + if (++matches != 1 || !(wanted & bit) || (seen & bit)) + goto put_node; + seen |= bit; + for (u64 off = 0; off < size; off += DCP_DART_PAGE) { + unsigned int root = ((iova + off) >> 25) & 2047; + unsigned int leaf = ((iova + off) >> 14) & 2047; + + if (apple_dart_dcp_translate(dcp, iova + off) != res.start + off || + !test_bit(root, dcp->foreign) || + !dcp_dart_pte_frontend(READ_ONCE(dcp->table[root][leaf]))) + goto put_node; + } + } + ret = matches == 1 ? 0 : -EINVAL; +put_node: + of_node_put(node); + if (ret) + return ret; + } + return seen == wanted && apple_dart_dcp_registers(dcp) ? 0 : -EINVAL; +} + +/* + * J713: adopt an external display DART's SID16, the PIODMA stream its + * dcpext's shared buffers are mapped in, by the internal frontend's rules: + * the boot loader's root and tables are imported and kept (their region + * reserved), nothing is written now, and later only free slices are + * claimed, through the shared-buffer API and apple_dart_dcp_dispext_map(); + * SID0 (scanout) the same way, if it can be. The consumer is the dcpext's + * piodma child, see apple_dart_dispext_validate(). + */ +static int apple_dart_dispext_prepare(struct apple_dart *dart) +{ + struct dcp_adt_tables firmware, bounds; + struct dcp_adt_frontend frontend; + struct apple_dart_dcp *borrowed; + int ret; + + dev_dbg(dart->dev, "external display DART: enabled streams %08x, protect %08x\n", + readl(dart->regs + DART_T8110_ENABLE_STREAMS), + readl(dart->regs + DART_T8110_PROTECT)); + if (dart->hw->type != DART_T8110 || dart->num_streams != 32) + return -ENODEV; + ret = apple_dart_dcp_read_adt(dart, &firmware, &frontend); + if (ret) + return dev_err_probe(dart->dev, ret, "tables not described\n"); + if (!(frontend.seen & 4) || frontend.end[1] <= frontend.start[1]) + return dev_err_probe(dart->dev, -ENODEV, "no SID16 table region\n"); + bounds = firmware; + bounds.start = frontend.start[1]; + bounds.end = frontend.end[1]; + ret = apple_dart_dcp_import(dart, 16, bounds, &borrowed); + if (ret) + return ret; + if (!apple_dart_dcp_registers(borrowed)) + return -EBUSY; + dart->frontend[1] = borrowed; + /* The TTBR/TCR lock as adopted: a later check compares against it. */ + dart->dispext_protect = readl(dart->regs + DART_T8110_PROTECT); + dart->dispext_lock = readl(dart->regs + DART_T8110_PROTECT_LOCK); + dev_dbg(dart->dev, "SID16 (PIODMA) adopted: %u foreign 32-MiB slots kept; protect %08x lock %08x\n", + bitmap_weight(borrowed->foreign, DCP_DART_ENTRIES), dart->dispext_protect, + dart->dispext_lock); + /* + * SID0 (scanout) the same way, for the firmware's D201 with flag 0; + * its free slices are claimed only through + * apple_dart_dcp_dispext_map(). Optional: SID16 goes on without. + */ + if (!(frontend.seen & 2) || frontend.end[0] <= frontend.start[0]) { + dev_info(dart->dev, "no SID0 table region: SID0 left alone\n"); + return 0; + } + bounds = firmware; + bounds.start = frontend.start[0]; + bounds.end = frontend.end[0]; + ret = apple_dart_dcp_import(dart, 0, bounds, &borrowed); + if (ret || !apple_dart_dcp_registers(borrowed)) { + dev_warn(dart->dev, "SID0 not adopted (%d); SID16 is\n", ret); + return 0; + } + dart->frontend[0] = borrowed; + dev_dbg(dart->dev, "SID0 (scanout) adopted: %u foreign 32-MiB slots kept\n", + bitmap_weight(borrowed->foreign, DCP_DART_ENTRIES)); + return 0; +} + +/* + * An adopted external display DART's SID16 consumer: the piodma child + * of its own dcpext, naming this DART's SID16 only; and the inherited root + * and tables unchanged since their import. + */ +static int apple_dart_dispext_validate(struct apple_dart_dcp *dcp, struct device_node *consumer) +{ + struct device_node *owner = of_find_node_by_path(dcp->dart->dcp_instance->dcp); + struct device_node *parent = of_get_parent(consumer); + struct of_phandle_args spec; + unsigned long flags; + int ret = 0; + + if (!owner || parent != owner || !of_node_name_eq(consumer, "piodma") || + !of_device_is_available(consumer) || dcp->sid != 16) + ret = -ENODEV; + of_node_put(parent); + of_node_put(owner); + if (ret) + return ret; + if (!apple_dart_dcp_registers(dcp)) + return -EBUSY; + /* No fault IRQ is attached for this DART: look for a pending fault. */ + if (apple_dart_fault_pending(dcp->dart)) { + apple_dart_dcp_fail(dcp, "a DART fault is pending"); + return -EIO; + } + spin_lock_irqsave(&dcp->lock, flags); + for (unsigned int i = 0; i < DCP_DART_ENTRIES && !ret; i++) { + if (READ_ONCE(dcp->root[i]) != dcp->entry[i]) + ret = -EBUSY; + else if (dcp->table[i]) + for (unsigned int j = 0; j < DCP_DART_ENTRIES; j++) + if (READ_ONCE(dcp->table[i][j]) != dcp->expected[i][j]) { + ret = -EBUSY; + break; + } + } + if (ret) + apple_dart_dcp_fail(dcp, "validation failed: %d", ret); + spin_unlock_irqrestore(&dcp->lock, flags); + if (ret) + return ret; + if (of_count_phandle_with_args(consumer, "iommus", "#iommu-cells") != 1) + return -EINVAL; + ret = of_parse_phandle_with_args(consumer, "iommus", "#iommu-cells", 0, &spec); + if (ret) + return ret; + ret = spec.np == dcp->dart->dev->of_node && spec.args_count == 1 && + spec.args[0] == 16 ? 0 : -EINVAL; + of_node_put(spec.np); + return ret; +} + +/* Read-only admission; SID0 later maps only free slices, SID16 only via its pair API. */ +static int apple_dart_frontend_prepare(struct apple_dart *dart) +{ + struct apple_dart_dcp *borrowed[2]; + struct dcp_adt_tables firmware; + struct dcp_adt_frontend frontend; + int ret = apple_dart_dcp_read_adt(dart, &firmware, &frontend); + + if (ret) + return ret; + for (unsigned int i = 0; i < 2; i++) { + struct dcp_adt_tables bounds = firmware; + struct device_node *node; + + bounds.start = frontend.start[i]; + bounds.end = frontend.end[i]; + ret = apple_dart_dcp_import(dart, i ? 16 : 0, bounds, &borrowed[i]); + if (ret) + return ret; + node = of_find_node_by_path(i ? "disp0_piodma" : "disp0"); + ret = node ? apple_dart_frontend_validate(borrowed[i], node) : -ENODEV; + of_node_put(node); + if (ret) + return dev_err_probe(dart->dev, ret, + "SID%u inherited frontend mapping validation failed\n", i ? 16 : 0); + } + /* Neither stream is published before both imports and all mappings pass. */ + if (!apple_dart_dcp_registers(borrowed[0]) || !apple_dart_dcp_registers(borrowed[1])) + return -EBUSY; + dart->frontend[0] = borrowed[0]; + dart->frontend[1] = borrowed[1]; + dev_info(dart->dev, "Inherited SID0/SID16 frontend admitted; SID0 strict DMA outside %u foreign and %u retained slices\n", + bitmap_weight(borrowed[0]->foreign, DCP_DART_ENTRIES), DCP_DART_SCANOUT_SLOTS); + return 0; +} + +static ssize_t j713_display_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + int status[2]; + + if (!dart->frontend[0] || !dart->frontend[1] || !READ_ONCE(dart->iommu.ready)) + return -ENODEV; + for (unsigned int i = 0; i < 2; i++) { + struct device_node *node = of_find_node_by_path(i ? "disp0_piodma" : "disp0"); + + status[i] = node ? apple_dart_frontend_validate(dart->frontend[i], node) : -ENODEV; + of_node_put(node); + if (status[i]) + apple_dart_dcp_fail(dart->frontend[i], "frontend validation failed: %d", + status[i]); + } + return sysfs_emit(buf, "sid0=%d sid16=%d host_mapping=%u\n", status[0], status[1], + READ_ONCE(dart->frontend[1]->shared_mapping)); +} +static DEVICE_ATTR(j713_display_state, 0400, j713_display_state_show, NULL); + +static struct attribute *apple_dart_frontend_attrs[] = { + &dev_attr_j713_display_state.attr, + NULL, +}; + +static const struct attribute_group apple_dart_frontend_group = { + .attrs = apple_dart_frontend_attrs, +}; + +static int apple_dart_dcp_validate(struct apple_dart_dcp *dcp, struct device *dev) +{ + struct device_node *node; + const char *uuid; + unsigned int count = 0; + unsigned int oslogs = 0; + u32 seen = 0; + int nr; + int ret = 0; + + if (dcp->sid != 23) { + if (dcp->owner && dcp->owner != dev) + return -EBUSY; + ret = dcp->dart->dispext ? + apple_dart_dispext_validate(dcp, dev->of_node) : + apple_dart_frontend_validate(dcp, dev->of_node); + if (!ret) + dcp->owner = dev; + return ret; + } + + if (!of_device_is_compatible(dev->of_node, + dcp->dart->dcp_instance == &dcp_adt_internal ? + "apple,t8132-dcp" : "apple,t8132-dcpext") || + of_property_read_string(dev->of_node, "apple,firmware-uuid", &uuid) || + strcmp(uuid, "81BAF476-679D-3CC0-8C6A-E6B6FA659D5E") || + (dcp->owner && dcp->owner != dev)) + return -ENODEV; + /* Do not silently skip malformed ranges or allocation failures in a list helper. */ + nr = of_property_count_u32_elems(dev->of_node, "memory-region"); + if (nr <= 0 || nr > 64) + return -EINVAL; + for (int i = 0; i < nr; i++) { + struct resource res; + const __be32 *maps; + unsigned int matches = 0; + int bytes; + + node = of_parse_phandle(dev->of_node, "memory-region", i); + if (!node) + return -EINVAL; + if (!of_device_is_compatible(node, "apple,asc-mem") || + !of_property_read_bool(node, "no-map") || + of_address_to_resource(node, 0, &res)) { + ret = -EINVAL; + goto put_node; + } + maps = of_get_property(node, "iommu-addresses", &bytes); + if (of_property_read_bool(node, "apple,dcp-os-log")) { + struct reserved_mem *rmem = of_reserved_mem_lookup(node); + + if (++oslogs != 1 || maps || resource_size(&res) != 0x24000 || + (res.start & (DCP_DART_PAGE - 1)) || !rmem || + rmem->base != res.start || rmem->size != resource_size(&res)) + ret = -EINVAL; + /* Explicitly physical and possibly relocated, not a DART map. */ + goto put_node; + } + if (!maps || bytes <= 0 || bytes % (5 * sizeof(*maps))) { + ret = -EINVAL; + goto put_node; + } + for (int j = 0; j < bytes / sizeof(*maps); j += 5) { + const struct dcp_adt_segment *segment; + struct device_node *owner = of_find_node_by_phandle(be32_to_cpu(maps[j])); + bool ours = owner == dev->of_node; + u64 iova = of_read_number(maps + j + 1, 2); + u64 size = of_read_number(maps + j + 3, 2); + bool cells = owner && of_n_addr_cells(owner) == 2 && of_n_size_cells(owner) == 2; + + of_node_put(owner); + if (!cells) { + ret = -EINVAL; + goto put_node; + } + if (!ours) + continue; + if (++matches != 1 || size != resource_size(&res) || + ((res.start | iova | size) & (DCP_DART_PAGE - 1)) || + !dcp_dart_contains(DCP_DART_ALIAS, DCP_DART_ALIAS + DCP_DART_APERTURE, + iova, size)) { + ret = -EINVAL; + goto put_node; + } + segment = dcp_adt_segment_match(&dcp->firmware, res.start, iova, size); + if (!segment || (seen & BIT(segment - dcp->firmware.segment))) { + dev_err(dev, "Inherited DCP mapping lacks matching ADT segment at %llx\n", + iova); + ret = -EINVAL; + goto put_node; + } + seen |= BIT(segment - dcp->firmware.segment); + for (u64 off = 0; off < size; off += DCP_DART_PAGE) { + unsigned int root = ((iova + off) >> 25) & 2047; + unsigned int leaf = ((iova + off) >> 14) & 2047; + + if (apple_dart_dcp_translate(dcp, iova + off) != res.start + off || + !test_bit(root, dcp->foreign) || + !dcp_dart_pte_firmware(READ_ONCE(dcp->table[root][leaf]), + segment->kind)) { + dev_err(dev, "Inherited DCP mapping mismatch at %llx\n", iova + off); + ret = -EINVAL; + goto put_node; + } + } + count++; + } + if (!matches) + ret = -EINVAL; +put_node: + of_node_put(node); + if (ret) + return ret; + } + if (!ret && (!count || oslogs != 1)) + ret = -ENODEV; + for (u32 i = 0; !ret && i < dcp->firmware.num_segments; i++) + if (dcp->firmware.segment[i].kind != 10 && !(seen & BIT(i))) + ret = -EINVAL; + if (!ret) + dcp->owner = dev; + return ret; +} + +static void apple_dart_dcp_reserve(struct apple_dart_dcp *dcp, struct list_head *head) +{ + struct iommu_resv_region *region, *next; + unsigned int start = 0, end; + LIST_HEAD(slices); + + /* SID0: whole foreign slices and the retained arena; generic DMA gets the rest. */ + while (dcp->sid == 0) { + while (start < DCP_DART_ENTRIES && !test_bit(start, dcp->foreign) && + !test_bit(start, dcp->arena)) + start++; + if (start == DCP_DART_ENTRIES) { + list_splice_tail(&slices, head); + return; + } + for (end = start; end < DCP_DART_ENTRIES && (test_bit(end, dcp->foreign) || + test_bit(end, dcp->arena)); end++) + ; + region = iommu_alloc_resv_region(DCP_DART_ALIAS + start * DCP_DART_SLICE, + (end - start) * DCP_DART_SLICE, 0, + IOMMU_RESV_RESERVED, GFP_KERNEL); + if (!region) + break; + list_add_tail(®ion->list, &slices); + start = end; + } + /* Never return a partial SID0 list: fall back to the whole aperture, failed. */ + list_for_each_entry_safe(region, next, &slices, list) + kfree(region); + if (dcp->sid == 0) + apple_dart_dcp_fail(dcp, "no SID0 reservation list"); + /* SID23/SID16 domains: only the retained allocators may allocate there. */ + region = iommu_alloc_resv_region(DCP_DART_ALIAS, DCP_DART_APERTURE, 0, + IOMMU_RESV_RESERVED, GFP_KERNEL); + if (!region) { + apple_dart_dcp_fail(dcp, "no memory for its reservation"); + return; + } + list_add_tail(®ion->list, head); +} + +/* Pin a page an uncertain SID0 translation may still reach; only reboot drops it. */ +static unsigned int apple_dart_dcp_retain(u64 pte) +{ + unsigned long pfn = PHYS_PFN(dcp_dart_pte_phys(pte)); + + return pte && pfn_valid(pfn) && folio_try_get(page_folio(pfn_to_page(pfn))); +} + +/* SID0 only: FLUSH_SID word 0x100 at +0x80 with bounded busy waits; failure is sticky. */ +static int apple_dart_dcp_flush(struct apple_dart_dcp *dcp) +{ + struct apple_dart_stream_map map = { .dart = dcp->dart }; + + if (dcp->sid != 0 || !apple_dart_dcp_registers(dcp)) + return -EIO; + bitmap_zero(map.sidmap, DART_MAX_STREAMS); + set_bit(0, map.sidmap); + dma_wmb(); + /* Raw: inside the caller's use of the DART (or a DART with no such gate). */ + return apple_dart_dcp_invalidate_tlb(&map); +} + +static int apple_dart_dcp_map_owned(struct apple_dart_dcp *dcp, u64 iova, phys_addr_t phys, + size_t size, size_t count, int prot, gfp_t gfp, + size_t *mapped, bool shared) +{ + unsigned int root = (iova >> 25) & 2047; + unsigned int leaf = (iova >> 14) & 2047; + u64 flags = 1 | (0xfffULL << 40); + u64 *fresh = NULL, *expected = NULL; + unsigned long irqflags; + bool generic = !shared && dcp->sid == 0; + int ret = 0; + + /* Generic SID0 DMA keeps the display's no-cache leaf policy. */ + if ((dcp->sid != 23 && !generic && !(shared && (dcp->sid == 16 || dcp->sid == 0) && + test_bit(root, dcp->claimed) && prot == (IOMMU_READ | IOMMU_WRITE))) || + (generic && (prot & ~(IOMMU_READ | IOMMU_WRITE | IOMMU_NOEXEC))) || + (shared && dcp->sid == 0 && !test_bit(root, dcp->arena)) || + !apple_dart_dcp_iova(iova) || size != DCP_DART_PAGE || !count || + ((iova | phys) & (DCP_DART_PAGE - 1)) || + !(prot & (IOMMU_READ | IOMMU_WRITE)) || phys >> 42 || + test_bit(root, dcp->foreign)) + return -EINVAL; + /* Private slots have one writer: the paired or SID0 scanout allocation API. */ + if (shared != !!test_bit(root, dcp->shared_roots) || + (generic && test_bit(root, dcp->arena))) + return -EPERM; + count = min_t(size_t, count, DCP_DART_ENTRIES - leaf); + /* Failed SID0 unmap must be able to pin every reachable allocation. A + * folio reference cannot stop slab objects being reused within that page. + * Generic DMA therefore admits only whole normal page-backed allocations. + */ + if (generic) + for (size_t i = 0; i < count; i++) { + unsigned long pfn = PHYS_PFN(phys + i * size); + struct page *page; + + if (!pfn_valid(pfn)) + return -EOPNOTSUPP; + page = pfn_to_page(pfn); + if (PageReserved(page) || folio_test_slab(page_folio(page)) || + !page_count(page)) + return -EOPNOTSUPP; + } + if (count * size > BIT_ULL(42) - phys) + return -ERANGE; + if (!(prot & IOMMU_READ)) + flags |= BIT(3); + if (!(prot & IOMMU_WRITE)) + flags |= BIT(2); + if (!(prot & IOMMU_CACHE)) + flags |= BIT(1); + if (!READ_ONCE(dcp->table[root])) { + fresh = (void *)get_zeroed_page(gfp); + if (!fresh) + return -ENOMEM; + expected = (void *)get_zeroed_page(gfp); + if (!expected) { + free_page((unsigned long)fresh); + return -ENOMEM; + } + if ((virt_to_phys(fresh) & (DCP_DART_PAGE - 1)) || virt_to_phys(fresh) >> 42) { + free_page((unsigned long)fresh); + free_page((unsigned long)expected); + return -ERANGE; + } + } + spin_lock_irqsave(&dcp->lock, irqflags); + /* A paired reservation may have won while this call allocated a leaf table. */ + if (shared != !!test_bit(root, dcp->shared_roots)) { + ret = -EPERM; + goto out; + } + if (dcp->failed || READ_ONCE(dcp->root[root]) != dcp->entry[root]) { + apple_dart_dcp_fail(dcp, "map: root %u changed: %016llx, adopted %016llx", root, + READ_ONCE(dcp->root[root]), dcp->entry[root]); + ret = -EIO; + goto out; + } + if (!dcp->table[root]) { + u64 entry = (virt_to_phys(fresh) >> 4) | 1; + + /* Publish a zeroed, host-owned table; never replace a foreign entry. */ + dma_wmb(); + if (cmpxchg64(&dcp->root[root], 0, entry)) { + apple_dart_dcp_fail(dcp, "map: root %u was taken meanwhile", root); + ret = -EBUSY; + goto out; + } + dcp->entry[root] = entry; + dcp->table[root] = fresh; + dcp->expected[root] = expected; + fresh = NULL; /* Tables remain pinned until reboot, including empty ones. */ + expected = NULL; + } + for (size_t i = 0; i < count; i++) { + if (READ_ONCE(dcp->table[root][leaf + i]) != dcp->expected[root][leaf + i]) { + apple_dart_dcp_fail(dcp, "map: leaf %u/%zu changed", root, leaf + i); + ret = -EIO; + goto out; + } + if (READ_ONCE(dcp->table[root][leaf + i])) { + ret = -EEXIST; + goto out; + } + } + for (size_t i = 0; i < count; i++) { + u64 pte = ((phys + i * size) >> 4) | flags; + + dcp->expected[root][leaf + i] = pte; + WRITE_ONCE(dcp->table[root][leaf + i], pte); + } + dma_wmb(); + if (mapped) + *mapped += count * size; +out: + spin_unlock_irqrestore(&dcp->lock, irqflags); + if (fresh) + free_page((unsigned long)fresh); + if (expected) + free_page((unsigned long)expected); + return ret; +} + +static int apple_dart_dcp_map(struct apple_dart_dcp *dcp, u64 iova, phys_addr_t phys, + size_t size, size_t count, int prot, gfp_t gfp, size_t *mapped) +{ + return apple_dart_dcp_map_owned(dcp, iova, phys, size, count, prot, gfp, mapped, false); +} + +/* + * Called with dcp->lock held after a refused SID0 unmap. DMA callers release + * the whole IOVA range and their backing, so retain every generic leaf before + * reporting logical retirement. Failed domains reject all subsequent maps; + * their hardware translations and retained pages are never reused. + */ +static bool apple_dart_dcp_retain_span(struct apple_dart_dcp *dcp, u64 iova, + size_t count, unsigned long *retained) +{ + u64 end = DCP_DART_ALIAS + DCP_DART_APERTURE; + bool complete = true; + + if (count < (end - iova) / DCP_DART_PAGE) + end = iova + count * DCP_DART_PAGE; + while (iova < end) { + unsigned int root = (iova >> 25) & 2047; + unsigned int leaf = (iova >> 14) & 2047; + u64 next = min_t(u64, end, (iova | (DCP_DART_SLICE - 1)) + 1); + + /* A generic DMA unmap must not retire a reserved/private slice. */ + if (test_bit(root, dcp->foreign) || test_bit(root, dcp->arena) || + test_bit(root, dcp->shared_roots)) { + complete = false; + } else if (dcp->table[root]) { + for (u64 dva = iova; dva < next; dva += DCP_DART_PAGE, leaf++) { + u64 pte = dcp->expected[root][leaf]; + + if (!pte) + continue; + if (apple_dart_dcp_retain(pte)) + (*retained)++; + else + complete = false; + } + } + iova = next; + } + return complete; +} + +static size_t __apple_dart_dcp_unmap(struct apple_dart_dcp *dcp, u64 iova, + size_t size, size_t count, struct iommu_iotlb_gather *gather); + +static size_t apple_dart_dcp_unmap(struct apple_dart_dcp *dcp, u64 iova, + size_t size, size_t count, struct iommu_iotlb_gather *gather) +{ + size_t unmapped; + + /* + * On an external display DART, only inside a use of its registers. + * Returning 0 (nothing unmapped) is not safe for general DMA teardown: + * the caller would take the IOVA as unmapped and reuse it, or free + * pages still mapped. It is unreachable here: this generic path serves + * only SID23 and SID0 DMA domains, and an external display DART's SID0 + * has no generic DMA user (its host mappings are the private window and + * scanout APIs, which never come through here). + */ + if (!apple_dart_dispext_get(dcp->dart)) { + dev_err_ratelimited(dcp->dart->dev, "SID%u unmap at %llx while unconfirmed since sleep: refused\n", + dcp->sid, iova); + return 0; + } + unmapped = __apple_dart_dcp_unmap(dcp, iova, size, count, gather); + apple_dart_dispext_put(dcp->dart); + return unmapped; +} + +static size_t __apple_dart_dcp_unmap(struct apple_dart_dcp *dcp, u64 iova, + size_t size, size_t count, struct iommu_iotlb_gather *gather) +{ + unsigned int root = (iova >> 25) & 2047; + unsigned int leaf = (iova >> 14) & 2047; + unsigned long flags, retained = 0; + size_t unmapped = 0, cleared = 0, span = count; /* Whole remaining core range. */ + + if ((dcp->sid != 23 && dcp->sid != 0) || !apple_dart_dcp_iova(iova) || + size != DCP_DART_PAGE || !count || (iova & (size - 1)) || + count > (DCP_DART_ALIAS + DCP_DART_APERTURE - iova) / size || + test_bit(root, dcp->foreign) || test_bit(root, dcp->shared_roots) || + test_bit(root, dcp->arena)) + return 0; + spin_lock_irqsave(&dcp->lock, flags); + if (test_bit(root, dcp->shared_roots)) + goto out; + count = min_t(size_t, count, DCP_DART_ENTRIES - leaf); + if (!dcp->table[root]) { + if (dcp->failed) + goto refuse; + goto out; + } + if (dcp->failed || READ_ONCE(dcp->root[root]) != dcp->entry[root]) + goto refuse; + for (size_t i = 0; i < count; i++) { + if (READ_ONCE(dcp->table[root][leaf + i]) != dcp->expected[root][leaf + i]) { + apple_dart_dcp_fail(dcp, "unmap: leaf %u/%zu changed", root, leaf + i); + goto refuse; + } + } + while (cleared < count && dcp->table[root][leaf + cleared]) + WRITE_ONCE(dcp->table[root][leaf + cleared++], 0); + dma_wmb(); + /* Strict SID0: invalidate before the IOVA or its pages can be reused. */ + if (cleared && dcp->sid == 0 && apple_dart_dcp_flush(dcp)) { + /* A stale translation may survive: republish the leaves and keep the pages. */ + for (size_t i = 0; i < cleared; i++) + WRITE_ONCE(dcp->table[root][leaf + i], dcp->expected[root][leaf + i]); + dma_wmb(); + goto refuse; + } + for (size_t i = 0; i < cleared; i++) + dcp->expected[root][leaf + i] = 0; + unmapped = cleared * size; + goto out; +refuse: + apple_dart_dcp_fail(dcp, "unmap at %llx refused (state changed or flush failed)", iova); + /* Claim logical retirement only once every possible DMA backing page + * is retained. IOVA reuse cannot publish a new translation: failed is + * sticky and every map path checks it under this same lock. + */ + if (dcp->sid == 0 && apple_dart_dcp_retain_span(dcp, iova, span, &retained)) + unmapped = span * size; + dcp->retained += retained; +out: + spin_unlock_irqrestore(&dcp->lock, flags); + if (retained) + dev_err_ratelimited(dcp->dart->dev, + "SID0 hardware unmap at %llx quarantined; retaining %lu pages until reboot\n", + iova, retained); + if (unmapped) + iommu_iotlb_gather_add_range(gather, iova, unmapped); + return unmapped; +} diff --git a/drivers/iommu/apple-dart-test.c b/drivers/iommu/apple-dart-test.c index 51e504321798d..5f512e18bfd50 100644 --- a/drivers/iommu/apple-dart-test.c +++ b/drivers/iommu/apple-dart-test.c @@ -940,7 +940,31 @@ static void apple_dart_test_resume_reset_retry(struct kunit *test) dart->hw->ttbr_valid | 0x1000); } +static void apple_dart_test_strict_engines(struct kunit *test) +{ + static const char * const strict[] = { + "apple,t8103-avd", "apple,t6000-avd", "apple,t8112-avd", "apple,t6041-avd", + "apple,t8132-avd", "apple,t8140-avd", "apple,t6000-ave", "apple,t8112-ave", + "apple,t6041-ave", "apple,t8103-ane", "apple,t6021-ane", "apple,t6041-ane", + "apple,t8112-isp", "apple,t6041-isp", "apple,isp", + }; + static const char * const lazy[] = { + "apple,t6021-ane-mailbox", "apple,asc-mailbox-v4", "apple,t8103-display-pipe", + "apple,nvme-ans2", "apple,t8103-dart", "apple,t6000-display-crossbar", + "brcm,bcm4378-fmac", "apple,", "avd", "foo,t8112-avd", "apple,t8112-avdx", + }; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(strict); i++) + KUNIT_EXPECT_TRUE_MSG(test, apple_dart_compat_is_strict_engine(strict[i]), + "%s", strict[i]); + for (i = 0; i < ARRAY_SIZE(lazy); i++) + KUNIT_EXPECT_FALSE_MSG(test, apple_dart_compat_is_strict_engine(lazy[i]), + "%s", lazy[i]); +} + static struct kunit_case apple_dart_test_cases[] = { + KUNIT_CASE(apple_dart_test_strict_engines), KUNIT_CASE(apple_dart_test_iommu_specifiers), KUNIT_CASE(apple_dart_test_independent_remap_roots), KUNIT_CASE(apple_dart_test_before_attach), diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 73af536772fc2..8fa1a1f39b5b8 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -19,11 +19,14 @@ #include #include #include +#include #include #include #include #include #include +#include +#include #include #include #include @@ -38,7 +41,10 @@ #include #include #include +#include +#include #include +#include #include #include "dma-iommu.h" @@ -133,6 +139,8 @@ #define DART_T8110_TLB_CMD_V2 BIT(15) #define DART_T8110_TLB_CMD_RANGE BIT(14) #define DART_T8110_TLB_CMD_STT BIT(13) +/* A second TLB busy flag (bit 0), polled on the J713 external display DARTs */ +#define DART_T8110_TLB_BUSY2 0x604 #define DART_T8110_TLB_START 0x98 #define DART_T8110_TLB_END 0xa0 #define DART_T8110_FLUSH_BUSY_TIMEOUT 2000 @@ -202,6 +210,7 @@ ((idx) << 2)) struct apple_dart_stream_map; +struct apple_dart_dcp; enum dart_type { DART_T8020, @@ -253,6 +262,21 @@ struct apple_dart_fw_root { u64 *accepted; }; +/* + * The subtrees behind the slots firmware holds in a four-level locked root + * (fw_mirror), recorded when the root is mapped: per slot, the root entry and + * a mapping of its mid-level table, and per mid entry the entry and a mapping + * of its leaf table. Leaf entries are read live. A table entry that no longer + * matches what was recorded resolves nothing. Only lookups use it: Linux + * still never publishes into those slots. + */ +struct apple_dart_fw_tree { + u64 *entry; + u64 **mid; + u64 **mid_entry; + u64 ***leaf; +}; + /* * Private structure associated with each DART device. * @@ -293,6 +317,15 @@ struct apple_dart { u32 power_retained : 1; /* Locked roots may hold firmware slots that mirror Linux mappings. */ u32 fw_mirror : 1; + /* + * fw_mirror's four-level locked roots hold firmware's own subtrees, which + * are recorded and reserved (apple_dart_snapshot_fw_tree()). Set only for + * apple,t6050-dart, where it was tested (the DCP's log and crash log). Before + * widening it, check on T6041/T8140/T8142 whether any locked stream is + * four-level and what its foreign slots hold: the reservation moves DMA + * placement for every client of that stream. + */ + u32 fw_tree : 1; u32 fw_handoff : 1; u32 tunnel_state_saved : 1; u32 tunnel_state_restored : 1; @@ -302,6 +335,42 @@ struct apple_dart { struct apple_tunable *tunables; u16 version; + /* J713 internal display DARTs, borrowed from the running DCP. */ + struct apple_dart_dcp *dcp; + struct apple_dart_dcp *frontend[2]; + /* The display coprocessor whose firmware DART (SID23) this is. */ + const struct dcp_adt_instance *dcp_instance; + bool preserve_display; + /* A J713 external display DART, borrowed from its external DCP */ + bool dispext; + /* + * J713 external display DART: its power confirmation across + * sleep, under dispext_pm_lock (a spinlock: only these fields, never + * across a register access): unverified from suspend until a state + * check at a D572(1) of the current generation, which every suspend + * and resume moves on; sleeping between suspend and resume; and the + * register uses in flight (apple_dart_dispext_get()), which an + * invalidation waits for (dispext_wq). + */ + spinlock_t dispext_pm_lock; + bool dispext_unverified; + bool dispext_sleeping; + u64 dispext_gen; + unsigned int dispext_users; + wait_queue_head_t dispext_wq; + u32 dispext_protect, dispext_lock; /* PROTECT and PROTECT_LOCK as adopted */ + /* Retained radio/display state left in place across suspend-to-idle. */ + bool s2idle_kept; + /* Resources reachable by a retained domain outlive device removal. */ + bool retained; + bool retained_mtp; + /* Cold ANE owns SID0 only; all other streams remain inherited. */ + bool cold_ane; + u16 managed_sids; + bool managed_failed; + /* Serialize owned TCR publication against IRQ quarantine. */ + spinlock_t managed_lock; + DECLARE_BITMAP(remapped, DART_MAX_STREAMS); u8 remap_target[DART_MAX_STREAMS]; @@ -320,14 +389,175 @@ struct apple_dart { u64 *locked_owned[DART_MAX_STREAMS][DART_MAX_TTBR]; /* Leaf tables behind the root slots firmware held when we attached. */ struct apple_dart_fw_root *locked_fw[DART_MAX_STREAMS][DART_MAX_TTBR]; + /* Firmware's subtrees in a four-level locked root (TTBR 0 only). */ + struct apple_dart_fw_tree *locked_fw_tree[DART_MAX_STREAMS]; /* * First DVA of the window of the domain using each locked stream. The * window can be the client's own (apple,dma-range on the client node) * rather than the DART's. */ u64 locked_window[DART_MAX_STREAMS]; + /* + * First error of a TLB invalidate that failed while retiring a locked + * root, per stream. The hardware may still cache translations through + * the retired page tables, so apple_dart_domain_free() keeps the page + * tables of every domain on such a stream. It is never cleared: only + * the hardware could show that the TLB no longer holds those entries. + * The first error of each stream is logged. + */ + int locked_retire_err[DART_MAX_STREAMS]; +}; + +#ifndef MODULE +static struct kexec_blocker apple_dart_radio_kexec = { + .users = ATOMIC_INIT(0), + .reason = "Apple radio DMA has no qualified takeover/reset boundary", +}; +#endif + +static int apple_dart_retain_radio(struct apple_dart *dart) +{ +#ifndef MODULE + int ret = kexec_block(&apple_dart_radio_kexec); + + if (ret) + return ret; + get_device(dart->dev); + WRITE_ONCE(dart->retained, true); + return 0; +#else + return -EOPNOTSUPP; +#endif +} + +#if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) +static struct kexec_blocker apple_dart_display_kexec = { + .users = ATOMIC_INIT(0), + .reason = "retained Apple display DART tables and firmware DMA", +}; +#endif + +static void apple_dart_release(void *data) +{ + struct apple_dart *dart = data; + + if (!READ_ONCE(dart->retained)) + kfree(dart); +} + +/* + * devres still removes IRQs, sysfs and registrations on unexpected removal. + * State and mappings reachable from pinned radio or display domains must survive it. + * Before publication these actions perform ordinary probe rollback. + */ +struct apple_dart_retained_resource { + struct apple_dart *dart; + void (*release)(void *ptr, resource_size_t size); + void *ptr; + resource_size_t size; }; +static void apple_dart_release_retained_resource(void *data) +{ + struct apple_dart_retained_resource *r = data; + + if (!READ_ONCE(r->dart->retained)) + r->release(r->ptr, r->size); + kfree(r); +} + +static int apple_dart_manage_retained_resource(struct apple_dart *dart, + void *ptr, resource_size_t size, + void (*release)(void *, resource_size_t)) +{ + struct apple_dart_retained_resource *r = kmalloc(sizeof(*r), GFP_KERNEL); + + if (!r) { + release(ptr, size); + return -ENOMEM; + } + *r = (struct apple_dart_retained_resource) { + .dart = dart, .release = release, .ptr = ptr, .size = size, + }; + return devm_add_action_or_reset(dart->dev, + apple_dart_release_retained_resource, r); +} + +static void apple_dart_free_retained_memory(void *ptr, resource_size_t size) +{ + kfree(ptr); +} + +static void *apple_dart_retained_alloc(struct apple_dart *dart, size_t size) +{ + void *ptr = kzalloc(size, GFP_KERNEL); + + if (!ptr || apple_dart_manage_retained_resource(dart, ptr, size, + apple_dart_free_retained_memory)) + return NULL; + return ptr; +} + +static void apple_dart_unmap_retained_memory(void *ptr, resource_size_t size) +{ + memunmap(ptr); +} + +static void *apple_dart_retained_memremap(struct apple_dart *dart, + phys_addr_t start, size_t size) +{ + void *ptr = memremap(start, size, MEMREMAP_WB); + + if (!ptr || apple_dart_manage_retained_resource(dart, ptr, size, + apple_dart_unmap_retained_memory)) + return ERR_PTR(-ENOMEM); + return ptr; +} + +static void apple_dart_release_retained_region(void *ptr, resource_size_t size) +{ + struct resource *res = ptr; + + release_mem_region(res->start, resource_size(res)); +} + +static int apple_dart_retained_request_region(struct apple_dart *dart, + resource_size_t start, resource_size_t size, const char *name) +{ + struct resource *res = request_mem_region(start, size, name); + + if (!res) + return -EBUSY; + return apple_dart_manage_retained_resource(dart, res, size, + apple_dart_release_retained_region); +} + +static void apple_dart_unmap_retained_mmio(void *ptr, resource_size_t size) +{ + iounmap((__force void __iomem *)ptr); +} + +static void __iomem *apple_dart_retained_ioremap(struct apple_dart *dart, + struct resource *res) +{ + void __iomem *ptr; + int ret; + + ret = apple_dart_retained_request_region(dart, res->start, + resource_size(res), dev_name(dart->dev)); + if (ret) + return IOMEM_ERR_PTR(ret); + /* Match devm_ioremap_resource(): Apple SoC MMIO is non-posted. */ + if (res->flags & IORESOURCE_MEM_NONPOSTED) + ptr = ioremap_np(res->start, resource_size(res)); + else + ptr = ioremap(res->start, resource_size(res)); + if (!ptr || apple_dart_manage_retained_resource(dart, (__force void *)ptr, + resource_size(res), apple_dart_unmap_retained_mmio)) + return IOMEM_ERR_PTR(-ENOMEM); + return ptr; +} + /* * PCIe-C is a cable-powered aperture and can report a failed transaction as * an asynchronous external abort unless each access is completed before the @@ -417,6 +647,7 @@ struct apple_dart_atomic_stream_map { */ struct apple_dart_domain { struct io_pgtable_ops *pgtbl_ops; + struct apple_dart_dcp *dcp; bool finalized; bool has_dma_range; @@ -469,6 +700,20 @@ struct apple_dart_master_cfg { static struct platform_driver apple_dart_driver; static const struct iommu_ops apple_dart_iommu_ops; +static bool apple_dart_fault_pending(struct apple_dart *dart); + +/* READY bring-up only: userspace must arm its watchdog before probing SID23. */ +static bool apple_dart_dcp_enabled; +module_param_named(j713_dcp, apple_dart_dcp_enabled, bool, 0600); +MODULE_PARM_DESC(j713_dcp, "Enable explicit J713 DCP DART probe after arming watchdog"); + +static bool apple_dart_frontend_enabled; +module_param_named(j713_display, apple_dart_frontend_enabled, bool, 0600); +MODULE_PARM_DESC(j713_display, "Enable explicit J713 display DART admission after arming watchdog"); + +static int apple_dart_dcp_invalidate_tlb(struct apple_dart_stream_map *stream_map); + +#include "apple-dart-dcp.h" static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) { @@ -485,7 +730,8 @@ static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) */ static bool apple_dart_remaps_streams(struct apple_dart *dart) { - return dart->hw->enable_remap && !dart->tunneled && !dart->per_stream; + return dart->hw->enable_remap && !dart->tunneled && + !dart->per_stream && !dart->managed_sids; } static void @@ -612,15 +858,22 @@ static bool apple_dart_fw_handoff_enabled(const struct apple_dart *dart) handoff == 1; } +/* A 16K table page, aligned and inside the output address space. */ +static bool apple_dart_fw_table_page(const struct apple_dart *dart, + phys_addr_t phys, phys_addr_t *end) +{ + return dart->pgsize == SZ_16K && IS_ALIGNED(phys, dart->pgsize) && + !check_add_overflow(phys, (phys_addr_t)dart->pgsize - 1, end) && + dart->oas < 64 && *end < BIT_ULL(dart->oas); +} + static bool apple_dart_fw_table_ram(const struct apple_dart *dart, phys_addr_t phys) { unsigned long first, last, pfn; phys_addr_t end; - if (dart->pgsize != SZ_16K || !IS_ALIGNED(phys, dart->pgsize) || - check_add_overflow(phys, (phys_addr_t)dart->pgsize - 1, &end) || - dart->oas >= 64 || end >= BIT_ULL(dart->oas)) + if (!apple_dart_fw_table_page(dart, phys, &end)) return false; first = PHYS_PFN(phys); last = PHYS_PFN(end); @@ -630,14 +883,41 @@ static bool apple_dart_fw_table_ram(const struct apple_dart *dart, return true; } -static bool apple_dart_fw_table_decode(const struct apple_dart *dart, - u64 pte, phys_addr_t *phys) +/* A valid table entry with no bits but the next table's address. */ +static bool apple_dart_fw_table_pte(const struct apple_dart *dart, u64 pte, + phys_addr_t *phys) { if (!(pte & APPLE_DART_PTE_VALID) || (pte & ~(GENMASK_ULL(37, 10) | APPLE_DART_PTE_VALID))) return false; *phys = apple_dart_pte_to_paddr(dart, pte); - return apple_dart_fw_table_ram(dart, *phys); + return true; +} + +static bool apple_dart_fw_table_decode(const struct apple_dart *dart, + u64 pte, phys_addr_t *phys) +{ + return apple_dart_fw_table_pte(dart, pte, phys) && + apple_dart_fw_table_ram(dart, *phys); +} + +/* + * A table page of firmware's four-level subtrees, which is mapped write-back: + * reserved RAM as for apple_dart_fw_table_ram(), or memory no resource + * claims. T6050's boot loader places these tables outside the memory node, + * so they are not RAM to Linux. An entry naming Linux's own RAM or a + * device's registers is refused, and its DVAs resolve nothing. + */ +static bool apple_dart_fw_tree_table(const struct apple_dart *dart, + phys_addr_t phys) +{ + phys_addr_t end; + + if (apple_dart_fw_table_ram(dart, phys)) + return true; + return apple_dart_fw_table_page(dart, phys, &end) && + region_intersects(phys, dart->pgsize, IORESOURCE_MEM, + IORES_DESC_NONE) == REGION_DISJOINT; } static bool apple_dart_fw_slot_owned(const struct apple_dart_fw_root *fw, @@ -668,6 +948,24 @@ static bool apple_dart_fw_mapping_matches(phys_addr_t phys, int prot, return phys == expected && (prot & mask) == (required & mask); } +/* + * Whether the firmware leaf behind root @slot of a three-level stream already + * translates the page at @addr to @paddr with the protection Linux asks for. + */ +static bool apple_dart_fw_page_matches(const struct apple_dart *dart, + const struct apple_dart_fw_root *fw, + u64 slot, u64 addr, phys_addr_t paddr, + int prot) +{ + u64 leaf = (addr / dart->pgsize) % (dart->pgsize / sizeof(u64)); + phys_addr_t phys; + int fw_prot; + + return apple_dart_fw_leaf_decode(READ_ONCE(fw->leaf[slot][leaf]), addr, + &phys, &fw_prot) && + apple_dart_fw_mapping_matches(phys, fw_prot, paddr, prot); +} + /* * A root slot firmware already uses can still back Linux mappings when the * firmware's leaf table translates every page Linux mapped there to the same @@ -846,6 +1144,118 @@ apple_dart_snapshot_fw_root(struct apple_dart *dart, int sid, const u64 *live) return fw; } +static void apple_dart_free_fw_tree(struct apple_dart *dart, + struct apple_dart_fw_tree *tree) +{ + size_t i, j, entries = dart->pgsize / sizeof(u64); + + if (!tree) + return; + /* Any of the four arrays may be missing after a failed allocation. */ + for (i = 0; i < entries; i++) { + if (tree->leaf && tree->leaf[i]) { + for (j = 0; j < entries; j++) + if (tree->leaf[i][j]) + memunmap(tree->leaf[i][j]); + kfree(tree->leaf[i]); + } + if (tree->mid_entry) + kfree(tree->mid_entry[i]); + if (tree->mid && tree->mid[i]) + memunmap(tree->mid[i]); + } + kfree(tree->leaf); + kfree(tree->mid_entry); + kfree(tree->mid); + kfree(tree->entry); + kfree(tree); +} + +/* + * Record firmware's subtrees in the four-level root @live of @sid, so that + * firmware-placed buffers (an RTKit co-processor's shared segments, which + * it asks the AP to map at its own DVA) resolve through iova_to_phys. A + * table that cannot be mapped, or that apple_dart_fw_tree_table() refuses, + * is left out; its DVAs resolve nothing. A slot can map up to one leaf table + * per mid-level entry (2048 with 16K pages); firmware holds a few slots. + */ +static struct apple_dart_fw_tree * +apple_dart_snapshot_fw_tree(struct apple_dart *dart, int sid, const u64 *live) +{ + size_t i, j, entries = dart->pgsize / sizeof(u64); + struct apple_dart_fw_tree *tree; + + if (!dart->fw_tree || !dart->fw_mirror || dart->fw_handoff || + dart->hw->fmt != APPLE_DART2 || + !dart->hw->tcr_4level || + !(apple_dart_readl(dart, DART_TCR(dart, sid)) & dart->hw->tcr_4level)) + return NULL; + + tree = kzalloc(sizeof(*tree), GFP_KERNEL); + if (!tree) + return NULL; + tree->entry = kcalloc(entries, sizeof(*tree->entry), GFP_KERNEL); + tree->mid = kcalloc(entries, sizeof(*tree->mid), GFP_KERNEL); + tree->mid_entry = kcalloc(entries, sizeof(*tree->mid_entry), GFP_KERNEL); + tree->leaf = kcalloc(entries, sizeof(*tree->leaf), GFP_KERNEL); + if (!tree->entry || !tree->mid || !tree->mid_entry || !tree->leaf) { + apple_dart_free_fw_tree(dart, tree); + return NULL; + } + + for (i = 0; i < entries; i++) { + u64 pte = READ_ONCE(live[i]); + phys_addr_t phys; + + if (!apple_dart_fw_table_pte(dart, pte, &phys) || + !apple_dart_fw_tree_table(dart, phys)) + continue; + tree->mid_entry[i] = kcalloc(entries, sizeof(u64), GFP_KERNEL); + tree->leaf[i] = kcalloc(entries, sizeof(u64 *), GFP_KERNEL); + tree->mid[i] = memremap(phys, dart->pgsize, MEMREMAP_WB); + if (!tree->mid_entry[i] || !tree->leaf[i] || !tree->mid[i]) + continue; + tree->entry[i] = pte; + for (j = 0; j < entries; j++) { + u64 mid = READ_ONCE(tree->mid[i][j]); + + if (!apple_dart_fw_table_pte(dart, mid, &phys) || + !apple_dart_fw_tree_table(dart, phys)) + continue; + tree->leaf[i][j] = memremap(phys, dart->pgsize, MEMREMAP_WB); + if (tree->leaf[i][j]) + tree->mid_entry[i][j] = mid; + } + } + return tree; +} + +/* What firmware's recorded four-level subtree translates @addr to, or 0. */ +static phys_addr_t apple_dart_fw_tree_phys(const struct apple_dart *dart, + const struct apple_dart_fw_tree *tree, + const u64 *live, u64 addr, int *prot_out) +{ + u32 pgshift = ilog2(dart->pgsize); + u32 bits = ilog2(dart->pgsize / sizeof(u64)); + u64 mask = BIT_ULL(bits) - 1; + u64 slot = addr >> (pgshift + 2 * bits); + u64 mid = (addr >> (pgshift + bits)) & mask; + u64 page = (addr >> pgshift) & mask; + phys_addr_t phys; + int prot; + + if (!tree || !live || slot > mask || !tree->entry[slot] || + READ_ONCE(live[slot]) != tree->entry[slot] || + !tree->mid_entry[slot][mid] || + READ_ONCE(tree->mid[slot][mid]) != tree->mid_entry[slot][mid] || + !apple_dart_fw_leaf_decode(READ_ONCE(tree->leaf[slot][mid][page]), + addr, &phys, &prot)) + return 0; + if (prot_out) + *prot_out = prot; + return phys; +} + static int apple_dart_fw_invalidate(struct apple_dart_stream_map *stream_map, u64 lo, u64 hi); @@ -856,6 +1266,7 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) int sid; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + struct apple_dart_fw_tree *tree = NULL; struct apple_dart_fw_root *fw; unsigned long flags; u32 ttbr; @@ -902,48 +1313,94 @@ apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) devm_memunmap(dart->dev, l1_tbl); return PTR_ERR(fw); } + if (!idx) + tree = apple_dart_snapshot_fw_tree(dart, sid, l1_tbl); spin_lock_irqsave(&dart->lock, flags); dart->locked_owned[sid][idx] = owned; dart->locked_ttbr[sid][idx] = l1_tbl; dart->locked_fw[sid][idx] = fw; + if (!idx) + dart->locked_fw_tree[sid] = tree; spin_unlock_irqrestore(&dart->lock, flags); } return 0; } +/* + * Retire the locked roots @idx of @stream_map's streams. Returns the first + * error of the TLB invalidate that retiring needs, which is also remembered + * per stream for apple_dart_domain_free(): a consumer release or an attach + * rollback can retire the root before the domain is freed. + */ static int apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx) { struct apple_dart *dart = stream_map->dart; - int sid; + int sid, ret = 0; for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { u64 *live = dart->locked_ttbr[sid][idx]; u64 *owned = dart->locked_owned[sid][idx]; + struct apple_dart_fw_tree *tree = NULL; struct apple_dart_fw_root *fw; unsigned long flags; + bool first = false; + int err = 0; if (!live) continue; apple_dart_retire_root(live, owned, dart->pgsize / sizeof(*live)); - if (dart->fw_mirror && dart->version >= 0x0202 && !idx) { + if ((dart->fw_mirror || dart->fw_handoff) && + dart->version >= 0x0202 && !idx) { struct apple_dart_stream_map stream = { .dart = dart }; __set_bit(sid, stream.sidmap); - (void)apple_dart_fw_invalidate(&stream, 0, U64_MAX); + err = apple_dart_fw_invalidate(&stream, 0, U64_MAX); + ret = ret ?: err; } spin_lock_irqsave(&dart->lock, flags); + if (err && !dart->locked_retire_err[sid]) { + dart->locked_retire_err[sid] = err; + first = true; + } fw = dart->locked_fw[sid][idx]; dart->locked_fw[sid][idx] = NULL; dart->locked_ttbr[sid][idx] = NULL; + if (!idx) { + tree = dart->locked_fw_tree[sid]; + dart->locked_fw_tree[sid] = NULL; + } spin_unlock_irqrestore(&dart->lock, flags); + /* Once per stream: the error is never cleared. */ + if (first) + dev_warn(dart->dev, + "stream %d: TLB invalidate failed retiring its root (%d); keeping the page tables of its domains until reboot\n", + sid, err); apple_dart_free_fw_root(dart, fw); + apple_dart_free_fw_tree(dart, tree); devm_memunmap(dart->dev, live); devm_kfree(dart->dev, owned); dart->locked_owned[sid][idx] = NULL; } - return 0; + return ret; +} + +/* The remembered retire-time invalidate error of @stream_map's streams, or 0. */ +static int apple_dart_retire_error(struct apple_dart_stream_map *stream_map) +{ + struct apple_dart *dart = stream_map->dart; + unsigned long flags; + int sid, ret = 0; + + spin_lock_irqsave(&dart->lock, flags); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + ret = dart->locked_retire_err[sid]; + if (ret) + break; + } + spin_unlock_irqrestore(&dart->lock, flags); + return ret; } static int @@ -1237,10 +1694,86 @@ apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE); } +/* + * Borrowed J713 display streams use the exact command the cleanroom DCP + * profile was qualified with: FLUSH_SID without the v2 or range flags, and + * only for SID23, strict SID0 or the private SID16 publisher. A refused or + * timed-out command fails the whole retained session. + */ +static int apple_dart_dcp_invalidate_tlb(struct apple_dart_stream_map *stream_map) +{ + struct apple_dart *dart = stream_map->dart; + const u32 command = DART_T8110_TLB_CMD_OP_FLUSH_SID; + unsigned long flags; + u32 status; + int sid, ret = 0; + + spin_lock_irqsave(&dart->lock, flags); + for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { + struct apple_dart_dcp *borrowed = apple_dart_borrowed_sid(dart, sid); + u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | + FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); + + if (!borrowed || (borrowed->sid == 16 && + !READ_ONCE(borrowed->shared_flushing)) || + !apple_dart_dcp_registers(borrowed)) { + ret = -EIO; + break; + } + ret = readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_STREAM_COMMAND_BUSY_TIMEOUT); + /* An external display DART also reports busy at 0x604 bit 0. */ + if (!ret && dart->dispext) + ret = readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_BUSY2, status, + !(status & BIT(0)), 1, + DART_STREAM_COMMAND_BUSY_TIMEOUT); + if (ret) + break; + writel(val, dart->regs + DART_T8110_TLB_CMD); + ret = readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_CMD, status, + !(status & DART_T8110_TLB_CMD_BUSY), 1, + DART_STREAM_COMMAND_BUSY_TIMEOUT); + if (!ret && dart->dispext) + ret = readl_poll_timeout_atomic(dart->regs + DART_T8110_TLB_BUSY2, status, + !(status & BIT(0)), 1, + DART_STREAM_COMMAND_BUSY_TIMEOUT); + if (ret) + break; + } + spin_unlock_irqrestore(&dart->lock, flags); + + if (ret) { + apple_dart_fail_borrowed(dart, "busy bit did not clear after TLB command %x for stream %d", + command, sid); + } + return ret; +} + +/* + * The IOMMU core's TLB command on a borrowed DART: on an external display + * DART only with its power confirmed (a use of its own); the borrowed + * streams' own operations run theirs inside their use, raw. + */ +static int apple_dart_dcp_invalidate_tlb_gated(struct apple_dart_stream_map *stream_map) +{ + int ret; + + if (!apple_dart_dispext_get(stream_map->dart)) + return -EAGAIN; + ret = apple_dart_dcp_invalidate_tlb(stream_map); + apple_dart_dispext_put(stream_map->dart); + return ret; +} + static int apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) { - if (stream_map->dart->fw_mirror && stream_map->dart->version >= 0x0202) + if (apple_dart_has_borrowed(stream_map->dart)) + return apple_dart_dcp_invalidate_tlb_gated(stream_map); + /* Keep out of firmware-held slots; see apple_dart_fw_invalidate(). */ + if ((stream_map->dart->fw_mirror || stream_map->dart->fw_handoff) && + stream_map->dart->version >= 0x0202) return apple_dart_fw_invalidate(stream_map, 0, U64_MAX); return apple_dart_t8110_hw_tlb_command_range( stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID, false, 0, 0); @@ -1248,6 +1781,16 @@ apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) static const char *apple_dart_t8110_v2_fault_name(u32 exception); +/* + * Exceptions can latch on any stream the hardware has. A tunnel DART runs with + * a fixed 64-stream topology, but a PCIe-C port also uses its reserved streams + * 253..255 (for example for MSIs), so scan all of them there. + */ +static unsigned int apple_dart_exception_sids(struct apple_dart *dart) +{ + return dart->tunneled ? DART_MAX_STREAMS : dart->num_streams; +} + static void apple_dart_drain_stale_exceptions(struct apple_dart *dart) { unsigned int i, drained = 0; @@ -1255,7 +1798,7 @@ static void apple_dart_drain_stale_exceptions(struct apple_dart *dart) if (dart->hw->type != DART_T8110 || dart->version < 0x0202) return; - for (i = 0; i < DIV_ROUND_UP(dart->num_streams, 32) && i < 8; i++) { + for (i = 0; i < DIV_ROUND_UP(apple_dart_exception_sids(dart), 32) && i < 8; i++) { u32 pending = readl(dart->regs + DART_T8110_EXCEPTION(i)); unsigned long bits = pending; unsigned int bit; @@ -1268,7 +1811,7 @@ static void apple_dart_drain_stale_exceptions(struct apple_dart *dart) u32 status; u64 addr; - if (sid >= dart->num_streams) + if (sid >= apple_dart_exception_sids(dart)) break; status = readl(dart->regs + @@ -1310,6 +1853,8 @@ static int apple_dart_t8110_v2_unlock(struct apple_dart *dart) KUNIT_STATIC_STUB_REDIRECT(apple_dart_t8110_v2_unlock, dart); + if (dart->managed_sids) + return -EPERM; if (!(readl(dart->regs + DART_T8110_DIAG_LOCK) & DART_T8110_DIAG_LOCK_ON_ERR)) return 0; @@ -1410,6 +1955,24 @@ static int apple_dart_hw_finish_reset(struct apple_dart *dart) if (dart->hw->type == DART_T8110) apple_dart_writel(dart, 0, DART_T8110_ERROR_MASK); + /* + * A PCIe-C port issues its own writes, such as MSIs turned into + * interrupt controller writes, on reserved streams outside the + * translated ones. Those must bypass. + */ + if (dart->tunneled && dart->hw->type == DART_T8110) { + u32 sid; + + of_property_for_each_u32(dart->dev->of_node, "apple,bypass-sids", sid) { + if (sid >= DART_MAX_STREAMS) + continue; + apple_dart_writel(dart, dart->hw->tcr_bypass, + dart->hw->tcr + 4 * sid); + apple_dart_writel(dart, BIT(sid % 32), + dart->hw->enable_streams + 4 * (sid / 32)); + } + } + bitmap_set(stream_map.sidmap, 0, dart->num_streams); ret = dart->hw->invalidate_tlb(&stream_map); if (ret) @@ -1418,38 +1981,292 @@ static int apple_dart_hw_finish_reset(struct apple_dart *dart) return ret; } -static int apple_dart_hw_reset(struct apple_dart *dart) +static bool apple_dart_managed_static_bypass(u32 tcr) { - struct apple_dart_stream_map stream_map; - int i; + /* T8132 does not retain the DAPF bypass request in the readable TCR. */ + return (tcr & ~DART_T8110_TCR_BYPASS_DAPF) == DART_T8110_TCR_BYPASS_DART; +} - stream_map.dart = dart; - bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); - bitmap_set(stream_map.sidmap, 0, dart->num_streams); +static ssize_t j713_radio_fault_state_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct apple_dart *dart = dev_get_drvdata(dev); + unsigned long flags; + ssize_t len = 0; + unsigned int sid, word; - apple_dart_hw_disable_dma(&stream_map); - apple_dart_hw_clear_all_ttbrs(&stream_map); - apple_dart_hw_apply_remaps(dart); + if (!device_is_bound(dev) || !dart || !READ_ONCE(dart->iommu.ready) || + !dart->managed_sids) + return -ENODEV; - /* enable all streams globally since TCR is used to control isolation */ - for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) - apple_dart_writel(dart, U32_MAX, - dart->hw->enable_streams + 4 * i); + /* + * A live, passive snapshot can race hardware updates and IRQ acknowledgements. + * Never acknowledge a latch, unmask errors, or change an owned/foreign stream. + */ + spin_lock_irqsave(&dart->managed_lock, flags); + len += sysfs_emit_at(buf, len, "version=%#x managed=%#x failed=%u\n", + dart->version, dart->managed_sids, dart->managed_failed); + len += sysfs_emit_at(buf, len, "error_mask=%#x error=%#x\n", + apple_dart_readl(dart, DART_T8110_ERROR_MASK), + apple_dart_readl(dart, DART_T8110_ERROR)); + for (word = 0; word < BITS_TO_U32(dart->num_streams); word++) { + u32 streams = apple_dart_readl(dart, DART_T8110_ERROR_STREAMS + 4 * word); + u32 exceptions = apple_dart_readl(dart, DART_T8110_EXCEPTION(word)); + + len += sysfs_emit_at(buf, len, "error_streams[%u]=%#x exceptions[%u]=%#x\n", + word, streams, word, exceptions); + } + for (sid = 1; sid < 16; sid++) { + u32 status; + u64 addr; - return apple_dart_hw_finish_reset(dart); + if (!(dart->managed_sids & BIT(sid))) + continue; + status = apple_dart_readl(dart, DART_T8110_ERR_SID_STATUS(sid)); + addr = apple_dart_readl(dart, DART_T8110_EXCEPTION_ADDR_LO(sid)) | + (u64)apple_dart_readl(dart, DART_T8110_EXCEPTION_ADDR_HI(sid)) << 32; + len += sysfs_emit_at(buf, len, "sid%u status=%#x addr=%#llx\n", sid, status, addr); + } + spin_unlock_irqrestore(&dart->managed_lock, flags); + return len; } +static DEVICE_ATTR_ADMIN_RO(j713_radio_fault_state); -static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, - bool range, u64 first, u64 last) -{ - struct apple_dart_atomic_stream_map *map; - struct apple_dart_stream_map stream; - struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - const u32 op = DART_T8110_TLB_CMD_OP_FLUSH_SID; - int i, j, ret, err = 0; - u64 start, end; +static struct attribute *apple_dart_managed_attrs[] = { + &dev_attr_j713_radio_fault_state.attr, + NULL, +}; - /* +static const struct attribute_group apple_dart_managed_group = { + .attrs = apple_dart_managed_attrs, +}; + +/* Ordinary-stream admission does not authorize a controller-wide reset. */ +static int apple_dart_managed_admit(struct apple_dart *dart, bool *init_bypass) +{ + unsigned int sid; + int idx; + bool initialize = false; + u32 reserved_tcr, protect, protect_lock, enabled, command, error, error_mask; + + if (!dart->managed_sids) { + *init_bypass = false; + return 0; + } + if (dart->hw->type != DART_T8110 || dart->locked || dart->num_streams < 19 || + dart->version < 0x0202) + return -EINVAL; + protect = apple_dart_readl(dart, DART_T8110_PROTECT); + protect_lock = apple_dart_readl(dart, DART_T8110_PROTECT_LOCK); + enabled = apple_dart_readl(dart, DART_T8110_ENABLE_STREAMS); + command = apple_dart_readl(dart, DART_T8110_TLB_CMD); + error = apple_dart_readl(dart, DART_T8110_ERROR); + error_mask = apple_dart_readl(dart, DART_T8110_ERROR_MASK); + dev_dbg(dart->dev, + "managed radio: PROTECT %#x LOCK %#x ENABLE %#x TLB %#x ERROR %#x MASK %#x\n", + protect, protect_lock, enabled, command, error, error_mask); + /* Preserve global policy; owned-stream quarantine requires unmasked errors. */ + if (error_mask) { + dev_err(dart->dev, "managed radio requires inherited unmasked errors (mask %#x)\n", + error_mask); + return -EBUSY; + } + /* LOCK freezes protection policy; ENABLE is not an allocation receipt. */ + if ((protect & DART_T8110_PROTECT_TTBR_TCR) || + (command & DART_T8110_TLB_CMD_BUSY) || + (error & DART_T8110_ERROR_FLAG) || apple_dart_fault_pending(dart)) + return -EBUSY; + for (idx = 0; idx < BITS_TO_U32(dart->num_streams); idx++) { + u32 errors = apple_dart_readl(dart, DART_T8110_ERROR_STREAMS + 4 * idx); + + if (errors) { + dev_err(dart->dev, "inherited error-stream word%d %#x\n", idx, errors); + return -EBUSY; + } + } + /* A requester can reach an ordinary target through another stream. */ + for (sid = 0; sid < (unsigned int)dart->num_streams; sid++) { + u32 tcr = apple_dart_readl(dart, DART_TCR(dart, sid)); + u32 target = FIELD_GET(DART_T8110_TCR_REMAP, tcr); + + if ((tcr & DART_T8110_TCR_REMAP_ENABLE) && + (dart->managed_sids & BIT(target))) { + dev_err(dart->dev, "inherited SID%u remaps into ordinary SID%u\n", + sid, target); + return -EBUSY; + } + } + /* ADT assigns SID16 a static bypass role, independently of radio data. */ + reserved_tcr = apple_dart_readl(dart, DART_TCR(dart, 16)); + if (!apple_dart_managed_static_bypass(reserved_tcr)) { + if (reserved_tcr != dart->hw->tcr_disabled || !dart->supports_bypass) + return -EBUSY; + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + if (apple_dart_readl(dart, DART_TTBR(dart, 16, idx)) & + dart->hw->ttbr_valid) + return -EBUSY; + initialize = true; + } + for (sid = 1; sid < 16; sid++) { + u32 tcr; + + if (!(dart->managed_sids & BIT(sid))) + continue; + tcr = apple_dart_readl(dart, DART_TCR(dart, sid)); + if (tcr != dart->hw->tcr_disabled && tcr != DART_T8110_TCR_TRANSLATE_ENABLE) { + dev_err(dart->dev, "ordinary SID%u has an inherited active mode\n", sid); + return -EBUSY; + } + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + if (apple_dart_readl(dart, DART_TTBR(dart, sid, idx)) & + dart->hw->ttbr_valid) { + dev_err(dart->dev, "ordinary SID%u still has a valid root\n", sid); + return -EBUSY; + } + } + *init_bypass = initialize; + return 0; +} + +/* The host may start its cold link only before any owned root is published. */ +int apple_dart_radio_check_idle(struct device *dev) +{ + struct apple_dart *dart; + unsigned long flags; + bool init_bypass = false; + int ret; + + device_lock(dev); + if (dev->driver != &apple_dart_driver.driver || !device_is_bound(dev)) { + ret = -EPROBE_DEFER; + goto out_unlock; + } + dart = dev_get_drvdata(dev); + if (!dart || !READ_ONCE(dart->iommu.ready)) { + ret = -EPROBE_DEFER; + goto out_unlock; + } + if (dart->managed_sids != (BIT(1) | BIT(2))) { + ret = -EINVAL; + goto out_unlock; + } + spin_lock_irqsave(&dart->managed_lock, flags); + ret = dart->managed_failed || apple_dart_fault_pending(dart) ? -EIO : + apple_dart_managed_admit(dart, &init_bypass); + if (!ret && init_bypass) + ret = -EBUSY; + spin_unlock_irqrestore(&dart->managed_lock, flags); +out_unlock: + device_unlock(dev); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_radio_check_idle); + +static void apple_dart_managed_quarantine_locked(struct apple_dart *dart) +{ + struct apple_dart_stream_map stream = { .dart = dart }; + unsigned int sid; + + lockdep_assert_held(&dart->managed_lock); + WRITE_ONCE(dart->managed_failed, true); + for (sid = 1; sid < 16; sid++) + if (dart->managed_sids & BIT(sid)) + __set_bit(sid, stream.sidmap); + apple_dart_hw_disable_dma(&stream); +} + +static void apple_dart_managed_quarantine(struct apple_dart *dart) +{ + unsigned long flags; + + spin_lock_irqsave(&dart->managed_lock, flags); + apple_dart_managed_quarantine_locked(dart); + spin_unlock_irqrestore(&dart->managed_lock, flags); +} + +static int apple_dart_hw_reset(struct apple_dart *dart) +{ + struct apple_dart_stream_map stream_map; + int i; + + if (dart->managed_sids || dart->cold_ane) + return -EBUSY; + stream_map.dart = dart; + bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); + bitmap_set(stream_map.sidmap, 0, dart->num_streams); + + apple_dart_hw_disable_dma(&stream_map); + apple_dart_hw_clear_all_ttbrs(&stream_map); + apple_dart_hw_apply_remaps(dart); + + /* enable all streams globally since TCR is used to control isolation */ + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + apple_dart_writel(dart, U32_MAX, + dart->hw->enable_streams + 4 * i); + + return apple_dart_hw_finish_reset(dart); +} + +static int __apple_dart_dcp_domain_flush(struct apple_dart_domain *domain); + +/* + * Borrowed DCP domains have no io-pgtable; their flush ignores the range. + * On an external display DART only with its power confirmed: -EAGAIN + * (nothing retired) from sleep until its next confirmed power-up. + */ +static int apple_dart_dcp_domain_flush(struct apple_dart_domain *domain) +{ + struct apple_dart *dart = domain->dcp->dart; + int ret; + + if (!apple_dart_dispext_get(dart)) + return -EAGAIN; + ret = __apple_dart_dcp_domain_flush(domain); + apple_dart_dispext_put(dart); + return ret; +} + +static int __apple_dart_dcp_domain_flush(struct apple_dart_domain *domain) +{ + struct apple_dart_dcp *dcp = domain->dcp; + struct apple_dart *dart = dcp->dart; + struct apple_dart_stream_map stream_map; + int ret; + + /* Not as adopted: apple_dart_dcp_registers() retired it, saying why. */ + if (!apple_dart_dcp_registers(dcp)) + return -EIO; + /* SID0 generic DMA is strict: map sync and unmap use the real 0x100 flush. */ + if (dcp->sid == 0) + return apple_dart_dcp_flush(dcp); + /* Only the private shared-buffer API can publish SID16 leaves. */ + if (dcp->sid != 23) + return 0; + stream_map.dart = dart; + bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); + set_bit(dcp->sid, stream_map.sidmap); + dma_wmb(); + ret = apple_dart_dcp_invalidate_tlb(&stream_map); + if (ret) + apple_dart_dcp_fail(dcp, "domain flush failed: %d", ret); + return ret; +} + +static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, + bool range, u64 first, u64 last) +{ + struct apple_dart_atomic_stream_map *map; + struct apple_dart_stream_map stream; + struct io_pgtable_cfg *cfg; + const u32 op = DART_T8110_TLB_CMD_OP_FLUSH_SID; + int i, j, ret, err = 0; + u64 start, end; + + if (domain->dcp) + return apple_dart_dcp_domain_flush(domain); + cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; + + /* * After an unmap, the caller frees the pages once this returns. Every * DART that can be reached must therefore be invalidated, even if * publishing the page table or powering another DART failed. @@ -1485,6 +2302,8 @@ static int apple_dart_domain_flush_tlb_range(struct apple_dart_domain *domain, start, end); else ret = stream.dart->hw->invalidate_tlb(&stream); + if (ret && stream.dart->managed_sids) + apple_dart_managed_quarantine(stream.dart); pm_runtime_put(stream.dart->dev); err = err ?: ret; } @@ -1535,6 +2354,15 @@ static bool apple_dart_domain_uses_fw(struct apple_dart_domain *domain, int i, /* * Hardware continues to use every firmware-owned slot. Its entire IOVA * span is excluded from Linux allocation, including currently unmapped holes. + * + * Linux may still repeat a page the firmware leaf already translates, with + * the same physical address and protection. That happens when one domain + * spans several DARTs and only some of them inherited a reserved region: + * the J700 display subsystem uses disp0 and dispext0, and only disp0 scans + * out the boot framebuffer. apple_dart_inherited_phys() cannot report the + * region as mapped for the whole domain, so the core maps it, and the copy + * must be accepted on disp0 while dispext0 gets it in a vacant slot. + * apple_dart_publish_root() never writes the firmware slot itself. */ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, phys_addr_t paddr, size_t size, int prot) @@ -1556,6 +2384,7 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, u64 addr = dva + off; for (sid = 0; sid < dart->num_streams; sid++) { + const struct apple_dart_fw_root *fw; bool four; u64 idx, slot; const u64 *live, *owned; @@ -1563,12 +2392,6 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, if (!apple_dart_domain_uses_fw(domain, i, map, sid)) continue; if (!dart->fw_handoff) { - const struct apple_dart_fw_root *fw; - phys_addr_t phys; - u64 pte; - int fw_prot; - u64 leaf = (addr >> pgshift) & (entries - 1); - slot = (addr >> (pgshift + bits)) & (entries - 1); idx = addr >> (pgshift + 2 * bits); if (idx >= DART_MAX_TTBR) @@ -1577,10 +2400,8 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, live = dart->locked_ttbr[sid][idx]; if (!live || !apple_dart_fw_slot_owned(fw, live, slot)) continue; - pte = READ_ONCE(fw->leaf[slot][leaf]); - if (!apple_dart_fw_leaf_decode(pte, addr, &phys, &fw_prot) || - !apple_dart_fw_mapping_matches(phys, fw_prot, - paddr + off, prot)) { + if (!apple_dart_fw_page_matches(dart, fw, slot, addr, + paddr + off, prot)) { ret = -EBUSY; goto unlock; } @@ -1595,8 +2416,17 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, continue; live = dart->locked_ttbr[sid][idx]; owned = dart->locked_owned[sid][idx]; - /* All pages of an inherited slot remain firmware's. */ - if (live && apple_dart_fw_slot_foreign(live, owned, slot)) { + if (!live || !apple_dart_fw_slot_foreign(live, owned, slot)) + continue; + /* + * All pages of an inherited slot remain firmware's; + * only a page its firmware leaf already maps the + * same way may be repeated. + */ + fw = dart->locked_fw[sid][idx]; + if (four || !apple_dart_fw_slot_owned(fw, live, slot) || + !apple_dart_fw_page_matches(dart, fw, slot, addr, + paddr + off, prot)) { ret = -EBUSY; goto unlock; } @@ -1610,11 +2440,18 @@ static int apple_dart_check_fw_map(struct apple_dart_domain *domain, u64 dva, return 0; } +/* + * What firmware's own retained translations in @domain map @address to, or 0. + * With @prot_out, also the protection every stream grants (IOMMU_READ and + * IOMMU_WRITE as the leaves say). + */ static phys_addr_t -apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) +apple_dart_inherited_phys_prot(struct apple_dart_domain *domain, dma_addr_t address, + int *prot_out) { struct apple_dart_atomic_stream_map *map; phys_addr_t result = 0; + int granted = IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE; int i, sid; for_each_stream_map(i, domain, map) { @@ -1624,31 +2461,90 @@ apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) size_t page = (address / dart->pgsize) % entries; size_t slot = address / dart->pgsize / entries; - if (!dart->locked || (!dart->fw_handoff && !dart->fw_mirror) || slot >= entries) + if (!dart->locked || (!dart->fw_handoff && !dart->fw_mirror)) return 0; spin_lock_irqsave(&dart->lock, flags); for (sid = 0; sid < dart->num_streams; sid++) { + const struct apple_dart_fw_tree *tree; struct apple_dart_fw_root *fw; - phys_addr_t phys; + phys_addr_t phys = 0; int prot; if (!apple_dart_domain_uses_fw(domain, i, map, sid)) continue; + tree = dart->locked_fw_tree[sid]; fw = dart->locked_fw[sid][0]; - if (!apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || - !apple_dart_fw_leaf_decode(READ_ONCE(fw->leaf[slot][page]), - address, &phys, &prot) || - (result && result != phys)) { + prot = 0; + if (tree) + phys = apple_dart_fw_tree_phys(dart, tree, + dart->locked_ttbr[sid][0], + address, &prot); + else if (slot >= entries || + !apple_dart_fw_slot_owned(fw, dart->locked_ttbr[sid][0], slot) || + !apple_dart_fw_leaf_decode(READ_ONCE(fw->leaf[slot][page]), + address, &phys, &prot)) + phys = 0; + if (!phys || (result && result != phys)) { spin_unlock_irqrestore(&dart->lock, flags); return 0; } result = phys; + granted &= prot; } spin_unlock_irqrestore(&dart->lock, flags); } + if (prot_out) + *prot_out = result ? granted : 0; return result; } +static phys_addr_t +apple_dart_inherited_phys(struct apple_dart_domain *domain, dma_addr_t address) +{ + return apple_dart_inherited_phys_prot(domain, address, NULL); +} + +/* + * Resolves a DVA a co-processor's firmware names in a request of its own, for + * a host copy to or from that buffer. Only firmware's retained mappings count: + * a DVA Linux mapped itself (a framebuffer, an imported or DMA_TO_DEVICE + * buffer) is refused, so the host never writes what the device may only read. + * @write requires the leaf to grant writes. -EACCES when refused, -ENXIO when + * unmapped. + */ +int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, + phys_addr_t *phys) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct apple_dart_domain *dart_domain; + struct io_pgtable_ops *ops; + struct io_pgtable_cfg *cfg; + dma_addr_t address; + phys_addr_t result; + int prot; + + if (!domain || domain->owner != &apple_dart_iommu_ops) + return -ENODEV; + dart_domain = to_dart_domain(domain); + ops = dart_domain->pgtbl_ops; + if (dart_domain->dcp || !ops) + return -ENODEV; + address = (iova + dart_domain->dma_offset) & dart_domain->mask; + if (ops->iova_to_phys(ops, address)) + return -EACCES; + cfg = &io_pgtable_ops_to_pgtable(ops)->cfg; + if (cfg->apple_dart_cfg.n_levels != 3 && cfg->apple_dart_cfg.n_levels != 4) + return -ENXIO; + result = apple_dart_inherited_phys_prot(dart_domain, address, &prot); + if (!result) + return -ENXIO; + if (!(prot & IOMMU_READ) || (write && !(prot & IOMMU_WRITE))) + return -EACCES; + *phys = result; + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_fw_remote); + static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { @@ -1658,6 +2554,8 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, phys_addr_t phys; dma_addr_t address; + if (dart_domain->dcp) + return apple_dart_dcp_translate(dart_domain->dcp, iova); if (!ops) return 0; @@ -1666,7 +2564,7 @@ static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, if (phys) return phys; cfg = &io_pgtable_ops_to_pgtable(ops)->cfg; - if (cfg->apple_dart_cfg.n_levels != 3) + if (cfg->apple_dart_cfg.n_levels != 3 && cfg->apple_dart_cfg.n_levels != 4) return 0; /* Existing firmware translations need no replacement software mapping. */ return apple_dart_inherited_phys(dart_domain, address); @@ -1683,8 +2581,18 @@ static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, u64 dva; int ret; + if (dart_domain->dcp) { + /* No publication while iommu_device_register() can still roll back. */ + if (!READ_ONCE(dart_domain->dcp->dart->iommu.ready)) + return -EAGAIN; + return apple_dart_dcp_map(dart_domain->dcp, iova, paddr, pgsize, + pgcount, prot, gfp, mapped); + } if (!ops) return -ENODEV; + if (dart_domain->stream_maps[0].dart && + READ_ONCE(dart_domain->stream_maps[0].dart->managed_failed)) + return -EIO; dva = (iova + dart_domain->dma_offset) & dart_domain->mask; ret = apple_dart_check_fw_map(dart_domain, dva, paddr, pgsize * pgcount, @@ -1704,13 +2612,15 @@ static size_t apple_dart_unmap_pages(struct iommu_domain *domain, struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; + if (dart_domain->dcp) + return apple_dart_dcp_unmap(dart_domain->dcp, iova, pgsize, pgcount, gather); return ops->unmap_pages(ops, (iova + dart_domain->dma_offset) & dart_domain->mask, pgsize, pgcount, gather); } -static void +static int apple_dart_setup_translation(struct apple_dart_domain *domain, struct apple_dart_stream_map *stream_map) { @@ -1719,10 +2629,22 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, struct io_pgtable_cfg *pgtbl_cfg = &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; - /* enable all streams globally since TCR is used to control isolation */ - for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) - apple_dart_writel(dart, U32_MAX, - dart->hw->enable_streams + 4 * i); + if (dart->cold_ane) { + if (bitmap_weight(stream_map->sidmap, dart->num_streams) != 1 || + !test_bit(0, stream_map->sidmap)) + return -EPERM; + apple_dart_writel(dart, apple_dart_readl(dart, dart->hw->enable_streams) | BIT(0), + dart->hw->enable_streams); + } else if (dart->managed_sids) { + /* Admission owns only ordinary SIDs 1/2, both in the first word. */ + apple_dart_writel(dart, stream_map->sidmap[0] & dart->managed_sids, + dart->hw->enable_streams); + } else { + /* Other profiles use TCR to control isolation. */ + for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) + apple_dart_writel(dart, U32_MAX, + dart->hw->enable_streams + 4 * i); + } for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) { u64 ttbr = virt_to_phys(pgtbl_cfg->apple_dart_cfg.ttbr[i]); @@ -1733,7 +2655,7 @@ apple_dart_setup_translation(struct apple_dart_domain *domain, apple_dart_hw_enable_translation(stream_map, pgtbl_cfg->apple_dart_cfg.n_levels); - stream_map->dart->hw->invalidate_tlb(stream_map); + return dart->hw->invalidate_tlb(stream_map); } /* Record the DVA window of @domain for invalidates of @stream_map's streams. */ @@ -1810,8 +2732,33 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, mutex_lock(&dart_domain->init_lock); - if (dart_domain->finalized) + if (dart_domain->finalized) { + if (dart_domain->dcp && (apple_dart_borrowed_cfg(cfg) != dart_domain->dcp || + READ_ONCE(dart_domain->dcp->failed))) + ret = -EIO; + goto done; + } + + /* A borrowed J713 display stream uses its live root, not an io-pgtable. */ + if (apple_dart_has_borrowed(dart)) { + struct apple_dart_dcp *dcp = apple_dart_borrowed_cfg(cfg); + + if (!dcp || cfg->stream_maps[1].dart || !dcp->owner || dcp->domain || + READ_ONCE(dcp->failed) || + bitmap_weight(cfg->stream_maps[0].sidmap, dart->num_streams) != 1 || + !test_bit(dcp->sid, cfg->stream_maps[0].sidmap)) { + ret = -EBUSY; + goto done; + } + dart_domain->dcp = dcp; + dcp->domain = dart_domain; + dart_domain->domain.pgsize_bitmap = SZ_16K; + dart_domain->domain.geometry.aperture_start = DCP_DART_ALIAS; + dart_domain->domain.geometry.aperture_end = DCP_DART_ALIAS + DCP_DART_APERTURE - 1; + dart_domain->domain.geometry.force_aperture = true; + dart_domain->finalized = true; goto done; + } /* Reserved mappings may be installed before any device is attached. */ for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { @@ -1955,6 +2902,7 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, struct apple_dart_stream_map *stream_map; struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct apple_dart_domain *dart_domain = to_dart_domain(domain); + bool streams_added = false; for_each_stream_map(i, cfg, stream_map) WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); @@ -1963,14 +2911,35 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, if (ret) goto err; + /* Live root is already validated and borrowed; attach must not publish it. */ + if (dart_domain->dcp) + goto err; + ret = apple_dart_domain_add_streams(dart_domain, cfg); if (ret) goto err; + streams_added = true; for_each_stream_map(i, cfg, stream_map) { - if (!stream_map->dart->locked) - apple_dart_setup_translation(dart_domain, stream_map); - else { + if (stream_map->dart->managed_sids) { + struct apple_dart *dart = stream_map->dart; + unsigned long flags; + + spin_lock_irqsave(&dart->managed_lock, flags); + if (dart->managed_failed) + ret = -EIO; + else + ret = apple_dart_setup_translation(dart_domain, stream_map); + if (ret) + apple_dart_managed_quarantine_locked(dart); + spin_unlock_irqrestore(&dart->managed_lock, flags); + if (ret) + goto err; + } else if (!stream_map->dart->locked) { + ret = apple_dart_setup_translation(dart_domain, stream_map); + if (ret) + goto err; + } else { ret = apple_dart_setup_translation_locked(dart_domain, stream_map); if (ret) goto err; @@ -1978,6 +2947,8 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain, } err: + if (ret && streams_added && cfg->stream_maps[0].dart->managed_sids) + apple_dart_mod_streams(dart_domain->stream_maps, cfg->stream_maps, false); for_each_stream_map(i, cfg, stream_map) pm_runtime_put(stream_map->dart->dev); return ret; @@ -1991,7 +2962,7 @@ static int apple_dart_attach_dev_identity(struct iommu_domain *domain, struct apple_dart_stream_map *stream_map; int i; - if (!cfg->supports_bypass) + if (!cfg->supports_bypass || cfg->stream_maps[0].dart->managed_sids) return -EINVAL; if (cfg->locked) @@ -2001,7 +2972,8 @@ static int apple_dart_attach_dev_identity(struct iommu_domain *domain, WARN_ON(pm_runtime_get_sync(stream_map->dart->dev) < 0); for_each_stream_map(i, cfg, stream_map) - apple_dart_hw_enable_bypass(stream_map); + if (!stream_map->dart->retained_mtp) + apple_dart_hw_enable_bypass(stream_map); for_each_stream_map(i, cfg, stream_map) pm_runtime_put(stream_map->dart->dev); @@ -2057,6 +3029,21 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev) if (!dev_iommu_fwspec_get(dev) || !cfg) return ERR_PTR(-ENODEV); + if (apple_dart_has_borrowed(cfg->stream_maps[0].dart)) { + struct apple_dart_dcp *borrowed = apple_dart_borrowed_cfg(cfg); + int ret = -EINVAL; + + /* An external display DART's registers only inside a use. */ + if (borrowed && !apple_dart_dispext_get(borrowed->dart)) { + ret = -EAGAIN; + } else if (borrowed) { + ret = apple_dart_dcp_validate(borrowed, dev); + apple_dart_dispext_put(borrowed->dart); + } + if (ret) + return ERR_PTR(ret); + } + for_each_stream_map(i, cfg, stream_map) device_link_add(dev, stream_map->dart->dev, DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER | @@ -2072,7 +3059,8 @@ static void apple_dart_release_device(struct device *dev) struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); for_each_stream_map(j, cfg, stream_map) { - if (!stream_map->dart->locked) + /* Borrowed display roots stay exactly as the DCP left them. */ + if (!stream_map->dart->locked || apple_dart_has_borrowed(stream_map->dart)) continue; ret = pm_runtime_resume_and_get(stream_map->dart->dev); if (ret < 0) { @@ -2131,9 +3119,21 @@ static void apple_dart_domain_free(struct iommu_domain *domain) struct apple_dart_domain *dart_domain = to_dart_domain(domain); struct apple_dart_atomic_stream_map *map; struct apple_dart_stream_map stream; - int i, j, idx; + int i, j, idx, ret; + if (dart_domain->dcp) { + /* Registration rollback: no consumer could publish host DMA yet. */ + if (!READ_ONCE(dart_domain->dcp->dart->iommu.ready)) { + dart_domain->dcp->domain = NULL; + kfree(dart_domain); + return; + } + dev_info(dart_domain->dcp->dart->dev, "Pinning DCP domain until reboot\n"); + return; + } for_each_stream_map(i, dart_domain, map) { + if (map->dart->managed_sids) + return; if (!map->dart->locked) continue; stream.dart = map->dart; @@ -2147,14 +3147,22 @@ static void apple_dart_domain_free(struct iommu_domain *domain) "retaining page tables after power failure: %d\n", j); return; } - for (idx = 0; idx < stream.dart->hw->ttbr_count; idx++) - apple_dart_hw_unmap_locked_ttbr(&stream, idx); + ret = 0; + for (idx = 0; idx < stream.dart->hw->ttbr_count; idx++) { + j = apple_dart_hw_unmap_locked_ttbr(&stream, idx); + ret = ret ?: j; + } j = stream.dart->hw->invalidate_tlb(&stream); + ret = ret ?: j; pm_runtime_put(stream.dart->dev); - /* Do not free page tables that hardware may still cache. */ - if (j) { + /* + * Do not free page tables that hardware may still cache, also + * when a consumer release or an attach rollback retired the root. + */ + ret = ret ?: apple_dart_retire_error(&stream); + if (ret) { dev_err_ratelimited(stream.dart->dev, - "retaining page tables after TLB invalidation failure: %d\n", j); + "retaining page tables after TLB invalidation failure: %d\n", ret); return; } } @@ -2231,6 +3239,15 @@ static int apple_dart_of_xlate(struct device *dev, return i; sid = args->args[0]; + if (dart->cold_ane && sid != 0) + return -EPERM; + if (dart->managed_sids && + (sid <= 0 || sid >= 16 || !(dart->managed_sids & BIT(sid)))) + return -EINVAL; + if (apple_dart_has_borrowed(dart) && + (sid < 0 || sid >= dart->num_streams || !apple_dart_borrowed_sid(dart, sid) || + (cfg && cfg->stream_maps[0].dart))) + return -EINVAL; if (!cfg) { cfg = kzalloc_obj(*cfg); @@ -2263,7 +3280,7 @@ static int apple_dart_of_xlate(struct device *dev, return -EINVAL; } - cfg->supports_bypass &= dart->supports_bypass; + cfg->supports_bypass &= dart->supports_bypass && !dart->managed_sids && !dart->cold_ane; cfg->locked |= dart->locked; for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { @@ -2283,6 +3300,245 @@ static int apple_dart_of_xlate(struct device *dev, return -EINVAL; } +#if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) +/* + * The check, inside a use of the DART's registers (on an external display + * DART, apple_dart_dispext_get() by the caller). + */ +static int __apple_dart_dcp_check(struct device *dev) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct apple_dart_domain *dd; + struct apple_dart_dcp *dcp; + unsigned long flags; + bool drift = false; + int ret; + + if (!domain || domain->owner != &apple_dart_iommu_ops || !iommu_is_dma_domain(domain)) + return -ENODEV; + dd = to_dart_domain(domain); + if (!dd->dcp || dd->dcp->owner != dev || dd->dcp->domain != dd || + !READ_ONCE(dd->dcp->dart->iommu.ready)) + return -ENODEV; + if (READ_ONCE(dd->dcp->failed)) + return -EIO; + dcp = dd->dcp; + if (!apple_dart_dcp_registers(dcp)) + goto failed; + spin_lock_irqsave(&dcp->lock, flags); + for (unsigned int i = 0; i < DCP_DART_ENTRIES; i++) { + if (READ_ONCE(dcp->root[i]) != READ_ONCE(dcp->entry[i])) { + drift = true; + break; + } + /* SID23 may update leaves under an inherited, stable root. Check + * their physical ownership too, not just known firmware segments. + */ + if (dcp->sid == 23 && test_bit(i, dcp->foreign) && dcp->table[i]) + for (unsigned int j = 0; j < DCP_DART_ENTRIES; j++) + if (!apple_dart_dcp_leaf_safe(dcp, READ_ONCE(dcp->table[i][j]))) { + drift = true; + break; + } + if (drift) + break; + } + spin_unlock_irqrestore(&dcp->lock, flags); + if (drift) + goto failed; + ret = apple_dart_dcp_validate(dcp, dev); + if (ret) + goto failed; + return 0; +failed: + /* On an external display DART, both frontends: the DART is retired. */ + if (dcp->dart->dispext) + apple_dart_fail_borrowed(dcp->dart, "check: %s", drift ? "a root or leaf changed" : + "registers or ownership not as adopted"); + else + apple_dart_dcp_fail(dcp, "check: %s", drift ? "a root or leaf changed" : + "registers or ownership not as adopted"); + return -EIO; +} + +/* + * On an external display DART, only with its power confirmed (a use of its + * registers): -EAGAIN, nothing retired, from sleep until its firmware's next + * confirmed power-up. + */ +int apple_dart_dcp_check(struct device *dev) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct apple_dart *dart; + int ret; + + if (!domain || domain->owner != &apple_dart_iommu_ops || !to_dart_domain(domain)->dcp) + return __apple_dart_dcp_check(dev); + dart = to_dart_domain(domain)->dcp->dart; + if (!apple_dart_dispext_get(dart)) + return -EAGAIN; + ret = __apple_dart_dcp_check(dev); + apple_dart_dispext_put(dart); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_check); + +int apple_dart_dcp_check_buffer(struct device *dev, dma_addr_t iova, size_t size) +{ + struct apple_dart_domain *dd; + struct apple_dart_dcp *dcp; + u64 end; + int ret = apple_dart_dcp_check(dev); + + if (ret) + return ret; + if (size > SZ_16M || !dcp_dart_contains(DCP_DART_ALIAS, + DCP_DART_ALIAS + DCP_DART_APERTURE, iova, size)) + return -EINVAL; + dd = to_dart_domain(iommu_get_domain_for_dev(dev)); + dcp = dd->dcp; + if (dcp->sid != 23) + return -ENODEV; + end = iova + size; + for (u64 page = iova & ~(DCP_DART_PAGE - 1); page < end; page += DCP_DART_PAGE) { + unsigned int root = (page >> 25) & 2047; + unsigned int leaf = (page >> 14) & 2047; + + /* A readable firmware TEXT mapping is never writable RTKit shared memory. */ + if (!apple_dart_dcp_translate(dcp, page) || !test_bit(root, dcp->foreign) || + !dcp_dart_pte_full_rw(READ_ONCE(dcp->table[root][leaf]), true)) { + apple_dart_dcp_fail(dcp, "buffer at %llx is not writable shared memory", + page); + return -EACCES; + } + } + return 0; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_check_buffer); + +void *apple_dart_dcp_alloc(struct device *dev, size_t size, dma_addr_t *dva) +{ + struct apple_dart_dcp_allocation *allocation; + struct apple_dart_domain *dd; + struct apple_dart_dcp *dcp; + unsigned int root, count, allocated = 0; + bool coherent = dev_is_dma_coherent(dev); + int prot = IOMMU_READ | IOMMU_WRITE | (coherent ? IOMMU_CACHE : 0); + int ret = apple_dart_dcp_check(dev); + void *result; + + if (ret) + return ERR_PTR(ret); + if (!size || size > SZ_16M) + return ERR_PTR(-EINVAL); + size = PAGE_ALIGN(size); + count = size >> PAGE_SHIFT; + dd = to_dart_domain(iommu_get_domain_for_dev(dev)); + dcp = dd->dcp; + mutex_lock(&dcp->alloc_lock); + if (dcp->sid != 23) { + result = ERR_PTR(-ENODEV); + goto unlock; + } + if (READ_ONCE(dcp->failed) || dcp->allocated > SZ_64M - size) { + result = ERR_PTR(-ENOSPC); + goto unlock; + } + /* One empty 32-MiB slice per object. Never alias a foreign or previous object. */ + for (root = 0; root < DCP_DART_ENTRIES; root++) + if (!test_bit(root, dcp->claimed) && + !dcp->entry[root] && !READ_ONCE(dcp->root[root])) + break; + if (root == DCP_DART_ENTRIES) { + result = ERR_PTR(-ENOSPC); + goto unlock; + } + set_bit(root, dcp->claimed); /* Includes failures before a root PTE appears. */ + allocation = kzalloc(struct_size(allocation, pages, count), GFP_KERNEL); + if (!allocation) { + result = ERR_PTR(-ENOMEM); + goto unlock; + } + allocation->size = size; + allocation->dva = DCP_DART_ALIAS + root * DCP_DART_SLICE; + for (allocated = 0; allocated < count; allocated++) { + allocation->pages[allocated] = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!allocation->pages[allocated]) + goto free_unpublished; + if (!coherent) + arch_dma_prep_coherent(allocation->pages[allocated], PAGE_SIZE); + } + allocation->cpu = dma_common_pages_remap(allocation->pages, size, + dma_pgprot(dev, PAGE_KERNEL, 0), + __builtin_return_address(0)); + if (!allocation->cpu) + goto free_unpublished; + /* Own payload pages BEFORE the first PTE, including every rollback case. */ + list_add_tail(&allocation->link, &dcp->allocations); + dcp->allocated += size; + for (unsigned int i = 0; i < count; i++) { + ret = iommu_map(&dd->domain, allocation->dva + i * PAGE_SIZE, + page_to_phys(allocation->pages[i]), PAGE_SIZE, prot, GFP_KERNEL); + if (ret) + goto pin_failure; + } + ret = apple_dart_dcp_check(dev); + if (ret) + goto pin_failure; + for (unsigned int i = 0; i < count; i++) { + if (apple_dart_dcp_translate(dcp, allocation->dva + i * PAGE_SIZE) != + page_to_phys(allocation->pages[i])) { + ret = -EIO; + goto pin_failure; + } + } + *dva = allocation->dva; + result = allocation->cpu; + goto unlock; +pin_failure: + apple_dart_dcp_fail(dcp, "payload mapping failed: %d", ret); + dev_err(dev, "Retaining %zu payload bytes after DCP mapping failure %d\n", size, ret); + result = ERR_PTR(ret); + goto unlock; +free_unpublished: + /* Safe only because no page address or PTE has been exposed yet. */ + while (allocated) + __free_page(allocation->pages[--allocated]); + kfree(allocation); + result = ERR_PTR(-ENOMEM); +unlock: + mutex_unlock(&dcp->alloc_lock); + return result; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_alloc); + +static int apple_dart_dcp_shared_flush(struct apple_dart_dcp *dcp) +{ + struct apple_dart_stream_map map = { .dart = dcp->dart }; + int ret; + + if (dcp->sid != 16 || !apple_dart_dcp_registers(dcp)) + return -EIO; + /* An external display DART has no fault IRQ attached: look first. */ + if (dcp->dart->dispext && apple_dart_fault_pending(dcp->dart)) { + apple_dart_fail_borrowed(dcp->dart, "a DART fault is pending (SID16 flush)"); + return -EIO; + } + bitmap_zero(map.sidmap, DART_MAX_STREAMS); + set_bit(16, map.sidmap); + /* Generic SID16 flush is a no-op; authorize only this synchronous call. */ + WRITE_ONCE(dcp->shared_flushing, true); + dma_wmb(); + /* Raw: inside the caller's use of the DART (no gate taken again). */ + ret = apple_dart_dcp_invalidate_tlb(&map); + WRITE_ONCE(dcp->shared_flushing, false); + return ret; +} + +#include "apple-dart-dcp-shared.h" +#include "apple-dart-dcp-scanout.h" +#endif /* CONFIG_APPLE_DART_J713_DCP */ + static DEFINE_MUTEX(apple_dart_groups_lock); static void apple_dart_release_group(void *iommu_data) @@ -2423,16 +3679,79 @@ static bool apple_dart_capable(struct device *dev, enum iommu_cap cap) } } -static int apple_dart_def_domain_type(struct device *dev) -{ - struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); - - if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE) +/* + * Media and ML engines (video decode and encode, neural engine, camera ISP) + * are run by their own coprocessor firmware, which can still have DMA in + * flight after the host driver has given up on a job: a timeout, a reset, + * a stream torn down mid-frame. With deferred flushing the DART keeps + * translating a freed buffer's addresses until the flush queue drains, so + * such a late write lands silently in pages that already went back to the + * allocator. Strict flushing makes it fault at the DART instead. These + * engines map their buffers once per stream or model, so the per-unmap + * invalidate is not on any hot path. + * + * The client is identified by its device tree compatible, "apple,-avd" + * and so on, so every SoC generation is covered without a per-SoC table. + */ +static const char * const apple_dart_strict_engines[] = { + "avd", "ave", "ane", "isp", +}; + +static bool apple_dart_compat_is_strict_engine(const char *compat) +{ + const char *engine; + unsigned int i; + + if (!str_has_prefix(compat, "apple,")) + return false; + engine = strrchr(compat, '-'); + engine = engine ? engine + 1 : compat + strlen("apple,"); + for (i = 0; i < ARRAY_SIZE(apple_dart_strict_engines); i++) + if (!strcmp(engine, apple_dart_strict_engines[i])) + return true; + return false; +} + +static bool apple_dart_master_needs_strict(struct device *dev) +{ + struct property *prop; + const char *compat; + + if (!dev->of_node) + return false; + of_property_for_each_string(dev->of_node, "compatible", prop, compat) + if (apple_dart_compat_is_strict_engine(compat)) + return true; + return false; +} + +static int apple_dart_def_domain_type(struct device *dev) +{ + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); + + /* Retained radio and borrowed display streams require strict DMA. */ + if (cfg->stream_maps[0].dart->managed_sids || + apple_dart_has_borrowed(cfg->stream_maps[0].dart)) + return IOMMU_DOMAIN_DMA; + if (cfg->stream_maps[0].dart->retained_mtp) + return IOMMU_DOMAIN_IDENTITY; + if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE) + return IOMMU_DOMAIN_IDENTITY; + /* + * Tunneled PCIe is untrusted, and the PCIe-C DART bypasses only its + * reserved streams: translate everything behind a USB4 port. + */ + if (cfg->stream_maps[0].dart->tunneled) + return IOMMU_DOMAIN_DMA; + if (dev_is_pci(dev) && of_machine_is_compatible("apple,j713") && + cfg->supports_bypass && !cfg->locked) return IOMMU_DOMAIN_IDENTITY; if (!cfg->supports_bypass) return IOMMU_DOMAIN_DMA; if (cfg->locked) return IOMMU_DOMAIN_DMA; + if (apple_dart_master_needs_strict(dev)) + return IOMMU_DOMAIN_DMA; return 0; } @@ -2480,7 +3799,12 @@ static void apple_dart_get_fw_resv_regions(struct device *dev, struct apple_dart *dart = stream->dart; size_t entries = dart->pgsize / sizeof(u64); - if (!dart->fw_handoff || entries > DART_MAX_ROOT_ENTRIES) + /* + * Only the handoff and fw_tree reserve anything: leave the other + * fw_mirror DARTs (T6041, T8140, T8142) untouched, TCR reads included. + */ + if ((!dart->fw_handoff && !dart->fw_tree) || + entries > DART_MAX_ROOT_ENTRIES) continue; for_each_set_bit(sid, stream->sidmap, dart->num_streams) { for (idx = 0; idx < dart->hw->ttbr_count; idx++) { @@ -2504,6 +3828,13 @@ static void apple_dart_get_fw_resv_regions(struct device *dev, __set_bit(slot, slots); spin_unlock_irqrestore(&dart->lock, flags); + /* + * A three-level fw_mirror slot may repeat firmware's + * pages (apple_dart_check_fw_map()); a four-level one + * is firmware's alone, so keep the allocator out of it. + */ + if (!dart->fw_handoff && !(four && dart->fw_tree)) + continue; if (!four && check_add_overflow(window, ((u64)idx << (ilog2(dart->pgsize) + 2 * ilog2(entries))), &window)) @@ -2530,8 +3861,18 @@ static void apple_dart_get_fw_resv_regions(struct device *dev, static void apple_dart_get_resv_regions(struct device *dev, struct list_head *head) { + struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); struct iommu_resv_region *region; + if (cfg && apple_dart_has_borrowed(cfg->stream_maps[0].dart)) { + struct apple_dart_dcp *borrowed = apple_dart_borrowed_cfg(cfg); + + /* Already reconciled against live leaves; never request reconstruction. */ + if (borrowed) + apple_dart_dcp_reserve(borrowed, head); + return; + } + if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) { int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; @@ -2701,6 +4042,11 @@ static const char *apple_dart_t8110_v2_fault_name(u32 exception) return "unknown"; } +static void apple_dart_quarantine_borrowed(struct apple_dart *dart, const char *why) +{ + apple_dart_fail_borrowed(dart, "%s", why); +} + static bool apple_dart_t8110_v2_block_pending(u16 version, u32 block_error) { return version >= 0x0203 && @@ -2711,7 +4057,7 @@ static irqreturn_t apple_dart_t8110_v2_irq(int irq, void *dev) { struct apple_dart *dart = dev; irqreturn_t ret = IRQ_NONE; - unsigned int words = DIV_ROUND_UP(dart->num_streams, 32); + unsigned int words = DIV_ROUND_UP(apple_dart_exception_sids(dart), 32); unsigned int i; u32 block_error; @@ -2728,13 +4074,16 @@ static irqreturn_t apple_dart_t8110_v2_irq(int irq, void *dev) if (!pending) continue; + if (dart->managed_sids) + apple_dart_managed_quarantine(dart); + apple_dart_quarantine_borrowed(dart, "a stream exception is pending"); for_each_set_bit(bit, &bits, 32) { unsigned int sid = i * 32 + bit; u32 status; u64 addr; - if (sid >= dart->num_streams) + if (sid >= apple_dart_exception_sids(dart)) break; status = readl(dart->regs + @@ -2770,6 +4119,9 @@ static irqreturn_t apple_dart_t8110_v2_irq(int irq, void *dev) if (dart->version >= 0x0203) { block_error = readl(dart->regs + DART_T8110_BLOCK_ERROR); if (apple_dart_t8110_v2_block_pending(dart->version, block_error)) { + if (dart->managed_sids) + apple_dart_managed_quarantine(dart); + apple_dart_quarantine_borrowed(dart, "a block error is pending"); dev_err_ratelimited(dart->dev, "block exception: status:0x%08x\n", block_error); @@ -2778,18 +4130,92 @@ static irqreturn_t apple_dart_t8110_v2_irq(int irq, void *dev) ret = IRQ_HANDLED; } } - if (ret == IRQ_HANDLED) + /* Retained radio and borrowed display streams stay quarantined. */ + if (ret == IRQ_HANDLED && !dart->managed_sids && !apple_dart_has_borrowed(dart)) apple_dart_t8110_v2_unlock(dart); return ret; } +/* + * Borrowed J713 display DARTs share one IRQ. Any fault quarantines the + * retained session. Linux is the IRQ handler for the entire block, including + * inherited streams: acknowledge their diagnostic latch without changing any + * translation or disabling the shared level-triggered interrupt. + */ +static irqreturn_t apple_dart_dcp_irq(struct apple_dart *dart) +{ + const char *fault_name = NULL; + u32 error = readl(dart->regs + DART_T8110_ERROR); + u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error); + u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO); + u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI); + u64 addr = addr_lo | (((u64)addr_hi) << 32); + u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error); + + if (!(error & DART_T8110_ERROR_FLAG)) + return IRQ_NONE; + /* Quarantine the session, but never disable the shared display IRQ. */ + apple_dart_fail_borrowed(dart, "DART error %08x", error); + if (!apple_dart_borrowed_sid(dart, stream_idx)) { + dev_err_ratelimited(dart->dev, + "Unowned display fault: status=%08x SID%u at %llx; retained state pinned\n", + error, stream_idx, addr); + /* No other Linux handler owns this DART's inherited stream. */ + writel(error, dart->regs + DART_T8110_ERROR); + if (stream_idx < 32) + writel(BIT(stream_idx), dart->regs + DART_T8110_ERROR_STREAMS); + return IRQ_HANDLED; + } + + if (error_code == DART_T8110_ERROR_READ_FAULT) + fault_name = "READ FAULT"; + else if (error_code == DART_T8110_ERROR_WRITE_FAULT) + fault_name = "WRITE FAULT"; + else if (error_code == DART_T8110_ERROR_NO_PTE) + fault_name = "NO PTE FOR IOVA"; + else if (error_code == DART_T8110_ERROR_NO_PMD) + fault_name = "NO PMD FOR IOVA"; + else if (error_code == DART_T8110_ERROR_NO_PGD) + fault_name = "NO PGD FOR IOVA"; + else if (error_code == DART_T8110_ERROR_NO_TTBR) + fault_name = "NO TTBR FOR IOVA"; + else + fault_name = "unknown"; + + dev_err_ratelimited(dart->dev, + "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", + error, stream_idx, error_code, fault_name, addr); + + writel(error, dart->regs + DART_T8110_ERROR); + writel(BIT(stream_idx), dart->regs + DART_T8110_ERROR_STREAMS); + return IRQ_HANDLED; +} + +/* Read-only admission: do not clear a pending inherited fault before ownership. */ +static bool apple_dart_fault_pending(struct apple_dart *dart) +{ + if (dart->version >= 0x0202) { + for (unsigned int i = 0; i < DIV_ROUND_UP(dart->num_streams, 32); i++) + if (readl(dart->regs + DART_T8110_EXCEPTION(i))) + return true; + return dart->version >= 0x0203 && + apple_dart_t8110_v2_block_pending(dart->version, + readl(dart->regs + DART_T8110_BLOCK_ERROR)); + } + return (readl(dart->regs + DART_T8110_ERROR) & DART_T8110_ERROR_FLAG) || + readl(dart->regs + DART_T8110_ERROR_STREAMS); +} + static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) { struct apple_dart *dart = dev; + /* Borrowed T8132 is v2.2 too: use its per-SID exception ABI. */ if (dart->version >= 0x0202) return apple_dart_t8110_v2_irq(irq, dev); + if (apple_dart_has_borrowed(dart)) + return apple_dart_dcp_irq(dart); return apple_dart_t8110_legacy_irq(irq, dev); } @@ -2814,6 +4240,20 @@ static bool apple_dart_is_locked(struct apple_dart *dart) return !!(apple_dart_readl(dart, dart->hw->lock) & dart->hw->lock_bit); } +static bool apple_dart_streams_bypassed(struct apple_dart *dart) +{ + unsigned int sid; + + for (sid = 0; sid < dart->num_streams; sid++) { + if ((apple_dart_readl(dart, DART_TCR(dart, sid)) & + (DART_T8110_TCR_BYPASS_DART | DART_T8110_TCR_TRANSLATE_ENABLE)) != + DART_T8110_TCR_BYPASS_DART || + (apple_dart_readl(dart, DART_TTBR(dart, sid, 0)) & dart->hw->ttbr_valid)) + return false; + } + return true; +} + static void apple_dart_iounmap_np(void *data) { iounmap((__force void __iomem *)data); @@ -2850,6 +4290,111 @@ apple_dart_platform_ioremap_np_resource(struct platform_device *pdev, return regs; } +struct apple_dart_available { + struct list_head list; + struct device_node *np; +}; + +static DEFINE_MUTEX(apple_dart_available_lock); +static LIST_HEAD(apple_dart_available_list); + +void apple_dart_make_available(struct device_node *np) +{ + struct apple_dart_available *a; + + guard(mutex)(&apple_dart_available_lock); + list_for_each_entry(a, &apple_dart_available_list, list) + if (a->np == np) + return; + a = kzalloc(sizeof(*a), GFP_KERNEL); + if (!a) + return; + a->np = of_node_get(np); + list_add(&a->list, &apple_dart_available_list); +} +EXPORT_SYMBOL_GPL(apple_dart_make_available); + +static bool apple_dart_is_available(struct device_node *np) +{ + struct apple_dart_available *a; + + if (!of_property_read_bool(np, "apple,manual-availability")) + return true; + guard(mutex)(&apple_dart_available_lock); + list_for_each_entry(a, &apple_dart_available_list, list) + if (a->np == np) + return true; + return false; +} + +static DEFINE_MUTEX(apple_dart_display_irq_lock); +static struct apple_dart *apple_dart_display_irq_peers[2]; +static int apple_dart_display_shared_irq = -1; + +static irqreturn_t apple_dart_display_pair_irq(int irq, void *data) +{ + irqreturn_t result = IRQ_NONE; + + /* Both admitted suppliers are retained. Removal frees/synchronizes this + * action before changing either pointer; the handler takes no pair mutex. + */ + for (unsigned int i = 0; i < ARRAY_SIZE(apple_dart_display_irq_peers); i++) + if (apple_dart_irq(irq, apple_dart_display_irq_peers[i]) == IRQ_HANDLED) + result = IRQ_HANDLED; + return result; +} + +static int apple_dart_display_irq_request(struct apple_dart *dart) +{ + unsigned int peer = dart->dcp ? 1 : 0; + int ret; + + guard(mutex)(&apple_dart_display_irq_lock); + if (apple_dart_display_irq_peers[peer]) + return -EBUSY; + apple_dart_display_irq_peers[peer] = dart; + if (!apple_dart_display_irq_peers[!peer]) + return 0; /* Never enable this level IRQ for just one admitted block. */ + if (dart->irq != apple_dart_display_irq_peers[!peer]->irq || + apple_dart_fault_pending(dart) || + apple_dart_fault_pending(apple_dart_display_irq_peers[!peer])) { + ret = -EIO; + goto err; + } + ret = request_irq(dart->irq, apple_dart_display_pair_irq, IRQF_SHARED, + "apple-dart display fault handlers", apple_dart_display_irq_peers); + if (ret) + goto err; + apple_dart_display_shared_irq = dart->irq; + return 0; +err: + apple_dart_display_irq_peers[peer] = NULL; + return ret; +} + +static void apple_dart_free_irq(struct apple_dart *dart) +{ + if (dart->irq < 0) + return; + if (!dart->preserve_display) { + free_irq(dart->irq, dart); + return; + } + guard(mutex)(&apple_dart_display_irq_lock); + if (apple_dart_display_shared_irq >= 0) { + /* Neither peer may publish DMA after losing shared fault observation. */ + for (unsigned int i = 0; i < ARRAY_SIZE(apple_dart_display_irq_peers); i++) + if (apple_dart_display_irq_peers[i]) + apple_dart_quarantine_borrowed(apple_dart_display_irq_peers[i], + "the shared fault IRQ went"); + free_irq(apple_dart_display_shared_irq, apple_dart_display_irq_peers); + apple_dart_display_shared_irq = -1; + } + for (unsigned int i = 0; i < ARRAY_SIZE(apple_dart_display_irq_peers); i++) + if (apple_dart_display_irq_peers[i] == dart) + apple_dart_display_irq_peers[i] = NULL; +} + static int apple_dart_probe(struct platform_device *pdev) { int ret, sid; @@ -2862,16 +4407,129 @@ static int apple_dart_probe(struct platform_device *pdev) bool tunneled = dev->of_node->parent && of_node_name_prefix(dev->of_node->parent, "usb4-pcie-tunnel-"); + bool j713_display, j713_dcpext, j713_dispext, j713_radio; + + /* Its registers answer only once its client is up */ + if (!apple_dart_is_available(dev->of_node)) + return dev_err_probe(dev, -EPROBE_DEFER, "waiting for its client\n"); - dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); + /* + * The J713 internal display DARTs are live: DCP firmware (SID23) and + * the scanout/PIODMA frontend (SID0/SID16) keep using them. Leave both + * untouched until userspace, with its watchdog armed, opts in. + */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + j713_display = res && !tunneled && of_machine_is_compatible("apple,j713") && + of_device_is_compatible(dev->of_node, "apple,t8132-dart") && + (res->start == 0x30a300000ULL || res->start == 0x30a340000ULL); + /* The external DCP firmware DARTs (SID23) of dcpext0 and dcpext1. */ + j713_dcpext = res && !tunneled && of_machine_is_compatible("apple,j713") && + of_device_is_compatible(dev->of_node, "apple,t8132-dart") && + (res->start == 0x286340000ULL || res->start == 0x48a340000ULL); + /* The external display DARTs (SID0/SID16) of dispext0 and dispext1. */ + j713_dispext = res && !tunneled && of_machine_is_compatible("apple,j713") && + of_device_is_compatible(dev->of_node, "apple,t8132-dart") && + (res->start == 0x286300000ULL || res->start == 0x48a300000ULL); + /* The J713 radio DART's fault interrupt is not described by its loader. */ + j713_radio = res && !tunneled && of_machine_is_compatible("apple,j713") && + of_machine_is_compatible("apple,t8132") && + of_device_is_compatible(dev->of_node, "apple,t8132-dart") && + res->start == 0x490000000ULL; + if (j713_display && !IS_ENABLED(CONFIG_APPLE_DART_J713_DCP)) + return dev_err_probe(dev, -EOPNOTSUPP, + "retained J713 display requires CONFIG_APPLE_DART_J713_DCP=y\n"); + if (j713_display && + ((res->start == 0x30a300000ULL && !READ_ONCE(apple_dart_frontend_enabled)) || + (res->start == 0x30a340000ULL && !READ_ONCE(apple_dart_dcp_enabled)))) { + dev_info(dev, "Leaving inherited display DART untouched; explicit guarded probe required\n"); + return -ENODEV; + } + if (j713_dcpext) { + /* Its block is powered only if the loader handed the DCP over. */ + struct device_node *owner = + of_find_node_by_path(res->start == 0x48a340000ULL ? "dcpext1" : "dcpext0"); + u32 ready = 0; + bool handed_over = owner && + !of_property_read_u32(owner, "apple,t8132-dcpext-firmware-ready", &ready) && + ready == 1; + + of_node_put(owner); + if (!IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) || !handed_over) { + dev_info(dev, "Leaving inherited external display DART untouched\n"); + return -ENODEV; + } + } + if (j713_dispext) { + /* + * Its block shares the DCP's CPU domain, powered only if the + * loader handed the DCP over. + */ + struct device_node *owner = + of_find_node_by_path(res->start == 0x48a300000ULL ? "dcpext1" : "dcpext0"); + u32 ready = 0; + bool handed_over = owner && + !of_property_read_u32(owner, "apple,t8132-dcpext-firmware-ready", &ready) && + ready == 1; + + struct platform_device *owner_dev = owner ? of_find_device_by_node(owner) : NULL; + /* + * Probed at boot in the loader's state. Not once the dcpext's + * driver has its session: its firmware may have the DART off. + */ + bool session = owner_dev && device_is_bound(&owner_dev->dev); + + if (owner_dev) + put_device(&owner_dev->dev); + of_node_put(owner); + if (session) { + dev_info(dev, "External DCP session already running: display DART left untouched\n"); + return -ENODEV; + } + if (!IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) || !handed_over) { + dev_info(dev, "Leaving inherited external display DART untouched\n"); + return -ENODEV; + } + } + + dart = kzalloc(sizeof(*dart), GFP_KERNEL); if (!dart) return -ENOMEM; + ret = devm_add_action_or_reset(dev, apple_dart_release, dart); + if (ret) + return ret; dart->dev = dev; dart->hw = of_device_get_match_data(dev); dart->tunneled = tunneled; + dart->cold_ane = of_device_is_compatible(dev->of_node, "apple,t8140-ane-dart"); + if (dart->cold_ane && !IS_BUILTIN(CONFIG_APPLE_DART)) + return -EOPNOTSUPP; dart->per_stream = of_device_is_compatible(dev->of_node, "apple,t8140-dart") || of_device_is_compatible(dev->of_node, "apple,t8142-dart"); + if (of_property_present(dev->of_node, "apple,managed-sids")) { + u32 ids[2]; + + if (!IS_BUILTIN(CONFIG_APPLE_DART)) + return dev_err_probe(dev, -EOPNOTSUPP, + "managed radio requires built-in DART support\n"); + if (!j713_radio || + of_property_count_u32_elems(dev->of_node, "apple,managed-sids") != 2 || + of_property_read_u32_array(dev->of_node, "apple,managed-sids", ids, 2) || + ids[0] != 1 || ids[1] != 2) + return -EINVAL; + dart->managed_sids = BIT(1) | BIT(2); + } + spin_lock_init(&dart->managed_lock); + dart->preserve_display = j713_display || j713_dcpext || j713_dispext; + dart->dispext = j713_dispext; + spin_lock_init(&dart->dispext_pm_lock); + init_waitqueue_head(&dart->dispext_wq); + if (j713_dcpext) + dart->dcp_instance = &dcp_adt_external[res->start == 0x48a340000ULL]; + else if (j713_dispext) + dart->dcp_instance = &dcp_adt_external[res->start == 0x48a300000ULL]; + else if (j713_display) + dart->dcp_instance = &dcp_adt_internal; if (tunneled) { pd_np = of_parse_phandle(dev->of_node, "power-domains", 0); if (pd_np) { @@ -2895,6 +4553,8 @@ static int apple_dart_probe(struct platform_device *pdev) if (tunneled) dart->regs = apple_dart_platform_ioremap_np_resource(pdev, 0, &res); + else if (dart->managed_sids || dart->preserve_display || dart->cold_ane) + dart->regs = apple_dart_retained_ioremap(dart, res); else dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); @@ -2926,14 +4586,36 @@ static int apple_dart_probe(struct platform_device *pdev) } } - dart->irq = platform_get_irq(pdev, 0); - if (dart->irq < 0) - return -ENODEV; + if (j713_dcpext || j713_dispext) { + /* + * Its fault IRQ is shared with the external display DART, which is + * not admitted: never enable that level IRQ for one block alone. + */ + dart->irq = -1; + } else if (dart->preserve_display || j713_radio) { + dart->irq = platform_get_irq_optional(pdev, 0); + if (dart->irq == -ENXIO) + dart->irq = -1; + else if (dart->irq < 0) + return dart->irq; + } else { + dart->irq = platform_get_irq(pdev, 0); + if (dart->irq < 0) + return -ENODEV; + } + + if (dart->managed_sids && dart->irq < 0) + return dev_err_probe(dev, -ENODEV, "managed radio requires a fault IRQ\n"); ret = devm_clk_bulk_get_all(dev, &dart->clks); if (ret < 0) return ret; dart->num_clks = ret; + if (dart->managed_sids && dart->num_clks) + return -EINVAL; + /* This inherited profile has no managed clocks; do not acquire new ones. */ + if (dart->preserve_display && dart->num_clks) + return -ENODEV; ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks); if (ret) @@ -2969,6 +4651,9 @@ static int apple_dart_probe(struct platform_device *pdev) dart->oas = DART_PCIEC_ADDR_WIDTH; dart->num_streams = DART_PCIEC_STREAMS; dart->four_level = true; + /* T8132 PCIe-C DARTs keep per-stream exception records. */ + if (of_device_is_compatible(dev->of_node, "apple,t8132-dart")) + dart->version = 0x0202; } else { dart->pgsize = 1 << DART_PCIEC_PAGE_SHIFT; dart->supports_bypass = false; @@ -3088,6 +4773,19 @@ static int apple_dart_probe(struct platform_device *pdev) goto err_clk_disable; } + /* + * v2.2 moved the exception bitmap to 0x4000 and the per-stream + * exception records to 0x8000. A node whose reg stops short of them + * would fault on the first exception read during reset. + */ + if (dart->hw->type == DART_T8110 && dart->version >= 0x0202 && + resource_size(res) < DART_T8110_ERR_SID(dart->num_streams)) { + ret = dev_err_probe(dev, -EINVAL, + "MMIO region %pR too small for v2.2 exception records (need %#x); fix the DT reg\n", + res, DART_T8110_ERR_SID(dart->num_streams)); + goto err_clk_disable; + } + ret = apple_dart_read_remaps(dart); if (ret) goto err_clk_disable; @@ -3099,34 +4797,184 @@ static int apple_dart_probe(struct platform_device *pdev) dart->locked = apple_dart_is_locked(dart); /* Reserved table-page lifetime requires the matched three-level loader. */ dart->fw_handoff = apple_dart_fw_handoff_enabled(dart); - /* Preserve our retained-root path when no reserved-table handoff exists. */ + /* + * Preserve our retained-root path when no reserved-table handoff exists. + * T6050's boot loader leaves every display DART locked on four-level + * roots of its own (dart-dcpextN SID 23, dart-dispextN SIDs 0 and 16). + */ dart->fw_mirror = dart->locked && !dart->fw_handoff && - (of_device_is_compatible(dev->of_node, "apple,t8140-dart") || + (of_device_is_compatible(dev->of_node, "apple,t6041-dart") || + of_device_is_compatible(dev->of_node, "apple,t6050-dart") || + of_device_is_compatible(dev->of_node, "apple,t8140-dart") || of_device_is_compatible(dev->of_node, "apple,t8142-dart")); - if (!dart->locked) { + /* + * J614s may arrive with the MTP live after the boot keyboard used it. + * Do not reset its DART unless there is no advertised handoff. + */ + if (of_machine_is_compatible("apple,j614s") && + res->start == 0x514800000ULL) { + struct device_node *mtp, *iommu; + const char *handoff; + bool live, advertised; + + mtp = of_find_node_by_path("/mtp@514600000"); + iommu = mtp ? of_parse_phandle(mtp, "iommus", 0) : NULL; + advertised = mtp && of_find_property(mtp, "apple,rtkit-handoff", NULL); + live = advertised && of_device_is_compatible(mtp, "apple,t6041-mtp") && + !of_property_read_string(mtp, "apple,rtkit-handoff", &handoff) && + !strcmp(handoff, "mtp-live-v1"); + if (advertised && (!live || iommu != dev->of_node || dart->hw->type != DART_T8110 || + dart->locked || tunneled || !dart->supports_bypass || + !apple_dart_streams_bypassed(dart))) { + of_node_put(iommu); + of_node_put(mtp); + dev_err(dev, "Refusing invalid J614s live MTP DART handoff\n"); + ret = -EINVAL; + goto err_clk_disable; + } + if (live) { + dart->retained_mtp = true; + dev_info(dev, "Retaining the live J614s MTP bypass\n"); + } + of_node_put(iommu); + of_node_put(mtp); + } + dart->fw_tree = dart->fw_mirror && + of_device_is_compatible(dev->of_node, "apple,t6050-dart"); + if (of_machine_is_compatible("apple,j713") && + of_machine_is_compatible("apple,t8132") && + res->start == 0x394800000ULL && dart->hw->type == DART_T8110 && + !dart->locked && !tunneled && dart->supports_bypass && + apple_dart_streams_bypassed(dart)) { + dart->retained_mtp = true; + dev_info(dev, "Retaining the live J713 MTP bypass\n"); + } + if (dart->preserve_display) { + if (dart->hw->type != DART_T8110 || !bitmap_empty(dart->remapped, DART_MAX_STREAMS)) { + ret = -EINVAL; + goto err_clk_disable; + } + dev_info(dev, "J713 display DART version %04x, locked %d\n", + dart->version, dart->locked); + ret = res->start == 0x30a340000ULL || j713_dcpext ? + apple_dart_dcp_prepare(dart) : + dart->dispext ? apple_dart_dispext_prepare(dart) : + apple_dart_frontend_prepare(dart); + if (ret) + goto err_clk_disable; + } + if (dart->cold_ane) { + /* No reset, TCR/TTBR clear, global enable, remap or IRQ ACK. + * Cold acquisition is checked by the staged ANE backend before + * it attaches a private SID0 domain. Even bypass SIDs stay intact. + */ + if (dart->locked || dart->version != 0x203 || dart->num_streams != 16 || + !bitmap_empty(dart->remapped, DART_MAX_STREAMS)) { + ret = -EPERM; + goto err_clk_disable; + } + dart->irq = -1; + dart->retained = true; + get_device(dev); + dev_info(dev, "cold ANE DART: inherited streams preserved; SID0 only\n"); + } else if (dart->managed_sids) { + bool init_bypass = false; + + ret = apple_dart_fault_pending(dart) ? -EBUSY : + apple_dart_managed_admit(dart, &init_bypass); + if (ret) + goto err_clk_disable; + /* Acquire the takeover guard before the first managed register write. */ + ret = apple_dart_retain_radio(dart); + if (ret) + goto err_clk_disable; + if (init_bypass) { + struct apple_dart_stream_map stream = { .dart = dart }; + u32 bypass_tcr; + + __set_bit(16, stream.sidmap); + apple_dart_hw_enable_bypass(&stream); + bypass_tcr = apple_dart_readl(dart, DART_TCR(dart, 16)); + if (!apple_dart_managed_static_bypass(bypass_tcr)) { + ret = dev_err_probe(dev, -EIO, + "static SID16 bypass readback failed\n"); + goto err_clk_disable; + } + apple_dart_writel(dart, BIT(16), dart->hw->enable_streams); + } + /* ENABLE is write-one-to-set; no foreign stream changes state. */ + apple_dart_writel(dart, dart->managed_sids, dart->hw->enable_streams); + } else if (!dart->locked && !dart->retained_mtp && !dart->preserve_display) { ret = apple_dart_hw_reset(dart); if (ret) goto err_clk_disable; } - if (dart->locked) + /* Borrowed display state: never acknowledge someone else's fault. */ + if (dart->locked && !dart->preserve_display) apple_dart_drain_stale_exceptions(dart); - ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, - "apple-dart fault handler", dart); - if (ret) + /* Refuse an already pending inherited fault before joining the shared IRQ. */ + if (dart->preserve_display && + (apple_dart_fault_pending(dart) || + readl(dart->regs + DART_T8110_ERROR_MASK))) { + ret = dev_err_probe(dev, -EIO, + "pending or masked display DART faults; no IRQ attachment\n"); goto err_clk_disable; + } + + if (dart->irq >= 0) { + ret = dart->preserve_display ? apple_dart_display_irq_request(dart) : + request_irq(dart->irq, apple_dart_irq, IRQF_SHARED, + "apple-dart fault handler", dart); + if (ret) + goto err_clk_disable; + } ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s", dev_name(&pdev->dev)); if (ret) goto err_free_irq; + if (dart->managed_sids) { + ret = devm_device_add_group(dev, &apple_dart_managed_group); + if (ret) + goto err_sysfs_remove; + } + /* An external display DART's state is not read from sysfs. */ + if (dart->frontend[0] && !dart->dispext) { + ret = devm_device_add_group(dev, &apple_dart_frontend_group); + if (ret) + goto err_sysfs_remove; + } +#if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) + if (apple_dart_has_borrowed(dart)) { + /* iommu_device_register() may attach consumers and publish host + * tables. From this point neither successful nor failed removal + * returns ownership of any hardware-visible memory. + */ + /* Pin the guard and immutable reason before the generic API can + * retain their addresses. No successful acquisition is released: + * the display has no proven DMA-stop/reset boundary before reboot. + */ + __module_get(THIS_MODULE); + ret = kexec_block(&apple_dart_display_kexec); + if (ret) { + module_put(THIS_MODULE); + goto err_sysfs_remove; + } + get_device(dev); + WRITE_ONCE(dart->retained, true); + } +#endif ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev); if (ret) goto err_sysfs_remove; - pm_runtime_put(dev); + /* Retain supplier references; generic suspend/resume is not applicable. */ + if (!dart->retained_mtp && !dart->managed_sids && !dart->cold_ane && + !apple_dart_has_borrowed(dart)) + pm_runtime_put(dev); dev_dbg( &pdev->dev, @@ -3136,11 +4984,24 @@ static int apple_dart_probe(struct platform_device *pdev) return 0; err_sysfs_remove: + /* Failed registration can already have published retained DMA state. */ + if (dart->retained) { + if (dart->managed_sids) + apple_dart_managed_quarantine(dart); + apple_dart_quarantine_borrowed(dart, "registration failed"); + } iommu_device_sysfs_remove(&dart->iommu); err_free_irq: - free_irq(dart->irq, dart); + if (dart->irq >= 0) + apple_dart_free_irq(dart); err_clk_disable: - pm_runtime_put(dev); + if (dart->managed_sids && dart->retained && !READ_ONCE(dart->managed_failed)) + apple_dart_managed_quarantine(dart); + /* Do not schedule suspend on failed radio or display adoption. */ + if (dart->retained || dart->preserve_display) + pm_runtime_put_noidle(dev); + else + pm_runtime_put(dev); clk_bulk_disable_unprepare(dart->num_clks, dart->clks); return ret; @@ -3248,15 +5109,43 @@ static void apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); + if (dart->cold_ane) { + dev_warn(&pdev->dev, "cold ANE DART retained until reset\n"); + return; + } + if (dart->managed_sids) { + apple_dart_managed_quarantine(dart); + if (dart->irq >= 0) + apple_dart_free_irq(dart); + iommu_device_unregister(&dart->iommu); + iommu_device_sysfs_remove(&dart->iommu); + return; + } + if (apple_dart_has_borrowed(dart)) { + /* Unbind is suppressed and the module is pinned. If the device + * itself is removed, detach callbacks before devres runs, while + * retaining all state a domain or running firmware can reach. + */ + apple_dart_fail_borrowed(dart, "the DART device is removed"); + if (dart->irq >= 0) + apple_dart_free_irq(dart); + iommu_device_unregister(&dart->iommu); + iommu_device_sysfs_remove(&dart->iommu); + dev_warn(&pdev->dev, "display DART removed; live state retained until hardware reset\n"); + return; + } + /* * Cable removal gates the port clock before this device is removed. * A command issued then never completes. The next probe resets the * block after that clock is running again. */ - if (!dart->locked && !READ_ONCE(dart->commands_gated)) + if (!dart->locked && !dart->retained_mtp && + !READ_ONCE(dart->commands_gated)) apple_dart_hw_reset(dart); - free_irq(dart->irq, dart); + if (dart->irq >= 0) + apple_dart_free_irq(dart); iommu_device_unregister(&dart->iommu); iommu_device_sysfs_remove(&dart->iommu); @@ -3264,6 +5153,21 @@ static void apple_dart_remove(struct platform_device *pdev) clk_bulk_disable_unprepare(dart->num_clks, dart->clks); } +static void apple_dart_shutdown(struct platform_device *pdev) +{ + struct apple_dart *dart = platform_get_drvdata(pdev); + + if (dart->managed_sids) + apple_dart_managed_quarantine(dart); + if (!apple_dart_has_borrowed(dart)) + return; + /* Stop host publication; firmware mappings and backing stay intact + * until the platform reset. The retained kexec guard prevents a new + * kernel from reusing any of this memory without that reset. + */ + apple_dart_fail_borrowed(dart, "shutdown"); +} + static const struct apple_dart_hw apple_dart_hw_t8103 = { .type = DART_T8020, .irq_handler = apple_dart_t8020_irq, @@ -3372,31 +5276,68 @@ static const struct apple_dart_hw apple_dart_hw_t8110 = { .ttbr_count = 1, }; +static void apple_dart_save_state(struct apple_dart *dart) +{ + unsigned int sid, idx; + + for (sid = 0; sid < dart->num_streams; sid++) { + dart->save_tcr[sid] = apple_dart_readl(dart, + DART_TCR(dart, sid)); + for (idx = 0; idx < dart->hw->ttbr_count; idx++) + dart->save_ttbr[sid][idx] = + apple_dart_readl(dart, + DART_TTBR(dart, sid, idx)); + } +} + static __maybe_unused int apple_dart_suspend(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); - unsigned int sid, idx; dart->tunnel_state_restored = false; if (READ_ONCE(dart->commands_gated)) return 0; + /* + * The cold ANE DARTs belong to the ANE backend, which refuses system + * sleep up front while it runs the firmware. Otherwise they hold + * nothing to keep: neither save nor reset them. + */ + if (dart->cold_ane) + return 0; + /* + * An adopted external display DART may be off while its display is: no + * register is read for sleep, and none until its firmware powers it up + * again. + */ + if (dart->dispext) { + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + return -EBUSY; + if (apple_dart_has_borrowed(dart)) + apple_dart_dispext_invalidate(dart, true); + return 0; + } + /* + * Retained radio and display DMA has no suspend boundary of its own. + * Suspend-to-idle leaves it running in place: these DARTs have no + * power domain here, and the DCP keeps the display domains on. The + * registers are saved only for resume to check. + */ + if (dart->managed_sids || dart->preserve_display || apple_dart_has_borrowed(dart)) { + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE) + return -EBUSY; + apple_dart_save_state(dart); + dart->s2idle_kept = true; + return 0; + } /* The tunneled DART has no state to save when its domain stays on. */ - if (dart->power_retained) + if (dart->power_retained || dart->retained_mtp) return 0; /* Locked DARTs can't be restored so skip saving their registers. */ if (dart->locked) return 0; - for (sid = 0; sid < dart->num_streams; sid++) { - dart->save_tcr[sid] = apple_dart_readl(dart, - DART_TCR(dart, sid)); - for (idx = 0; idx < dart->hw->ttbr_count; idx++) - dart->save_ttbr[sid][idx] = - apple_dart_readl(dart, - DART_TTBR(dart, sid, idx)); - } - + apple_dart_save_state(dart); return 0; } @@ -3460,12 +5401,54 @@ static int apple_dart_invalidate_each_stream(struct apple_dart *dart) return 0; } +/* + * Resume of a DART whose live DMA stayed in place through suspend-to-idle. It + * cannot be restored from the saved copy. If the translations changed, the + * consumers' DMA is no longer what they set up: refuse their further use + * through the same paths as a translation fault before failing resume, since + * a resume error does not stop the PM core from resuming them. + */ +static int apple_dart_resume_retained(struct apple_dart *dart) +{ + if (!dart->s2idle_kept) + return -EBUSY; + dart->s2idle_kept = false; + if (apple_dart_state_retained(dart)) + return 0; + + dev_err(dart->dev, "retained DMA state lost in suspend-to-idle\n"); + if (dart->managed_sids) + apple_dart_managed_quarantine(dart); + apple_dart_quarantine_borrowed(dart, "retained DMA state lost in suspend-to-idle"); + return -EIO; +} + static __maybe_unused int apple_dart_resume(struct device *dev) { struct apple_dart *dart = dev_get_drvdata(dev); unsigned int sid, idx; int ret; + /* No reset or restore: see apple_dart_suspend(). */ + if (dart->cold_ane) + return 0; + /* + * An adopted external display DART may be off now, its display with it, + * and nothing confirms otherwise: no register of it is read or written + * (apple_dart_dispext_unconfirmed()) until its firmware powers it up + * again; the state check then (apple_dart_dcp_dispext_state()) runs + * first: kept, it goes on; lost, it is retired, saying so. + */ + if (dart->dispext) { + if (apple_dart_has_borrowed(dart)) { + apple_dart_dispext_invalidate(dart, false); + dev_dbg(dev, "external display DART unconfirmed after sleep\n"); + } + return 0; + } + if (dart->managed_sids || dart->preserve_display || apple_dart_has_borrowed(dart)) + return apple_dart_resume_retained(dart); + /* PCIe-C restored these mappings before starting its downstream link. */ if (dart->tunnel_state_restored) { dart->tunnel_state_restored = false; @@ -3477,7 +5460,7 @@ static __maybe_unused int apple_dart_resume(struct device *dev) * state. Resetting one here is unnecessary; DART access must remain * ordered after PCIe-C port resume. */ - if (dart->power_retained || READ_ONCE(dart->commands_gated)) + if (dart->power_retained || dart->retained_mtp || READ_ONCE(dart->commands_gated)) return 0; /* Locked DARTs can't be restored, and they should not need it */ @@ -3532,11 +5515,119 @@ static const struct of_device_id apple_dart_of_match[] = { { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 }, { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 }, + { .compatible = "apple,t8132-dart", .data = &apple_dart_hw_t8110 }, { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 }, {}, }; MODULE_DEVICE_TABLE(of, apple_dart_of_match); +#if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) +/* + * J713 external DCP: whether an adopted external display DART kept its + * state, its power on (the caller's to know): the TTBR/TCR lock (PROTECT + * and PROTECT_LOCK bit 0) as adopted, and the adopted streams' TCR and TTBR + * as adopted. Register reads only. Kept: no longer unverified after sleep. + * Lost (it is not rebuilt): every borrowed stream is retired, saying why. + * @when names the occasion for the log. + */ +static int apple_dart_dispext_verify(struct apple_dart *dart, const char *when) +{ + bool kept = true, was, current_gen = false; + unsigned long flags; + u32 protect, lock; + u64 gen; + + /* + * A use of its registers of its own, of the generation it starts in: + * not while asleep (between suspend and resume nothing confirms its + * power), and an invalidation meanwhile waits for it to end. + */ + spin_lock_irqsave(&dart->dispext_pm_lock, flags); + if (dart->dispext_sleeping) { + spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + dev_dbg(dart->dev, "external display DART %s during sleep: not checked\n", when); + return -EAGAIN; + } + gen = dart->dispext_gen; + was = dart->dispext_unverified; + dart->dispext_users++; + spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + + protect = readl(dart->regs + DART_T8110_PROTECT); + lock = readl(dart->regs + DART_T8110_PROTECT_LOCK); + for (unsigned int i = 0; i < ARRAY_SIZE(dart->frontend); i++) { + struct apple_dart_dcp *dcp = dart->frontend[i]; + u32 tcr, ttbr; + + if (!dcp) + continue; + tcr = readl(dart->regs + DART_TCR(dart, dcp->sid)); + ttbr = readl(dart->regs + DART_TTBR(dart, dcp->sid, 0)); + kept &= tcr == DART_T8110_TCR_TRANSLATE_ENABLE && ttbr == dcp->ttbr; + dev_dbg(dart->dev, "external display DART %s: SID%u TCR %08x TTBR0 %08x (adopted %08x)\n", + when, dcp->sid, tcr, ttbr, dcp->ttbr); + } + /* The lock as adopted: set then, it must still be set. */ + kept &= (protect & lock & DART_T8110_PROTECT_TTBR_TCR) == + (dart->dispext_protect & dart->dispext_lock & DART_T8110_PROTECT_TTBR_TCR); + + spin_lock_irqsave(&dart->dispext_pm_lock, flags); + /* Only a check of the current generation confirms: a sleep since voids it. */ + current_gen = gen == dart->dispext_gen; + if (kept && current_gen) + dart->dispext_unverified = false; + if (!--dart->dispext_users) + wake_up_all(&dart->dispext_wq); + spin_unlock_irqrestore(&dart->dispext_pm_lock, flags); + + dev_dbg(dart->dev, "external display DART %s: protect %08x lock %08x (adopted %08x/%08x): state %s%s%s\n", + when, protect, lock, dart->dispext_protect, dart->dispext_lock, + kept ? "kept" : "lost", + was ? " (unconfirmed since sleep until now)" : "", + kept && !current_gen ? "; a sleep came meanwhile: not confirmed" : ""); + if (!kept) { + apple_dart_quarantine_borrowed(dart, "state lost (TTBR, TCR or the lock not as adopted)"); + return -EIO; + } + return current_gen ? 0 : -EAGAIN; +} + +/* + * J713 external DCP: after its firmware switched the display DART on again, + * check that the DART kept its state: the TTBR/TCR lock (PROTECT and + * PROTECT_LOCK bit 0) still set, and the adopted streams' TCR and TTBR as + * adopted. Register reads only. A lost state is not rebuilt: the borrowed + * streams are retired. @consumer is the dcpext's piodma node; -ENODEV if + * its DART is not an adopted external display DART. + */ +int apple_dart_dcp_dispext_state(struct device_node *consumer) +{ + struct of_phandle_args spec; + struct platform_device *pdev; + struct apple_dart *dart; + int ret = -ENODEV; + + if (!consumer || + of_parse_phandle_with_args(consumer, "iommus", "#iommu-cells", 0, &spec)) + return -ENODEV; + pdev = of_find_device_by_node(spec.np); + of_node_put(spec.np); + if (!pdev) + return -ENODEV; + /* Only a bound, adopted one: a failed probe leaves no live state. */ + if (!device_is_bound(&pdev->dev) || pdev->dev.driver != &apple_dart_driver.driver) + goto out; + dart = platform_get_drvdata(pdev); + if (!dart || !dart->dispext || !dart->frontend[1]) + goto out; + ret = apple_dart_dispext_verify(dart, "after power-up"); +out: + put_device(&pdev->dev); + return ret; +} +EXPORT_SYMBOL_GPL(apple_dart_dcp_dispext_state); +#endif + static struct platform_driver apple_dart_driver = { .driver = { .name = "apple-dart", @@ -3546,6 +5637,7 @@ static struct platform_driver apple_dart_driver = { }, .probe = apple_dart_probe, .remove = apple_dart_remove, + .shutdown = apple_dart_shutdown, }; module_platform_driver(apple_dart_driver); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index c624d36cd106a..d71802242992f 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1233,9 +1233,9 @@ dma_addr_t iommu_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, * If both the physical buffer start address and size are page aligned, * we don't need to use a bounce page. */ - if (dev_use_swiotlb(dev, size, dir) && + if (!(attrs & DMA_ATTR_MMIO) && dev_use_swiotlb(dev, size, dir) && iova_unaligned(iovad, phys, size)) { - if (attrs & (DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT)) + if (attrs & DMA_ATTR_REQUIRE_COHERENT) return DMA_MAPPING_ERROR; phys = iommu_dma_map_swiotlb(dev, phys, size, dir, attrs); diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 608b617fb817f..ee36789a7f0d8 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -2895,8 +2895,13 @@ static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key } } - if (*ivmode && !strcmp(*ivmode, "lmk")) + if (*ivmode && !strcmp(*ivmode, "lmk")) { + if (crypt_integrity_aead(cc)) { + ti->error = "AEAD transforms not supported for LMK"; + return -EINVAL; + } cc->tfms_count = 64; + } if (*ivmode && !strcmp(*ivmode, "essiv")) { if (!*ivopts) { diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 86fc35e76a0f7..4bda7b5dba1a6 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -2721,7 +2721,7 @@ static void dm_integrity_inline_recheck(struct work_struct *w) outgoing_data = dio->integrity_payload + PAGE_SIZE; while (dio->bio_details.bi_iter.bi_size) { - char digest[HASH_MAX_DIGESTSIZE]; + char digest[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; int r; struct bio_integrity_payload *bip; struct bio_vec bv; @@ -2788,7 +2788,7 @@ static inline bool dm_integrity_check(struct dm_integrity_c *ic, struct dm_integ unsigned pos = 0; while (dio->bio_details.bi_iter.bi_size) { - char digest[HASH_MAX_DIGESTSIZE]; + char digest[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); char *mem = integrity_kmap(ic, bv.bv_page); integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest); diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c index 61af2a437a050..d3430825acad1 100644 --- a/drivers/md/dm-ioctl.c +++ b/drivers/md/dm-ioctl.c @@ -658,10 +658,16 @@ static void *get_result_buffer(struct dm_ioctl *param, size_t param_size, { param->data_start = align_ptr(param + 1) - (void *) param; - if (param->data_start < param_size) + if (param->data_start < param_size) { *len = param_size - param->data_start; - else + } else { + /* + * The buffer can be as small as offsetof(*param, data), which + * is less than sizeof(*param), so don't run off the end of it. + */ *len = 0; + param->data_start = param_size; + } return ((void *) param) + param->data_start; } @@ -1289,6 +1295,7 @@ static int do_resume(struct dm_ioctl *param) /* Do we need to load a new map ? */ if (new_map) { sector_t old_size, new_size; + blk_mode_t new_map_mode; dm_ima_context_table_op(md, ima_context, DM_IMA_TABLE_SAVE); /* Suspend if it isn't already suspended */ @@ -1321,6 +1328,7 @@ static int do_resume(struct dm_ioctl *param) } } + new_map_mode = dm_table_get_mode(new_map); old_size = dm_get_size(md); old_map = dm_swap_table(md, new_map); if (IS_ERR(old_map)) { @@ -1336,7 +1344,7 @@ static int do_resume(struct dm_ioctl *param) if (old_size && new_size && old_size != new_size) need_resize_uevent = true; - if (dm_table_get_mode(new_map) & BLK_OPEN_WRITE) + if (new_map_mode & BLK_OPEN_WRITE) set_disk_ro(dm_disk(md), 0); else set_disk_ro(dm_disk(md), 1); diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index 1489fda9d24a0..02e3cc4730267 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -1294,6 +1294,20 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv) goto bad_store; } + /* + * snapshot-merge requires the optional prepare_merge/commit_merge + * callbacks; the transient exception store does not implement them. + * The handover path checks this in __validate_exception_handover(), + * but a directly constructed snapshot-merge never goes through it. + */ + if (dm_target_is_snapshot_merge(ti) && + (!s->store->type->prepare_merge || + !s->store->type->commit_merge)) { + ti->error = "Snapshot exception store does not support snapshot-merge."; + r = -EINVAL; + goto bad_hash_tables; + } + argv += args_used; argc -= args_used; diff --git a/drivers/media/platform/apple/avd/avd-drv.c b/drivers/media/platform/apple/avd/avd-drv.c index 270ffb6996596..986855879c7bf 100644 --- a/drivers/media/platform/apple/avd/avd-drv.c +++ b/drivers/media/platform/apple/avd/avd-drv.c @@ -23,6 +23,8 @@ #include "avd-inst.h" #include "avd-regs.h" +#define AVD_CODEC_INFO_MAX_SIZE 0x40 + static void calc_tile_meta(u32 w, u32 h, u32 bpb, u32 tile_dim, u32 meta_hdr_bytes, u32 *tile, u32 *meta) { @@ -833,6 +835,7 @@ static int avd_probe(struct platform_device *pdev) { struct avd_dev *avd; const struct of_device_id *match; + const char *codec_info = NULL; int ret, irq; avd = devm_kzalloc(&pdev->dev, sizeof(*avd), GFP_KERNEL); @@ -877,6 +880,30 @@ static int avd_probe(struct platform_device *pdev) return ret; } + /* Determine firmware and HW variant capabilities */ + if (avd->fw->size >= AVD_CODEC_INFO_MAX_SIZE) + codec_info = avd->fw->data + avd->fw->size - AVD_CODEC_INFO_MAX_SIZE; + if (codec_info && strncmp(codec_info, "CODECS:", 7) == 0) { + unsigned int caps = 0; + unsigned int variant_caps = avd->variant->capabilities; + + if (strnstr(codec_info, "av1", AVD_CODEC_INFO_MAX_SIZE)) + caps |= variant_caps & AVD_CAPABILITY_AV1; + if (strnstr(codec_info, "h264", AVD_CODEC_INFO_MAX_SIZE)) + caps |= variant_caps & AVD_CAPABILITY_H264; + if (strnstr(codec_info, "h265", AVD_CODEC_INFO_MAX_SIZE)) + caps |= variant_caps & AVD_CAPABILITY_HEVC; + if (strnstr(codec_info, "vp9", AVD_CODEC_INFO_MAX_SIZE)) + caps |= variant_caps & AVD_CAPABILITY_VP9; + + dev_info(avd->dev, "Firmware codecs: %.27s\n", codec_info + 7); + + avd->capabilities = caps; + } else { + /* Backward compatibility to old firmware, do not upstream. */ + avd->capabilities = avd->variant->capabilities; + } + ret = dma_set_mask_and_coherent(avd->dev, DMA_BIT_MASK((avd->variant->quirks & AVD_QUIRK_LSR) ? 38 : 64)); diff --git a/drivers/media/platform/apple/avd/avd-v4l2.c b/drivers/media/platform/apple/avd/avd-v4l2.c index 9ab137883478a..4622e92055486 100644 --- a/drivers/media/platform/apple/avd/avd-v4l2.c +++ b/drivers/media/platform/apple/avd/avd-v4l2.c @@ -457,7 +457,7 @@ static const struct avd_coded_fmt_desc avd_coded_fmts[] = { static bool avd_is_capable(struct avd_ctx *ctx, unsigned int capability) { - return (ctx->dev->variant->capabilities & capability) == capability; + return (ctx->dev->capabilities & capability) == capability; } static const struct avd_coded_fmt_desc * diff --git a/drivers/media/platform/apple/avd/avd.h b/drivers/media/platform/apple/avd/avd.h index 09a607b6a57a2..fd77a0163e8c5 100644 --- a/drivers/media/platform/apple/avd/avd.h +++ b/drivers/media/platform/apple/avd/avd.h @@ -232,6 +232,7 @@ struct avd_dev { struct reset_control *rstc; const struct avd_variant *variant; + unsigned int capabilities; }; struct avd_segment { diff --git a/drivers/media/platform/apple/isp/Kconfig b/drivers/media/platform/apple/isp/Kconfig index b51d19ee115de..3967e51d472cf 100644 --- a/drivers/media/platform/apple/isp/Kconfig +++ b/drivers/media/platform/apple/isp/Kconfig @@ -8,15 +8,18 @@ config VIDEO_APPLE_ISP select APPLE_PMP_REPORT depends on ARCH_APPLE || COMPILE_TEST depends on OF_ADDRESS + depends on PM depends on V4L_PLATFORM_DRIVERS depends on VIDEO_DEV help Driver for the image signal processor that runs the built-in camera of Apple silicon machines. The ISP firmware is loaded by the bootloader. When a stream starts, the driver loads the - calibration file for the detected camera module, + sensor tuning file for the detected camera model, apple/isp__XX.dat, from the firmware search - path; if it is missing, the stream runs without calibration. + path; if it is missing, the stream runs without sensor tuning. + Invalid files and firmware upload errors fail the stream setup. + These setfiles do not contain per-module factory calibration. config VIDEO_APPLE_ISP_KUNIT_TEST bool "KUnit tests for the Apple ISP driver" if !KUNIT_ALL_TESTS @@ -24,9 +27,10 @@ config VIDEO_APPLE_ISP_KUNIT_TEST depends on KUNIT=y || VIDEO_APPLE_ISP=m default KUNIT_ALL_TESTS help - Builds unit tests for the videobuf2 queue set-up and for the - shared memory the firmware frees into the Apple ISP driver. They - need no hardware. Only useful for kernel developers running KUnit. + Builds unit tests for queue setup, P010 layouts and buffer identity, + firmware sensor geometry, the capture command stream, shared memory + retirement and the per-SoC DART address filter windows. The command + tests stub the IPC transport and need no hardware. Only useful for kernel developers running KUnit. If unsure, say N. diff --git a/drivers/media/platform/apple/isp/isp-cam-test.c b/drivers/media/platform/apple/isp/isp-cam-test.c new file mode 100644 index 0000000000000..191d8ada0d265 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-cam-test.c @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Synthetic geometry checks: no camera firmware or setfile data. */ +#include +#include +#include +#include "isp-ipc.h" + +static void isp_test_sensor_geometry(struct kunit *test) +{ + struct cmd_ch_camera_config config = { + .in_width = 1920, .in_height = 2160, + .out_width = 1920, .out_height = 1080, + .in_width2 = 1920, .in_height2 = 2160, + .out_width2 = 1920, .out_height2 = 1080, + }; + + KUNIT_EXPECT_TRUE(test, isp_sensor_config_usable(&config)); + /* A future larger output is accepted only when both descriptions agree. */ + config.in_width = 4096; + config.in_width2 = 4096; + config.in_height = 3072; + config.in_height2 = 3072; + config.out_width = 4096; + config.out_width2 = 4096; + config.out_height = 3072; + config.out_height2 = 3072; + KUNIT_EXPECT_TRUE(test, isp_sensor_config_usable(&config)); + config.out_height2 = 2160; + KUNIT_EXPECT_FALSE(test, isp_sensor_config_usable(&config)); + config.out_height2 = 3072; + config.in_height2 = 2160; + KUNIT_EXPECT_FALSE(test, isp_sensor_config_usable(&config)); + config.in_height2 = 3072; + config.out_width = 4097; + config.out_width2 = 4097; + KUNIT_EXPECT_FALSE(test, isp_sensor_config_usable(&config)); + config.out_width = 0; + config.out_width2 = 0; + KUNIT_EXPECT_FALSE(test, isp_sensor_config_usable(&config)); + config.out_width = 8192; + config.out_width2 = 8192; + KUNIT_EXPECT_FALSE(test, isp_sensor_config_usable(&config)); +} + +struct isp_test_command { + u32 words[39]; + u32 size; +}; + +/* Derived from the proxy command fixture, with contiguous V4L2 plane sizes. */ +static const struct isp_test_command isp_test_preview_commands[] = { + {{0, 0x24, 0}, 0xc}, + {{0, 0x13b, 0, 1}, 0x10}, + {{0, 0x107, 0, 5}, 0x10}, + {{0, 0x10e, 0, 1}, 0x10}, + {{0, 0x10f, 0}, 0xc}, + {{0, 0x80c, 0, 0, 0, 1280, 720}, 0x1c}, + {{0, 0xb09, 0, 1280, 720, 1, 0x12, 2560, 2560, 0, 0, 720, 0, 1280}, 0x38}, + {{0, 0x208, 0, 7680}, 0x10}, + {{0, 0x20a, 0, 3840}, 0x10}, + {{[1] = 0x117, [3] = 8 << 16, [4] = 0x8000, [5] = 0x8000, [37] = 1}, 0x9c}, + {{[1] = 0x117, [3] = 9 | 2 << 16, [4] = 1843200, [5] = 2560, + [12] = 921600, [13] = 2560, [37] = 2}, 0x9c}, + {{0, 0x125, 0, 1}, 0x10}, + {{0, 0xb0d, 0, 1}, 0x10}, + {{0, 0x138, 0, 0}, 0x10}, +}; + +struct isp_test_command_trace { + unsigned int count; + unsigned int fail_at; + int setfile_error; + unsigned int setfile_calls; + u32 last_opcode; +}; + +static int isp_test_send(struct apple_isp *isp, void *args, u32 insize, + u32 outsize, int timeout) +{ + struct kunit *test = kunit_get_current_test(); + struct isp_test_command_trace *trace = test->priv; + const struct isp_test_command *expected; + u32 expected_outsize; + + if (trace->count >= ARRAY_SIZE(isp_test_preview_commands)) { + KUNIT_FAIL(test, "unexpected extra command"); + return -EIO; + } + expected = &isp_test_preview_commands[trace->count++]; + KUNIT_EXPECT_EQ(test, insize, expected->size); + expected_outsize = expected->words[1] == 0x117 || + (expected->words[1] == 0x24 && !isp_is_p010(isp)) ? insize : 0; + KUNIT_EXPECT_EQ(test, outsize, expected_outsize); + if (insize == expected->size) + KUNIT_EXPECT_MEMEQ(test, args, expected->words, insize); + if (trace->count == trace->fail_at) + return -ETIME; + return 0; +} + +static void isp_test_free_controls(void *ptr) +{ + v4l2_ctrl_handler_free(ptr); +} + +static struct apple_isp *isp_test_capture_alloc(struct kunit *test) +{ + static const struct apple_isp_hw hw = { + .gen = ISP_GEN_T6040, .fw_abi = ISP_FW_ABI_H17, + .scl1 = true, .meta_size = ISP_META_SIZE_T8140, + }; + static struct isp_preset preset = { + .index = 5, .input_dim = {1920, 2160}, + .output_dim = {1280, 720}, .crop_size = {1280, 720}, + }; + struct apple_isp *isp = kunit_kzalloc(test, sizeof(*isp), GFP_KERNEL); + struct isp_format *fmt; + int err; + + KUNIT_ASSERT_NOT_NULL(test, isp); + isp->dev = kunit_device_register(test, "apple-isp-camera-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp->dev); + isp->hw = &hw; + isp->frame_rate = 30; + fmt = isp_get_current_format(isp); + fmt->version = 0x958; + fmt->pixelformat = V4L2_PIX_FMT_P010; + fmt->preset = &preset; + fmt->strides[0] = 2560; + fmt->strides[1] = 2560; + fmt->plane_size[0] = 1843200; + fmt->plane_size[1] = 921600; + v4l2_ctrl_handler_init(&isp->ctrl_handler, 1); + err = kunit_add_action_or_reset(test, isp_test_free_controls, &isp->ctrl_handler); + KUNIT_ASSERT_EQ(test, err, 0); + isp->exposure_priority = + v4l2_ctrl_new_std(&isp->ctrl_handler, NULL, + V4L2_CID_EXPOSURE_AUTO_PRIORITY, 0, 1, 1, 1); + KUNIT_ASSERT_EQ(test, isp->ctrl_handler.error, 0); + kunit_activate_static_stub(test, isp_cmd_send, isp_test_send); + return isp; +} + +static void isp_test_preview_stream(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_test_command_trace trace = {}; + + test->priv = &trace; + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), 0); + KUNIT_EXPECT_EQ(test, trace.count, ARRAY_SIZE(isp_test_preview_commands)); +} + +static void isp_test_preview_command_failures(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_test_command_trace trace; + + test->priv = &trace; + for (unsigned int i = 1; i <= ARRAY_SIZE(isp_test_preview_commands); i++) { + trace = (struct isp_test_command_trace){ .fail_at = i }; + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), -ETIME); + KUNIT_EXPECT_EQ(test, trace.count, i); + } +} + +static int isp_test_load_setfile(struct apple_isp *isp, u32 ch) +{ + struct kunit *test = kunit_get_current_test(); + struct isp_test_command_trace *trace = test->priv; + + trace->setfile_calls++; + return trace->setfile_error; +} + +static void isp_test_setfile_failure_stops_setup(struct kunit *test) +{ + static const int errors[] = { -EINVAL, -EIO, -ETIME, -EINTR, -ENOMEM }; + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_test_command_trace trace; + + isp_get_current_format(isp)->pixelformat = V4L2_PIX_FMT_NV12; + kunit_activate_static_stub(test, isp_ch_load_setfile, isp_test_load_setfile); + test->priv = &trace; + for (unsigned int i = 0; i < ARRAY_SIZE(errors); i++) { + trace = (struct isp_test_command_trace){ .setfile_error = errors[i] }; + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), errors[i]); + /* Only the flicker command preceded the rejected tuning. */ + KUNIT_EXPECT_EQ(test, trace.count, 1); + KUNIT_EXPECT_EQ(test, trace.setfile_calls, 1); + } +} + +/* Count NV12 setup commands and fail the chosen one, without a fixture. */ +static int isp_test_count_send(struct apple_isp *isp, void *args, u32 insize, + u32 outsize, int timeout) +{ + struct kunit *test = kunit_get_current_test(); + struct isp_test_command_trace *trace = test->priv; + + trace->last_opcode = ((const u32 *)args)[1]; + if (++trace->count == trace->fail_at) + return -ETIME; + return 0; +} + +static void isp_test_optional_setfile_continues_setup(struct kunit *test) +{ + static const int results[] = { 0, -ENOENT }; + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_test_command_trace trace; + + isp_get_current_format(isp)->pixelformat = V4L2_PIX_FMT_NV12; + kunit_activate_static_stub(test, isp_ch_load_setfile, isp_test_load_setfile); + kunit_activate_static_stub(test, isp_cmd_send, isp_test_count_send); + test->priv = &trace; + for (unsigned int i = 0; i < ARRAY_SIZE(results); i++) { + trace = (struct isp_test_command_trace){ + .setfile_error = results[i], .fail_at = 2, + }; + /* The mains frequency after tuning, rather than tuning, fails. */ + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), -ETIME); + KUNIT_EXPECT_EQ(test, trace.count, 2); + KUNIT_EXPECT_EQ(test, trace.last_opcode, 0x8212); + KUNIT_EXPECT_EQ(test, trace.setfile_calls, 1); + } +} + +struct isp_test_nv12_trace { + unsigned int count; + u32 opcode[64]; + u32 size[64]; + u32 word[64][3]; +}; + +/* Accept every NV12 setup command and record its opcode and first fields. */ +static int isp_test_record(struct apple_isp *isp, void *args, u32 insize, + u32 outsize, int timeout) +{ + struct kunit *test = kunit_get_current_test(); + struct isp_test_nv12_trace *trace = test->priv; + const u32 *words = args; + unsigned int i = trace->count; + + if (i >= ARRAY_SIZE(trace->opcode)) { + KUNIT_FAIL(test, "unexpected extra command"); + return -EIO; + } + trace->opcode[i] = words[1]; + trace->size[i] = insize; + for (unsigned int j = 0; j < 3 && 12 + 4 * j < insize; j++) + trace->word[i][j] = words[3 + j]; + trace->count++; + return 0; +} + +static int isp_test_find(const struct isp_test_nv12_trace *trace, u32 opcode) +{ + for (unsigned int i = 0; i < trace->count; i++) + if (trace->opcode[i] == opcode) + return i; + return -1; +} + +static int isp_test_no_setfile(struct apple_isp *isp, u32 ch) +{ + return 0; +} + +static struct apple_isp *isp_test_nv12_alloc(struct kunit *test, + enum isp_sensor_id id) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_format *fmt = isp_get_current_format(isp); + + fmt->pixelformat = V4L2_PIX_FMT_NV12; + fmt->id = id; + fmt->version = isp_setfiles[id].version; + isp->temporal_filter = 1; + kunit_activate_static_stub(test, isp_ch_load_setfile, isp_test_no_setfile); + kunit_activate_static_stub(test, isp_cmd_send, isp_test_record); + return isp; +} + +/* Expect exactly @n commands, in this order, from the first of them on. */ +static void isp_test_expect_run(struct kunit *test, + const struct isp_test_nv12_trace *trace, + const u32 (*run)[2], unsigned int n) +{ + int first = isp_test_find(trace, run[0][0]); + + KUNIT_ASSERT_GE(test, first, 0); + KUNIT_ASSERT_LE(test, first + n, trace->count); + for (unsigned int i = 0; i < n; i++) { + KUNIT_EXPECT_EQ(test, trace->opcode[first + i], run[i][0]); + KUNIT_EXPECT_EQ(test, trace->size[first + i], run[i][1]); + } +} + +static void isp_test_temporal_filter_band0(struct kunit *test) +{ + static const u32 run[][2] = { + { 0xc100, 0x18 }, { 0xc120, 0x10 }, { 0xc11c, 0x10 }, + { 0xc11d, 0x14 }, { 0xc102, 0x0c }, { 0xc113, 0x0c }, + }; + struct apple_isp *isp = isp_test_nv12_alloc(test, ISP_IMX958_2224_01); + struct isp_test_nv12_trace *trace; + int i; + + trace = kunit_kzalloc(test, sizeof(*trace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, trace); + test->priv = trace; + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), 0); + isp_test_expect_run(test, trace, run, ARRAY_SIZE(run)); + /* The start record is (1, 0, 0) whatever the DT property says. */ + i = isp_test_find(trace, 0xc100); + KUNIT_EXPECT_EQ(test, trace->word[i][0], 1); + KUNIT_EXPECT_EQ(test, trace->word[i][1], 0); + KUNIT_EXPECT_EQ(test, trace->word[i][2], 0); + i = isp_test_find(trace, 0xc11d); + KUNIT_EXPECT_EQ(test, trace->word[i][0], 0x7800); + KUNIT_EXPECT_EQ(test, trace->word[i][1], 0x6400); + KUNIT_EXPECT_LT(test, isp_test_find(trace, 0xc114), 0); +} + +static void isp_test_temporal_filter_off(struct kunit *test) +{ + static const u32 run[][2] = { + { 0xc114, 0x0c }, { 0xc103, 0x0c }, { 0xc101, 0x0c }, + }; + struct apple_isp *isp = isp_test_nv12_alloc(test, ISP_IMX558_1925_03); + struct isp_test_nv12_trace *trace; + + trace = kunit_kzalloc(test, sizeof(*trace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, trace); + test->priv = trace; + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), 0); + isp_test_expect_run(test, trace, run, ARRAY_SIZE(run)); + KUNIT_EXPECT_LT(test, isp_test_find(trace, 0xc100), 0); + KUNIT_EXPECT_LT(test, isp_test_find(trace, 0xc102), 0); + KUNIT_EXPECT_LT(test, isp_test_find(trace, 0xc113), 0); +} + +static void isp_test_power_line_frequency(struct kunit *test) +{ + struct apple_isp *isp = isp_test_nv12_alloc(test, ISP_IMX958_2224_01); + struct isp_test_nv12_trace *trace; + int i; + + trace = kunit_kzalloc(test, sizeof(*trace), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, trace); + test->priv = trace; + + /* No control: 50 Hz, in Hz 16.16 */ + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), 0); + i = isp_test_find(trace, 0x8212); + KUNIT_ASSERT_GE(test, i, 0); + KUNIT_EXPECT_EQ(test, trace->size[i], 0x10); + KUNIT_EXPECT_EQ(test, trace->word[i][0], 0x00320000); + /* After the tuning, before the sensor configuration is selected */ + KUNIT_EXPECT_LT(test, i, isp_test_find(trace, 0x107)); + + isp->power_line_frequency = + v4l2_ctrl_new_std_menu(&isp->ctrl_handler, NULL, + V4L2_CID_POWER_LINE_FREQUENCY, + V4L2_CID_POWER_LINE_FREQUENCY_60HZ, + BIT(V4L2_CID_POWER_LINE_FREQUENCY_DISABLED), + V4L2_CID_POWER_LINE_FREQUENCY_50HZ); + KUNIT_ASSERT_NOT_NULL(test, isp->power_line_frequency); + KUNIT_ASSERT_EQ(test, v4l2_ctrl_s_ctrl(isp->power_line_frequency, + V4L2_CID_POWER_LINE_FREQUENCY_60HZ), 0); + memset(trace, 0, sizeof(*trace)); + KUNIT_EXPECT_EQ(test, isp_ch_configure_capture(isp, 0), 0); + i = isp_test_find(trace, 0x8212); + KUNIT_ASSERT_GE(test, i, 0); + KUNIT_EXPECT_EQ(test, trace->word[i][0], 0x003c0000); +} + +static int isp_test_config_reply(struct apple_isp *isp, void *args, u32 insize, + u32 outsize, int timeout) +{ + struct kunit *test = kunit_get_current_test(); + u32 *count = test->priv; + struct cmd_ch_camera_config_h17 *reply = isp->cmd_virt; + struct cmd_ch_camera_config *config = &reply->config; + + KUNIT_EXPECT_EQ(test, insize, 0x120); + KUNIT_EXPECT_EQ(test, outsize, 0x120); + memcpy(reply, args, sizeof(*reply)); + KUNIT_EXPECT_EQ(test, config->preset, (*count)++); + config->in_width = 1920; + config->in_height = 2160; + config->out_width = 1920; + config->out_height = 1080; + if (config->preset == 1) { + config->in_width = 4096; + config->in_height = 3072; + config->out_width = 4096; + config->out_height = 3072; + } + config->in_width2 = config->in_width; + config->in_height2 = config->in_height; + config->out_width2 = config->out_width; + config->out_height2 = config->out_height; + if (config->preset == 2) + config->out_height2 = 2160; + reply->unknown[sizeof(reply->unknown) - 1] = 0x5a; + return 0; +} + +static void isp_test_free_configs(void *ptr) +{ + kfree(ptr); +} + +static void isp_test_sensor_config_list(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_preset presets[] = { + { .index = 5, .input_dim = {1920, 2160}, + .output_dim = {1920, 1080}, .crop_size = {1920, 1080} }, + { .index = 5, .input_dim = {1920, 2160}, + .output_dim = {1280, 720}, .crop_size = {1280, 720} }, + }; + u32 count = 0; + int err; + + isp->cmd_virt = kunit_kzalloc(test, 0x120, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp->cmd_virt); + isp->presets = presets; + isp->num_presets = ARRAY_SIZE(presets); + test->priv = &count; + kunit_activate_static_stub(test, isp_cmd_send, isp_test_config_reply); + KUNIT_ASSERT_EQ(test, isp_ch_cache_sensor_configs(isp, 0, 6), 0); + err = kunit_add_action_or_reset(test, isp_test_free_configs, isp->sensor_configs); + KUNIT_ASSERT_EQ(test, err, 0); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(isp->sensor_configs[5].reply + 0xc), 5); + KUNIT_EXPECT_EQ(test, isp->sensor_configs[5].reply[ISP_SENSOR_CONFIG_SIZE - 1], 0x5a); + KUNIT_EXPECT_EQ(test, count, 6); + KUNIT_EXPECT_EQ(test, isp->num_sensor_configs, 6); + KUNIT_EXPECT_EQ(test, isp->num_presets, 3); + KUNIT_EXPECT_EQ(test, isp->presets[0].index, 5); + KUNIT_EXPECT_EQ(test, isp->presets[0].output_dim.y, 1080); + KUNIT_EXPECT_TRUE(test, isp->presets[1].p010); + KUNIT_EXPECT_TRUE(test, isp->presets[2].p010); + KUNIT_EXPECT_EQ(test, isp->presets[2].index, 1); + KUNIT_EXPECT_EQ(test, isp->presets[2].output_dim.x, 4096); + KUNIT_EXPECT_EQ(test, isp->presets[2].crop_size.y, 3072); + KUNIT_EXPECT_FALSE(test, isp->sensor_configs[2].usable); + KUNIT_EXPECT_EQ(test, isp_ch_cache_sensor_configs(isp, 0, 0), -EINVAL); + KUNIT_EXPECT_EQ(test, isp_ch_cache_sensor_configs(isp, 0, 65), -EINVAL); + KUNIT_EXPECT_EQ(test, count, 6); +} + +static int isp_test_bad_config_reply(struct apple_isp *isp, void *args, u32 insize, + u32 outsize, int timeout) +{ + struct cmd_ch_camera_config_h17 *reply = isp->cmd_virt; + + isp_test_config_reply(isp, args, insize, outsize, timeout); + reply->config.preset = 65; + return 0; +} + +static void isp_test_sensor_config_failure(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_preset preset = {}; + u32 count = 0; + + isp->cmd_virt = kunit_kzalloc(test, ISP_SENSOR_CONFIG_SIZE, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp->cmd_virt); + isp->presets = &preset; + isp->num_presets = 1; + test->priv = &count; + kunit_activate_static_stub(test, isp_cmd_send, isp_test_bad_config_reply); + KUNIT_EXPECT_EQ(test, isp_ch_cache_sensor_configs(isp, 0, 6), -EIO); + KUNIT_EXPECT_EQ(test, count, 1); + KUNIT_EXPECT_EQ(test, isp->num_sensor_configs, 0); + KUNIT_EXPECT_PTR_EQ(test, isp->sensor_configs, NULL); + KUNIT_EXPECT_PTR_EQ(test, isp->presets, &preset); +} + +struct isp_test_ack_state { + unsigned int fault, calls; +}; + +static int isp_test_ack_reply(struct apple_isp *isp, struct isp_channel *chan, + unsigned long timeout) +{ + struct kunit *test = kunit_get_current_test(); + struct isp_test_ack_state *state = test->priv; + + state->calls++; + chan->rsp = (struct isp_message){ .arg0 = chan->req.arg0 | ISP_IPC_FLAG_ACK, + .arg1 = chan->req.arg2 }; + switch (state->fault) { + case 1: + chan->rsp.arg2 = 1; + break; + case 2: + put_unaligned_le32(1, isp->cmd_virt); + break; + case 3: + chan->rsp.arg1 += 4; + break; + case 4: + return -ETIME; + } + return 0; +} + +static void isp_test_command_ack_failures(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_test_ack_state state = {}; + + isp->cmd_virt = kunit_kzalloc(test, 0x400, GFP_KERNEL); + isp->chan_io = kunit_kzalloc(test, sizeof(*isp->chan_io), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp->cmd_virt); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_io); + isp->chan_io->name = "IO"; + test->priv = &state; + kunit_deactivate_static_stub(test, isp_cmd_send); + kunit_activate_static_stub(test, ipc_chan_send, isp_test_ack_reply); + KUNIT_EXPECT_EQ(test, isp_cmd_ch_camera_config_select(isp, 0, 5), 0); + for (state.fault = 1; state.fault <= 4; state.fault++) { + unsigned int fault = state.fault; + + state.calls = 0; + KUNIT_EXPECT_EQ(test, isp_cmd_ch_camera_config_select(isp, 0, 5), + fault == 4 ? -ETIME : -EIO); + /* A failed command does not take the next one with it. */ + KUNIT_EXPECT_FALSE(test, isp->ipc_failed); + state.fault = 0; + KUNIT_EXPECT_EQ(test, isp_cmd_ch_camera_config_select(isp, 0, 5), 0); + KUNIT_EXPECT_EQ(test, state.calls, 2); + state.fault = fault; + } + + /* Firmware that never answered gets no further command. */ + isp->ipc_failed = true; + state.calls = 0; + KUNIT_EXPECT_EQ(test, isp_cmd_ch_camera_config_select(isp, 0, 5), -EIO); + KUNIT_EXPECT_EQ(test, state.calls, 0); +} + +static void isp_test_call_crop(struct kunit *test) +{ + struct isp_preset source = { + .index = 7, .input_dim = { 1936, 1096 }, + .output_dim = { 1920, 1080 }, .crop_offset = { 8, 8 }, + .crop_size = { 1920, 1080 }, .p010 = true, + }; + struct isp_preset result; + + KUNIT_ASSERT_TRUE(test, + isp_size_from_crop(&source, (struct coord){ 640, 480 }, &result)); + KUNIT_EXPECT_EQ(test, result.index, 7); + KUNIT_EXPECT_TRUE(test, result.p010); + KUNIT_EXPECT_EQ(test, result.crop_offset.x, 248); + KUNIT_EXPECT_EQ(test, result.crop_offset.y, 8); + KUNIT_EXPECT_EQ(test, result.crop_size.x, 1440); + KUNIT_EXPECT_EQ(test, result.crop_size.y, 1080); + KUNIT_EXPECT_EQ(test, result.output_dim.x, 640); + KUNIT_EXPECT_EQ(test, result.output_dim.y, 480); +} + +static void isp_test_call_crop_bounds(struct kunit *test) +{ + struct isp_preset source = { + .input_dim = { 1920, 2160 }, .output_dim = { 1920, 1080 }, + .crop_size = { 1920, 1080 }, + }; + struct isp_preset result; + static const struct coord rejected[] = { + { 1080, 1920 }, { 1760, 1328 }, { 1552, 1552 }, + { 1922, 1080 }, { 0, 480 }, { 641, 480 }, + }; + + /* Stacked input rows do not make a portrait or a larger square. */ + for (unsigned int i = 0; i < ARRAY_SIZE(rejected); i++) + KUNIT_EXPECT_FALSE(test, isp_size_from_crop(&source, rejected[i], &result)); + source.crop_offset.x = 2; + KUNIT_EXPECT_FALSE(test, + isp_size_from_crop(&source, (struct coord){ 640, 480 }, &result)); + source.crop_offset.x = U32_MAX; + KUNIT_EXPECT_FALSE(test, + isp_size_from_crop(&source, (struct coord){ 640, 480 }, &result)); +} + +static void isp_test_call_size_table(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_preset source = { + .index = 7, .input_dim = { 1920, 1920 }, + .output_dim = { 1920, 1920 }, .crop_size = { 1920, 1920 }, + }; + + isp->presets = &source; + isp->num_presets = 1; + KUNIT_ASSERT_EQ(test, isp_add_call_sizes(isp), 0); + KUNIT_EXPECT_EQ(test, isp->num_presets, ARRAY_SIZE(isp_call_sizes) + 1); + KUNIT_EXPECT_EQ(test, isp->presets[0].output_dim.x, 1920); + KUNIT_EXPECT_EQ(test, isp->presets[0].output_dim.y, 1920); + for (unsigned int i = 0; i < ARRAY_SIZE(isp_call_sizes); i++) { + struct isp_preset *preset = &isp->presets[i + 1]; + + KUNIT_EXPECT_EQ(test, preset->output_dim.x, isp_call_sizes[i].x); + KUNIT_EXPECT_EQ(test, preset->output_dim.y, isp_call_sizes[i].y); + KUNIT_EXPECT_EQ(test, preset->index, 7); + KUNIT_EXPECT_FALSE(test, preset->p010); + KUNIT_EXPECT_EQ(test, preset->crop_size.x * preset->output_dim.y, + preset->crop_size.y * preset->output_dim.x); + KUNIT_EXPECT_EQ(test, (preset->crop_offset.x | preset->crop_offset.y | + preset->crop_size.x | preset->crop_size.y) & 1, 0); + KUNIT_EXPECT_LE(test, preset->crop_offset.x + preset->crop_size.x, 1920); + KUNIT_EXPECT_LE(test, preset->crop_offset.y + preset->crop_size.y, 1920); + } + /* Rebuilding preserves order and never duplicates an existing size. */ + KUNIT_ASSERT_EQ(test, isp_add_call_sizes(isp), 0); + KUNIT_EXPECT_EQ(test, isp->num_presets, ARRAY_SIZE(isp_call_sizes) + 1); +} + +static void isp_test_call_size_capabilities(struct kunit *test) +{ + struct apple_isp *isp = isp_test_capture_alloc(test); + struct isp_preset sources[] = { + { .index = 5, .input_dim = { 1920, 2160 }, + .output_dim = { 1920, 1080 }, .crop_size = { 1920, 1080 }, .p010 = true }, + { .index = 5, .input_dim = { 1920, 2160 }, + .output_dim = { 1280, 720 }, .crop_size = { 1280, 720 }, .p010 = true }, + { .index = 9, .input_dim = { 1536, 3024 }, + .output_dim = { 1536, 3024 }, .crop_size = { 1536, 3024 }, .p010 = true }, + }; + + isp->presets = sources; + isp->num_presets = ARRAY_SIZE(sources); + KUNIT_ASSERT_EQ(test, isp_add_call_sizes(isp), 0); + KUNIT_EXPECT_EQ(test, isp->num_presets, 9); + for (unsigned int i = 0; i < ARRAY_SIZE(sources); i++) + KUNIT_EXPECT_MEMEQ(test, &isp->presets[i], &sources[i], sizeof(sources[i])); + for (unsigned int i = ARRAY_SIZE(sources); i < isp->num_presets; i++) { + struct isp_preset *preset = &isp->presets[i]; + + KUNIT_EXPECT_TRUE(test, preset->p010); + KUNIT_EXPECT_FALSE(test, preset->output_dim.x == 1760 && + preset->output_dim.y == 1328); + KUNIT_EXPECT_FALSE(test, preset->output_dim.x == 1552 && + preset->output_dim.y == 1552); + } + KUNIT_EXPECT_EQ(test, isp->presets[8].output_dim.x, 1536); + KUNIT_EXPECT_EQ(test, isp->presets[8].output_dim.y, 1536); +} + +static struct kunit_case isp_cam_test_cases[] = { + KUNIT_CASE(isp_test_call_crop), + KUNIT_CASE(isp_test_call_crop_bounds), + KUNIT_CASE(isp_test_call_size_table), + KUNIT_CASE(isp_test_call_size_capabilities), + KUNIT_CASE(isp_test_setfile_failure_stops_setup), + KUNIT_CASE(isp_test_optional_setfile_continues_setup), + KUNIT_CASE(isp_test_command_ack_failures), + KUNIT_CASE(isp_test_sensor_geometry), + KUNIT_CASE(isp_test_sensor_config_list), + KUNIT_CASE(isp_test_sensor_config_failure), + KUNIT_CASE(isp_test_preview_stream), + KUNIT_CASE(isp_test_preview_command_failures), + KUNIT_CASE(isp_test_temporal_filter_band0), + KUNIT_CASE(isp_test_temporal_filter_off), + KUNIT_CASE(isp_test_power_line_frequency), + {} +}; + +static struct kunit_suite isp_cam_test_suite = { + .name = "apple-isp-camera", + .test_cases = isp_cam_test_cases, +}; + +kunit_test_suite(isp_cam_test_suite); diff --git a/drivers/media/platform/apple/isp/isp-cam.c b/drivers/media/platform/apple/isp/isp-cam.c index 5090ed2ed5b98..a9480ad236730 100644 --- a/drivers/media/platform/apple/isp/isp-cam.c +++ b/drivers/media/platform/apple/isp/isp-cam.c @@ -1,16 +1,33 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright 2023 Eileen Yoon */ +#include #include +#include +#include +#include +#include #include +#include #include "isp-cam.h" #include "isp-cmd.h" #include "isp-fw.h" #include "isp-iommu.h" +#include "isp-ipc.h" #include "isp-regs.h" -#define ISP_MAX_PRESETS 32 +/* + * How the ISP's own temporal noise filter is set up for a sensor. macOS + * picks this per sensor on the H17 interface: the IMX558 (J700) runs + * without it and the IMX958 (J616s) runs it with explicit strength + * controls. Everything else keeps the original start sequence. + */ +enum isp_temporal_filter { + ISP_TEMPORAL_FILTER_START, + ISP_TEMPORAL_FILTER_OFF, + ISP_TEMPORAL_FILTER_BAND0, +}; struct isp_setfile { u32 version; @@ -19,6 +36,7 @@ struct isp_setfile { size_t size; /* bytes present in the file if fewer than size; the rest is zero */ size_t file_size; + enum isp_temporal_filter temporal_filter; }; // clang-format off @@ -48,7 +66,8 @@ static const struct isp_setfile isp_setfiles[] = { [ISP_IMX514_2820_04] = {0x514, 0x28200405, "apple/isp_2820_04XX.dat", 0xa198}, [ISP_IMX558_1921_01] = {0x558, 0x19210106, "apple/isp_1921_01XX.dat", 0xad40}, [ISP_IMX558_1922_02] = {0x558, 0x19220201, "apple/isp_1922_02XX.dat", 0xad40}, - [ISP_IMX558_1925_03] = {0x558, 0x19250306, "apple/isp_1925_03XX.dat", 0xcbc0, 0xcba6}, + [ISP_IMX558_1925_03] = {0x558, 0x19250306, "apple/isp_1925_03XX.dat", 0xcbc0, 0xcba6, + ISP_TEMPORAL_FILTER_OFF}, [ISP_IMX603_7920_01] = {0x603, 0x79200109, "apple/isp_7920_01XX.dat", 0xad2c}, [ISP_IMX603_7920_02] = {0x603, 0x79200205, "apple/isp_7920_02XX.dat", 0xad2c}, [ISP_IMX603_7921_01] = {0x603, 0x79210104, "apple/isp_7921_01XX.dat", 0xad90}, @@ -70,6 +89,10 @@ static const struct isp_setfile isp_setfiles[] = { [ISP_IMX814_2123_01] = {0x814, 0x21230101, "apple/isp_2123_01XX.dat", 0xed54}, [ISP_IMX853_7622_01] = {0x853, 0x76220112, "apple/isp_7622_01XX.dat", 0x247f8}, [ISP_IMX913_7523_01] = {0x913, 0x75230107, "apple/isp_7523_01XX.dat", 0x247f8}, + [ISP_IMX958_2224_01] = {0x958, 0x22240106, "apple/isp_2224_01XX.dat", 0x6ff4, 0, + ISP_TEMPORAL_FILTER_BAND0}, + [ISP_IMX958_2226_02] = {0x958, 0x22260201, "apple/isp_2226_02XX.dat", 0x6ff4, 0, + ISP_TEMPORAL_FILTER_BAND0}, [ISP_VD56G0_6221_01] = {0xd56, 0x62210102, "apple/isp_6221_01XX.dat", 0x1b80}, [ISP_VD56G0_6222_01] = {0xd56, 0x62220102, "apple/isp_6222_01XX.dat", 0x1b80}, }; @@ -158,6 +181,13 @@ static int isp_ch_get_sensor_id(struct apple_isp *isp, u32 ch) case 0x913: id = ISP_IMX913_7523_01; break; + case 0x958: + /* + * Section 0x50f of 2224 and 2226 names sensor 0x958; 2924's + * names 0x614. + */ + id = isp->hw->imx958_2226 ? ISP_IMX958_2226_02 : ISP_IMX958_2224_01; + break; case 0xd56: id = ISP_VD56G0_6221_01; break; @@ -188,6 +218,23 @@ static int isp_check_presets(struct apple_isp *isp, u32 num_fw_presets) if (!num_fw_presets) return 0; + /* + * A deferred start detects the camera after the presets were + * advertised: keep them, and refuse a stream on one the firmware + * lacks instead. + */ + if (isp->hw->deferred_start) { + isp->num_fw_presets = num_fw_presets; + for (int i = 0; i < isp->num_presets; i++) + if (isp->presets[i].index >= num_fw_presets) + dev_warn(isp->dev, + "%ux%u preset: config %u, firmware has %u\n", + isp->presets[i].output_dim.x, + isp->presets[i].output_dim.y, + isp->presets[i].index, num_fw_presets); + return 0; + } + for (int i = 0; i < isp->num_presets; i++) { struct isp_preset *preset = &isp->presets[i]; @@ -212,6 +259,225 @@ static int isp_check_presets(struct apple_isp *isp, u32 num_fw_presets) return 0; } +/* Do not mistake stacked sensor input rows for a processed output mode. */ +static bool isp_sensor_config_usable(const struct cmd_ch_camera_config *config) +{ + return config->in_width && config->in_height && + config->out_width && config->out_height && + config->out_width <= 16384 && config->out_height <= 16384 && + !(config->out_width & 1) && !(config->out_height & 1) && + config->out_width <= config->in_width && + config->out_height <= config->in_height && + config->in_width == config->in_width2 && + config->in_height == config->in_height2 && + config->out_width == config->out_width2 && + config->out_height == config->out_height2; +} + +static int isp_ch_cache_sensor_configs(struct apple_isp *isp, u32 ch, u32 count) +{ + struct cmd_ch_camera_config_h17 args; + struct isp_sensor_config *configs; + struct isp_preset *presets; + unsigned int num = isp->num_presets; + int err; + + /* The extended reply is established only for the IMX958 T6040 path. */ + if (!isp_supports_p010(isp)) + return 0; + if (!count || count > ISP_MAX_SENSOR_CONFIGS) + return -EINVAL; + + presets = devm_kcalloc(isp->dev, num + count, sizeof(*presets), GFP_KERNEL); + if (!presets) + return -ENOMEM; + memcpy(presets, isp->presets, num * sizeof(*presets)); + configs = kcalloc(count, sizeof(*configs), GFP_KERNEL); + if (!configs) + return -ENOMEM; + + for (u32 i = 0; i < count; i++) { + struct cmd_ch_camera_config *config = &args.config; + struct isp_sensor_config *cached = &configs[i]; + bool duplicate = false; + + err = isp_cmd_ch_camera_config_get(isp, ch, i, &args); + if (err) + goto free_configs; + if (config->opcode != (u64)CISP_CMD_CH_CAMERA_CONFIG_GET << 32 || + config->chan != ch || config->preset != i) { + err = -EIO; + goto free_configs; + } + + memcpy(cached->reply, &args, sizeof(args)); + cached->input_dim = (struct coord){config->in_width, config->in_height}; + cached->output_dim = (struct coord){config->out_width, config->out_height}; + cached->sensor_clk = config->sensor_clk; + cached->usable = isp_sensor_config_usable(config); + dev_dbg(isp->dev, "sensor config %u: input %ux%u output %ux%u clock %u%s\n", + i, config->in_width, config->in_height, + config->out_width, config->out_height, config->sensor_clk, + cached->usable ? "" : " (unverified geometry)"); + print_hex_dump_debug("apple-isp config: ", DUMP_PREFIX_OFFSET, + 16, 4, &args, sizeof(args), false); + /* Both processed preview sizes are recorded in the proxy source. */ + for (unsigned int j = 0; j < num; j++) { + struct isp_preset *preset = &presets[j]; + bool preview = i == 5 && config->in_width == 1920 && + config->in_height == 2160 && + ((preset->output_dim.x == 1280 && + preset->output_dim.y == 720) || + (preset->output_dim.x == 1920 && + preset->output_dim.y == 1080)); + bool reported = cached->usable && + preset->output_dim.x == config->out_width && + preset->output_dim.y == config->out_height; + + if (preset->index == i && + preset->input_dim.x == config->in_width && + preset->input_dim.y == config->in_height && + !preset->crop_offset.x && !preset->crop_offset.y && + /* Same aspect ratio, never enlarged: the scaler may shrink it. */ + preset->crop_size.x >= preset->output_dim.x && + preset->crop_size.x * preset->output_dim.y == + preset->crop_size.y * preset->output_dim.x) + preset->p010 = preview || reported; + } + if (!cached->usable) + continue; + + /* Keep the board's config-5 presets, including the default, first. */ + for (unsigned int j = 0; j < num; j++) { + if (presets[j].output_dim.x == config->out_width && + presets[j].output_dim.y == config->out_height) { + duplicate = true; + break; + } + } + if (duplicate) + continue; + + presets[num++] = (struct isp_preset){ + .index = i, + .input_dim = cached->input_dim, + .output_dim = cached->output_dim, + .crop_size = cached->output_dim, + .p010 = true, + }; + } + isp->sensor_configs = configs; + isp->num_sensor_configs = count; + isp->presets = presets; + isp->num_presets = num; + return 0; + +free_configs: + kfree(configs); + return err; +} + +/* Common call sizes, offered only when a known processed crop contains them. */ +static const struct coord isp_call_sizes[] = { + { 640, 480 }, { 1280, 720 }, { 1920, 1080 }, { 1080, 1920 }, + { 1760, 1328 }, { 1328, 1760 }, { 1552, 1552 }, + { 1280, 960 }, { 960, 1280 }, +}; + +/* + * CH_CROP[_SCL1]_SET takes x/y/width/height independently of the output + * dimensions in CH_OUTPUT_CONFIG[_SCL1]_SET. Stay within an established + * processed crop, rather than the sensor input (which may contain stacked + * rows). Use the largest exact-aspect crop, centred on the chroma grid, + * with no upscaling. Neither command reports a scaler limit or rotation. + */ +static bool isp_size_from_crop(const struct isp_preset *source, + struct coord size, struct isp_preset *preset) +{ + u32 divisor, x, y, scale; + + if (!size.x || !size.y || (size.x | size.y) & 1 || + !source->crop_size.x || !source->crop_size.y || + (source->crop_offset.x | source->crop_offset.y) & 1 || + source->crop_size.x > 16384 || source->crop_size.y > 16384 || + source->crop_offset.x > source->input_dim.x || + source->crop_size.x > source->input_dim.x - source->crop_offset.x || + source->crop_offset.y > source->input_dim.y || + source->crop_size.y > source->input_dim.y - source->crop_offset.y) + return false; + + divisor = gcd(size.x, size.y); + x = size.x / divisor; + y = size.y / divisor; + scale = round_down(min(source->crop_size.x / x, + source->crop_size.y / y), 2); + if (scale < divisor) + return false; + + *preset = *source; + preset->output_dim = size; + preset->crop_size = (struct coord){ x * scale, y * scale }; + preset->crop_offset.x += round_down((source->crop_size.x - x * scale) / 2, 2); + preset->crop_offset.y += round_down((source->crop_size.y - y * scale) / 2, 2); + return true; +} + +static unsigned int isp_append_size(struct isp_preset *presets, unsigned int num, + unsigned int sources, struct coord size) +{ + struct isp_preset candidate, best; + u32 area, best_area = 0; + + for (unsigned int i = 0; i < num; i++) + if (presets[i].output_dim.x == size.x && + presets[i].output_dim.y == size.y) + return num; + + /* Never use a generated size as evidence for another generated size. */ + for (unsigned int i = 0; i < sources; i++) { + if (!isp_size_from_crop(&presets[i], size, &candidate)) + continue; + area = candidate.crop_size.x * candidate.crop_size.y; + if (area > best_area) { + best = candidate; + best_area = area; + } + } + if (best_area) + presets[num++] = best; + return num; +} + +static int isp_add_call_sizes(struct apple_isp *isp) +{ + unsigned int sources = isp->num_presets, num = sources; + struct isp_preset *presets; + struct isp_preset candidate; + u32 square = 0; + + presets = devm_kcalloc(isp->dev, sources + ARRAY_SIZE(isp_call_sizes) + 1, + sizeof(*presets), GFP_KERNEL); + if (!presets) + return -ENOMEM; + memcpy(presets, isp->presets, sources * sizeof(*presets)); + + for (unsigned int i = 0; i < ARRAY_SIZE(isp_call_sizes); i++) + num = isp_append_size(presets, num, sources, isp_call_sizes[i]); + + /* A smaller sensor can still offer a square without enlarging pixels. */ + for (unsigned int i = 0; i < sources; i++) { + u32 side = round_down(min(presets[i].crop_size.x, + presets[i].crop_size.y), 2); + + if (isp_size_from_crop(&presets[i], (struct coord){ side, side }, &candidate)) + square = max(square, side); + } + num = isp_append_size(presets, num, sources, (struct coord){ square, square }); + isp->presets = presets; + isp->num_presets = num; + return 0; +} + static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); @@ -228,10 +494,14 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) dev_dbg(isp->dev, "found sensor %x on ch %d\n", args->version, ch); - /* The metadata buffers are allocated for the sizes the driver knows. */ + /* + * The metadata buffers are allocated for the sizes the driver knows; + * without a capture metadata pool its size is not used. + */ if (isp->hw->fw_abi == ISP_FW_ABI_H17 && (args->unk_68 != isp->hw->meta_size || - args->unk_78 != isp->hw->capture_meta_size)) { + (isp->hw->capture_meta_size && + args->unk_78 != isp->hw->capture_meta_size))) { dev_err(isp->dev, "metadata sizes 0x%x/0x%x, expected 0x%x/0x%x\n", args->unk_68, args->unk_78, isp->hw->meta_size, @@ -245,7 +515,8 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) err = isp_ch_get_sensor_id(isp, ch); if (err || (fmt->id != ISP_IMX248_1820_01 && fmt->id != ISP_IMX558_1921_01 && - fmt->id != ISP_IMX558_1925_03 && fmt->id != ISP_IMX364_8720_01)) { + fmt->id != ISP_IMX558_1925_03 && fmt->id != ISP_IMX364_8720_01 && + fmt->id != ISP_IMX958_2224_01 && fmt->id != ISP_IMX958_2226_02)) { dev_err(isp->dev, "ch %d: unsupported sensor. Please file a bug report with hardware info & dmesg trace.\n", ch); @@ -254,6 +525,10 @@ static int isp_ch_cache_sensor_info(struct apple_isp *isp, u32 ch) } err = isp_check_presets(isp, args->num_presets); + if (!err) + err = isp_ch_cache_sensor_configs(isp, ch, args->num_presets); + if (!err) + err = isp_add_call_sizes(isp); exit: kfree(args); @@ -311,8 +586,12 @@ int apple_isp_detect_camera(struct apple_isp *isp) dev_err(isp->dev, "failed to boot firmware for initial sensor detection: %d\n", err); - /* Resident firmware would fail a deferred retry the same way. */ - return isp->hw->resident_fw ? err : -EPROBE_DEFER; + /* + * Resident firmware would fail a deferred retry the same way, + * and a deferred start is not a probe to defer. + */ + return isp->hw->resident_fw || isp->hw->deferred_start ? + err : -EPROBE_DEFER; } err = isp_detect_camera(isp); @@ -321,6 +600,8 @@ int apple_isp_detect_camera(struct apple_isp *isp) if (isp->hw->resident_fw) { if (err) apple_isp_firmware_halt(isp); + else + isp->camera_detected = true; return err; } @@ -331,9 +612,153 @@ int apple_isp_detect_camera(struct apple_isp *isp) apple_isp_firmware_shutdown(isp); + if (!err) + isp->camera_detected = true; return err; } +/* + * A setfile is a chain of { le32 size, le32 0, le32 tag, payload } sections + * from 0x14 on, size counting what follows it. The lens shading sections + * hold 18-byte table records after a 10-byte header: { ?, 0x20, 0x20, + * offset lo, offset hi, 0, 0, width, height }. The xx24 setfiles keep the + * row stride (2 * width) in "offset hi"; the H17 firmware reads it as the + * offset's high half ("baseHi") and faults far past the file. Their xx25 + * twins are the same tables with it zeroed, as macOS hands them over. + */ +static void isp_setfile_fixup_h17(void *data, size_t len) +{ + size_t off = 0x14; + + while (off + 12 <= len) { + u32 size = get_unaligned_le32(data + off); + size_t end = off + 4 + size; + size_t rec; + int i; + + if (size < 8 || end > len) + break; + + switch (get_unaligned_le32(data + off + 8)) { + case 0xc003: + case 0xc01b: + case 0xc01c: + case 0xc01f: + case 0xc021: + rec = off + 12 + 10; + for (i = 0; i < 16 && rec + 18 <= end; i++, rec += 18) { + if (get_unaligned_le16(data + rec + 2) != 0x20 || + get_unaligned_le16(data + rec + 4) != 0x20) + break; + if (get_unaligned_le16(data + rec + 8) == + 2 * get_unaligned_le16(data + rec + 14)) + put_unaligned_le16(0, data + rec + 8); + } + break; + } + off = end; + } +} + +/* + * The noise reduction radial gains (setfile record 0xc404) from the lens + * shading table (0xc003), as macOS's appleh16camerad computes them before + * loading the setfile: per radius bin, 4096 over the mean shading gain of + * the four Bayer planes along the rays to the four image corners. + */ +static u8 *isp_setfile_record(u8 *data, size_t len, u32 tag, size_t *size) +{ + size_t off = 0x14; + + while (off + 12 <= len) { + u32 sz = get_unaligned_le32(data + off); + + if (get_unaligned_le32(data + off + 4) || sz < 8 || off + 4 + sz > len) + return NULL; + if (get_unaligned_le32(data + off + 8) == tag) { + *size = sz; + return data + off + 4; + } + off += 4 + sz; + } + return NULL; +} + +#define LSC_PLANES 4 +#define LSC_TABLE 0x5c +#define RG_BINS 257 + +static u32 isp_lsc_gain(const u8 *lsc, size_t lsc_size, int plane, s64 x, s64 y) +{ + const u8 *h = lsc + 0x12 + plane * 0x12; + u32 sx = get_unaligned_le16(h + 2), sy = get_unaligned_le16(h + 4); + u32 base = get_unaligned_le32(h + 6); + u32 gx = get_unaligned_le16(h + 14), gy = get_unaligned_le16(h + 16); + s64 fx, fy, ax, ay; + s64 ix, iy; + u32 t00, t10, t01, t11; + size_t i00; + + /* x and y in 1/256 half-resolution pixels; fx, fy in 1/256 cells */ + fx = div_s64(x, sx); + fy = div_s64(y, sy); + ix = clamp_t(s64, fx >> 8, 0, gx - 2); + iy = clamp_t(s64, fy >> 8, 0, gy - 2); + ax = clamp_t(s64, fx - (ix << 8), 0, 256); + ay = clamp_t(s64, fy - (iy << 8), 0, 256); + i00 = LSC_TABLE + 2 * (base + iy * gx + ix); + if (i00 + 2 * (gx + 2) > lsc_size + 4) + return 4096; + t00 = get_unaligned_le16(lsc + i00); + t10 = get_unaligned_le16(lsc + i00 + 2); + t01 = get_unaligned_le16(lsc + i00 + 2 * gx); + t11 = get_unaligned_le16(lsc + i00 + 2 * gx + 2); + return ((t00 * (256 - ax) + t10 * ax) * (256 - ay) + + (t01 * (256 - ax) + t11 * ax) * ay) >> 16; +} + +static void isp_setfile_radial_gain(u8 *data, size_t len) +{ + size_t lsc_size, rg_size; + u8 *lsc = isp_setfile_record(data, len, 0xc003, &lsc_size); + u8 *rg = isp_setfile_record(data, len, 0xc404, &rg_size); + s64 cx, cy, dx[4], dy[4], dist[4], rmax = 0; + u32 w, h, shift; + int i, c, p; + + if (!lsc || !rg || lsc_size < LSC_TABLE || rg_size < 0xe + 2 * RG_BINS - 4) + return; + w = get_unaligned_le16(lsc + 0xe); + h = get_unaligned_le16(lsc + 0x10); + if (!w || !h) + return; + + /* full-resolution pixels in 1/256; the optical centre sits 156 rows up */ + cx = (s64)w << 7; + cy = ((s64)h << 7) - (156 << 8); + for (c = 0; c < 4; c++) { + dx[c] = ((c & 1) ? (s64)w << 8 : 0) - cx; + dy[c] = ((c & 2) ? (s64)h << 8 : 0) - cy; + dist[c] = int_sqrt64(dx[c] * dx[c] + dy[c] * dy[c]); + rmax = max(rmax, dist[c]); + } + shift = order_base_2(DIV_ROUND_UP(rmax, 256 << 8)); + put_unaligned_le16(shift, rg + 0xc); + + for (i = 0; i < RG_BINS; i++) { + s64 t = min_t(s64, (s64)i << (shift + 8), rmax); + u64 sum = 0; + + for (p = 0; p < LSC_PLANES; p++) + for (c = 0; c < 4; c++) + sum += isp_lsc_gain(lsc, lsc_size, p, + (cx + div64_s64(t * dx[c], dist[c])) / 2, + (cy + div64_s64(t * dy[c], dist[c])) / 2); + put_unaligned_le16(min_t(u64, div64_u64(16ULL * 4096 * 4096, sum), 0xffff), + rg + 0xe + 2 * i); + } +} + static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); @@ -343,14 +768,20 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) u32 magic; int err; + KUNIT_STATIC_STUB_REDIRECT(isp_ch_load_setfile, isp, ch); + if (WARN_ON_ONCE(setfile->size > isp->data_surf->size)) return -ENOSPC; err = firmware_request_nowarn(&fw, setfile->path, isp->dev); if (err) { - dev_warn_once(isp->dev, - "setfile '%s' not loaded (%d), streaming without calibration\n", - setfile->path, err); + if (err == -ENOENT) + dev_warn_once(isp->dev, + "setfile '%s' missing, streaming without sensor tuning\n", + setfile->path); + else + dev_err(isp->dev, "failed to read setfile '%s': %d\n", + setfile->path, err); return err; } @@ -372,9 +803,29 @@ static int isp_ch_load_setfile(struct apple_isp *isp, u32 ch) memcpy(isp->data_surf->virt, fw->data, len); memset(isp->data_surf->virt + len, 0, setfile->size - len); release_firmware(fw); + /* Seen on T6040 only; T8140 runs its setfiles as they are. */ + if (isp->hw->gen == ISP_GEN_T6040 || isp->hw->setfile_stride_fixup) + isp_setfile_fixup_h17(isp->data_surf->virt, len); + if (isp->hw->setfile_radial_gain) + isp_setfile_radial_gain(isp->data_surf->virt, len); + + if (isp->hw->lsc_ideal_percent) { + err = isp_cmd_ch_lsc_ideal_percent_set(isp, ch, + isp->hw->lsc_ideal_percent); + if (err) + return err; + } - return isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, - setfile->size); + err = isp_cmd_ch_set_file_load(isp, ch, isp->data_surf->iova, + setfile->size); + if (err) + dev_err(isp->dev, "ch %u: setfile '%s' load failed: %d\n", + ch, setfile->path, err); + else + dev_dbg(isp->dev, "ch %u: loaded sensor tuning '%s' (0x%zx bytes)\n", + ch, setfile->path, setfile->size); + + return err; } static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) @@ -397,26 +848,103 @@ static int isp_ch_configure_frame_rate(struct apple_isp *isp, u32 ch) return isp_cmd_ch_ae_frame_rate_min_set(isp, ch, min_rate); } -static int isp_ch_configure_pools(struct apple_isp *isp, u32 ch) +/* + * Tell auto exposure the mains frequency, so that it picks exposure times + * that do not beat with lights on that supply. The firmware takes it in Hz, + * 16.16 fixed point. macOS sends it on the H17 interface for every camera. + */ +static int isp_ch_configure_power_line(struct apple_isp *isp, u32 ch) +{ + u32 hz = 50; + + if (isp->hw->fw_abi != ISP_FW_ABI_H17) + return 0; + + if (isp->power_line_frequency && + v4l2_ctrl_g_ctrl(isp->power_line_frequency) == + V4L2_CID_POWER_LINE_FREQUENCY_60HZ) + hz = 60; + + return isp_cmd_apple_ch_ae_flicker_freq_update_current_set(isp, ch, + hz << 16); +} + +static int isp_ch_configure_temporal_filter(struct apple_isp *isp, u32 ch) { struct isp_format *fmt = isp_get_format(isp, ch); int err; - err = isp_cmd_ch_buffer_pool_config_set(isp, ch, CISP_POOL_TYPE_META); + switch (isp_setfiles[fmt->id].temporal_filter) { + case ISP_TEMPORAL_FILTER_OFF: + /* The firmware keeps its state between streams; stop it. */ + err = isp_cmd_apple_ch_temporal_filter_disable(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_motion_history_stop(isp, ch); + if (err) + return err; + + return isp_cmd_apple_ch_temporal_filter_stop(isp, ch); + + case ISP_TEMPORAL_FILTER_BAND0: + err = isp_cmd_apple_ch_tnr_start_h17(isp, ch); + if (err) + return err; + + err = isp_cmd_apple_ch_tnr_high_strength_set(isp, ch, 0); + if (err) + return err; + + err = isp_cmd_apple_ch_tnr_band0_mod_set(isp, ch, 0); + if (err) + return err; + + /* 120 and 100, in 1/256 steps */ + err = isp_cmd_apple_ch_tnr_band0_threshold_set(isp, ch, 120 << 8, + 100 << 8); + if (err) + return err; + break; + + default: + err = isp_cmd_apple_ch_temporal_filter_start(isp, ch, + isp->temporal_filter); + if (err) + return err; + break; + } + + err = isp_cmd_apple_ch_motion_history_start(isp, ch); if (err) return err; - err = isp_cmd_ch_buffer_pool_config_set(isp, ch, - CISP_POOL_TYPE_META_CAPTURE); + return isp_cmd_apple_ch_temporal_filter_enable(isp, ch); +} + +static int isp_ch_configure_pools(struct apple_isp *isp, u32 ch) +{ + struct isp_format *fmt = isp_get_format(isp, ch); + int err; + + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, CISP_POOL_TYPE_META); if (err) return err; + /* H17 firmware without capture metadata has no such pool. */ + if (isp->hw->fw_abi != ISP_FW_ABI_H17 || isp->hw->capture_meta_size) { + err = isp_cmd_ch_buffer_pool_config_set(isp, ch, + CISP_POOL_TYPE_META_CAPTURE); + if (err) + return err; + } + /* The H17 firmware also takes the geometry of the output buffers. */ if (isp->hw->fw_abi != ISP_FW_ABI_H17) return 0; return isp_cmd_ch_buffer_pool_config_set_rendered(isp, ch, - ISP_MAX_BUFFERS, + isp_num_rendered(isp), fmt->plane_size[0], fmt->strides[0], fmt->plane_size[1], @@ -429,14 +957,21 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) bool h17 = isp->hw->fw_abi == ISP_FW_ABI_H17; int err; - isp_cmd_flicker_sensor_set(isp, 0); + err = isp_cmd_flicker_sensor_set(isp, 0); + if (isp_is_p010(isp) && err) + return err; + + /* The proxy-qualified P010 route uses the firmware defaults. */ + if (isp_is_p010(isp)) + goto select_config; /* - * The setfile isn't requisite but then we don't get calibration. If - * loading it was interrupted by a signal, give up on the stream. + * Missing sensor tuning is optional. A present but invalid setfile or + * a failed firmware upload must not silently select firmware defaults. + * This model-wide tuning is separate from per-module factory data. */ err = isp_ch_load_setfile(isp, ch); - if (err == -EINTR) + if (err && err != -ENOENT) return err; if (isp->hw->lpdp) { @@ -445,6 +980,11 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) return err; } + err = isp_ch_configure_power_line(isp, ch); + if (err) + return err; + +select_config: err = isp_cmd_ch_sbs_enable(isp, ch, 1); if (err) return err; @@ -472,7 +1012,8 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) err = isp_cmd_ch_output_config_set(isp, ch, fmt->preset->output_dim.x, fmt->preset->output_dim.y, fmt->strides, CISP_COLORSPACE_REC709, - CISP_OUTPUT_FORMAT_YUV_2PLANE); + isp_is_p010(isp) ? CISP_OUTPUT_FORMAT_P010 : + CISP_OUTPUT_FORMAT_YUV_2PLANE); if (err) return err; @@ -500,6 +1041,9 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) if (err) return err; + if (isp_is_p010(isp)) + return isp_cmd_ch_master_slave_sync_mode_set(isp, ch, 0); + err = isp_cmd_ch_cnr_start(isp, ch); if (err) return err; @@ -508,8 +1052,8 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) if (err) return err; - /* The H17 firmware faults on this face-detection metering setup. */ - if (!h17) { + /* Most H17 firmware faults on this face-detection metering setup. */ + if (!h17 || isp->hw->ae_fd_scene_metering) { err = isp_cmd_apple_ch_ae_fd_scene_metering_config_set(isp, ch); if (err) @@ -538,15 +1082,7 @@ static int isp_ch_configure_capture(struct apple_isp *isp, u32 ch) return err; } - err = isp_cmd_apple_ch_temporal_filter_start(isp, ch, isp->temporal_filter); - if (err) - return err; - - err = isp_cmd_apple_ch_motion_history_start(isp, ch); - if (err) - return err; - - err = isp_cmd_apple_ch_temporal_filter_enable(isp, ch); + err = isp_ch_configure_temporal_filter(isp, ch); if (err) return err; @@ -563,18 +1099,38 @@ static int isp_configure_capture(struct apple_isp *isp) int apple_isp_start_camera(struct apple_isp *isp) { + struct isp_format *fmt = isp_get_current_format(isp); int err; + /* + * A deferred start detects the camera now. Resident firmware is then + * left running; firmware that starts again is stopped after the + * detection like at probe, and booted again here. + */ + if (isp->hw->deferred_start && !isp->camera_detected) { + err = apple_isp_detect_camera(isp); + if (err < 0) { + dev_err(isp->dev, "failed to detect the camera: %d\n", err); + return err; + } + } err = apple_isp_firmware_boot(isp); if (err < 0) { dev_err(isp->dev, "failed to boot firmware: %d\n", err); return err; } + if (isp->num_fw_presets && fmt->preset->index >= isp->num_fw_presets) { + dev_err(isp->dev, "preset config %u: firmware has %u\n", + fmt->preset->index, isp->num_fw_presets); + apple_isp_firmware_shutdown(isp); + return -EINVAL; + } + err = isp_configure_capture(isp); if (err) { dev_err(isp->dev, "failed to configure capture: %d\n", err); - apple_isp_firmware_shutdown(isp); + apple_isp_stop_camera(isp); return err; } @@ -583,7 +1139,15 @@ int apple_isp_start_camera(struct apple_isp *isp) void apple_isp_stop_camera(struct apple_isp *isp) { - apple_isp_firmware_shutdown(isp); + /* + * A timed-out request is resolved before anything the firmware may + * still use for it is released. Firmware that does not answer even + * late is stopped first. + */ + if (isp_ipc_resync(isp)) + apple_isp_firmware_halt(isp); + else + apple_isp_firmware_shutdown(isp); } /* @@ -607,6 +1171,8 @@ static enum hrtimer_restart apple_isp_wdt_timer(struct hrtimer *timer) struct apple_isp *isp = container_of(timer, struct apple_isp, wdt_timer); + if (isp->hw->camera_light && !isp->cil_lit) + return HRTIMER_NORESTART; apple_isp_wdt_kick(isp); hrtimer_forward_now(timer, ns_to_ktime(ISP_WDT_PERIOD_NS)); @@ -623,6 +1189,8 @@ static void apple_isp_wdt_start(struct apple_isp *isp) { if (!isp->wdt || isp->wdt_running) return; + if (isp->hw->camera_light && !isp->cil_lit) + return; isp->wdt_running = true; apple_isp_wdt_kick(isp); @@ -639,25 +1207,95 @@ void apple_isp_wdt_stop(struct apple_isp *isp) isp->wdt_running = false; } +#define ISP_GPIO_DATA BIT(0) +#define ISP_GPIO_MODE GENMASK(3, 1) +#define ISP_GPIO_INPUT_ENABLE BIT(9) +/* The mode and the pin's level, as the input buffer reads it */ +#define ISP_GPIO_CHECK (ISP_GPIO_MODE | ISP_GPIO_DATA) + +/* + * Light the camera light before the sensor may stream: drive its pin as an + * output at the on level, with the input buffer on so the pin's level reads + * back, and refuse to stream unless it does. + */ +static int apple_isp_light_on(struct apple_isp *isp) +{ + u32 val, on = isp->cil_on_high ? ISP_GPIO_DATA : 0; + + if (!isp->hw->camera_light) + return 0; + val = (isp->cil_config & ~ISP_GPIO_DATA) | ISP_GPIO_INPUT_ENABLE | on; + writel(val, isp->cil); + udelay(10); + if ((readl(isp->cil) & ISP_GPIO_CHECK) != (val & ISP_GPIO_CHECK)) { + dev_err(isp->dev, "camera light did not come on (0x%x, wrote 0x%x), not streaming\n", + readl(isp->cil), val); + writel(isp->cil_saved, isp->cil); + return -EIO; + } + isp->cil_lit = true; + return 0; +} + +/* + * Only once the sensor has stopped streaming. The pin goes back to how the + * boot loader left it, where a powered sensor still lights the light. + */ +void apple_isp_light_off(struct apple_isp *isp) +{ + if (!isp->hw->camera_light || !isp->cil_lit) + return; + apple_isp_wdt_stop(isp); + isp->cil_lit = false; + writel(isp->cil_saved, isp->cil); +} + +/* + * Once the firmware is halted no frame can reach the host, but a sensor + * the dead firmware left powered would keep the light on until reset: + * drive it off. A halted resident firmware does not start again. + */ +void apple_isp_light_force_off(struct apple_isp *isp) +{ + if (!isp->hw->camera_light) + return; + apple_isp_light_off(isp); + writel((isp->cil_config & ~ISP_GPIO_DATA) | (isp->cil_on_high ? 0 : ISP_GPIO_DATA), + isp->cil); +} + int apple_isp_start_capture(struct apple_isp *isp) { int err; + err = apple_isp_light_on(isp); + if (err) + return err; + /* Serviced from before the channel starts... */ apple_isp_wdt_start(isp); err = isp_cmd_ch_start(isp, 0); // TODO channel mask - if (err) + if (err) { apple_isp_wdt_stop(isp); + apple_isp_light_off(isp); + } return err; } void apple_isp_stop_capture(struct apple_isp *isp) { + if (isp->ipc_failed || isp->fw_state == ISP_FW_DEAD) + return; isp_cmd_ch_stop(isp, 0); // TODO channel mask isp_cmd_ch_buffer_return(isp, isp->current_ch); /* ... until its buffers are back. */ apple_isp_wdt_stop(isp); + apple_isp_light_off(isp); } + +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) +#include "isp-cam-test.c" +#endif diff --git a/drivers/media/platform/apple/isp/isp-cam.h b/drivers/media/platform/apple/isp/isp-cam.h index 51c2f2ec6c079..8cdcd17e2088e 100644 --- a/drivers/media/platform/apple/isp/isp-cam.h +++ b/drivers/media/platform/apple/isp/isp-cam.h @@ -22,6 +22,8 @@ int apple_isp_start_capture(struct apple_isp *isp); void apple_isp_stop_capture(struct apple_isp *isp); void apple_isp_wdt_init(struct apple_isp *isp); +void apple_isp_light_off(struct apple_isp *isp); +void apple_isp_light_force_off(struct apple_isp *isp); void apple_isp_wdt_stop(struct apple_isp *isp); #endif /* __ISP_CAM_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-cmd.c b/drivers/media/platform/apple/isp/isp-cmd.c index 42c448fd0f30c..3e5a4a732a7a0 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.c +++ b/drivers/media/platform/apple/isp/isp-cmd.c @@ -5,30 +5,58 @@ #include "isp-drv.h" #include "isp-iommu.h" #include "isp-ipc.h" +#include + +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) +#include +#endif #define CISP_OPCODE_SHIFT 32UL #define CISP_OPCODE(x) (((u64)(x)) << CISP_OPCODE_SHIFT) #define CISP_OPCODE_GET(x) (((u64)(x)) >> CISP_OPCODE_SHIFT) #define CISP_TIMEOUT msecs_to_jiffies(3000) -#define CISP_SEND_IN(x, a) (cisp_send((x), &(a), sizeof(a), 0, CISP_TIMEOUT)) -#define CISP_SEND_INOUT(x, a) (cisp_send((x), &(a), sizeof(a), sizeof(a), CISP_TIMEOUT)) +#define CISP_SEND_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, CISP_TIMEOUT)) +#define CISP_SEND_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), CISP_TIMEOUT)) #define CISP_SEND_OUT(x, a) (cisp_send_read((x), (a), sizeof(*a), sizeof(*a))) -#define CISP_POST_IN(x, a) (cisp_send((x), &(a), sizeof(a), 0, 0)) -#define CISP_POST_INOUT(x, a) (cisp_send((x), &(a), sizeof(a), sizeof(a), 0)) +#define CISP_POST_IN(x, a) (isp_cmd_send((x), &(a), sizeof(a), 0, 0)) +#define CISP_POST_INOUT(x, a) (isp_cmd_send((x), &(a), sizeof(a), sizeof(a), 0)) -static int cisp_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) +int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, int timeout) { - struct isp_channel *chan = isp->chan_io; - struct isp_message *req = &chan->req; + struct isp_channel *chan; + struct isp_message *req; int err; +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(isp_cmd_send, isp, args, insize, outsize, timeout); +#endif + /* Before the command area is overwritten */ + err = isp_ipc_resync(isp); + if (err) + return err; + chan = isp->chan_io; + req = &chan->req; + req->arg0 = isp->cmd_iova; req->arg1 = insize; req->arg2 = outsize; memcpy(isp->cmd_virt, args, insize); err = ipc_chan_send(isp, chan, timeout); + /* + * An answered request leaves the ring in step, so a malformed answer + * fails only this command. A timeout is resolved before the next one. + */ + if (isp_is_p010(isp)) { + struct isp_message *rsp = &chan->rsp; + + if (!err && timeout && + (rsp->arg1 != outsize || rsp->arg2 || rsp->arg3 || + rsp->arg4 || rsp->arg5 || rsp->arg6 || rsp->arg7 || + get_unaligned_le64(isp->cmd_virt) != get_unaligned_le64(args))) + err = -EIO; + } if (err) { u64 opcode; memcpy(&opcode, args, sizeof(opcode)); @@ -45,7 +73,7 @@ static int cisp_send_read(struct apple_isp *isp, void *args, u32 insize, u32 outsize) { /* TODO do I need to lock the iova space? */ - int err = cisp_send(isp, args, insize, outsize, CISP_TIMEOUT); + int err = isp_cmd_send(isp, args, insize, outsize, CISP_TIMEOUT); if (err) return err; @@ -151,16 +179,24 @@ int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, return CISP_SEND_IN(isp, args); } -int isp_cmd_set_dsid_clr_multi_bc(struct apple_isp *isp, u64 dsid_clr_base, - u32 dsid_clr_range) +int isp_cmd_set_dsid_clr_multi_bc(struct apple_isp *isp, const u64 *bases, + unsigned int count, u32 dsid_clr_range) { struct cmd_set_dsid_clr_multi_bc_reg_base args = { .opcode = CISP_OPCODE(CISP_CMD_SET_DSID_CLR_MULTI_BC_REG_BASE), - .count = 1, + .count = count, .dsid_clr_range = dsid_clr_range, - .dsid_clr_base = dsid_clr_base, }; - return CISP_SEND_IN(isp, args); + + /* The caller, isp_set_dsid_clr(), bounds count by the array. */ + if (!count) + return -EINVAL; + memcpy(args.dsid_clr_base, bases, count * sizeof(*bases)); + + return isp_cmd_send(isp, &args, + offsetof(typeof(args), dsid_clr_base) + + count * sizeof(*bases), 0, + CISP_TIMEOUT); } int isp_cmd_pmp_ctrl_set(struct apple_isp *isp, u64 clock_scratch, @@ -224,6 +260,8 @@ int isp_cmd_flicker_sensor_set(struct apple_isp *isp, u32 mode) .opcode = CISP_OPCODE(CISP_CMD_FLICKER_SENSOR_SET), .mode = mode, }; + if (isp_is_p010(isp)) + return CISP_SEND_IN(isp, args); return CISP_SEND_INOUT(isp, args); } @@ -252,6 +290,16 @@ int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, return err; } +int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 index, + struct cmd_ch_camera_config_h17 *args) +{ + memset(args, 0, sizeof(*args)); + args->config.opcode = CISP_OPCODE(CISP_CMD_CH_CAMERA_CONFIG_GET); + args->config.chan = chan; + args->config.preset = index; + return CISP_SEND_OUT(isp, args); +} + int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, struct cmd_ch_camera_config *args) { @@ -460,9 +508,9 @@ int isp_cmd_ch_buffer_pool_config_set(struct apple_isp *isp, u32 chan, u16 type) .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_CONFIG_SET), .chan = chan, .type = type, - .count = ISP_MAX_BUFFERS, - .meta_size0 = size, - .meta_size1 = size, + .count = isp_num_meta(isp), + .meta_size0 = isp_is_p010(isp) ? 0x8000 : size, + .meta_size1 = isp_is_p010(isp) ? 0x8000 : size, .unk0 = 0, .unk1 = 0, .unk2 = 0, @@ -481,7 +529,8 @@ int isp_cmd_ch_buffer_pool_config_set_rendered(struct apple_isp *isp, u32 chan, struct cmd_ch_buffer_pool_config_set_rendered args = { .opcode = CISP_OPCODE(CISP_CMD_CH_BUFFER_POOL_CONFIG_SET), .chan = chan, - .type = CISP_POOL_TYPE_RENDERED, + .type = isp->hw->scl1 ? CISP_POOL_TYPE_RENDERED_SCL1 : + CISP_POOL_TYPE_RENDERED, .count = count, .plane0_size = plane0_size, .plane0_stride = plane0_stride, @@ -558,6 +607,49 @@ int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan) return CISP_SEND_IN(isp, args); } +int isp_cmd_apple_ch_tnr_start_h17(struct apple_isp *isp, u32 chan) +{ + struct cmd_apple_ch_temporal_filter_start_h17 args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_START), + .chan = chan, + .unk_c = 1, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_tnr_high_strength_set(struct apple_isp *isp, u32 chan, + u32 enable) +{ + struct cmd_apple_ch_temporal_filter_enable_set args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_HIGH_STRENGTH), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_tnr_band0_mod_set(struct apple_isp *isp, u32 chan, + u32 enable) +{ + struct cmd_apple_ch_temporal_filter_enable_set args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_MODULATION), + .chan = chan, + .enable = enable, + }; + return CISP_SEND_IN(isp, args); +} + +int isp_cmd_apple_ch_tnr_band0_threshold_set(struct apple_isp *isp, u32 chan, + u32 first, u32 second) +{ + struct cmd_apple_ch_temporal_filter_band0_threshold args = { + .opcode = CISP_OPCODE(CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_THRESHOLD), + .chan = chan, + .threshold = { first, second }, + }; + return CISP_SEND_IN(isp, args); +} + int isp_cmd_ch_ae_stability_set(struct apple_isp *isp, u32 chan, u32 stability) { struct cmd_ch_ae_stability_set args = { @@ -637,6 +729,17 @@ int isp_cmd_apple_ch_ae_metering_mode_set(struct apple_isp *isp, u32 chan, return CISP_SEND_IN(isp, args); } +int isp_cmd_ch_lsc_ideal_percent_set(struct apple_isp *isp, u32 chan, + u16 percent) +{ + struct cmd_ch_lsc_ideal_percent_set args = { + .opcode = CISP_OPCODE(CISP_CMD_CH_LSC_IDEAL_PERCENT_SET), + .chan = chan, + .percent = percent, + }; + return CISP_SEND_IN(isp, args); +} + int isp_cmd_apple_ch_ae_flicker_freq_update_current_set(struct apple_isp *isp, u32 chan, u32 freq) { diff --git a/drivers/media/platform/apple/isp/isp-cmd.h b/drivers/media/platform/apple/isp/isp-cmd.h index ab0c85023bebf..acbd864816d28 100644 --- a/drivers/media/platform/apple/isp/isp-cmd.h +++ b/drivers/media/platform/apple/isp/isp-cmd.h @@ -70,6 +70,7 @@ #define CISP_CMD_CH_ALS_DISABLE 0x0a1d #define CISP_CMD_CH_CNR_START 0x0a2f #define CISP_CMD_CH_MBNR_ENABLE 0x0a3a +#define CISP_CMD_CH_LSC_IDEAL_PERCENT_SET 0x0a4e #define CISP_CMD_CH_OUTPUT_CONFIG_SET 0x0b01 #define CISP_CMD_CH_OUTPUT_CONFIG_SCL1_SET 0x0b09 #define CISP_CMD_CH_PREVIEW_STREAM_SET 0x0b0d @@ -97,6 +98,9 @@ #define CISP_CMD_APPLE_CH_MOTION_HISTORY_STOP 0xc103 #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_ENABLE 0xc113 #define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_DISABLE 0xc114 +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_MODULATION 0xc11c +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_BAND0_THRESHOLD 0xc11d +#define CISP_CMD_APPLE_CH_TEMPORAL_FILTER_HIGH_STRENGTH 0xc120 #define CISP_POOL_TYPE_META 0x0 #define CISP_POOL_TYPE_RENDERED 0x1 @@ -124,6 +128,7 @@ #define CISP_COLORSPACE_REC709 0x1 #define CISP_OUTPUT_FORMAT_YUV_2PLANE 0x0 #define CISP_OUTPUT_FORMAT_YUV_1PLANE 0x1 +#define CISP_OUTPUT_FORMAT_P010 0x12 #define CISP_OUTPUT_FORMAT_RGB 0x2 #define CISP_BUFFER_RECYCLE_MODE_EMPTY_ONLY 0x1 @@ -197,15 +202,20 @@ struct cmd_set_dsid_clr_req_base { } __packed; static_assert(sizeof(struct cmd_set_dsid_clr_req_base) == 0x14); -/* H17: a count of broadcast-clear windows and the first window */ +/* + * H17: a count of broadcast-clear windows, their common range and the + * windows; the command carries only the @count windows. + */ +#define ISP_DSID_MULTI_BC_MAX 8 + struct cmd_set_dsid_clr_multi_bc_reg_base { u64 opcode; u8 count; u8 pad_9[3]; u32 dsid_clr_range; - u64 dsid_clr_base; + u64 dsid_clr_base[ISP_DSID_MULTI_BC_MAX]; } __packed; -static_assert(sizeof(struct cmd_set_dsid_clr_multi_bc_reg_base) == 0x18); +static_assert(sizeof(struct cmd_set_dsid_clr_multi_bc_reg_base) == 0x50); struct cmd_pmp_ctrl_set { u64 opcode; @@ -250,6 +260,8 @@ struct cmd_flicker_sensor_set { } __packed; static_assert(sizeof(struct cmd_flicker_sensor_set) == 0xc); +int isp_cmd_send(struct apple_isp *isp, void *args, u32 insize, u32 outsize, + int timeout); int isp_cmd_start(struct apple_isp *isp, u32 mode); int isp_cmd_stop(struct apple_isp *isp, u32 mode); int isp_cmd_power_down(struct apple_isp *isp); @@ -260,8 +272,8 @@ int isp_cmd_config_get(struct apple_isp *isp, struct cmd_config_get *args); int isp_cmd_set_isp_pmu_base(struct apple_isp *isp, u64 pmu_base); int isp_cmd_set_dsid_clr_req_base(struct apple_isp *isp, u64 dsid_clr_base, u32 dsid_clr_range); -int isp_cmd_set_dsid_clr_multi_bc(struct apple_isp *isp, u64 dsid_clr_base, - u32 dsid_clr_range); +int isp_cmd_set_dsid_clr_multi_bc(struct apple_isp *isp, const u64 *bases, + unsigned int count, u32 dsid_clr_range); int isp_cmd_set_dsid_clr_req_base2(struct apple_isp *isp, u64 dsid_clr_base0, u64 dsid_clr_base1, u64 dsid_clr_base2, u64 dsid_clr_base3, u32 dsid_clr_range0, @@ -375,6 +387,15 @@ struct cmd_ch_camera_config { } __packed; static_assert(sizeof(struct cmd_ch_camera_config) == 0xdc); +/* H17 appends 0x44 bytes to the common camera configuration record. */ +struct cmd_ch_camera_config_h17 { + struct cmd_ch_camera_config config; + u8 unknown[0x44]; +} __packed; +static_assert(sizeof(struct cmd_ch_camera_config_h17) == 0x120); +static_assert(offsetof(struct cmd_ch_camera_config, in_width) == 0x10); +static_assert(offsetof(struct cmd_ch_camera_config, out_width2) == 0xd4); + struct cmd_ch_camera_config_select { u64 opcode; u32 chan; @@ -511,6 +532,8 @@ int isp_cmd_ch_start(struct apple_isp *isp, u32 chan); int isp_cmd_ch_stop(struct apple_isp *isp, u32 chan); int isp_cmd_ch_info_get(struct apple_isp *isp, u32 chan, struct cmd_ch_info *args); +int isp_cmd_ch_camera_config_get(struct apple_isp *isp, u32 chan, u32 index, + struct cmd_ch_camera_config_h17 *args); int isp_cmd_ch_camera_config_current_get(struct apple_isp *isp, u32 chan, struct cmd_ch_camera_config *args); int isp_cmd_ch_camera_config_select(struct apple_isp *isp, u32 chan, @@ -624,6 +647,32 @@ struct cmd_apple_ch_temporal_filter_start { } __packed; static_assert(sizeof(struct cmd_apple_ch_temporal_filter_start) == 0x14); +/* The H17 firmware's start record carries one more word, sent as zero. */ +struct cmd_apple_ch_temporal_filter_start_h17 { + u64 opcode; + u32 chan; + u32 unk_c; + u32 unk_10; + u32 unk_14; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_start_h17) == 0x18); + +/* High-strength filtering (0xc120) and band 0 modulation (0xc11c) */ +struct cmd_apple_ch_temporal_filter_enable_set { + u64 opcode; + u32 chan; + u32 enable; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_enable_set) == 0x10); + +/* Band 0 strength modulation thresholds, in 1/256 steps */ +struct cmd_apple_ch_temporal_filter_band0_threshold { + u64 opcode; + u32 chan; + u32 threshold[2]; +} __packed; +static_assert(sizeof(struct cmd_apple_ch_temporal_filter_band0_threshold) == 0x14); + struct cmd_apple_ch_temporal_filter_stop { u64 opcode; u32 chan; @@ -660,6 +709,13 @@ int isp_cmd_apple_ch_motion_history_start(struct apple_isp *isp, u32 chan); int isp_cmd_apple_ch_motion_history_stop(struct apple_isp *isp, u32 chan); int isp_cmd_apple_ch_temporal_filter_enable(struct apple_isp *isp, u32 chan); int isp_cmd_apple_ch_temporal_filter_disable(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_tnr_start_h17(struct apple_isp *isp, u32 chan); +int isp_cmd_apple_ch_tnr_high_strength_set(struct apple_isp *isp, u32 chan, + u32 enable); +int isp_cmd_apple_ch_tnr_band0_mod_set(struct apple_isp *isp, u32 chan, + u32 enable); +int isp_cmd_apple_ch_tnr_band0_threshold_set(struct apple_isp *isp, u32 chan, + u32 first, u32 second); struct cmd_ch_ae_stability_set { u64 opcode; @@ -709,6 +765,15 @@ struct cmd_apple_ch_ae_fd_scene_metering_config_set { static_assert(sizeof(struct cmd_apple_ch_ae_fd_scene_metering_config_set) == 0x24); +/* The lens shading falloff the firmware corrects to, in percent at the corner */ +struct cmd_ch_lsc_ideal_percent_set { + u64 opcode; + u32 chan; + u16 percent; + u16 pad; +} __packed; +static_assert(sizeof(struct cmd_ch_lsc_ideal_percent_set) == 0x10); + struct cmd_apple_ch_ae_metering_mode_set { u64 opcode; u32 chan; @@ -737,6 +802,8 @@ int isp_cmd_apple_ch_ae_fd_scene_metering_config_set(struct apple_isp *isp, u32 chan); int isp_cmd_apple_ch_ae_metering_mode_set(struct apple_isp *isp, u32 chan, u32 mode); +int isp_cmd_ch_lsc_ideal_percent_set(struct apple_isp *isp, u32 chan, + u16 percent); int isp_cmd_apple_ch_ae_flicker_freq_update_current_set(struct apple_isp *isp, u32 chan, u32 freq); diff --git a/drivers/media/platform/apple/isp/isp-drv-test.c b/drivers/media/platform/apple/isp/isp-drv-test.c new file mode 100644 index 0000000000000..dbddb079a0f57 --- /dev/null +++ b/drivers/media/platform/apple/isp/isp-drv-test.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for the per-SoC data of the Apple ISP driver, built into + * isp-drv.c: the DART address filter windows and the firmware start. They + * need no hardware. + */ + +#include +#include +#include + +/* Every address filter window must lie inside a window the firmware is mapped. */ +static void isp_test_dapf_in_fw_mmio(struct kunit *test) +{ + const struct of_device_id *match; + + for (match = apple_isp_of_match; match->compatible[0]; match++) { + const struct apple_isp_hw *hw = match->data; + + for (unsigned int i = 0; i < hw->num_dapf; i++) { + const struct isp_dapf_window *w = &hw->dapf[i]; + bool inside = false; + + KUNIT_EXPECT_LE_MSG(test, w->start, w->end, "%s window %u", + match->compatible, i); + for (unsigned int j = 0; j < hw->num_fw_mmio; j++) { + const struct isp_mmio_window *m = &hw->fw_mmio[j]; + + if (w->start >= m->base && w->end < m->base + m->size) + inside = true; + } + KUNIT_EXPECT_TRUE_MSG(test, inside, + "%s window %u 0x%llx-0x%llx is not in fw_mmio", + match->compatible, i, w->start, w->end); + } + } +} + +#define ISP_TEST_DAPF_FILL 0xa5 + +/* + * Each window fills its own 0x40-byte slot with exactly these words, and + * leaves every other byte alone. The bank is plain memory here: the driver + * is arm64 only, where the MMIO accessors work on it unchanged. + */ +static void isp_test_dapf_program(struct kunit *test) +{ + static const struct isp_dapf_window windows[] = { + { 0x380704000, 0x380704063 }, + { 0x382824000, 0x382824004 }, + }; + static const struct apple_isp_hw hw = { + .dapf = windows, + .num_dapf = ARRAY_SIZE(windows), + }; + size_t size = (ARRAY_SIZE(windows) + 1) * DAPF_WINDOW_STRIDE; + u8 expect[DAPF_WINDOW_STRIDE]; + struct apple_isp *isp; + u8 *bank; + + isp = kunit_kzalloc(test, sizeof(*isp), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp); + isp->dev = kunit_device_register(test, "apple-isp-dapf-test"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp->dev); + bank = kunit_kmalloc(test, size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, bank); + memset(bank, ISP_TEST_DAPF_FILL, size); + isp->hw = &hw; + isp->dapf = (__force void __iomem *)bank; + + KUNIT_ASSERT_EQ(test, apple_isp_program_dapf(isp), 0); + + for (unsigned int i = 0; i < ARRAY_SIZE(windows); i++) { + memset(expect, ISP_TEST_DAPF_FILL, sizeof(expect)); + put_unaligned_le32(0x31, &expect[0x00]); + put_unaligned_le32(0, &expect[0x04]); + put_unaligned_le64(windows[i].start, &expect[0x08]); + put_unaligned_le64(windows[i].end, &expect[0x10]); + put_unaligned_le32(1, &expect[0x20]); + KUNIT_EXPECT_MEMEQ_MSG(test, bank + i * DAPF_WINDOW_STRIDE, expect, + sizeof(expect), "window %u", i); + } + + /* The slot after the last window is untouched. */ + KUNIT_EXPECT_NULL(test, memchr_inv(bank + ARRAY_SIZE(windows) * DAPF_WINDOW_STRIDE, + ISP_TEST_DAPF_FILL, DAPF_WINDOW_STRIDE)); +} + +static struct kunit_case isp_dapf_test_cases[] = { + KUNIT_CASE(isp_test_dapf_in_fw_mmio), + KUNIT_CASE(isp_test_dapf_program), + {} +}; + +static struct kunit_suite isp_dapf_test_suite = { + .name = "apple-isp-dapf", + .test_cases = isp_dapf_test_cases, +}; + +kunit_test_suite(isp_dapf_test_suite); + +/* + * A deferred start detects the camera once: its firmware must keep running + * or start again. Firmware that starts again is not resident, and only it + * is quiesced before the power-down. + */ +static void isp_test_firmware_lifetime(struct kunit *test) +{ + const struct of_device_id *match; + + for (match = apple_isp_of_match; match->compatible[0]; match++) { + const struct apple_isp_hw *hw = match->data; + + if (hw->deferred_start) + KUNIT_EXPECT_TRUE_MSG(test, hw->resident_fw || hw->fw_restore, + "%s", match->compatible); + KUNIT_EXPECT_FALSE_MSG(test, hw->resident_fw && hw->fw_restore, "%s", + match->compatible); + if (hw->coproc_quiesce) + KUNIT_EXPECT_TRUE_MSG(test, hw->fw_restore, "%s", + match->compatible); + } +} + +static struct kunit_case isp_hw_test_cases[] = { + KUNIT_CASE(isp_test_firmware_lifetime), + {} +}; + +static struct kunit_suite isp_hw_test_suite = { + .name = "apple-isp-hw", + .test_cases = isp_hw_test_cases, +}; + +kunit_test_suite(isp_hw_test_suite); diff --git a/drivers/media/platform/apple/isp/isp-drv.c b/drivers/media/platform/apple/isp/isp-drv.c index 45e73995657fe..b1c9929c794b7 100644 --- a/drivers/media/platform/apple/isp/isp-drv.c +++ b/drivers/media/platform/apple/isp/isp-drv.c @@ -5,10 +5,12 @@ * Copyright (C) 2023 The Asahi Linux Contributors */ +#include #include #include #include #include +#include #include #include #include @@ -20,6 +22,41 @@ #include "isp-regs.h" #include "isp-v4l2.h" +static void apple_isp_kvfree(void *data) +{ + kvfree(data); +} + +/* + * Keep a copy of the firmware data segment as the boot loader loaded it + * (hw->fw_restore). This must run before the firmware first starts. + */ +static int apple_isp_copy_fw_data(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + struct resource res; + int err; + + err = of_reserved_mem_region_to_resource_byname(dev->of_node, "data", &res); + if (err) + return err; + + isp->fw_data_size = resource_size(&res); + isp->fw_data = devm_memremap(dev, res.start, isp->fw_data_size, MEMREMAP_WB); + if (IS_ERR(isp->fw_data)) + return PTR_ERR(isp->fw_data); + + isp->fw_data_copy = kvmalloc(isp->fw_data_size, GFP_KERNEL); + if (!isp->fw_data_copy) + return -ENOMEM; + err = devm_add_action_or_reset(dev, apple_isp_kvfree, isp->fw_data_copy); + if (err) + return err; + + memcpy(isp->fw_data_copy, isp->fw_data, isp->fw_data_size); + return 0; +} + static void apple_isp_detach_genpd(struct apple_isp *isp) { if (isp->pd_count <= 1) @@ -144,17 +181,117 @@ static int apple_isp_map_fw_mmio(struct apple_isp *isp) return 0; } +/* The IOVA range of a memory-region, from its last iommu-addresses entry. */ +static int isp_of_region_iova(struct device *dev, const char *name, + phys_addr_t *base, size_t *size) +{ + const __be32 *maps, *end; + int idx, len; + + idx = of_property_match_string(dev->of_node, "memory-region-names", + name); + if (idx < 0) + return -ENOENT; + + struct device_node *np __free(device_node) = + of_parse_phandle(dev->of_node, "memory-region", idx); + if (!np) + return -ENODEV; + + maps = of_get_property(np, "iommu-addresses", &len); + if (!maps || !len) + return -ENODEV; + + end = maps + len / sizeof(__be32); + + while (maps < end) { + maps++; + maps = of_translate_dma_region(dev->of_node, maps, base, size); + } + + return 0; +} + +/* + * The firmware reaches its fixed MMIO windows through the first DART, whose + * address filter the ADT describes for them. Its registers are on that DART. + */ +static int apple_isp_map_dapf(struct apple_isp *isp) +{ + struct device *dev = isp->dev; + /* The first iommus entry, whatever its #iommu-cells */ + struct device_node *dart __free(device_node) = + of_parse_phandle(dev->of_node, "iommus", 0); + struct resource res; + int idx, err; + + if (!dart) + return dev_err_probe(dev, -ENODEV, "no ISP DART\n"); + + /* DAPF_* describe the T8110 DART's address filter */ + if (!of_device_is_compatible(dart, "apple,t8110-dart")) + return dev_err_probe(dev, -ENODEV, + "%pOF has no known address filter\n", dart); + + idx = of_property_match_string(dart, "reg-names", "dapf"); + if (idx < 0) + return dev_err_probe(dev, idx, + "%pOF has no address filter registers\n", dart); + + err = of_address_to_resource(dart, idx, &res); + if (err) + return dev_err_probe(dev, err, "%pOF: bad address filter registers\n", + dart); + + if (resource_size(&res) < isp->hw->num_dapf * DAPF_WINDOW_STRIDE) + return dev_err_probe(dev, -EINVAL, "%pOF: address filter too small\n", + dart); + + isp->dapf = devm_ioremap_resource(dev, &res); + return PTR_ERR_OR_ZERO(isp->dapf); +} + +/* Program the address filter windows, then check that they took. */ +int apple_isp_program_dapf(struct apple_isp *isp) +{ + const struct apple_isp_hw *hw = isp->hw; + u32 ctrl2 = hw->dapf_ctrl2 ?: DAPF_CTRL2_ISP; + void __iomem *w; + unsigned int i; + + for (i = 0; i < hw->num_dapf; i++) { + w = isp->dapf + i * DAPF_WINDOW_STRIDE; + writel(0, w + DAPF_CTRL1); + writeq(hw->dapf[i].start, w + DAPF_START); + writeq(hw->dapf[i].end, w + DAPF_END); + writel(DAPF_CTRL0_ISP, w + DAPF_CTRL0); + writel(ctrl2, w + DAPF_CTRL2); + } + + /* Only bits 7:0 of CTRL0 and bits 63:2 of END read back. */ + for (i = 0; i < hw->num_dapf; i++) { + w = isp->dapf + i * DAPF_WINDOW_STRIDE; + if (readl(w + DAPF_CTRL1) || + readq(w + DAPF_START) != hw->dapf[i].start || + (readq(w + DAPF_END) | 3) != (hw->dapf[i].end | 3) || + (readl(w + DAPF_CTRL0) & 0xff) != DAPF_CTRL0_ISP || + readl(w + DAPF_CTRL2) != ctrl2) { + dev_err(isp->dev, "DAPF window %u (0x%llx-0x%llx) failed readback\n", + i, hw->dapf[i].start, hw->dapf[i].end); + return -EIO; + } + } + + return 0; +} + static int apple_isp_init_iommu(struct apple_isp *isp) { struct device *dev = isp->dev; - phys_addr_t heap_base; - size_t heap_size; + phys_addr_t heap_base, data_base; + size_t heap_size, data_size; u64 vm_size; int err; - int idx; - int size; - struct device_node *mem_node; - const __be32 *maps, *end; isp->domain = iommu_get_domain_for_dev(isp->dev); if (!isp->domain) @@ -162,30 +299,38 @@ static int apple_isp_init_iommu(struct apple_isp *isp) isp->shift = __ffs(isp->domain->pgsize_bitmap); isp->fw_iova_mask = isp->hw->fw_iova_mask ?: U64_MAX; - idx = of_property_match_string(dev->of_node, "memory-region-names", - "heap"); - mem_node = of_parse_phandle(dev->of_node, "memory-region", idx); - if (!mem_node) { - dev_err(dev, "No memory-region found for heap\n"); - return -ENODEV; - } - - maps = of_get_property(mem_node, "iommu-addresses", &size); - if (!maps || !size) { + err = isp_of_region_iova(dev, "heap", &heap_base, &heap_size); + if (err) { dev_err(dev, "No valid iommu-addresses found for heap\n"); return -ENODEV; } - end = maps + size / sizeof(__be32); + isp->fw.heap_base = heap_base; + isp->fw.heap_top = heap_base + heap_size; - while (maps < end) { - maps++; - maps = of_translate_dma_region(dev->of_node, maps, &heap_base, - &heap_size); + /* + * Where the bootloader loads the firmware's data, it backs the + * firmware's main memory only up to the end of it. The rest, up to + * the heap top, is the host's to back. + */ + err = -ENOENT; + if (isp->hw->gen == ISP_GEN_T6040) + err = isp_of_region_iova(dev, "data", &data_base, &data_size); + if (!err) { + isp->fw.data_top = data_base + data_size; + if (isp->fw.data_top <= isp->fw.heap_base || + isp->fw.data_top > isp->fw.heap_top || + !IS_ALIGNED(isp->fw.data_top, 1UL << isp->shift)) { + dev_err(dev, "data 0x%llx outside the heap 0x%llx-0x%llx\n", + isp->fw.data_top, isp->fw.heap_base, + isp->fw.heap_top); + return -EINVAL; + } + } else if (err != -ENOENT) { + dev_err(dev, "No valid iommu-addresses found for data\n"); + return -ENODEV; } - isp->fw.heap_top = heap_base + heap_size; - err = of_property_read_u64(dev->of_node, "apple,dart-vm-size", &vm_size); if (err) { @@ -195,11 +340,11 @@ static int apple_isp_init_iommu(struct apple_isp *isp) // FIXME: refactor this, maybe use regular iova stuff? isp->iova_size = vm_size - (heap_base & 0xffffffff); - drm_mm_init(&isp->iovad, isp->fw.heap_top, isp->iova_size); + mt_init_flags(&isp->iovas, MT_FLAGS_ALLOC_RANGE); err = apple_isp_map_fw_mmio(isp); if (err) { - drm_mm_takedown(&isp->iovad); + mtree_destroy(&isp->iovas); return err; } @@ -209,7 +354,8 @@ static int apple_isp_init_iommu(struct apple_isp *isp) static void apple_isp_free_iommu(struct apple_isp *isp) { apple_isp_unmap_fw_mmio(isp, isp->hw->num_fw_mmio); - drm_mm_takedown(&isp->iovad); + WARN_ON(!mtree_empty(&isp->iovas)); + mtree_destroy(&isp->iovas); } static int isp_of_read_coord(struct device *dev, struct device_node *np, @@ -421,6 +567,10 @@ static int apple_isp_probe(struct platform_device *pdev) if (err) isp->temporal_filter = 0; + /* the sensor connection layout, for firmware that takes it at boot */ + of_property_read_u32(dev->of_node, "apple,cam-connections-scheme", + &isp->cam_connections_scheme); + err = apple_isp_init_presets(isp); if (err) { dev_err(dev, "failed to initialize presets\n"); @@ -469,6 +619,12 @@ static int apple_isp_probe(struct platform_device *pdev) } } + if (isp->hw->num_dapf) { + err = apple_isp_map_dapf(isp); + if (err) + goto detach_genpd; + } + /* Capture watchdog, where the ISP gates its frames on one */ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "wdt"); if (res) { @@ -480,6 +636,25 @@ static int apple_isp_probe(struct platform_device *pdev) apple_isp_wdt_init(isp); } + if (isp->hw->camera_light) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cil"); + if (!res || of_property_read_u32(dev->of_node, "apple,cil-config", + &isp->cil_config)) { + dev_err(dev, "no camera light described, not using the camera\n"); + err = -ENODEV; + goto detach_genpd; + } + /* Not requested: the pin sits in a GPIO block Linux does not own. */ + isp->cil = devm_ioremap(dev, res->start, resource_size(res)); + if (!isp->cil) { + err = -ENOMEM; + goto detach_genpd; + } + isp->cil_on_high = of_property_read_bool(dev->of_node, + "apple,cil-active-high"); + isp->cil_saved = readl(isp->cil); + } + isp->irq = platform_get_irq(pdev, 0); if (isp->irq < 0) { err = isp->irq; @@ -490,7 +665,6 @@ static int apple_isp_probe(struct platform_device *pdev) goto detach_genpd; } - mutex_init(&isp->iovad_lock); mutex_init(&isp->video_lock); spin_lock_init(&isp->buf_lock); init_waitqueue_head(&isp->wait); @@ -499,6 +673,7 @@ static int apple_isp_probe(struct platform_device *pdev) mutex_init(&isp->gc_lock); INIT_LIST_HEAD(&isp->bufs_pending); INIT_LIST_HEAD(&isp->bufs_submitted); + INIT_LIST_HEAD(&isp->bufs_returned); isp->wq = alloc_workqueue("apple-isp-wq", WQ_UNBOUND, 0); if (!isp->wq) { dev_err(dev, "failed to create workqueue\n"); @@ -518,12 +693,23 @@ static int apple_isp_probe(struct platform_device *pdev) goto free_iommu; } + if (isp->hw->fw_restore) { + err = apple_isp_copy_fw_data(isp); + if (err) { + dev_err(dev, "failed to copy the firmware data: %d\n", err); + apple_isp_free_firmware_surface(isp); + goto free_iommu; + } + } + pm_runtime_enable(dev); - err = apple_isp_detect_camera(isp); - if (err) { - dev_err(dev, "failed to detect camera: %d\n", err); - goto free_surface; + if (!isp->hw->deferred_start) { + err = apple_isp_detect_camera(isp); + if (err) { + dev_err(dev, "failed to detect camera: %d\n", err); + goto free_surface; + } } err = apple_isp_setup_video(isp); @@ -546,6 +732,7 @@ static int apple_isp_probe(struct platform_device *pdev) detach_genpd: apple_isp_detach_genpd(isp); free_isp: + kfree(isp->sensor_configs); /* Nothing else can reach it before the video device is registered. */ kfree(isp); return err; @@ -786,6 +973,278 @@ static const struct apple_isp_hw apple_isp_hw_t8140 = { .boot_mode = 1, .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, .resident_fw = true, + .meta_exposure = true, +}; + +/* The same for T6040, from the J616s ADT's DART address filter. */ +static const struct isp_mmio_window apple_isp_fw_mmio_t6040[] = { + { 0x220004000, 0x1d4000 }, /* DSID broadcast-clear windows */ + { 0x222004000, 0x1d4000 }, + { 0x224004000, 0x1d4000 }, + { 0x226004000, 0x1d4000 }, + { 0x485660000, 0x4000 }, + { 0x5003c0000, 0x28000 }, /* PMGR, with the PMP clock scratch */ + { 0x500e3c000, 0x4000 }, + { 0x502284000, 0x4000 }, /* PMGR, ISP power states */ + { 0x502880000, 0x8000 }, + { 0x504824000, 0x4000 }, /* PMP bandwidth scratch */ + { 0x51062c000, 0x20000 }, +}; + +/* + * T6040 runs the same coprocessor and firmware interface as T8140, booted + * as macOS boots it on the J616s (macOS 26.6.2 camera log, kext register + * map) and as aurora's m1n1 proxy captures with it: no coprocessor reset, + * four broadcast-clear windows in eight slots, the scaler 1 output path and + * no capture metadata pool or watchdog. + */ +static const struct apple_isp_hw apple_isp_hw_t6040 = { + .gen = ISP_GEN_T6040, + .fw_abi = ISP_FW_ABI_H17, + .pmu_base = 0x0, + + .dsid_count = 4, + .dsid_slots = 8, + .dsid_clr_base0 = 0x2201d4000, + .dsid_clr_base1 = 0x2221d4000, + .dsid_clr_base2 = 0x2241d4000, + .dsid_clr_base3 = 0x2261d4000, + .dsid_clr_range0 = 0x2fc, + + /* the PMGR and PMP scratch words of the ISP's DAPF windows */ + .clock_scratch = 0x5003d0fa8, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x504824000, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + .mbox_irq_enable = ISP_MBOX_IRQ_ENABLE_T6031, + + .scl1 = true, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8140, + .fw_mmio = apple_isp_fw_mmio_t6040, + .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t6040), + + .coproc_control = ISP_COPROC_CONTROL_T8140, + .boot_mode = 1, + .shared_size = 0xde600000, + .resident_fw = true, + .meta_exposure = true, +}; + +/* T8132: the firmware's MMIO windows, from the J713 ADT's DART address filter */ +static const struct isp_mmio_window apple_isp_fw_mmio_t8132[] = { + { 0x220004000, 0x1d4000 }, /* DSID broadcast-clear windows */ + { 0x222004000, 0x1d4000 }, + { 0x3803c0000, 0x28000 }, /* PMGR, with the PMP clock scratch */ + { 0x380704000, 0x4000 }, /* PMGR, ISP power states */ + { 0x380730000, 0x4000 }, + { 0x380e3c000, 0x4000 }, + { 0x382824000, 0x4000 }, /* PMP bandwidth scratch */ + { 0x39062c000, 0x20000 }, + { 0x501660000, 0x4000 }, +}; + +/* The ADT's DART address filter windows, in its order and with its ends */ +static const struct isp_dapf_window apple_isp_dapf_t8132[] = { + { 0x380704000, 0x380704063 }, /* PMGR, ISP power states */ + { 0x380730000, 0x380733fff }, + { 0x380e3c000, 0x380e3c003 }, + { 0x39062c000, 0x390648003 }, + { 0x501660000, 0x501660003 }, + { 0x220004000, 0x2200162fc }, /* DSID broadcast-clear windows */ + { 0x220044000, 0x2200562fc }, + { 0x2201c4000, 0x2201d62fc }, + { 0x222004000, 0x2220162fc }, + { 0x222044000, 0x2220562fc }, + { 0x2221c4000, 0x2221d62fc }, + { 0x3803c0000, 0x3803e4000 }, /* PMGR, with the PMP clock scratch */ + { 0x382824000, 0x382824004 }, /* PMP bandwidth scratch */ +}; + +/* + * T8132 runs T6040's coprocessor and firmware interface, booted the same + * way. Its two AMCCs (ADT mcc amcc_aperture_count) give two broadcast-clear + * windows, passed one per AMCC without padding slots. No PMP_CTRL_SET is + * sent: T8132's PMP scratch offsets are not established. Its firmware + * starts again from its data segment as loaded once the coprocessor was + * quiesced before the power-down, so the ISP is powered only while the + * camera is in use. + */ +static const struct apple_isp_hw apple_isp_hw_t8132 = { + .gen = ISP_GEN_T6040, + .fw_abi = ISP_FW_ABI_H17, + .pmu_base = 0x0, + + .dsid_count = 2, + .dsid_clr_base0 = 0x2201d4000, + .dsid_clr_base1 = 0x2221d4000, + .dsid_clr_range0 = 0x2fc, + + .mbox_irq_enable = ISP_MBOX_IRQ_ENABLE_T6031, + + .scl1 = true, + .lpdp = false, + .meta_size = ISP_META_SIZE_T8140, + .fw_mmio = apple_isp_fw_mmio_t8132, + .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t8132), + .dapf = apple_isp_dapf_t8132, + .num_dapf = ARRAY_SIZE(apple_isp_dapf_t8132), + + .coproc_control = ISP_COPROC_CONTROL_T8140, + .coproc_status = ISP_COPROC_STATUS_T8140, + .coproc_quiesce = ISP_COPROC_QUIESCE_T8140, + .boot_mode = 1, + .shared_size = 0xdec00000, + .fw_restore = true, + .deferred_start = true, +}; + +/* + * T8142 (M5, J813): the Neo's ISP17a. The J813 ADT's dart-isp + * dapf-instance-0 ranges, rounded out to 16 KiB pages; mostly the T8140 + * windows at +0x80000000, with one DSID window and two new ranges. + */ +static const struct isp_mmio_window apple_isp_fw_mmio_t8142[] = { + { 0x220104000, 0x14000 }, /* DSID broadcast-clear window */ + { 0x3803c0000, 0x28000 }, /* PMGR, with the PMP clock scratch */ + { 0x380704000, 0x4000 }, /* PMGR, ISP power states */ + { 0x380730000, 0x4000 }, + { 0x380e3c000, 0x4000 }, + { 0x382724000, 0x4000 }, /* PMP bandwidth scratch */ + { 0x39062c000, 0x30000 }, + { 0x3a1340000, 0x4000 }, + { 0x3a1358000, 0xc000 }, + { 0x421660000, 0x4000 }, +}; + +static const struct apple_isp_hw apple_isp_hw_t8142 = { + .gen = ISP_GEN_T8140, + .fw_abi = ISP_FW_ABI_H17, + .pmu_base = 0x0, + + .dsid_count = 1, + .dsid_clr_base0 = 0x220116000, + .dsid_clr_range0 = 0x2fc, + + .clock_scratch = 0x3803d0ca0, + .clock_base = 0x0, + .clock_bit = 0x0, + .clock_size = 0x8, + .bandwidth_scratch = 0x382724000, + .bandwidth_base = 0x0, + .bandwidth_bit = 0x0, + .bandwidth_size = 0x8, + .mbox_irq_enable = ISP_MBOX_IRQ_ENABLE_T6031, + + /* + * No BES scaler: the firmware asserts on CH_CROP_SET ("BesProc is + * removed/not present. Must config AuxSclPr") and takes the SCL1 set. + */ + .scl1 = true, + .lpdp = true, + .meta_size = ISP_META_SIZE_T8140, + + .fw_iova_mask = GENMASK_ULL(35, 0), + .mbox2_offset = 0x410, + .mbox_irq_route = true, + .coproc_control = ISP_COPROC_CONTROL_T8140, + .fw_mmio = apple_isp_fw_mmio_t8142, + .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t8142), + .boot_mode = 1, + .capture_meta_size = ISP_CAPTURE_META_SIZE_T8140, + .resident_fw = true, + /* The J813 IMX958 runs the xx24 setfile 2224_01, as T6040 does. */ + .setfile_stride_fixup = true, +}; + +/* + * T6050 (M5 Max, J714c): ISP17a, ASC version 0x110003. The J714c ADT's + * dart-isp0 dapf-instance-0 ranges, rounded out to 16 KiB pages; the T8140 + * windows at other bases, with eight DSID windows. + */ +static const struct isp_mmio_window apple_isp_fw_mmio_t6050[] = { + { 0x2220004000, 0x14000 }, /* DSID broadcast-clear windows */ + { 0x2222004000, 0x14000 }, + { 0x2224004000, 0x14000 }, + { 0x2226004000, 0x14000 }, + { 0x2228004000, 0x14000 }, + { 0x222a004000, 0x14000 }, + { 0x222c004000, 0x14000 }, + { 0x222e004000, 0x14000 }, + { 0x280680000, 0x8000 }, + { 0x280900000, 0x4000 }, /* PMGR, ISP power states */ + { 0x284240000, 0x28000 }, /* PMGR, as T8140's PMP clock scratch window */ + { 0x284e3c000, 0x4000 }, + { 0x286724000, 0x4000 }, /* PMP bandwidth scratch */ + { 0x29062c000, 0x20000 }, + { 0x509660000, 0x4000 }, +}; + +/* The J714c ADT's dart-isp0 address filter windows, in its order and with its ends */ +static const struct isp_dapf_window apple_isp_dapf_t6050[] = { + { 0x2809003b0, 0x2809003f3 }, /* PMGR, ISP power states */ + { 0x280680000, 0x280687fff }, + { 0x284e3c000, 0x284e3c003 }, + { 0x29062c000, 0x290648003 }, + { 0x509660000, 0x509660003 }, + { 0x286724000, 0x286724004 }, /* PMP bandwidth scratch */ + { 0x284240000, 0x284264000 }, /* PMGR, as T8140's PMP clock scratch window */ + { 0x2220004000, 0x22200162fc }, /* DSID broadcast-clear windows */ + { 0x2222004000, 0x22220162fc }, + { 0x2224004000, 0x22240162fc }, + { 0x2226004000, 0x22260162fc }, + { 0x2228004000, 0x22280162fc }, + { 0x222a004000, 0x222a0162fc }, + { 0x222c004000, 0x222c0162fc }, + { 0x222e004000, 0x222e0162fc }, +}; + +/* + * T6050 (M5 Max, J714c) boots as T6040: the same isp0 and dart-isp0 + * addresses, mailbox at +0x2924000, and 32 MiB of firmware main memory at + * IOVA 0 whose __TEXT and __DATA end at 0x1fec000. The broadcast-clear + * window is the first of the eight in its DAPF ranges. The PMP clock scratch + * word is not known yet; without it the driver sends no PMP control. + */ +static const struct apple_isp_hw apple_isp_hw_t6050 = { + .gen = ISP_GEN_T6040, + .fw_abi = ISP_FW_ABI_H17, + .pmu_base = 0x0, + + .dsid_count = 1, + .dsid_slots = 8, + .dsid_clr_base0 = 0x2220016000, + .dsid_clr_range0 = 0x2fc, + + .clock_scratch = 0x0, + .mbox_irq_enable = ISP_MBOX_IRQ_ENABLE_T6031, + + .scl1 = true, + /* The sensor link is LPDP ("SIF 0 (LPDP)"), as on T8142. */ + .lpdp = true, + .meta_size = ISP_META_SIZE_T8140, + .fw_mmio = apple_isp_fw_mmio_t6050, + .num_fw_mmio = ARRAY_SIZE(apple_isp_fw_mmio_t6050), + .dapf = apple_isp_dapf_t6050, + .num_dapf = ARRAY_SIZE(apple_isp_dapf_t6050), + .dapf_ctrl2 = 0xcc3, + + .coproc_control = ISP_COPROC_CONTROL_T8140, + .boot_mode = 1, + .shared_size = 0xde600000, + .resident_fw = true, + .meta_exposure = true, + .camera_light = true, + /* The J714c's IMX958 (build revision 2) is tuned by 2226_02 */ + .imx958_2226 = true, + .setfile_radial_gain = true, + /* As macOS's appleh16camerad sets them for the J714c's IMX958 */ + .lsc_ideal_percent = 84, + .ae_fd_scene_metering = true, }; static const struct of_device_id apple_isp_of_match[] = { @@ -797,6 +1256,10 @@ static const struct of_device_id apple_isp_of_match[] = { { .compatible = "apple,t6030-isp", .data = &apple_isp_hw_t6030 }, { .compatible = "apple,t6031-isp", .data = &apple_isp_hw_t6031 }, { .compatible = "apple,t8140-isp", .data = &apple_isp_hw_t8140 }, + { .compatible = "apple,t8142-isp", .data = &apple_isp_hw_t8142 }, + { .compatible = "apple,t6050-isp", .data = &apple_isp_hw_t6050 }, + { .compatible = "apple,t6041-isp", .data = &apple_isp_hw_t6040 }, + { .compatible = "apple,t8132-isp", .data = &apple_isp_hw_t8132 }, {}, }; MODULE_DEVICE_TABLE(of, apple_isp_of_match); @@ -837,8 +1300,8 @@ static __maybe_unused int apple_isp_freeze(struct device *dev) /* * Hibernation hands the system from one kernel to another, and * firmware left running would go on using the memory of the kernel - * that started it. Stop it cleanly instead; the camera is then - * unavailable until the next system boot. + * that started it. Stop it cleanly instead; if it ran, the camera is + * then unavailable until the next system boot. */ if (isp->hw->resident_fw) { mutex_lock(&isp->video_lock); @@ -870,6 +1333,23 @@ static const struct dev_pm_ops apple_isp_pm_ops = { RUNTIME_PM_OPS(apple_isp_runtime_suspend, apple_isp_runtime_resume, NULL) }; +/* + * Stop firmware that starts again (hw->fw_restore) before a reboot or a + * kexec, so that the next kernel finds its data segment as loaded. + */ +static void apple_isp_shutdown(struct platform_device *pdev) +{ + struct apple_isp *isp = platform_get_drvdata(pdev); + + if (!isp || !isp->hw->fw_restore) + return; + /* A stream that still runs is stopped cleanly first, as for hibernation */ + apple_isp_video_suspend(isp); + mutex_lock(&isp->video_lock); + apple_isp_firmware_halt(isp); + mutex_unlock(&isp->video_lock); +} + static struct platform_driver apple_isp_driver = { .driver = { .name = "apple-isp", @@ -878,9 +1358,14 @@ static struct platform_driver apple_isp_driver = { }, .probe = apple_isp_probe, .remove = apple_isp_remove, + .shutdown = apple_isp_shutdown, }; module_platform_driver(apple_isp_driver); MODULE_AUTHOR("Eileen Yoon "); MODULE_DESCRIPTION("Apple ISP driver"); MODULE_LICENSE("GPL v2"); + +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) +#include "isp-drv-test.c" +#endif diff --git a/drivers/media/platform/apple/isp/isp-drv.h b/drivers/media/platform/apple/isp/isp-drv.h index 8f945c26f00e6..ec8b80f3d3281 100644 --- a/drivers/media/platform/apple/isp/isp-drv.h +++ b/drivers/media/platform/apple/isp/isp-drv.h @@ -6,11 +6,11 @@ #include #include +#include #include #include #include -#include #include #include #include @@ -30,6 +30,8 @@ /* used to limit the user space buffers to the buffer_pool_config */ #define ISP_MAX_BUFFERS 16 +#define ISP_MAX_SENSOR_CONFIGS 64 +#define ISP_SENSOR_CONFIG_SIZE 0x120 /* capture metadata buffers, for firmware that has that pool */ #define ISP_CAPMETA_BUFFERS 8 @@ -51,6 +53,7 @@ enum isp_generation { ISP_GEN_T8112, ISP_GEN_T6031, ISP_GEN_T8140, + ISP_GEN_T6040, }; /* @@ -79,7 +82,6 @@ enum isp_firmware_version { }; struct isp_surf { - struct drm_mm_node *mm; struct list_head head; u64 size; u64 type; @@ -91,6 +93,7 @@ struct isp_surf { refcount_t refcount; bool gc; bool submitted; + bool iova_reserved; /* @iova comes from &apple_isp.iovas */ }; struct isp_message { @@ -115,6 +118,8 @@ struct isp_channel { void *virt; u32 doorbell; u32 cursor; + unsigned long deadline; + bool deadline_active; struct mutex lock; struct isp_message req; struct isp_message rsp; @@ -132,12 +137,23 @@ struct isp_mmio_window { u64 size; }; +/* + * A DART address filter window. Both bounds are inclusive and word + * granular, as the hardware takes them. + */ +struct isp_dapf_window { + u64 start; + u64 end; +}; + struct isp_preset { u32 index; struct coord input_dim; struct coord output_dim; struct coord crop_offset; struct coord crop_size; + /* Processed P010 geometry established by preview or a config reply. */ + bool p010; }; struct apple_isp_hw { @@ -146,6 +162,8 @@ struct apple_isp_hw { u64 pmu_base; int dsid_count; + /* windows of the H17 multi broadcast-clear command, the unused all ones */ + u8 dsid_slots; u64 dsid_clr_base0; u64 dsid_clr_base1; u64 dsid_clr_base2; @@ -178,22 +196,79 @@ struct apple_isp_hw { bool mbox_irq_route; /* coprocessor control register, 0 for ISP_COPROC_CONTROL */ u32 coproc_control; + /* coprocessor status register, 0 for ISP_COPROC_STATUS */ + u32 coproc_status; + /* + * coprocessor quiesce register, 0 for none. The CPU core domains of + * a coprocessor that stopped only power down cleanly once it has + * quiesced; otherwise they stay busy and never run again. + */ + u32 coproc_quiesce; /* windows mapped 1:1 for the firmware */ const struct isp_mmio_window *fw_mmio; unsigned int num_fw_mmio; + /* + * The windows the ADT gives the first DART's address filter, inside + * the fw_mmio windows. The filter loses them with ISP_SYS, so they + * are programmed after every power-up. + */ + const struct isp_dapf_window *dapf; + unsigned int num_dapf; + /* The ADT's DAPF r20 word, if not DAPF_CTRL2_ISP */ + u32 dapf_ctrl2; + /* ISP_GPIO_6 boot mode */ u32 boot_mode; + /* shared memory size in the H16 boot descriptor, 0 for the T8140 one */ + u64 shared_size; + /* size of a capture metadata buffer, 0 for no such pool */ u32 capture_meta_size; + /* metadata buffers carry the frame's exposure report (H17 firmware) */ + bool meta_exposure; + /* - * The firmware can be started only once per system boot, so it is - * started at probe and kept running while the driver is bound. + * The firmware leaves its data segment changed when it stops, and + * does not start from it again. Keep a copy of the segment as the + * boot loader loaded it, taken at probe before the firmware first + * runs, and put it back whenever the firmware has stopped. + */ + bool fw_restore; + + /* + * The firmware can be started only once per system boot, so once + * started it is kept running while the driver is bound. It starts at + * probe, or at the first stream with deferred_start. */ bool resident_fw; + + /* + * Start the firmware, and detect the camera, at the first stream + * instead of at probe, so the ISP stays off until then. The firmware + * must be resident or start again (fw_restore): the camera is + * detected only once. + */ + bool deferred_start; + /* + * The camera light is the driver's to drive: no stream starts unless + * it is lit, and the capture watchdog is fed only while it is. + */ + bool camera_light; + + /* The xx24 setfiles' lens shading strides need zeroing (T8142). */ + bool setfile_stride_fixup; + /* The IMX958 module takes setfile 2226_02, not 2224_01 */ + bool imx958_2226; + /* Compute the setfile's noise reduction radial gains from its lens shading */ + bool setfile_radial_gain; + /* The lens shading falloff to correct to, in percent at the corner (0: unset) */ + u16 lsc_ideal_percent; + /* H17 firmware that takes the face-detection scene metering setup */ + bool ae_fd_scene_metering; }; enum isp_sensor_id { @@ -244,13 +319,24 @@ enum isp_sensor_id { ISP_IMX814_2123_01, ISP_IMX853_7622_01, ISP_IMX913_7523_01, + ISP_IMX958_2224_01, + ISP_IMX958_2226_02, ISP_VD56G0_6221_01, ISP_VD56G0_6222_01, }; +struct isp_sensor_config { + struct coord input_dim; + struct coord output_dim; + u32 sensor_clk; + bool usable; + u8 reply[ISP_SENSOR_CONFIG_SIZE]; +}; + struct isp_format { enum isp_sensor_id id; u32 version; + u32 pixelformat; struct isp_preset *preset; unsigned int num_planes; u32 strides[VB2_MAX_PLANES]; @@ -264,8 +350,12 @@ struct apple_isp { enum isp_firmware_version fw_compat; u32 platform_id; u32 temporal_filter; + u32 cam_connections_scheme; struct isp_preset *presets; int num_presets; + /* Read-only sensor configuration replies, cached before video registration. */ + unsigned int num_sensor_configs; + struct isp_sensor_config *sensor_configs; int num_channels; struct isp_format fmts[ISP_MAX_CHANNELS]; @@ -276,10 +366,16 @@ struct apple_isp { struct v4l2_device v4l2_dev; struct v4l2_ctrl_handler ctrl_handler; struct v4l2_ctrl *exposure_priority; + struct v4l2_ctrl *power_line_frequency; + /* read-only exposure report of the last frame, NULL without one */ + struct v4l2_ctrl *gain; + struct v4l2_ctrl *exposure; + /* last reported total gain (1/256 steps) and exposure (us) */ + u32 frame_gain; + u32 frame_exposure_us; struct vb2_queue vbq; struct mutex video_lock; unsigned int sequence; - bool multiplanar; /* capture rate in frames per second, changes under video_lock */ unsigned int frame_rate; @@ -295,25 +391,43 @@ struct apple_isp { void __iomem *gpio; void __iomem *mbox2; void __iomem *wdt; + /* The camera light's GPIO register (hw->camera_light) */ + void __iomem *cil; + u32 cil_config; + u32 cil_saved; + bool cil_on_high; + bool cil_lit; + void __iomem *dapf; struct hrtimer wdt_timer; bool wdt_running; struct iommu_domain *domain; unsigned long shift; - struct drm_mm iovad; /* TODO iova.c can't allocate bottom-up */ - u64 iova_size; /* size of the iovad range */ + /* allocated IOVA ranges, in IOMMU pages, lowest free range first */ + struct maple_tree iovas; + u64 iova_size; /* size of the range @iovas allocates from */ u64 fw_iova_mask; /* see isp_fw_iova() */ - struct mutex iovad_lock; struct isp_firmware { + u64 heap_base; u64 heap_top; + u64 data_top; /* 0 unless the bootloader loads the data */ } fw; /* changes under video_lock once the video device is registered */ enum isp_fw_state fw_state; + /* the camera was detected; until then a deferred start is pending */ + bool camera_detected; + /* configurations the firmware has, once a deferred start detected them */ + u32 num_fw_presets; struct isp_surf *ipc_surf; struct isp_surf *extra_surf; struct isp_surf *data_surf; + struct isp_surf *main_tail_surf; + /* With hw->fw_restore: the firmware data segment and its copy */ + void *fw_data; + void *fw_data_copy; + size_t fw_data_size; struct isp_surf *log_surf; struct isp_surf *bt_surf; struct isp_surf *meta_surfs[ISP_MAX_BUFFERS]; @@ -347,16 +461,35 @@ struct apple_isp { spinlock_t buf_lock; struct list_head bufs_pending; struct list_head bufs_submitted; + struct list_head bufs_returned; + u64 buffer_tag; + unsigned long meta_returned; + /* + * The channel of a timed-out request that has not been answered yet, + * see isp_ipc_resync(). Sends are serialized by the callers, which + * share the command area. + */ + struct isp_channel *ipc_stale; + /* + * Not even a late answer came, so where the firmware is in the ring + * is unknown. Nothing is sent until the firmware is stopped, and + * resident firmware cannot be started again before the next boot. + */ + bool ipc_failed; }; struct isp_chan_ops { int (*handle)(struct apple_isp *isp, struct isp_channel *chan); + /* Called after the response slot and its doorbell are published. */ + void (*complete)(struct apple_isp *isp); }; struct isp_buffer { struct vb2_v4l2_buffer vb; struct list_head link; struct isp_surf surfs[VB2_MAX_PLANES]; + u64 tag; + enum vb2_buffer_state return_state; }; #define to_isp_buffer(x) container_of((x), struct isp_buffer, vb) @@ -367,6 +500,8 @@ enum { ISP_STATE_SLEEPING, }; +int apple_isp_program_dapf(struct apple_isp *isp); + #define isp_dbg(isp, fmt, ...) \ dev_dbg((isp)->dev, "[%s] " fmt, __func__, ##__VA_ARGS__) @@ -384,4 +519,37 @@ enum { ((isp)->hw->capture_meta_size ? ISP_CAPMETA_BUFFERS : 0) #define isp_get_current_format(isp) (isp_get_format(isp, isp->current_ch)) +static inline bool isp_supports_p010(struct apple_isp *isp) +{ + /* Its config discovery would change formats already advertised. */ + return isp->hw->gen == ISP_GEN_T6040 && !isp->hw->deferred_start && + isp_get_current_format(isp)->version == 0x958; +} + +static inline bool isp_has_p010(struct apple_isp *isp) +{ + if (!isp_supports_p010(isp)) + return false; + for (int i = 0; i < isp->num_presets; i++) { + if (isp->presets[i].p010) + return true; + } + return false; +} + +static inline bool isp_is_p010(struct apple_isp *isp) +{ + return isp_get_current_format(isp)->pixelformat == V4L2_PIX_FMT_P010; +} + +static inline unsigned int isp_num_rendered(struct apple_isp *isp) +{ + return isp_is_p010(isp) ? 2 : ISP_MAX_BUFFERS; +} + +static inline unsigned int isp_num_meta(struct apple_isp *isp) +{ + return isp_is_p010(isp) ? 8 : ISP_MAX_BUFFERS; +} + #endif /* __ISP_DRV_H__ */ diff --git a/drivers/media/platform/apple/isp/isp-fw.c b/drivers/media/platform/apple/isp/isp-fw.c index c96681173fd0f..bab0d7d9d8b8e 100644 --- a/drivers/media/platform/apple/isp/isp-fw.c +++ b/drivers/media/platform/apple/isp/isp-fw.c @@ -6,10 +6,12 @@ #include #include +#include #include #include #include #include +#include #include "isp-cam.h" #include "isp-cmd.h" @@ -52,6 +54,11 @@ static inline u32 isp_coproc_control(struct apple_isp *isp) return isp->hw->coproc_control ?: ISP_COPROC_CONTROL; } +static inline u32 isp_coproc_status(struct apple_isp *isp) +{ + return isp->hw->coproc_status ?: ISP_COPROC_STATUS; +} + /* Wait for the firmware to write @expected to a GPIO word. */ static int isp_gpio_wait(struct apple_isp *isp, u32 reg, u32 expected) { @@ -163,6 +170,7 @@ static_assert(sizeof(struct isp_firmware_bootargs) == 0x180); */ #define ISP_H16_DESCRIPTOR_SIZE 0x200 #define ISP_H16_SHARED_SIZE 0xee000000ULL +#define ISP_H16_DESC_CAM_SCHEME 0x60 #define ISP_H16_DESC_NO_OPTICAL_CARD_ID 0xb0 struct isp_firmware_bootargs_h16 { @@ -210,6 +218,7 @@ static const struct isp_chan_ops sm_ops = { static const struct isp_chan_ops bt_ops = { .handle = ipc_bt_handle, + .complete = ipc_bt_complete, }; static irqreturn_t apple_isp_isr(int irq, void *dev) @@ -315,6 +324,31 @@ static int isp_reset_coproc_t8140(struct apple_isp *isp) return 0; } +/* + * The T6040 firmware is started as the bootloader left it, without a reset: + * the coprocessor must never have run, and its reset vector must be the + * entry of the firmware text at the base of the heap region. + */ +static int isp_check_coproc_t6040(struct apple_isp *isp) +{ + u32 control = isp_coproc_read32(isp, isp_coproc_control(isp)); + u64 rvbar = readq(isp->coproc + ISP_COPROC_RVBAR); + + if (control) { + isp_err(isp, "coprocessor already started (control 0x%x)\n", + control); + return -EBUSY; + } + + if (rvbar != (isp->fw.heap_base | 1)) { + isp_err(isp, "reset vector 0x%llx is not the firmware at 0x%llx\n", + rvbar, isp->fw.heap_base); + return -ENODEV; + } + + return 0; +} + static int isp_reset_coproc(struct apple_isp *isp) { int retries; @@ -323,6 +357,8 @@ static int isp_reset_coproc(struct apple_isp *isp) if (isp->hw->gen == ISP_GEN_T8140) return isp_reset_coproc_t8140(isp); + if (isp->hw->gen == ISP_GEN_T6040) + return isp_check_coproc_t6040(isp); isp_coproc_write32(isp, ISP_COPROC_EDPRCR, 0x2); @@ -361,10 +397,47 @@ static int isp_reset_coproc(struct apple_isp *isp) return 0; } +/* + * With hw->fw_restore, put the firmware memory back as the boot loader left + * it: the data segment from its copy, and the main memory above it cleared. + */ +static void isp_restore_fw_memory(struct apple_isp *isp) +{ + struct isp_surf *tail = isp->main_tail_surf; + + if (!isp->fw_data_copy) + return; + + memcpy(isp->fw_data, isp->fw_data_copy, isp->fw_data_size); + if (tail) + for (u32 i = 0; i < tail->num_pages; i++) + clear_highpage(tail->pages[i]); +} + static void isp_firmware_shutdown_stage1(struct apple_isp *isp) { + /* Only a coprocessor that was started has to settle and quiesce */ + bool started = isp->hw->coproc_quiesce && + isp_coproc_read32(isp, isp_coproc_control(isp)); + u32 val; + + /* A coprocessor that powered down waits in WFI */ + if (started && + readl_poll_timeout(isp->coproc + isp_coproc_status(isp), val, + val & ISP_COPROC_IN_WFI, 100, 200 * USEC_PER_MSEC)) + dev_warn(isp->dev, "coproc not in WFI before power-down (status 0x%x)\n", + val); + isp_coproc_write32(isp, isp_coproc_control(isp), 0x0); + if (started) { + /* The request bit alone asks for it, and reads back once done */ + isp_coproc_write32(isp, isp->hw->coproc_quiesce, ISP_COPROC_QUIESCED); + if (readl_poll_timeout(isp->coproc + isp->hw->coproc_quiesce, val, + val & ISP_COPROC_QUIESCED, 10, 50 * USEC_PER_MSEC)) + dev_warn(isp->dev, "coproc did not quiesce (0x%x)\n", val); + } + apple_isp_power_down_domains(isp); } @@ -373,12 +446,20 @@ static int isp_firmware_boot_stage1(struct apple_isp *isp) int err; // u32 val; + isp_restore_fw_memory(isp); + err = apple_isp_power_up_domains(isp); if (err < 0) return err; - /* ISP17a has no clock enable word */ - if (isp->hw->gen != ISP_GEN_T8140) + if (isp->dapf) { + err = apple_isp_program_dapf(isp); + if (err) + goto shutdown; + } + + /* ISP17a and T6040 have no clock enable word */ + if (isp->hw->gen != ISP_GEN_T8140 && isp->hw->gen != ISP_GEN_T6040) isp_gpio_write32(isp, ISP_GPIO_CLOCK_EN, 0x1); #if 0 @@ -445,11 +526,27 @@ int apple_isp_alloc_firmware_surface(struct apple_isp *isp) dev_dbg(isp->dev, "Data surface iova: 0x%llx\n", (long long)isp->data_surf->iova); + /* Without it, the firmware faults on its first writes there. */ + if (isp->fw.data_top && isp->fw.data_top < isp->fw.heap_top) { + isp->main_tail_surf = + isp_alloc_surface_at(isp, isp->fw.data_top, + isp->fw.heap_top - isp->fw.data_top); + if (!isp->main_tail_surf) { + isp_err(isp, "failed to back main memory 0x%llx-0x%llx\n", + isp->fw.data_top, isp->fw.heap_top); + isp_free_surface(isp, isp->data_surf); + isp_free_surface(isp, isp->ipc_surf); + return -ENOMEM; + } + } + return 0; } void apple_isp_free_firmware_surface(struct apple_isp *isp) { + if (isp->main_tail_surf) + isp_free_surface(isp, isp->main_tail_surf); isp_free_surface(isp, isp->data_surf); isp_free_surface(isp, isp->ipc_surf); } @@ -491,12 +588,14 @@ static void isp_write_bootargs_h16(struct apple_isp *isp, void *virt, memset(&args, 0, sizeof(args)); args.ipc_iova = isp->ipc_surf->iova; args.shared_base = isp->fw.heap_top & 0xffffffff; - args.shared_size = ISP_H16_SHARED_SIZE; + args.shared_size = isp->hw->shared_size ?: ISP_H16_SHARED_SIZE; args.extra_iova = isp->extra_surf->iova; args.extra_size = isp->extra_surf->size; args.platform_id = isp->platform_id; args.ipc_queue_size = (u64)args_offset + 1; args.unk_68 = 0x40; + put_unaligned_le64(isp->cam_connections_scheme, + &args.descriptor[ISP_H16_DESC_CAM_SCHEME]); args.descriptor[ISP_H16_DESC_NO_OPTICAL_CARD_ID] = 1; memcpy(virt, &args, sizeof(args)); } @@ -789,26 +888,26 @@ static int isp_firmware_boot_stage3(struct apple_isp *isp) static int isp_stop_command_processor(struct apple_isp *isp) { -#if 0 - int res = isp_cmd_stop(isp, 0); - if (res) { - isp_err(isp, "isp_cmd_stop() failed\n"); - return res; - } + int res, ret; - /* Wait for ISP_GPIO_0 to 0xf7fbdff9 -> 0x8042006 */ isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); - isp_cmd_power_down(isp); -#else - isp_gpio_write32(isp, ISP_GPIO_0, 0xf7fbdff9); - - int res = isp_cmd_suspend(isp); + /* + * Firmware that starts again (hw->fw_restore) is stopped for good: + * STOP also powers the camera module down, which SUSPEND leaves on, + * even across a reboot. POWER_DOWN follows even if STOP failed. + */ + if (isp->hw->fw_restore) { + res = isp_cmd_stop(isp, 0); + ret = isp_cmd_power_down(isp); + res = res ?: ret; + } else { + res = isp_cmd_suspend(isp); + } if (res) { - isp_err(isp, "isp_cmd_suspend() failed\n"); + isp_err(isp, "failed to stop the firmware: %d\n", res); return res; } -#endif if (isp_gpio_wait(isp, ISP_GPIO_0, 0x8042006)) { isp_err(isp, "never received magic number from firmware\n"); @@ -825,12 +924,26 @@ static int isp_set_dsid_clr(struct apple_isp *isp) int err; /* - * The H17 firmware takes no PMU base, and its single broadcast-clear - * window in the multi-window form. + * The H17 firmware takes no PMU base, and its broadcast-clear windows + * in the multi-window form. */ - if (hw->fw_abi == ISP_FW_ABI_H17) - return isp_cmd_set_dsid_clr_multi_bc(isp, hw->dsid_clr_base0, + if (hw->fw_abi == ISP_FW_ABI_H17) { + const u64 used[] = { hw->dsid_clr_base0, hw->dsid_clr_base1, + hw->dsid_clr_base2, hw->dsid_clr_base3 }; + u64 bases[ISP_DSID_MULTI_BC_MAX]; + unsigned int slots = max_t(unsigned int, hw->dsid_count, + hw->dsid_slots); + + if (WARN_ON(hw->dsid_count > ARRAY_SIZE(used) || + slots > ARRAY_SIZE(bases))) + return -EINVAL; + + for (unsigned int i = 0; i < slots; i++) + bases[i] = i < hw->dsid_count ? used[i] : U64_MAX; + + return isp_cmd_set_dsid_clr_multi_bc(isp, bases, slots, hw->dsid_clr_range0); + } err = isp_cmd_set_isp_pmu_base(isp, hw->pmu_base); if (err) @@ -909,6 +1022,9 @@ static void isp_firmware_stop(struct apple_isp *isp) isp_firmware_shutdown_stage3(isp); isp_firmware_shutdown_stage2(isp); isp_collect_gc_surface(isp); + + /* Also leave it as loaded for a rebound driver or another kernel. */ + isp_restore_fw_memory(isp); } static int isp_firmware_boot(struct apple_isp *isp) @@ -920,6 +1036,8 @@ static int isp_firmware_boot(struct apple_isp *isp) /* Stage 1 stops the coprocessor and powers it down itself. */ isp_err(isp, "failed firmware boot stage 1: %d\n", err); isp_collect_gc_surface(isp); + /* The firmware may have run: leave its memory as loaded. */ + isp_restore_fw_memory(isp); return err; } @@ -977,7 +1095,8 @@ static void isp_firmware_shutdown(struct apple_isp *isp) * Resident firmware (hw->resident_fw) is loaded by the bootloader and * does not start again once it has been stopped: after CISP_CMD_SUSPEND, * a coprocessor reset and a power cycle of its domains it never completes - * the first handshake. It is booted once, at probe, and runs until the + * the first handshake. It is booted once, at probe or with + * deferred_start at the first stream, and runs until the * driver is unbound, probe fails or the system hibernates, also across * suspend; after it has stopped the device stays unusable until the next * system boot. @@ -998,6 +1117,10 @@ int apple_isp_firmware_boot(struct apple_isp *isp) return -EIO; } + /* The firmware starts with new rings. */ + isp->ipc_stale = NULL; + isp->ipc_failed = false; + /* Needs to be power cycled for IOMMU to behave correctly */ err = pm_runtime_resume_and_get(isp->dev); if (err < 0) { @@ -1019,7 +1142,7 @@ int apple_isp_firmware_boot(struct apple_isp *isp) return 0; } -/* The end of a stream or of the camera detection at probe */ +/* The end of a stream or of the camera detection */ void apple_isp_firmware_shutdown(struct apple_isp *isp) { if (!isp->hw->resident_fw) { @@ -1049,5 +1172,7 @@ void apple_isp_firmware_halt(struct apple_isp *isp) isp_firmware_shutdown(isp); pm_runtime_put_sync(isp->dev); + apple_isp_light_force_off(isp); + isp->ipc_stale = NULL; isp->fw_state = isp->hw->resident_fw ? ISP_FW_DEAD : ISP_FW_OFF; } diff --git a/drivers/media/platform/apple/isp/isp-iommu.c b/drivers/media/platform/apple/isp/isp-iommu.c index ade5a4a31f499..12dbf6d47cccb 100644 --- a/drivers/media/platform/apple/isp/isp-iommu.c +++ b/drivers/media/platform/apple/isp/isp-iommu.c @@ -56,41 +56,35 @@ static void isp_surf_vunmap(struct apple_isp *isp, struct isp_surf *surf) static void isp_surf_unreserve_iova(struct apple_isp *isp, struct isp_surf *surf) { - if (surf->mm) { - mutex_lock(&isp->iovad_lock); - drm_mm_remove_node(surf->mm); - mutex_unlock(&isp->iovad_lock); - kfree(surf->mm); - } - surf->mm = NULL; + if (surf->iova_reserved) + mtree_erase(&isp->iovas, surf->iova >> isp->shift); + surf->iova_reserved = false; } +/* + * The IOVA range starts at the top of the firmware heap. Hand out the lowest + * free range of whole IOMMU pages above it. + */ static int isp_surf_reserve_iova(struct apple_isp *isp, struct isp_surf *surf) { + unsigned long first = DIV_ROUND_UP_ULL(isp->fw.heap_top, 1ULL << isp->shift); + unsigned long last = ((isp->fw.heap_top + isp->iova_size) >> isp->shift) - 1; + unsigned long pages = DIV_ROUND_UP_ULL(surf->size, 1ULL << isp->shift); + unsigned long index; int err; - surf->mm = kzalloc(sizeof(*surf->mm), GFP_KERNEL); - if (!surf->mm) - return -ENOMEM; - - mutex_lock(&isp->iovad_lock); - err = drm_mm_insert_node_generic(&isp->iovad, surf->mm, - ALIGN(surf->size, 1UL << isp->shift), - 1UL << isp->shift, 0, 0); - mutex_unlock(&isp->iovad_lock); + err = mtree_alloc_range(&isp->iovas, &index, surf, pages, first, last, + GFP_KERNEL); if (err < 0) { dev_err(isp->dev, "failed to reserve 0x%llx of iova space\n", surf->size); - goto mm_free; + return err; } - surf->iova = surf->mm->start; + surf->iova = (dma_addr_t)index << isp->shift; + surf->iova_reserved = true; return 0; -mm_free: - kfree(surf->mm); - surf->mm = NULL; - return err; } static void isp_surf_iommu_unmap(struct apple_isp *isp, struct isp_surf *surf) @@ -129,7 +123,7 @@ static int isp_surf_iommu_map(struct apple_isp *isp, struct isp_surf *surf) static void __isp_surf_init(struct apple_isp *isp, struct isp_surf *surf, u64 size, bool gc) { - surf->mm = NULL; + surf->iova_reserved = false; surf->virt = NULL; surf->size = ALIGN(size, 1UL << isp->shift); surf->num_pages = surf->size >> isp->shift; @@ -218,6 +212,35 @@ struct isp_surf *isp_alloc_surface_vmap(struct apple_isp *isp, u64 size) return surf; } +/* A surface at a fixed IOVA, outside the range the others come from. */ +struct isp_surf *isp_alloc_surface_at(struct apple_isp *isp, dma_addr_t iova, + u64 size) +{ + struct isp_surf *surf; + + surf = kzalloc_obj(*surf); + if (!surf) + return NULL; + + __isp_surf_init(isp, surf, size, false); + surf->iova = iova; + + if (isp_surf_alloc_pages(surf) < 0) + goto free_surf; + + if (isp_surf_iommu_map(isp, surf) < 0) + goto free_pages; + + refcount_set(&surf->refcount, 1); + return surf; + +free_pages: + isp_surf_free_pages(surf); +free_surf: + kfree(surf); + return NULL; +} + void isp_free_surface(struct apple_isp *isp, struct isp_surf *surf) { if (refcount_dec_and_test(&surf->refcount)) { @@ -262,7 +285,7 @@ int apple_isp_iommu_map_sgt(struct apple_isp *isp, struct isp_surf *surf, ssize_t mapped; // TODO userptr sends unaligned sizes - surf->mm = NULL; + surf->iova_reserved = false; surf->size = size; err = isp_surf_reserve_iova(isp, surf); diff --git a/drivers/media/platform/apple/isp/isp-iommu.h b/drivers/media/platform/apple/isp/isp-iommu.h index a7fce2713f584..3a999a8c5b3d5 100644 --- a/drivers/media/platform/apple/isp/isp-iommu.h +++ b/drivers/media/platform/apple/isp/isp-iommu.h @@ -10,6 +10,8 @@ struct isp_surf *__isp_alloc_surface(struct apple_isp *isp, u64 size, bool gc); #define isp_alloc_surface(isp, size) (__isp_alloc_surface(isp, size, false)) #define isp_alloc_surface_gc(isp, size) (__isp_alloc_surface(isp, size, true)) struct isp_surf *isp_alloc_surface_vmap(struct apple_isp *isp, u64 size); +struct isp_surf *isp_alloc_surface_at(struct apple_isp *isp, dma_addr_t iova, + u64 size); int isp_surf_vmap(struct apple_isp *isp, struct isp_surf *surf); void isp_free_surface(struct apple_isp *isp, struct isp_surf *surf); void isp_free_released_surfaces(struct apple_isp *isp); diff --git a/drivers/media/platform/apple/isp/isp-ipc-test.c b/drivers/media/platform/apple/isp/isp-ipc-test.c index 6521cecb4cfb6..d7ca456055cec 100644 --- a/drivers/media/platform/apple/isp/isp-ipc-test.c +++ b/drivers/media/platform/apple/isp/isp-ipc-test.c @@ -144,7 +144,139 @@ static void isp_test_sm_free_log_bt(struct kunit *test) KUNIT_EXPECT_NULL(test, t->isp->bt_surf); } +static void isp_test_command_ring_wrap(struct kunit *test) +{ + struct isp_sm_test *t = isp_sm_test_alloc(test); + struct isp_channel *chan = t->chan; + struct isp_message *slots; + + chan->num = 8; + chan->virt = kunit_kcalloc(test, chan->num, sizeof(*slots), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, chan->virt); + slots = chan->virt; + chan->req.arg0 = 0x5000000; + isp_get_current_format(t->isp)->pixelformat = V4L2_PIX_FMT_P010; + for (unsigned int i = 0; i < 100000; i++) { + unsigned int cursor = chan->cursor; + + /* Re-publication must clear the old ACK before waiting on a wrap. */ + slots[cursor].arg0 = chan->req.arg0; + KUNIT_ASSERT_FALSE(test, chan_tx_done(t->isp, chan)); + KUNIT_ASSERT_EQ(test, chan->cursor, cursor); + slots[cursor].arg0 |= ISP_IPC_FLAG_ACK; + KUNIT_ASSERT_TRUE(test, chan_tx_done(t->isp, chan)); + KUNIT_ASSERT_EQ(test, chan->cursor, (cursor + 1) % chan->num); + } + KUNIT_EXPECT_EQ(test, chan->cursor, 0); +} + +static void isp_test_late_ack_keeps_cursor(struct kunit *test) +{ + struct isp_sm_test *t = isp_sm_test_alloc(test); + struct isp_message response = { .arg0 = 0x5000001 }; + + isp_get_current_format(t->isp)->pixelformat = V4L2_PIX_FMT_P010; + t->chan->virt = &response; + t->chan->num = 8; + t->chan->req.arg0 = 0x5000000; + t->chan->deadline_active = true; + t->chan->deadline = jiffies; + KUNIT_EXPECT_FALSE(test, chan_tx_done(t->isp, t->chan)); + KUNIT_EXPECT_EQ(test, t->chan->cursor, 0); + t->chan->deadline = jiffies + HZ; + KUNIT_EXPECT_TRUE(test, chan_tx_done(t->isp, t->chan)); + KUNIT_EXPECT_EQ(test, t->chan->cursor, 1); +} + +/* A P010 command ring whose slots and doorbell are plain memory */ +static struct isp_message *isp_test_command_ring(struct kunit *test, + struct isp_sm_test *t) +{ + struct isp_channel *chan = t->chan; + struct isp_message *slots; + + t->isp->mbox2 = (__force void __iomem *) + kunit_kzalloc(test, ISP_MBOX2_SIZE, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, (__force void *)t->isp->mbox2); + init_waitqueue_head(&t->isp->wait); + isp_get_current_format(t->isp)->pixelformat = V4L2_PIX_FMT_P010; + + chan->name = "IO"; + chan->num = 8; + chan->doorbell = 1; + chan->virt = kunit_kcalloc(test, chan->num, sizeof(*slots), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, chan->virt); + chan->req.arg0 = 0x5000000; + + return chan->virt; +} + +static void isp_test_timeout_late_answer(struct kunit *test) +{ + struct isp_sm_test *t = isp_sm_test_alloc(test); + struct isp_message *slots = isp_test_command_ring(test, t); + struct isp_channel *chan = t->chan; + + KUNIT_EXPECT_EQ(test, ipc_chan_send(t->isp, chan, 1), -ETIME); + KUNIT_EXPECT_PTR_EQ(test, t->isp->ipc_stale, chan); + KUNIT_EXPECT_FALSE(test, t->isp->ipc_failed); + KUNIT_EXPECT_EQ(test, chan->cursor, 0); + + /* The answer comes after the deadline and is consumed by the resync. */ + slots[0].arg0 |= ISP_IPC_FLAG_ACK; + KUNIT_EXPECT_EQ(test, isp_ipc_resync(t->isp), 0); + KUNIT_EXPECT_NULL(test, t->isp->ipc_stale); + KUNIT_EXPECT_EQ(test, chan->cursor, 1); + + /* The next request takes the next slot; the old ACK does not answer it. */ + KUNIT_EXPECT_EQ(test, ipc_chan_send(t->isp, chan, 1), -ETIME); + KUNIT_EXPECT_EQ(test, slots[1].arg0, chan->req.arg0); + KUNIT_EXPECT_EQ(test, chan->cursor, 1); + slots[1].arg0 |= ISP_IPC_FLAG_ACK; + KUNIT_EXPECT_EQ(test, isp_ipc_resync(t->isp), 0); + KUNIT_EXPECT_EQ(test, chan->cursor, 2); + KUNIT_EXPECT_FALSE(test, t->isp->ipc_failed); +} + +static void isp_test_timeout_no_answer(struct kunit *test) +{ + struct isp_sm_test *t = isp_sm_test_alloc(test); + struct isp_message *slots = isp_test_command_ring(test, t); + struct isp_channel *chan = t->chan; + + KUNIT_EXPECT_EQ(test, ipc_chan_send(t->isp, chan, 1), -ETIME); + KUNIT_EXPECT_EQ(test, isp_ipc_resync(t->isp), -EIO); + KUNIT_EXPECT_TRUE(test, t->isp->ipc_failed); + KUNIT_EXPECT_NULL(test, t->isp->ipc_stale); + + /* An answer after the resync gave up admits nothing. */ + slots[0].arg0 |= ISP_IPC_FLAG_ACK; + KUNIT_EXPECT_EQ(test, ipc_chan_send(t->isp, chan, 1), -EIO); + KUNIT_EXPECT_EQ(test, isp_ipc_resync(t->isp), -EIO); + KUNIT_EXPECT_EQ(test, chan->cursor, 0); +} + +static void isp_test_message_payload_copy(struct kunit *test) +{ + struct isp_sm_test *t = isp_sm_test_alloc(test); + struct isp_message response = { + .arg0 = 0x5000001, .arg1 = 1, .arg2 = 2, .arg3 = 3, + .arg4 = 4, .arg5 = 5, .arg6 = 6, .arg7 = 7, + }; + struct isp_message copied = {}; + + isp_get_current_format(t->isp)->pixelformat = V4L2_PIX_FMT_P010; + t->chan->virt = &response; + chan_read_msg_index(t->isp, t->chan, &copied, 0); + KUNIT_EXPECT_MEMEQ(test, &copied, &response, sizeof(response)); +} + static struct kunit_case isp_ipc_test_cases[] = { + KUNIT_CASE(isp_test_message_payload_copy), + KUNIT_CASE(isp_test_late_ack_keeps_cursor), + KUNIT_CASE(isp_test_timeout_late_answer), + KUNIT_CASE(isp_test_timeout_no_answer), + KUNIT_CASE(isp_test_command_ring_wrap), KUNIT_CASE(isp_test_sm_free_releases), KUNIT_CASE(isp_test_sm_free_alias), KUNIT_CASE(isp_test_sm_free_unknown), diff --git a/drivers/media/platform/apple/isp/isp-ipc.c b/drivers/media/platform/apple/isp/isp-ipc.c index f709f6e305763..7b4dddf892a04 100644 --- a/drivers/media/platform/apple/isp/isp-ipc.c +++ b/drivers/media/platform/apple/isp/isp-ipc.c @@ -6,10 +6,16 @@ #include "isp-regs.h" #include "isp-fw.h" +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) +#include +#endif + #define ISP_IPC_FLAG_TERMINAL_ACK 0x3 /* longer firmware log lines are cut */ #define ISP_IPC_TERMINAL_MAX_LEN 512 #define ISP_IPC_BUFEXC_STAT_META_OFFSET 0x10 +/* how much longer a timed-out request may take to be answered */ +#define ISP_IPC_RESYNC_TIMEOUT msecs_to_jiffies(1000) struct isp_sm_deferred_work { struct work_struct work; @@ -66,7 +72,17 @@ static inline void chan_read_msg_index(struct apple_isp *isp, struct isp_channel *chan, struct isp_message *msg, u32 index) { - memcpy(msg, chan_msg_virt(chan, index), sizeof(*msg)); + if (isp_is_p010(isp)) { + u64 *slot = chan_msg_virt(chan, index); + u64 arg0 = READ_ONCE(*slot); + + /* Acquire validity before consuming the firmware-owned payload. */ + dma_rmb(); + memcpy(msg, slot, sizeof(*msg)); + msg->arg0 = arg0; + } else { + memcpy(msg, chan_msg_virt(chan, index), sizeof(*msg)); + } } static inline void chan_read_msg(struct apple_isp *isp, @@ -124,6 +140,8 @@ static int chan_handle_once(struct apple_isp *isp, struct isp_channel *chan) isp_mbox2_write32(isp, ISP_MBOX2_IRQ_DOORBELL, chan->doorbell); chan_update_cursor(chan); + if (chan->ops->complete) + chan->ops->complete(isp); return 0; } @@ -139,10 +157,12 @@ static inline bool chan_rx_done(struct apple_isp *isp, struct isp_channel *chan) int ipc_chan_handle(struct apple_isp *isp, struct isp_channel *chan) { + unsigned int budget = isp_is_p010(isp) ? chan->num : UINT_MAX; int err = 0; mutex_lock(&chan->lock); - while (1) { + /* A busy producer must not starve the other completion/control rings. */ + while (budget--) { chan_read_msg(isp, chan, &chan->req); if (chan_rx_done(isp, chan)) { err = 0; @@ -160,6 +180,9 @@ int ipc_chan_handle(struct apple_isp *isp, struct isp_channel *chan) static inline bool chan_tx_done(struct apple_isp *isp, struct isp_channel *chan) { + if (isp_is_p010(isp) && chan->deadline_active && + time_after_eq(jiffies, chan->deadline)) + return false; dma_rmb(); chan_read_msg(isp, chan, &chan->rsp); @@ -171,11 +194,66 @@ static inline bool chan_tx_done(struct apple_isp *isp, struct isp_channel *chan) return false; } +/* + * A request that timed out still owns its slot, and the firmware may answer + * it late. That answer must not be taken for the answer to the next request, + * which goes in the same slot with the same arg0, and the next request must + * not overwrite the command area while the firmware may still read the old + * one. So before anything else is written, wait once more for the late + * answer, for a bounded time, and consume it; the ring is then in step again. + * If it has not come yet, the doorbell is rung again in case the first one + * was missed. IO and DEBUG share a doorbell bit, so the firmware already + * copes with a doorbell that brings no new request. If the firmware still + * does not answer, where it is in the ring is unknown and nothing more is + * sent until it is stopped. + */ +int isp_ipc_resync(struct apple_isp *isp) +{ + struct isp_channel *chan = isp->ipc_stale; + + if (isp->ipc_failed) + return -EIO; + if (!chan) + return 0; + + isp->ipc_stale = NULL; + chan->deadline_active = false; + if (!chan_tx_done(isp, chan)) { + isp_mbox2_write32(isp, ISP_MBOX2_IRQ_DOORBELL, chan->doorbell); + if (!wait_event_timeout(isp->wait, chan_tx_done(isp, chan), + ISP_IPC_RESYNC_TIMEOUT)) { + isp->ipc_failed = true; + dev_err(isp->dev, + "%s: no answer to the timed-out request, sending no more commands\n", + chan->name); + return -EIO; + } + } + + dev_warn(isp->dev, + "%s: timed-out request answered late, channel back in step\n", + chan->name); + + return 0; +} + int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, unsigned long timeout) { long t; + int err; +#if IS_ENABLED(CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST) + KUNIT_STATIC_STUB_REDIRECT(ipc_chan_send, isp, chan, timeout); +#endif + /* Callers that fill the command area resync before doing so. */ + err = isp_ipc_resync(isp); + if (err) + return err; + if (isp_is_p010(isp) && timeout) { + chan->deadline = jiffies + timeout; + chan->deadline_active = true; + } chan_write_msg(isp, chan, &chan->req); dma_wmb(); @@ -186,6 +264,9 @@ int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, t = wait_event_timeout(isp->wait, chan_tx_done(isp, chan), timeout); if (t == 0) { + /* Resolved before the next request, see isp_ipc_resync(). */ + if (isp_is_p010(isp)) + isp->ipc_stale = chan; dev_err(isp->dev, "%s: timed out on request [0x%llx, 0x%llx, 0x%llx]\n", chan->name, chan->req.arg0, chan->req.arg1, @@ -193,6 +274,7 @@ int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, return -ETIME; } + chan->deadline_active = false; isp_dbg(isp, "%s: request success (%ld)\n", chan->name, t); return 0; diff --git a/drivers/media/platform/apple/isp/isp-ipc.h b/drivers/media/platform/apple/isp/isp-ipc.h index 7bc95a2005465..0c93100020baa 100644 --- a/drivers/media/platform/apple/isp/isp-ipc.h +++ b/drivers/media/platform/apple/isp/isp-ipc.h @@ -21,6 +21,7 @@ int ipc_chan_handle(struct apple_isp *isp, struct isp_channel *chan); int ipc_chan_send(struct apple_isp *isp, struct isp_channel *chan, unsigned long timeout); +int isp_ipc_resync(struct apple_isp *isp); int ipc_tm_handle(struct apple_isp *isp, struct isp_channel *chan); int ipc_sm_handle(struct apple_isp *isp, struct isp_channel *chan); diff --git a/drivers/media/platform/apple/isp/isp-regs.h b/drivers/media/platform/apple/isp/isp-regs.h index f2f6f1c9489f4..b7f9b2fb87faa 100644 --- a/drivers/media/platform/apple/isp/isp-regs.h +++ b/drivers/media/platform/apple/isp/isp-regs.h @@ -24,6 +24,10 @@ #define ISP_COPROC_IRQ_MASK_5 0x1400a14 #define ISP_COPROC_CONTROL_T8140 0x1600044 +#define ISP_COPROC_STATUS_T8140 0x1600048 +#define ISP_COPROC_QUIESCE_T8140 0x1600040 +/* Requests the quiesce when set, and reads back once it has happened */ +#define ISP_COPROC_QUIESCED BIT(25) /* * ISP17a (T8140) reset. The fourth fabric word is at 0xa50, off the 0x100 @@ -40,6 +44,20 @@ #define ISP_COPROC_RESET_ACK_0_T8140 0x1600818 #define ISP_COPROC_RESET_ACK_1_T8140 0x160081c +/* + * Address filter (DAPF) registers of a T8110-type DART: an array of windows, + * each with its inclusive start and end and three control words. Every ISP + * window takes the control values its ADT dapf-instance entry gives. + */ +#define DAPF_WINDOW_STRIDE 0x40 +#define DAPF_CTRL0 0x00 +#define DAPF_CTRL0_ISP 0x31 +#define DAPF_CTRL1 0x04 +#define DAPF_START 0x08 +#define DAPF_END 0x10 +#define DAPF_CTRL2 0x20 +#define DAPF_CTRL2_ISP 0x1 + #define ISP_MBOX_IRQ_INTERRUPT 0x00 #define ISP_MBOX_IRQ_ENABLE 0x04 #define ISP_MBOX_IRQ_ENABLE_T6031 0x08 diff --git a/drivers/media/platform/apple/isp/isp-v4l2-test.c b/drivers/media/platform/apple/isp/isp-v4l2-test.c index 895a6d1299c22..88f428b3b0c27 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2-test.c +++ b/drivers/media/platform/apple/isp/isp-v4l2-test.c @@ -7,6 +7,7 @@ #include #include +#include #include #define ISP_TEST_FRAME_SIZE 0x1000 @@ -19,6 +20,8 @@ static struct apple_isp *isp_test_alloc(struct kunit *test) KUNIT_ASSERT_NOT_ERR_OR_NULL(test, isp); mutex_init(&isp->video_lock); + spin_lock_init(&isp->buf_lock); + INIT_LIST_HEAD(&isp->bufs_returned); isp_get_current_format(isp)->total_size = ISP_TEST_FRAME_SIZE; return isp; @@ -34,6 +37,8 @@ static void isp_test_queue_init(struct kunit *test) KUNIT_ASSERT_EQ(test, isp_vb2_queue_init(isp), 0); KUNIT_EXPECT_GE(test, isp->vbq.max_num_buffers, VB2_MAX_FRAME); + KUNIT_EXPECT_EQ(test, isp->vbq.type, V4L2_BUF_TYPE_VIDEO_CAPTURE); + KUNIT_EXPECT_FALSE(test, isp->vbq.is_multiplanar); vb2_queue_release(&isp->vbq); } @@ -99,6 +104,12 @@ static void isp_test_create_bufs_limit(struct kunit *test) KUNIT_EXPECT_EQ(test, isp_test_setup(test, ISP_MAX_BUFFERS, &nbuffers, &num_planes, sizes), -ENOBUFS); + /* The contiguous API must refuse a second independent memory plane. */ + nbuffers = 1; + num_planes = 2; + KUNIT_EXPECT_EQ(test, isp_test_setup(test, 0, &nbuffers, &num_planes, + sizes), -EINVAL); + /* A plane smaller than a frame is still refused. */ nbuffers = 1; num_planes = 1; @@ -107,7 +118,289 @@ static void isp_test_create_bufs_limit(struct kunit *test) sizes), -EINVAL); } +static const struct apple_isp_hw isp_test_t6040 = { + .gen = ISP_GEN_T6040, + .fw_abi = ISP_FW_ABI_H17, +}; + +static void isp_test_p010_format(struct kunit *test) +{ + struct apple_isp *isp = isp_test_alloc(test); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_preset preset = { .output_dim = { 1920, 1080 }, .p010 = true }; + struct v4l2_format f = {}; + unsigned int sizes[VB2_MAX_PLANES] = {}; + unsigned int nbuffers = 16, num_planes = 0; + + isp->hw = &isp_test_t6040; + fmt->version = 0x958; + KUNIT_ASSERT_EQ(test, isp_set_preset(isp, fmt, &preset, V4L2_PIX_FMT_P010), 0); + isp_get_sp_pix_format(isp, &f, fmt); + KUNIT_EXPECT_EQ(test, f.fmt.pix.pixelformat, V4L2_PIX_FMT_P010); + KUNIT_EXPECT_EQ(test, f.fmt.pix.bytesperline, 3840); + KUNIT_EXPECT_EQ(test, f.fmt.pix.sizeimage, 6220800); + KUNIT_EXPECT_EQ(test, fmt->plane_size[0], (size_t)4147200); + KUNIT_EXPECT_EQ(test, fmt->plane_size[1], (size_t)2073600); + KUNIT_EXPECT_EQ(test, f.fmt.pix.quantization, V4L2_QUANTIZATION_LIM_RANGE); + + isp->vbq.drv_priv = isp; + isp_test_set_num_buffers(test, &isp->vbq, 0); + KUNIT_ASSERT_EQ(test, + isp_vb2_queue_setup(&isp->vbq, &nbuffers, &num_planes, sizes, NULL), 0); + KUNIT_EXPECT_EQ(test, nbuffers, 2); + KUNIT_EXPECT_EQ(test, num_planes, 1); + KUNIT_EXPECT_EQ(test, sizes[0], 6220800); + KUNIT_EXPECT_EQ(test, isp_num_meta(isp), 8); + + preset.output_dim = (struct coord){1280, 720}; + KUNIT_ASSERT_EQ(test, isp_set_preset(isp, fmt, &preset, V4L2_PIX_FMT_P010), 0); + KUNIT_EXPECT_EQ(test, fmt->strides[0], 2560); + KUNIT_EXPECT_EQ(test, fmt->total_size, (size_t)2764800); + /* NV12 remains the default and restores the existing pool capacity. */ + KUNIT_ASSERT_EQ(test, isp_set_preset(isp, fmt, &preset, V4L2_PIX_FMT_NV12), 0); + KUNIT_EXPECT_EQ(test, fmt->pixelformat, V4L2_PIX_FMT_NV12); + KUNIT_EXPECT_EQ(test, fmt->total_size, (size_t)1382400); + KUNIT_EXPECT_EQ(test, isp_num_rendered(isp), ISP_MAX_BUFFERS); + KUNIT_EXPECT_EQ(test, isp_num_meta(isp), ISP_MAX_BUFFERS); + +} + +static void isp_test_p010_admission(struct kunit *test) +{ + static const struct apple_isp_hw neo = { .gen = ISP_GEN_T8140 }; + struct apple_isp *isp = isp_test_alloc(test); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_preset preset = { .output_dim = {1280, 720}, .p010 = true }; + struct v4l2_fmtdesc desc = { .index = 0 }; + + isp->presets = &preset; + isp->num_presets = 1; + isp->hw = &isp_test_t6040; + fmt->version = 0x958; + KUNIT_ASSERT_EQ(test, isp_enum_pixelformat(isp, &desc), 0); + KUNIT_EXPECT_EQ(test, desc.pixelformat, V4L2_PIX_FMT_NV12); + desc.index = 1; + KUNIT_ASSERT_EQ(test, isp_enum_pixelformat(isp, &desc), 0); + KUNIT_EXPECT_EQ(test, desc.pixelformat, V4L2_PIX_FMT_P010); + desc.index = 2; + KUNIT_EXPECT_EQ(test, isp_enum_pixelformat(isp, &desc), -EINVAL); + desc.index = 1; + isp->hw = &neo; + KUNIT_EXPECT_EQ(test, isp_enum_pixelformat(isp, &desc), -EINVAL); + KUNIT_ASSERT_EQ(test, isp_set_preset(isp, fmt, &preset, V4L2_PIX_FMT_P010), 0); + KUNIT_EXPECT_EQ(test, fmt->pixelformat, V4L2_PIX_FMT_NV12); + KUNIT_EXPECT_EQ(test, fmt->strides[0], 1280); +} + +static void isp_test_p010_completion_identity(struct kunit *test) +{ + struct isp_format fmt = { .plane_size = {4147200, 2073600} }; + struct isp_buffer buf = { .surfs = {{ .iova = 0x10020000000ULL }}, .tag = 42 }; + struct isp_buflist_buffer desc = { + .iovas = {0x20000000, 0x203f4800}, + .num_planes = 2, .pool_type = CISP_POOL_TYPE_RENDERED_SCL1, .tag = 42, + }; + + KUNIT_EXPECT_TRUE(test, isp_p010_buffer_matches(&fmt, &buf, &desc)); + desc.tag--; + KUNIT_EXPECT_FALSE(test, isp_p010_buffer_matches(&fmt, &buf, &desc)); + desc.tag++; + desc.iovas[1]++; + KUNIT_EXPECT_FALSE(test, isp_p010_buffer_matches(&fmt, &buf, &desc)); + desc.iovas[1]--; + desc.iovas[0] |= BIT_ULL(40); + /* Reject mixed encodings, accept both exact full submitted addresses. */ + KUNIT_EXPECT_FALSE(test, isp_p010_buffer_matches(&fmt, &buf, &desc)); + desc.iovas[1] |= BIT_ULL(40); + KUNIT_EXPECT_TRUE(test, isp_p010_buffer_matches(&fmt, &buf, &desc)); + desc.iovas[0] |= BIT_ULL(41); + desc.iovas[1] |= BIT_ULL(41); + KUNIT_EXPECT_FALSE(test, isp_p010_buffer_matches(&fmt, &buf, &desc)); + desc.iovas[0] = (u32)desc.iovas[0]; + desc.iovas[1] = (u32)desc.iovas[1]; + desc.pool_type = CISP_POOL_TYPE_RENDERED; + KUNIT_EXPECT_FALSE(test, isp_p010_buffer_matches(&fmt, &buf, &desc)); + desc.pool_type = CISP_POOL_TYPE_RENDERED_SCL1; + desc.num_planes = 1; + KUNIT_EXPECT_FALSE(test, isp_p010_buffer_matches(&fmt, &buf, &desc)); +} + +static void isp_test_p010_report_defers_completion(struct kunit *test) +{ + struct apple_isp *isp = isp_test_alloc(test); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_buffer *buf = kunit_kzalloc(test, sizeof(*buf), GFP_KERNEL); + struct { + u64 type, count; + struct isp_buflist_buffer desc; + } report = { + .type = 2, .count = 1, + .desc = { .iovas = {0x20000000, 0x203f4800}, + .num_planes = 2, .pool_type = 9, .tag = 42 }, + }; + struct isp_surf report_surf = { + .iova = 0x30000000, .size = sizeof(report), .virt = &report, + }; + struct isp_channel chan = { + .req = { .arg0 = report_surf.iova, .arg1 = sizeof(report), + .arg2 = ISP_IPC_BUFEXC_FLAG_RENDER }, + }; + + KUNIT_ASSERT_NOT_NULL(test, buf); + fmt->pixelformat = V4L2_PIX_FMT_P010; + fmt->plane_size[0] = 4147200; + isp->fw_iova_mask = U64_MAX; + isp->bt_surf = &report_surf; + buf->surfs[0].iova = 0x10020000000ULL; + buf->tag = 42; + INIT_LIST_HEAD(&isp->bufs_submitted); + list_add(&buf->link, &isp->bufs_submitted); + /* There is no vb2 queue: completing it before the ACK would fault. */ + KUNIT_ASSERT_EQ(test, ipc_bt_handle(isp, &chan), 0); + KUNIT_EXPECT_TRUE(test, list_empty(&isp->bufs_submitted)); + KUNIT_EXPECT_PTR_EQ(test, + list_first_entry(&isp->bufs_returned, struct isp_buffer, link), buf); + KUNIT_EXPECT_EQ(test, buf->return_state, VB2_BUF_STATE_DONE); + KUNIT_EXPECT_EQ(test, chan.rsp.arg0, report_surf.iova | ISP_IPC_FLAG_ACK); +} + +static void isp_test_p010_mode_selection(struct kunit *test) +{ + struct apple_isp *isp = isp_test_alloc(test); + struct isp_preset presets[] = { + { .index = 5, .output_dim = {1920, 1080} }, + { .index = 5, .output_dim = {1280, 720}, .p010 = true }, + { .index = 1, .output_dim = {4096, 3072} }, + }; + + isp->hw = &isp_test_t6040; + isp_get_current_format(isp)->version = 0x958; + isp->presets = presets; + isp->num_presets = ARRAY_SIZE(presets); + KUNIT_EXPECT_PTR_EQ(test, isp_select_preset(isp, 0, 0, V4L2_PIX_FMT_P010), &presets[1]); + KUNIT_EXPECT_PTR_EQ(test, isp_select_preset(isp, 1920, 1080, V4L2_PIX_FMT_P010), + &presets[1]); + KUNIT_EXPECT_PTR_EQ(test, isp_select_preset(isp, 4096, 3072, V4L2_PIX_FMT_P010), + &presets[1]); + KUNIT_EXPECT_PTR_EQ(test, isp_select_preset(isp, 1920, 1080, V4L2_PIX_FMT_NV12), + &presets[0]); + /* Only a matching firmware output enables the larger P010 size. */ + presets[2].p010 = true; + KUNIT_EXPECT_PTR_EQ(test, isp_select_preset(isp, 4096, 3072, V4L2_PIX_FMT_P010), + &presets[2]); +} + +static int isp_test_buffer_ack(struct apple_isp *isp, struct isp_channel *chan, + unsigned long timeout) +{ + unsigned int *calls = kunit_get_current_test()->priv; + + (*calls)++; + return 0; +} + +static void isp_test_imx958_last_lease_requeue(struct kunit *test) +{ + struct apple_isp *isp = isp_test_alloc(test); + struct isp_format *fmt = isp_get_current_format(isp); + struct isp_buffer *buf = kunit_kzalloc(test, sizeof(*buf), GFP_KERNEL); + struct isp_buffer *submitted; + unsigned int calls = 0; + + KUNIT_ASSERT_NOT_NULL(test, buf); + isp->hw = &isp_test_t6040; + fmt->version = 0x958; + fmt->pixelformat = V4L2_PIX_FMT_P010; + fmt->num_planes = 2; + fmt->plane_size[0] = 1843200; + fmt->plane_size[1] = 921600; + INIT_LIST_HEAD(&isp->bufs_pending); + INIT_LIST_HEAD(&isp->bufs_submitted); + isp->cmd_virt = kunit_kzalloc(test, ISP_CMD_AREA_SIZE(0), GFP_KERNEL); + isp->chan_bh = kunit_kzalloc(test, sizeof(*isp->chan_bh), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp->cmd_virt); + KUNIT_ASSERT_NOT_NULL(test, isp->chan_bh); + for (unsigned int i = 0; i < ISP_MAX_BUFFERS; i++) { + isp->meta_surfs[i] = kunit_kzalloc(test, sizeof(struct isp_surf), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, isp->meta_surfs[i]); + isp->meta_surfs[i]->submitted = true; + } + isp->vbq.drv_priv = isp; + buf->vb.vb2_buf.vb2_queue = &isp->vbq; + buf->surfs[0].iova = 0x20000000; + set_bit(ISP_STATE_STREAMING, &isp->state); + test->priv = &calls; + kunit_activate_static_stub(test, ipc_chan_send, isp_test_buffer_ack); + for (unsigned int format = 0; format < 2; format++) { + fmt->pixelformat = format ? V4L2_PIX_FMT_NV12 : V4L2_PIX_FMT_P010; + calls = 0; + for (unsigned int i = 1; i <= 10000; i++) { + /* Both lists empty, as after the last completion was dequeued. */ + isp_vb2_buf_queue(&buf->vb.vb2_buf); + KUNIT_ASSERT_EQ(test, calls, i); + if (!format) + KUNIT_ASSERT_EQ(test, buf->tag, (u64)i); + KUNIT_ASSERT_TRUE(test, list_empty(&isp->bufs_pending)); + submitted = list_first_entry(&isp->bufs_submitted, struct isp_buffer, link); + KUNIT_ASSERT_PTR_EQ(test, submitted, buf); + list_del(&buf->link); + } + } +} + +/* The exposure report is found through the section table, never past it. */ +static void isp_test_meta_exposure(struct kunit *test) +{ + static const struct apple_isp_hw hw = { .meta_exposure = true }; + static const struct apple_isp_hw hw_none = { }; + struct apple_isp *isp = isp_test_alloc(test); + struct isp_surf meta = { .size = 0x2000 }; + u8 *virt = kunit_kzalloc(test, meta.size, GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, virt); + meta.virt = virt; + isp->hw = &hw; + isp->frame_gain = 256; + + /* fourth table entry: the section at 0x1870 */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le32(32475, virt + 0x1870); + put_unaligned_le16(0x0800, virt + 0x1874); + put_unaligned_le16(3253, virt + 0x1876); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* a section that would end past the buffer is ignored */ + put_unaligned_le32(meta.size - 4, virt + 0x1c); + put_unaligned_le32(1000, virt + meta.size - 4); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + + /* so is an unwritten section */ + put_unaligned_le32(0x1000, virt + 0x1c); + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_exposure_us, 32475); + + /* and firmware without the report */ + put_unaligned_le32(0x1870, virt + 0x1c); + put_unaligned_le16(600, virt + 0x1876); + isp->hw = &hw_none; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 3253); + isp->hw = &hw; + isp_meta_read_exposure(isp, &meta); + KUNIT_EXPECT_EQ(test, isp->frame_gain, 600); +} + static struct kunit_case isp_v4l2_test_cases[] = { + KUNIT_CASE(isp_test_meta_exposure), + KUNIT_CASE(isp_test_imx958_last_lease_requeue), + KUNIT_CASE(isp_test_p010_report_defers_completion), + KUNIT_CASE(isp_test_p010_mode_selection), + KUNIT_CASE(isp_test_p010_format), + KUNIT_CASE(isp_test_p010_admission), + KUNIT_CASE(isp_test_p010_completion_identity), KUNIT_CASE(isp_test_queue_init), KUNIT_CASE(isp_test_reqbufs_limit), KUNIT_CASE(isp_test_create_bufs_limit), diff --git a/drivers/media/platform/apple/isp/isp-v4l2.c b/drivers/media/platform/apple/isp/isp-v4l2.c index 072b8523279c2..81964ab1916c9 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.c +++ b/drivers/media/platform/apple/isp/isp-v4l2.c @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -21,24 +22,20 @@ #define ISP_MIN_FRAMES 2 #define ISP_MAX_PLANES 4 -#define ISP_MAX_PIX_FORMATS 2 #define ISP_BUFFER_TIMEOUT msecs_to_jiffies(1500) #define ISP_STRIDE_ALIGNMENT 64 +static void isp_vb2_set_min_buffers(struct apple_isp *isp); + /* Capture rates offered to userspace, in frames per second */ static const unsigned int isp_frame_rates[] = { 30, 25, 24, 20, 15 }; -static bool multiplanar = false; -module_param(multiplanar, bool, 0644); -MODULE_PARM_DESC(multiplanar, "Enable multiplanar API"); - struct isp_buflist_buffer { u64 iovas[ISP_MAX_PLANES]; u32 flags[ISP_MAX_PLANES]; u32 num_planes; u32 pool_type; - u32 tag; - u32 pad; + u64 tag; } __packed; static_assert(sizeof(struct isp_buflist_buffer) == ISP_BUFLIST_DESC_SIZE); @@ -57,6 +54,77 @@ static_assert(ISP_CMD_AREA_SIZE(0) >= ISP_IPC_BUFEXC_STAT_SIZE); */ #define ISP_BUFLIST_POOL_CAPTURE_META 2 +static bool isp_p010_buffer_matches(const struct isp_format *fmt, + const struct isp_buffer *buf, + const struct isp_buflist_buffer *desc) +{ + u64 y = buf->surfs[0].iova; + u64 uv = y + fmt->plane_size[0]; + bool full = desc->iovas[0] == y && desc->iovas[1] == uv; + bool low = desc->iovas[0] == (u32)y && desc->iovas[1] == (u32)uv; + + /* Accept the submitted pair or the zero-extended pair observed on H17. */ + return desc->num_planes == 2 && + desc->pool_type == CISP_POOL_TYPE_RENDERED_SCL1 && (full || low) && + !desc->iovas[2] && !desc->iovas[3] && desc->tag == buf->tag; +} + +/* + * A metadata buffer starts with a table of section offsets. On H17 firmware + * the fourth section opens with the frame's exposure: the integration time in + * microseconds (le32 at 0) and the total sensor gain in 1/256 steps, 256 for + * unity (le16 at 6). Observed on T6040 and T8140 with AE converging under a + * fixed scene and with the frame rate capping the integration time. + */ +#define ISP_META_SECTIONS 0x10 +#define ISP_META_SECTION_EXPOSURE 3 +#define ISP_META_EXPOSURE_TIME 0x0 +#define ISP_META_EXPOSURE_GAIN 0x6 +#define ISP_META_EXPOSURE_SIZE 0x8 + +static void isp_meta_read_exposure(struct apple_isp *isp, + const struct isp_surf *meta) +{ + const u8 *virt = meta->virt; + u32 offset, time; + u16 gain; + + if (!isp->hw->meta_exposure || !virt) + return; + + offset = get_unaligned_le32(virt + ISP_META_SECTIONS + + 4 * ISP_META_SECTION_EXPOSURE); + if (offset < ISP_META_SECTIONS || offset > meta->size || + meta->size - offset < ISP_META_EXPOSURE_SIZE) + return; + + time = get_unaligned_le32(virt + offset + ISP_META_EXPOSURE_TIME); + gain = get_unaligned_le16(virt + offset + ISP_META_EXPOSURE_GAIN); + /* An unwritten section or one this layout does not describe */ + if (!time || !gain) + return; + + WRITE_ONCE(isp->frame_exposure_us, time); + WRITE_ONCE(isp->frame_gain, gain); +} + +void ipc_bt_complete(struct apple_isp *isp) +{ + struct isp_buffer *buf, *tmp; + unsigned long flags, i; + + /* The firmware has received its ACK before userspace can reuse a lease. */ + spin_lock_irqsave(&isp->buf_lock, flags); + for_each_set_bit(i, &isp->meta_returned, ISP_MAX_BUFFERS) + isp->meta_surfs[i]->submitted = false; + isp->meta_returned = 0; + list_for_each_entry_safe(buf, tmp, &isp->bufs_returned, link) { + list_del(&buf->link); + vb2_buffer_done(&buf->vb.vb2_buf, buf->return_state); + } + spin_unlock_irqrestore(&isp->buf_lock, flags); +} + int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) { struct isp_message *req = &chan->req, *rsp = &chan->rsp; @@ -96,11 +164,15 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) bufd->pool_type); */ if (bufd->pool_type == 0) { - for (int j = 0; j < ARRAY_SIZE(isp->meta_surfs); j++) { + for (int j = 0; j < isp_num_meta(isp); j++) { struct isp_surf *meta = isp->meta_surfs[j]; if ((u32)bufd->iovas[0] == (u32)meta->iova) { WARN_ON(!meta->submitted); - meta->submitted = false; + isp_meta_read_exposure(isp, meta); + if (isp_is_p010(isp)) + __set_bit(j, &isp->meta_returned); + else + meta->submitted = false; } } } else if (bufd->pool_type == ISP_BUFLIST_POOL_CAPTURE_META && @@ -114,8 +186,10 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) } else { list_for_each_entry_safe_reverse( buf, tmp, &isp->bufs_submitted, link) { - if ((u32)buf->surfs[0].iova == - (u32)bufd->iovas[0]) { + if (isp_is_p010(isp) ? + isp_p010_buffer_matches(isp_get_current_format(isp), + buf, bufd) : + (u32)buf->surfs[0].iova == (u32)bufd->iovas[0]) { enum vb2_buffer_state state = VB2_BUF_STATE_ERROR; @@ -126,9 +200,13 @@ int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan) if (req->arg2 == ISP_IPC_BUFEXC_FLAG_RENDER) state = VB2_BUF_STATE_DONE; - vb2_buffer_done(&buf->vb.vb2_buf, - state); - list_del(&buf->link); + if (isp_is_p010(isp)) { + buf->return_state = state; + list_move_tail(&buf->link, &isp->bufs_returned); + } else { + vb2_buffer_done(&buf->vb.vb2_buf, state); + list_del(&buf->link); + } } } } @@ -161,12 +239,17 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) struct isp_buflist *bl = isp->cmd_virt; struct isp_buflist_buffer *bufd = &bl->buffers[0]; + /* Before the command area is overwritten */ + err = isp_ipc_resync(isp); + if (err) + return err; + /* Clear what earlier commands and batches left in the reserved fields. */ memset(bl, 0, ISP_CMD_AREA_SIZE(isp_num_capmeta(isp))); bl->type = 1; spin_lock_irqsave(&isp->buf_lock, flags); - for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { + for (int i = 0; i < isp_num_meta(isp); i++) { struct isp_surf *meta = isp->meta_surfs[i]; if (meta->submitted) @@ -205,9 +288,13 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) } /* One rendered pool per batch; the rest stays pending for the next. */ - while (num_rendered < ISP_MAX_BUFFERS && + while (num_rendered < isp_num_rendered(isp) && (buf = list_first_entry_or_null(&isp->bufs_pending, struct isp_buffer, link))) { + if (isp_is_p010(isp)) { + buf->tag = ++isp->buffer_tag; + bufd->tag = buf->tag; + } bufd->num_planes = fmt->num_planes; bufd->pool_type = isp->hw->scl1 ? CISP_POOL_TYPE_RENDERED_SCL1 : CISP_POOL_TYPE_RENDERED; @@ -249,6 +336,12 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) err = ipc_chan_send(isp, chan, ISP_BUFFER_TIMEOUT); if (err) { + /* + * The firmware may have taken the batch. Keep its leases owned + * until the timeout is resolved, see apple_isp_stop_camera(). + */ + if (isp_is_p010(isp)) + return err; /* If we fail, consider the buffer not submitted. */ dev_err(isp->dev, "%s: failed to send bufs: [0x%llx, 0x%llx, 0x%llx]\n", @@ -269,7 +362,7 @@ static int isp_submit_buffers(struct apple_isp *isp, bool pre_start) &isp->bufs_pending); } } - for (int j = 0; j < ARRAY_SIZE(isp->meta_surfs); j++) { + for (int j = 0; j < isp_num_meta(isp); j++) { struct isp_surf *meta = isp->meta_surfs[j]; if (bufd->iovas[0] == meta->iova) { meta->submitted = false; @@ -308,13 +401,13 @@ static int isp_vb2_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers, * REQBUFS frees the old buffers first, CREATE_BUFS adds to them, so * count the buffers the queue already has. */ - if (q_num_bufs >= ISP_MAX_BUFFERS) + if (q_num_bufs >= isp_num_rendered(isp)) return -ENOBUFS; *nbuffers = min_t(unsigned int, *nbuffers, - ISP_MAX_BUFFERS - q_num_bufs); + isp_num_rendered(isp) - q_num_bufs); if (*num_planes) { - if (sizes[0] < fmt->total_size) + if (*num_planes != 1 || sizes[0] < fmt->total_size) return -EINVAL; return 0; @@ -407,8 +500,23 @@ static void isp_vb2_buf_queue(struct vb2_buffer *vb) list_add_tail(&buf->link, &isp->bufs_pending); spin_unlock_irqrestore(&isp->buf_lock, flags); - if (test_bit(ISP_STATE_STREAMING, &isp->state) && !empty) - isp_submit_buffers(isp, false); + /* Restart an empty IMX958 queue when its last returned lease is queued. */ + if (test_bit(ISP_STATE_STREAMING, &isp->state) && + (isp_supports_p010(isp) || !empty)) { + if (isp_submit_buffers(isp, false) && isp_is_p010(isp)) { + /* + * After a timeout the leases stay owned until the + * stream stops. Only firmware that did not answer at + * all is stopped here, before its leases are released. + */ + if (isp->ipc_failed) { + apple_isp_firmware_halt(isp); + isp_vb2_release_buffers(isp, + VB2_BUF_STATE_ERROR); + } + vb2_queue_error(&isp->vbq); + } + } } static int apple_isp_start_streaming(struct apple_isp *isp) @@ -423,11 +531,12 @@ static int apple_isp_start_streaming(struct apple_isp *isp) } /* - * Resident firmware was not restarted, but the channel's pools are - * configured again for every stream. Metadata buffers still marked - * as submitted to the previous pools would never reach the new ones. + * The channel's pools are configured again for every stream, by + * resident firmware that kept running or by firmware that started + * afresh. Metadata buffers still marked as submitted to the previous + * pools, or to firmware that stopped, would never reach the new ones. */ - if (isp->hw->resident_fw) { + if (isp->hw->resident_fw || isp->hw->fw_restore) { spin_lock_irqsave(&isp->buf_lock, flags); for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) isp->meta_surfs[i]->submitted = false; @@ -484,12 +593,14 @@ static int isp_vb2_start_streaming(struct vb2_queue *q, unsigned int count) isp->sequence = 0; - /* The exposure priority is applied when the stream starts. */ + /* The exposure controls are applied when the stream starts. */ v4l2_ctrl_grab(isp->exposure_priority, true); + v4l2_ctrl_grab(isp->power_line_frequency, true); err = apple_isp_start_streaming(isp); if (err) { v4l2_ctrl_grab(isp->exposure_priority, false); + v4l2_ctrl_grab(isp->power_line_frequency, false); isp_vb2_release_buffers(isp, VB2_BUF_STATE_QUEUED); } @@ -503,6 +614,7 @@ static void isp_vb2_stop_streaming(struct vb2_queue *q) apple_isp_stop_streaming(isp); isp_vb2_release_buffers(isp, VB2_BUF_STATE_ERROR); v4l2_ctrl_grab(isp->exposure_priority, false); + v4l2_ctrl_grab(isp->power_line_frequency, false); } int apple_isp_video_suspend(struct apple_isp *isp) @@ -565,18 +677,23 @@ static const struct vb2_ops isp_vb2_ops = { }; static int isp_set_preset(struct apple_isp *isp, struct isp_format *fmt, - struct isp_preset *preset) + struct isp_preset *preset, u32 pixelformat) { + unsigned int bytes_per_sample; int i; size_t total_size; fmt->preset = preset; - /* I really fucking hope they all use NV12. */ + /* P010 has MSB-aligned 10-bit samples in little-endian 16-bit words. */ + fmt->pixelformat = pixelformat == V4L2_PIX_FMT_P010 && + isp_supports_p010(isp) && preset->p010 ? + V4L2_PIX_FMT_P010 : V4L2_PIX_FMT_NV12; + bytes_per_sample = fmt->pixelformat == V4L2_PIX_FMT_P010 ? 2 : 1; fmt->num_planes = 2; - fmt->strides[0] = ALIGN(preset->output_dim.x, ISP_STRIDE_ALIGNMENT); - /* UV subsampled interleaved */ - fmt->strides[1] = ALIGN(preset->output_dim.x, ISP_STRIDE_ALIGNMENT); + fmt->strides[0] = ALIGN(preset->output_dim.x * bytes_per_sample, + ISP_STRIDE_ALIGNMENT); + fmt->strides[1] = fmt->strides[0]; fmt->plane_size[0] = fmt->strides[0] * preset->output_dim.y; fmt->plane_size[1] = fmt->strides[1] * preset->output_dim.y / 2; @@ -589,17 +706,25 @@ static int isp_set_preset(struct apple_isp *isp, struct isp_format *fmt, } static struct isp_preset *isp_select_preset(struct apple_isp *isp, u32 width, - u32 height) + u32 height, u32 pixelformat) { struct isp_preset *preset, *best = &isp->presets[0]; int i, score, best_score = INT_MAX; - /* Default if no dimensions */ - if (width == 0 || height == 0) - return &isp->presets[0]; + bool p010 = pixelformat == V4L2_PIX_FMT_P010 && isp_has_p010(isp); + + /* An unspecified P010 size selects the processed 720p preview first. */ + if (width == 0 || height == 0) { + if (!p010) + return &isp->presets[0]; + width = 1280; + height = 720; + } for (i = 0; i < isp->num_presets; i++) { preset = &isp->presets[i]; + if (p010 && !preset->p010) + continue; score = abs((int)preset->output_dim.x - (int)width) + abs((int)preset->output_dim.y - (int)height); if (score < best_score) { @@ -623,44 +748,47 @@ static int isp_vidioc_querycap(struct file *file, void *priv, return 0; } -static int isp_vidioc_enum_format(struct file *file, void *fh, - struct v4l2_fmtdesc *f) +static int isp_enum_pixelformat(struct apple_isp *isp, struct v4l2_fmtdesc *f) { - struct apple_isp *isp = video_drvdata(file); - - if (f->index >= ISP_MAX_PIX_FORMATS) - return -EINVAL; - - switch (f->index) { - case 0: + /* Browser/WebRTC compatibility: the shipped first format remains NV12. */ + if (!f->index) f->pixelformat = V4L2_PIX_FMT_NV12; - break; - case 1: - if (!isp->multiplanar) - return -EINVAL; - f->pixelformat = V4L2_PIX_FMT_NV12M; - break; - default: + else if (f->index == 1 && isp_has_p010(isp)) + f->pixelformat = V4L2_PIX_FMT_P010; + else return -EINVAL; - } - return 0; } +static int isp_vidioc_enum_format(struct file *file, void *fh, + struct v4l2_fmtdesc *f) +{ + return isp_enum_pixelformat(video_drvdata(file), f); +} + static int isp_vidioc_enum_framesizes(struct file *file, void *fh, struct v4l2_frmsizeenum *f) { struct apple_isp *isp = video_drvdata(file); + struct isp_preset *preset = NULL; + u32 index = 0; - if (f->index >= isp->num_presets) + if (f->pixel_format != V4L2_PIX_FMT_NV12 && + !(f->pixel_format == V4L2_PIX_FMT_P010 && isp_has_p010(isp))) return -EINVAL; - if ((f->pixel_format != V4L2_PIX_FMT_NV12) && - (f->pixel_format != V4L2_PIX_FMT_NV12M)) + for (int i = 0; i < isp->num_presets; i++) { + if (f->pixel_format == V4L2_PIX_FMT_P010 && !isp->presets[i].p010) + continue; + if (index++ == f->index) { + preset = &isp->presets[i]; + break; + } + } + if (!preset) return -EINVAL; - - f->discrete.width = isp->presets[f->index].output_dim.x; - f->discrete.height = isp->presets[f->index].output_dim.y; + f->discrete.width = preset->output_dim.x; + f->discrete.height = preset->output_dim.y; f->type = V4L2_FRMSIZE_TYPE_DISCRETE; return 0; @@ -676,10 +804,12 @@ static int isp_vidioc_enum_frameintervals(struct file *file, void *fh, return -EINVAL; if (interval->pixel_format != V4L2_PIX_FMT_NV12 && - interval->pixel_format != V4L2_PIX_FMT_NV12M) + !(interval->pixel_format == V4L2_PIX_FMT_P010 && isp_has_p010(isp))) return -EINVAL; for (i = 0; i < isp->num_presets; i++) { + if (interval->pixel_format == V4L2_PIX_FMT_P010 && !isp->presets[i].p010) + continue; if (isp->presets[i].output_dim.x == interval->width && isp->presets[i].output_dim.y == interval->height) break; @@ -726,29 +856,12 @@ static inline void isp_get_sp_pix_format(struct apple_isp *isp, f->fmt.pix.sizeimage = fmt->total_size; f->fmt.pix.field = V4L2_FIELD_NONE; - f->fmt.pix.pixelformat = V4L2_PIX_FMT_NV12; + f->fmt.pix.pixelformat = fmt->pixelformat; f->fmt.pix.colorspace = V4L2_COLORSPACE_REC709; f->fmt.pix.ycbcr_enc = V4L2_YCBCR_ENC_709; f->fmt.pix.xfer_func = V4L2_XFER_FUNC_709; -} - -static inline void isp_get_mp_pix_format(struct apple_isp *isp, - struct v4l2_format *f, - struct isp_format *fmt) -{ - f->fmt.pix_mp.width = fmt->preset->output_dim.x; - f->fmt.pix_mp.height = fmt->preset->output_dim.y; - f->fmt.pix_mp.num_planes = fmt->num_planes; - for (int i = 0; i < fmt->num_planes; i++) { - f->fmt.pix_mp.plane_fmt[i].sizeimage = fmt->plane_size[i]; - f->fmt.pix_mp.plane_fmt[i].bytesperline = fmt->strides[i]; - } - - f->fmt.pix_mp.field = V4L2_FIELD_NONE; - f->fmt.pix_mp.pixelformat = V4L2_PIX_FMT_NV12M; - f->fmt.pix_mp.colorspace = V4L2_COLORSPACE_REC709; - f->fmt.pix_mp.ycbcr_enc = V4L2_YCBCR_ENC_709; - f->fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_709; + if (fmt->pixelformat == V4L2_PIX_FMT_P010) + f->fmt.pix.quantization = V4L2_QUANTIZATION_LIM_RANGE; } static int isp_vidioc_get_format(struct file *file, void *fh, @@ -774,14 +887,16 @@ static int isp_vidioc_set_format(struct file *file, void *fh, if (vb2_is_busy(&isp->vbq)) return -EBUSY; - preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); - err = isp_set_preset(isp, fmt, preset); + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height, + f->fmt.pix.pixelformat); + err = isp_set_preset(isp, fmt, preset, f->fmt.pix.pixelformat); if (err) return err; isp_get_sp_pix_format(isp, f, fmt); isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + isp_vb2_set_min_buffers(isp); return 0; } @@ -794,8 +909,9 @@ static int isp_vidioc_try_format(struct file *file, void *fh, struct isp_preset *preset; int err; - preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height); - err = isp_set_preset(isp, &fmt, preset); + preset = isp_select_preset(isp, f->fmt.pix.width, f->fmt.pix.height, + f->fmt.pix.pixelformat); + err = isp_set_preset(isp, &fmt, preset, f->fmt.pix.pixelformat); if (err) return err; @@ -804,69 +920,6 @@ static int isp_vidioc_try_format(struct file *file, void *fh, return 0; } -static int isp_vidioc_get_format_mplane(struct file *file, void *fh, - struct v4l2_format *f) -{ - struct apple_isp *isp = video_drvdata(file); - struct isp_format *fmt = isp_get_current_format(isp); - - if (!isp->multiplanar) - return -ENOTTY; - - isp_get_mp_pix_format(isp, f, fmt); - - return 0; -} - -static int isp_vidioc_set_format_mplane(struct file *file, void *fh, - struct v4l2_format *f) -{ - struct apple_isp *isp = video_drvdata(file); - struct isp_format *fmt = isp_get_current_format(isp); - struct isp_preset *preset; - int err; - - if (!isp->multiplanar) - return -ENOTTY; - - if (vb2_is_busy(&isp->vbq)) - return -EBUSY; - - preset = isp_select_preset(isp, f->fmt.pix_mp.width, - f->fmt.pix_mp.height); - err = isp_set_preset(isp, fmt, preset); - if (err) - return err; - - isp_get_mp_pix_format(isp, f, fmt); - - isp->vbq.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - - return 0; -} - -static int isp_vidioc_try_format_mplane(struct file *file, void *fh, - struct v4l2_format *f) -{ - struct apple_isp *isp = video_drvdata(file); - struct isp_format fmt = *isp_get_current_format(isp); - struct isp_preset *preset; - int err; - - if (!isp->multiplanar) - return -ENOTTY; - - preset = isp_select_preset(isp, f->fmt.pix_mp.width, - f->fmt.pix_mp.height); - err = isp_set_preset(isp, &fmt, preset); - if (err) - return err; - - isp_get_mp_pix_format(isp, f, &fmt); - - return 0; -} - static int isp_vidioc_enum_input(struct file *file, void *fh, struct v4l2_input *inp) { @@ -899,9 +952,7 @@ static int isp_vidioc_get_param(struct file *file, void *fh, { struct apple_isp *isp = video_drvdata(file); - if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && - (!isp->multiplanar || - a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) + if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) return -EINVAL; a->parm.capture.capability = V4L2_CAP_TIMEPERFRAME; @@ -917,9 +968,7 @@ static int isp_vidioc_set_param(struct file *file, void *fh, struct apple_isp *isp = video_drvdata(file); struct v4l2_fract *tpf = &a->parm.capture.timeperframe; - if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && - (!isp->multiplanar || - a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) + if (a->type != V4L2_BUF_TYPE_VIDEO_CAPTURE) return -EINVAL; /* The rate is configured when the stream starts. */ @@ -940,6 +989,27 @@ static int isp_vidioc_set_param(struct file *file, void *fh, return 0; } +static int isp_vidioc_log_status(struct file *file, void *fh) +{ + struct apple_isp *isp = video_drvdata(file); + + v4l2_ctrl_log_status(file, fh); + for (unsigned int i = 0; i < isp->num_sensor_configs; i++) { + const struct isp_sensor_config *config = &isp->sensor_configs[i]; + + v4l2_info(&isp->v4l2_dev, + "sensor config %u: input %ux%u output %ux%u clock %u%s\n", + i, config->input_dim.x, config->input_dim.y, + config->output_dim.x, config->output_dim.y, + config->sensor_clk, + config->usable ? "" : " (unverified geometry)"); + /* This explicit diagnostic ioctl replaces a probe-time parameter. */ + print_hex_dump(KERN_INFO, "apple-isp config: ", DUMP_PREFIX_OFFSET, + 16, 4, config->reply, sizeof(config->reply), false); + } + return 0; +} + static const struct v4l2_ioctl_ops isp_v4l2_ioctl_ops = { .vidioc_querycap = isp_vidioc_querycap, @@ -947,9 +1017,6 @@ static const struct v4l2_ioctl_ops isp_v4l2_ioctl_ops = { .vidioc_g_fmt_vid_cap = isp_vidioc_get_format, .vidioc_s_fmt_vid_cap = isp_vidioc_set_format, .vidioc_try_fmt_vid_cap = isp_vidioc_try_format, - .vidioc_g_fmt_vid_cap_mplane = isp_vidioc_get_format_mplane, - .vidioc_s_fmt_vid_cap_mplane = isp_vidioc_set_format_mplane, - .vidioc_try_fmt_vid_cap_mplane = isp_vidioc_try_format_mplane, .vidioc_enum_framesizes = isp_vidioc_enum_framesizes, .vidioc_enum_frameintervals = isp_vidioc_enum_frameintervals, @@ -969,7 +1036,7 @@ static const struct v4l2_ioctl_ops isp_v4l2_ioctl_ops = { .vidioc_streamon = vb2_ioctl_streamon, .vidioc_streamoff = vb2_ioctl_streamoff, - .vidioc_log_status = v4l2_ctrl_log_status, + .vidioc_log_status = isp_vidioc_log_status, .vidioc_subscribe_event = v4l2_ctrl_subscribe_event, .vidioc_unsubscribe_event = v4l2_event_unsubscribe, }; @@ -1001,6 +1068,7 @@ static void apple_isp_release(struct v4l2_device *v4l2_dev) v4l2_ctrl_handler_free(&isp->ctrl_handler); media_device_cleanup(&isp->mdev); + kfree(isp->sensor_configs); kfree(isp); } @@ -1019,6 +1087,22 @@ static void isp_free_meta_surfaces(struct apple_isp *isp) } } +/* + * vb2 allocates at least min_queued_buffers + 1 buffers, and the queue may + * hold no more than the rendered pool (isp_vb2_queue_setup()). The P010 pool + * has two slots, so a P010 stream starts with one queued buffer; otherwise + * REQBUFS can never get the three buffers vb2 asks for and fails with ENOMEM. + * Only called while the queue has no buffers. + */ +static void isp_vb2_set_min_buffers(struct apple_isp *isp) +{ + struct vb2_queue *vbq = &isp->vbq; + + vbq->min_queued_buffers = min_t(unsigned int, ISP_MIN_FRAMES, + isp_num_rendered(isp) - 1); + vbq->min_reqbufs_allocation = vbq->min_queued_buffers + 1; +} + static int isp_vb2_queue_init(struct apple_isp *isp) { struct vb2_queue *vbq = &isp->vbq; @@ -1037,6 +1121,53 @@ static int isp_vb2_queue_init(struct apple_isp *isp) return vb2_queue_init(vbq); } +static int isp_g_volatile_ctrl(struct v4l2_ctrl *ctrl) +{ + struct apple_isp *isp = + container_of(ctrl->handler, struct apple_isp, ctrl_handler); + + switch (ctrl->id) { + case V4L2_CID_GAIN: + ctrl->val = READ_ONCE(isp->frame_gain); + return 0; + case V4L2_CID_EXPOSURE_ABSOLUTE: + /* in 100 us units */ + ctrl->val = DIV_ROUND_CLOSEST(READ_ONCE(isp->frame_exposure_us), + 100); + return 0; + } + + return -EINVAL; +} + +static const struct v4l2_ctrl_ops isp_exposure_ctrl_ops = { + .g_volatile_ctrl = isp_g_volatile_ctrl, +}; + +/* + * Auto exposure picks the gain and exposure of each frame. Report the last + * frame's values read-only, so that userspace can follow the scene light, for + * example to decide when to denoise. The gain is the firmware's total sensor + * gain in 1/256 steps. + */ +static void isp_init_exposure_ctrls(struct apple_isp *isp) +{ + isp->frame_gain = 256; + isp->gain = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, V4L2_CID_GAIN, + 0, U16_MAX, 1, 256); + isp->exposure = v4l2_ctrl_new_std(&isp->ctrl_handler, + &isp_exposure_ctrl_ops, + V4L2_CID_EXPOSURE_ABSOLUTE, + 0, 10000, 1, 0); + if (isp->gain) + isp->gain->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; + if (isp->exposure) + isp->exposure->flags |= V4L2_CTRL_FLAG_READ_ONLY | + V4L2_CTRL_FLAG_VOLATILE; +} + int apple_isp_setup_video(struct apple_isp *isp) { struct video_device *vdev = &isp->vdev; @@ -1044,7 +1175,7 @@ int apple_isp_setup_video(struct apple_isp *isp) struct isp_format *fmt = isp_get_current_format(isp); int err; - err = isp_set_preset(isp, fmt, &isp->presets[0]); + err = isp_set_preset(isp, fmt, &isp->presets[0], V4L2_PIX_FMT_NV12); if (err) { dev_err(isp->dev, "failed to set default preset: %d\n", err); return err; @@ -1054,7 +1185,8 @@ int apple_isp_setup_video(struct apple_isp *isp) for (int i = 0; i < ARRAY_SIZE(isp->meta_surfs); i++) { isp->meta_surfs[i] = - isp_alloc_surface_vmap(isp, isp->hw->meta_size); + isp_alloc_surface_vmap(isp, isp_has_p010(isp) ? 0x8000 : + isp->hw->meta_size); if (!isp->meta_surfs[i]) { isp_err(isp, "failed to alloc meta surface\n"); err = -ENOMEM; @@ -1086,16 +1218,28 @@ int apple_isp_setup_video(struct apple_isp *isp) goto media_cleanup; } - isp->multiplanar = multiplanar; - /* * Auto exposure is always on. By default it holds the frame rate, as * V4L2 specifies; with exposure priority it may slow down in low light. */ - v4l2_ctrl_handler_init(&isp->ctrl_handler, 1); + v4l2_ctrl_handler_init(&isp->ctrl_handler, 4); isp->exposure_priority = v4l2_ctrl_new_std(&isp->ctrl_handler, NULL, V4L2_CID_EXPOSURE_AUTO_PRIORITY, 0, 1, 1, 0); + /* + * The H17 firmware's auto exposure avoids mains flicker at the given + * frequency. 50 Hz is the default, as on macOS when it does not know + * the country; userspace sets 60 Hz where the mains runs at 60 Hz. + */ + if (isp->hw->fw_abi == ISP_FW_ABI_H17) + isp->power_line_frequency = + v4l2_ctrl_new_std_menu(&isp->ctrl_handler, NULL, + V4L2_CID_POWER_LINE_FREQUENCY, + V4L2_CID_POWER_LINE_FREQUENCY_60HZ, + BIT(V4L2_CID_POWER_LINE_FREQUENCY_DISABLED), + V4L2_CID_POWER_LINE_FREQUENCY_50HZ); + if (isp->hw->meta_exposure) + isp_init_exposure_ctrls(isp); if (isp->ctrl_handler.error) { err = isp->ctrl_handler.error; dev_err(isp->dev, "failed to create controls: %d\n", err); @@ -1119,8 +1263,6 @@ int apple_isp_setup_video(struct apple_isp *isp) vdev->fops = &isp_v4l2_fops; vdev->ioctl_ops = &isp_v4l2_ioctl_ops; vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING; - if (isp->multiplanar) - vdev->device_caps |= V4L2_CAP_VIDEO_CAPTURE_MPLANE; vdev->entity.flags = MEDIA_ENT_FL_DEFAULT; vdev->v4l2_dev = &isp->v4l2_dev; vdev->vfl_type = VFL_TYPE_VIDEO; diff --git a/drivers/media/platform/apple/isp/isp-v4l2.h b/drivers/media/platform/apple/isp/isp-v4l2.h index 622288296aaae..55135594576b2 100644 --- a/drivers/media/platform/apple/isp/isp-v4l2.h +++ b/drivers/media/platform/apple/isp/isp-v4l2.h @@ -9,6 +9,7 @@ int apple_isp_setup_video(struct apple_isp *isp); void apple_isp_remove_video(struct apple_isp *isp); void apple_isp_free_video(struct apple_isp *isp); +void ipc_bt_complete(struct apple_isp *isp); int ipc_bt_handle(struct apple_isp *isp, struct isp_channel *chan); int apple_isp_video_suspend(struct apple_isp *isp); diff --git a/drivers/memstick/core/memstick.c b/drivers/memstick/core/memstick.c index e03989c4e99ea..0d15f48ef51d4 100644 --- a/drivers/memstick/core/memstick.c +++ b/drivers/memstick/core/memstick.c @@ -370,9 +370,7 @@ int memstick_set_rw_addr(struct memstick_dev *card) { card->next_request = h_memstick_set_rw_addr; memstick_new_req(card->host); - if (!wait_for_completion_timeout(&card->mrq_complete, - msecs_to_jiffies(500))) - card->current_mrq.error = -ETIMEDOUT; + wait_for_completion(&card->mrq_complete); return card->current_mrq.error; } @@ -405,9 +403,7 @@ static struct memstick_dev *memstick_alloc_card(struct memstick_host *host) card->next_request = h_memstick_read_dev_id; memstick_new_req(host); - if (!wait_for_completion_timeout(&card->mrq_complete, - msecs_to_jiffies(500))) - card->current_mrq.error = -ETIMEDOUT; + wait_for_completion(&card->mrq_complete); if (card->current_mrq.error) goto err_out; diff --git a/drivers/memstick/host/rtsx_usb_ms.c b/drivers/memstick/host/rtsx_usb_ms.c index beadc389f15f3..d5b3a96fc609a 100644 --- a/drivers/memstick/host/rtsx_usb_ms.c +++ b/drivers/memstick/host/rtsx_usb_ms.c @@ -27,7 +27,6 @@ struct rtsx_usb_ms { struct memstick_host *msh; struct memstick_request *req; - struct mutex host_mutex; struct work_struct handle_req; struct delayed_work poll_card; @@ -514,6 +513,13 @@ static void rtsx_usb_ms_handle_req(struct work_struct *work) struct memstick_host *msh = host->msh; int rc; + /* Fail requests after eject so their waiters are released. */ + if (host->eject) { + while (!memstick_next_req(msh, &host->req)) + host->req->error = -ENOMEDIUM; + return; + } + if (!host->req) { pm_runtime_get_sync(ms_dev(host)); do { @@ -547,8 +553,7 @@ static void rtsx_usb_ms_request(struct memstick_host *msh) dev_dbg(ms_dev(host), "--> %s\n", __func__); - if (!host->eject) - schedule_work(&host->handle_req); + schedule_work(&host->handle_req); } static int rtsx_usb_ms_set_param(struct memstick_host *msh, @@ -781,7 +786,6 @@ static int rtsx_usb_ms_drv_probe(struct platform_device *pdev) host->power_mode = MEMSTICK_POWER_OFF; platform_set_drvdata(pdev, host); - mutex_init(&host->host_mutex); INIT_WORK(&host->handle_req, rtsx_usb_ms_handle_req); INIT_DELAYED_WORK(&host->poll_card, rtsx_usb_ms_poll_card); @@ -812,27 +816,12 @@ static void rtsx_usb_ms_drv_remove(struct platform_device *pdev) { struct rtsx_usb_ms *host = platform_get_drvdata(pdev); struct memstick_host *msh = host->msh; - int err; host->eject = true; msh->removing = true; - cancel_work_sync(&host->handle_req); + flush_work(&host->handle_req); cancel_delayed_work_sync(&host->poll_card); - mutex_lock(&host->host_mutex); - if (host->req) { - dev_dbg(ms_dev(host), - "%s: Controller removed during transfer\n", - dev_name(&msh->dev)); - host->req->error = -ENOMEDIUM; - do { - err = memstick_next_req(msh, &host->req); - if (!err) - host->req->error = -ENOMEDIUM; - } while (!err); - } - mutex_unlock(&host->host_mutex); - /* Balance possible unbalanced usage count * e.g. unconditional module removal */ @@ -841,6 +830,8 @@ static void rtsx_usb_ms_drv_remove(struct platform_device *pdev) pm_runtime_disable(ms_dev(host)); memstick_remove_host(msh); + /* No card, no new requests; wait for the last failed one to finish. */ + cancel_work_sync(&host->handle_req); dev_dbg(ms_dev(host), ": Realtek USB Memstick controller has been removed\n"); memstick_free_host(msh); diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig index 4d7a7f0fe1ea5..9fd738df5db7b 100644 --- a/drivers/mfd/Kconfig +++ b/drivers/mfd/Kconfig @@ -351,7 +351,8 @@ config MFD_MACSMC_KUNIT_TEST Build tests for SMC key/value readers, request limits and metadata decoding. Static stubs replace mailbox commands and SRAM access, allowing malformed replies and output buffer limits to be checked - without a running SMC. + without a running SMC. Client driver tests can also replace + apple_smc_read() and apple_smc_write() with a model of their keys. config MFD_MADERA tristate "Cirrus Logic Madera codecs" diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 55d52c684d38b..fb9e1d7fa0ba7 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -190,6 +190,9 @@ static int apple_smc_rw_locked(struct apple_smc *smc, smc_key key, int apple_smc_read(struct apple_smc *smc, smc_key key, void *buf, size_t size) { + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_read, smc, key, buf, size); + guard(mutex)(&smc->mutex); return apple_smc_rw_locked(smc, key, NULL, 0, buf, size); @@ -198,6 +201,9 @@ EXPORT_SYMBOL(apple_smc_read); int apple_smc_write(struct apple_smc *smc, smc_key key, const void *buf, size_t size) { + if (IS_ENABLED(CONFIG_MFD_MACSMC_KUNIT_TEST)) + KUNIT_STATIC_STUB_REDIRECT(apple_smc_write, smc, key, buf, size); + guard(mutex)(&smc->mutex); return apple_smc_rw_locked(smc, key, buf, size, NULL, 0); diff --git a/drivers/mmc/host/cavium-octeon.c b/drivers/mmc/host/cavium-octeon.c index 8a0daddd9200e..1c219cf35a8e4 100644 --- a/drivers/mmc/host/cavium-octeon.c +++ b/drivers/mmc/host/cavium-octeon.c @@ -301,9 +301,12 @@ static void octeon_mmc_remove(struct platform_device *pdev) u64 dma_cfg; int i; - for (i = 0; i < CAVIUM_MAX_MMC; i++) + for (i = 0; i < CAVIUM_MAX_MMC; i++) { if (host->slot[i]) cvm_mmc_of_slot_remove(host->slot[i]); + if (host->slot_pdev[i]) + of_platform_device_destroy(&host->slot_pdev[i]->dev, NULL); + } dma_cfg = readq(host->dma_base + MIO_EMM_DMA_CFG(host)); dma_cfg &= ~MIO_EMM_DMA_CFG_EN; diff --git a/drivers/mmc/host/cavium-thunderx.c b/drivers/mmc/host/cavium-thunderx.c index 84ff6d82ae3c0..2635ac94ec5da 100644 --- a/drivers/mmc/host/cavium-thunderx.c +++ b/drivers/mmc/host/cavium-thunderx.c @@ -173,9 +173,12 @@ static void thunder_mmc_remove(struct pci_dev *pdev) u64 dma_cfg; int i; - for (i = 0; i < CAVIUM_MAX_MMC; i++) + for (i = 0; i < CAVIUM_MAX_MMC; i++) { if (host->slot[i]) cvm_mmc_of_slot_remove(host->slot[i]); + if (host->slot_pdev[i]) + of_platform_device_destroy(&host->slot_pdev[i]->dev, NULL); + } dma_cfg = readq(host->dma_base + MIO_EMM_DMA_CFG(host)); dma_cfg &= ~MIO_EMM_DMA_CFG_EN; diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c index b2680cc054bda..d70e37eb1cb59 100644 --- a/drivers/mmc/host/mtk-sd.c +++ b/drivers/mmc/host/mtk-sd.c @@ -3216,6 +3216,7 @@ static void msdc_drv_remove(struct platform_device *pdev) platform_set_drvdata(pdev, NULL); mmc_remove_host(mmc); + cancel_delayed_work_sync(&host->req_timeout); msdc_deinit_hw(host); msdc_gate_clock(host); diff --git a/drivers/mmc/host/sdhci-sprd.c b/drivers/mmc/host/sdhci-sprd.c index 3584a2b314a9f..0c8f389fe5847 100644 --- a/drivers/mmc/host/sdhci-sprd.c +++ b/drivers/mmc/host/sdhci-sprd.c @@ -890,6 +890,10 @@ static void sdhci_sprd_remove(struct platform_device *pdev) struct sdhci_host *host = platform_get_drvdata(pdev); struct sdhci_sprd_host *sprd_host = TO_SPRD_HOST(host); + pm_runtime_get_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + sdhci_remove_host(host, 0); clk_disable_unprepare(sprd_host->clk_sdio); diff --git a/drivers/net/amt.c b/drivers/net/amt.c index 7645ce3a29ad9..189b3a43e4e2a 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -80,15 +80,6 @@ static struct in6_addr mld2_all_node = MLD2_ALL_NODE_INIT; static struct mld2_grec mldv2_zero_grec; #endif -static struct amt_skb_cb *amt_skb_cb(struct sk_buff *skb) -{ - BUILD_BUG_ON(sizeof(struct amt_skb_cb) + sizeof(struct tc_skb_cb) > - sizeof_field(struct sk_buff, cb)); - - return (struct amt_skb_cb *)((void *)skb->cb + - sizeof(struct tc_skb_cb)); -} - static void __amt_source_gc_work(void) { struct amt_source_node *snode; @@ -789,6 +780,11 @@ static void amt_send_request(struct amt_dev *amt, bool v6) rcu_read_unlock(); } +static bool amt_send_membership_query(struct amt_dev *amt, + struct sk_buff *skb, + struct amt_tunnel_list *tunnel, + bool v6); + static void amt_send_igmp_gq(struct amt_dev *amt, struct amt_tunnel_list *tunnel) { @@ -798,8 +794,11 @@ static void amt_send_igmp_gq(struct amt_dev *amt, if (!skb) return; - amt_skb_cb(skb)->tunnel = tunnel; - dev_queue_xmit(skb); + skb_pull(skb, sizeof(struct ethhdr)); + if (amt_send_membership_query(amt, skb, tunnel, false)) { + amt->dev->stats.tx_dropped++; + kfree_skb(skb); + } } #if IS_ENABLED(CONFIG_IPV6) @@ -883,8 +882,11 @@ static void amt_send_mld_gq(struct amt_dev *amt, struct amt_tunnel_list *tunnel) if (!skb) return; - amt_skb_cb(skb)->tunnel = tunnel; - dev_queue_xmit(skb); + skb_pull(skb, sizeof(struct ethhdr)); + if (amt_send_membership_query(amt, skb, tunnel, true)) { + amt->dev->stats.tx_dropped++; + kfree_skb(skb); + } } #else static void amt_send_mld_gq(struct amt_dev *amt, struct amt_tunnel_list *tunnel) @@ -1183,7 +1185,6 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) #endif bool report = false; struct igmphdr *ih; - bool query = false; struct iphdr *iph; bool data = false; bool v6 = false; @@ -1201,9 +1202,6 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) case IGMP_HOST_MEMBERSHIP_REPORT: report = true; break; - case IGMP_HOST_MEMBERSHIP_QUERY: - query = true; - break; default: goto free; } @@ -1225,9 +1223,6 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) case ICMPV6_MLD2_REPORT: report = true; break; - case ICMPV6_MGM_QUERY: - query = true; - break; default: goto free; } @@ -1258,19 +1253,6 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) goto free; goto unlock; } else if (amt->mode == AMT_MODE_RELAY) { - if (query) { - tunnel = amt_skb_cb(skb)->tunnel; - if (!tunnel) { - WARN_ON(1); - goto free; - } - - /* Do not forward unexpected query */ - if (amt_send_membership_query(amt, skb, tunnel, v6)) - goto free; - goto unlock; - } - if (!data) goto free; list_for_each_entry_rcu(tunnel, &amt->tunnel_list, list) { diff --git a/drivers/net/bareudp.c b/drivers/net/bareudp.c index 5ef841c855269..2bfad8653be6a 100644 --- a/drivers/net/bareudp.c +++ b/drivers/net/bareudp.c @@ -355,7 +355,7 @@ static int bareudp_xmit_skb(struct sk_buff *skb, struct net_device *dev, if (err) goto free_dst; - skb_set_inner_protocol(skb, bareudp->ethertype); + skb_set_inner_protocol(skb, skb->protocol); udp_tunnel_xmit_skb(rt, sk, skb, saddr, info->key.u.ipv4.dst, tos, ttl, df, sport, bareudp->port, !net_eq(bareudp->net, dev_net(bareudp->dev)), @@ -423,6 +423,7 @@ static int bareudp6_xmit_skb(struct sk_buff *skb, struct net_device *dev, if (err) goto free_dst; + skb_set_inner_protocol(skb, skb->protocol); daddr = info->key.u.ipv6.dst; udp_tunnel6_xmit_skb(dst, sk, skb, dev, &saddr, &daddr, prio, ttl, diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index 34c5bbbed9aee..507d655eca21e 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -5738,12 +5738,12 @@ static int bond_xdp_set(struct net_device *dev, struct bpf_prog *prog, bpf_prog_inc(prog); } - if (prog) { + if (prog && !old_prog) static_branch_inc(&bpf_master_redirect_enabled_key); - } else if (old_prog) { - bpf_prog_put(old_prog); + else if (!prog && old_prog) static_branch_dec(&bpf_master_redirect_enabled_key); - } + if (old_prog) + bpf_prog_put(old_prog); return 0; diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c index 95fcdc1026f80..14edec5afb57e 100644 --- a/drivers/net/can/dev/skb.c +++ b/drivers/net/can/dev/skb.c @@ -210,6 +210,10 @@ static void init_can_skb(struct sk_buff *skb) { skb->pkt_type = PACKET_BROADCAST; skb->ip_summed = CHECKSUM_UNNECESSARY; + + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb_reset_transport_header(skb); } struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf) diff --git a/drivers/net/can/usb/f81604.c b/drivers/net/can/usb/f81604.c index f12318268e46b..4c147b9d6d690 100644 --- a/drivers/net/can/usb/f81604.c +++ b/drivers/net/can/usb/f81604.c @@ -169,7 +169,7 @@ struct f81604_int_data { u8 rxerr; u8 txerr; u8 val; -} __packed __aligned(4); +} __packed; struct f81604_sff { __be16 id; diff --git a/drivers/net/can/vxcan.c b/drivers/net/can/vxcan.c index e882250180ef6..f365deafaf542 100644 --- a/drivers/net/can/vxcan.c +++ b/drivers/net/can/vxcan.c @@ -78,6 +78,9 @@ static netdev_tx_t vxcan_xmit(struct sk_buff *oskb, struct net_device *dev) /* reset CAN GW hop counter */ csx->can_gw_hops = 0; + /* start with new CAN skb UID in the other namespace */ + csx->can_skb_uid = 0; + skb->pkt_type = PACKET_BROADCAST; skb->dev = peer; skb->ip_summed = CHECKSUM_UNNECESSARY; diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h index b4a5673ba365c..8f6ac13b5b182 100644 --- a/drivers/net/dsa/microchip/ksz_common.h +++ b/drivers/net/dsa/microchip/ksz_common.h @@ -778,7 +778,7 @@ static inline bool ksz_is_sgmii_port(struct ksz_device *dev, int port) #define KSZ88_FAMILY_ID 0x88 #define KSZ8895_FAMILY_ID 0x95 -#define KSZ8_PORT_STATUS_0 0x08 +#define KSZ8_PORT_STATUS_0 0x18 #define KSZ8_PORT_FIBER_MODE BIT(7) #define SW_CHIP_ID_M GENMASK(7, 4) diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index 2fc8c91f210c6..d4faf4503487e 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -2716,14 +2716,34 @@ static int airoha_qdma_set_trtcm_param(struct airoha_qdma *qdma, int channel, FIELD_PREP(TRTCM_METER_GROUP_MASK, group) | FIELD_PREP(TRTCM_PARAM_INDEX_MASK, idx) | FIELD_PREP(TRTCM_PARAM_RATE_TYPE_MASK, mode); + int i; - airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val); - airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); + /* + * HW might be in the middle of bucket handling and might + * require an additional try to actually apply the new TRTCM + * config. + */ + for (i = 0; i < TRTCM_CONFIG_RETRY_MAX; i++) { + u32 data = 0; - return read_poll_timeout(airoha_qdma_rr, val, - val & TRTCM_PARAM_RW_DONE_MASK, - USEC_PER_MSEC, 10 * USEC_PER_MSEC, true, - qdma, REG_TRTCM_CFG_PARAM(addr)); + airoha_qdma_wr(qdma, REG_TRTCM_DATA_LOW(addr), val); + airoha_qdma_wr(qdma, REG_TRTCM_CFG_PARAM(addr), config); + + if (read_poll_timeout(airoha_qdma_rr, data, + data & TRTCM_PARAM_RW_DONE_MASK, + USEC_PER_MSEC, 10 * USEC_PER_MSEC, + true, qdma, REG_TRTCM_CFG_PARAM(addr))) + continue; + + if (airoha_qdma_get_trtcm_param(qdma, channel, addr, param, + mode, &data, NULL)) + continue; + + if (data == val) + return 0; + } + + return -EBUSY; } static int airoha_qdma_set_trtcm_config(struct airoha_qdma *qdma, int channel, diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h index 70f16189b2582..dc0c29001f810 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.h +++ b/drivers/net/ethernet/airoha/airoha_eth.h @@ -171,6 +171,8 @@ enum trtcm_param { #define TRTCM_TOKEN_RATE_MASK GENMASK(23, 6) #define TRTCM_TOKEN_RATE_FRACTION_MASK GENMASK(5, 0) +#define TRTCM_CONFIG_RETRY_MAX 2 + enum airoha_dma_map_type { AIROHA_DMA_UNMAPPED, AIROHA_DMA_MAP_SINGLE, diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index ced998399efc3..78583cee3414f 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -486,7 +486,7 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) struct netdev_queue *txq; int i; dma_addr_t mapping; - unsigned int length, pad = 0; + unsigned int length; u32 len, free_size, vlan_tag_flags, cfa_action, flags; struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; struct pci_dev *pdev = bp->pdev; @@ -536,6 +536,16 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) return rc < 0 ? NETDEV_TX_BUSY : NETDEV_TX_OK; } + /* Pad after the SW USO branch: bnxt_sw_udp_gso_xmit() would + * otherwise account the padding as UDP payload. + * Must be done before skb_shinfo(skb)->nr_frags is sampled, + * because skb_put_padto() might linearize the skb. + */ + if (skb_put_padto(skb, BNXT_MIN_PKT_SIZE)) { + /* SKB already freed. */ + goto tx_kick_pending; + } + free_size = bnxt_tx_avail(bp, txr); if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { /* We must have raced with NAPI cleanup */ @@ -672,14 +682,6 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) } normal_tx: - if (length < BNXT_MIN_PKT_SIZE) { - pad = BNXT_MIN_PKT_SIZE - length; - if (skb_pad(skb, pad)) - /* SKB already freed. */ - goto tx_kick_pending; - length = BNXT_MIN_PKT_SIZE; - } - mapping = dma_map_single(&pdev->dev, skb->data, len, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(&pdev->dev, mapping))) @@ -759,10 +761,7 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) txbd->tx_bd_len_flags_type = cpu_to_le32(flags); } - flags &= ~TX_BD_LEN; - txbd->tx_bd_len_flags_type = - cpu_to_le32(((len + pad) << TX_BD_LEN_SHIFT) | flags | - TX_BD_FLAGS_PACKET_END); + txbd->tx_bd_len_flags_type |= cpu_to_le32(TX_BD_FLAGS_PACKET_END); netdev_tx_sent_queue(txq, skb->len); @@ -15506,7 +15505,7 @@ static void bnxt_fw_reset_task(struct work_struct *work) if (test_and_clear_bit(BNXT_STATE_FW_ACTIVATE_RESET, &bp->state) && !test_bit(BNXT_STATE_FW_ACTIVATE, &bp->state)) bnxt_dl_remote_reload(bp); - if (pci_enable_device(bp->pdev)) { + if (!pci_is_enabled(bp->pdev) && pci_enable_device(bp->pdev)) { netdev_err(bp->dev, "Cannot re-enable PCI device\n"); rc = -ENODEV; goto fw_reset_abort; @@ -17067,6 +17066,51 @@ void bnxt_print_device_info(struct bnxt *bp) pcie_print_link_status(bp->pdev); } +static void bnxt_clear_bars(struct pci_dev *pdev) +{ + int off; + + for (off = PCI_BASE_ADDRESS_0; off <= PCI_BASE_ADDRESS_5; off += 4) + pci_write_config_dword(pdev, off, 0); +} + +/* Clear any pending DMA transactions from crash kernel while loading driver in + * capture kernel. + */ +static int bnxt_kdump_reset(struct pci_dev *pdev) +{ + int rc, i; + u16 cmd; + + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + cmd &= ~(PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); + pci_write_config_word(pdev, PCI_COMMAND, cmd); + + if (pci_save_state(pdev)) + dev_warn(&pdev->dev, "Failed to save PCI state, PCI restore may be incomplete\n"); + + rc = pcie_flr(pdev); + if (rc) + dev_warn(&pdev->dev, "pcie_flr() failed (rc: %d), trying to continue\n", + rc); + + /* In case device is not returning CRS, wait 5 seconds longer */ + for (i = 0; i < 50; i++) { + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + if (!PCI_POSSIBLE_ERROR(cmd)) + break; + msleep(100); + } + if (PCI_POSSIBLE_ERROR(cmd)) { + dev_err(&pdev->dev, "PCI config space inaccessible after FLR, aborting\n"); + return -ENODEV; + } + + bnxt_clear_bars(pdev); + pci_restore_state(pdev); + return 0; +} + static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { struct bnxt_hw_resc *hw_resc; @@ -17082,12 +17126,10 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENODEV; } - /* Clear any pending DMA transactions from crash kernel - * while loading driver in capture kernel. - */ if (is_kdump_kernel()) { - pci_clear_master(pdev); - pcie_flr(pdev); + rc = bnxt_kdump_reset(pdev); + if (rc) + return rc; } max_irqs = bnxt_get_max_irq(pdev); @@ -17515,10 +17557,8 @@ static pci_ers_result_t bnxt_io_error_detected(struct pci_dev *pdev, * so we disable bus master to prevent any potential bad DMAs before * freeing kernel memory. */ - if (state == pci_channel_io_frozen) { - set_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state); + if (state == pci_channel_io_frozen) bnxt_fw_fatal_close(bp); - } if (netif_running(netdev)) __bnxt_close_nic(bp, true, true); @@ -17548,69 +17588,80 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) struct bnxt *bp = netdev_priv(netdev); int retry = 0; int err = 0; - int off; + u16 cmd; netdev_info(bp->dev, "PCI Slot Reset\n"); - if (test_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, &bp->state)) { - /* After DPC, the chip should return CRS when the vendor ID - * config register is read until it is ready. On all chips, - * this is not happening reliably so add a 5-second delay as a - * workaround. - */ - msleep(5000); - } + /* After a PCIe hot reset, the chip should return CRS when the + * vendor ID config register is read until it is ready. On all + * chips, this is not happening reliably so add a 5-second delay + * as a workaround. + */ + msleep(5000); netdev_lock(netdev); - if (pci_enable_device(pdev)) { + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + if (PCI_POSSIBLE_ERROR(cmd)) { dev_err(&pdev->dev, - "Cannot re-enable PCI device after reset.\n"); - } else { - pci_set_master(pdev); - /* Upon fatal error, our device internal logic that latches to - * BAR value is getting reset and will restore only upon - * rewriting the BARs. - * - * As pci_restore_state() does not re-write the BARs if the - * value is same as saved value earlier, driver needs to - * write the BARs to 0 to force restore, in case of fatal error. - */ - if (test_and_clear_bit(BNXT_STATE_PCI_CHANNEL_IO_FROZEN, - &bp->state)) { - for (off = PCI_BASE_ADDRESS_0; - off <= PCI_BASE_ADDRESS_5; off += 4) - pci_write_config_dword(bp->pdev, off, 0); - } - pci_restore_state(pdev); + "PCI config space inaccessible after reset\n"); + goto reset_exit; + } - bnxt_inv_fw_health_reg(bp); - bnxt_try_map_fw_health_reg(bp); + /* Upon PCIe error, our device internal logic that latches to + * BAR value is getting reset and will restore only upon + * rewriting the BARs. + * + * As pci_restore_state() does not re-write the BARs if the + * value is same as saved value earlier, driver needs to + * write the BARs to 0 to force restore. + */ + pci_clear_master(pdev); + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + cmd &= ~PCI_COMMAND_MEMORY; + pci_write_config_word(pdev, PCI_COMMAND, cmd); - /* In some PCIe AER scenarios, firmware may take up to - * 10 seconds to become ready in the worst case. - */ - do { - err = bnxt_try_recover_fw(bp); - if (!err) - break; - retry++; - } while (retry < BNXT_FW_SLOT_RESET_RETRY); + bnxt_clear_bars(pdev); + pci_restore_state(pdev); - if (err) { - dev_err(&pdev->dev, "Firmware not ready\n"); - goto reset_exit; + if (!pci_is_enabled(pdev) && pci_enable_device(pdev)) { + dev_err(&pdev->dev, + "Cannot re-enable PCI device after reset.\n"); + pci_read_config_word(pdev, PCI_COMMAND, &cmd); + if (!PCI_POSSIBLE_ERROR(cmd)) { + cmd &= ~(PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); + pci_write_config_word(pdev, PCI_COMMAND, cmd); } + goto reset_exit; + } + pci_set_master(pdev); - err = bnxt_hwrm_func_reset(bp); + bnxt_inv_fw_health_reg(bp); + bnxt_try_map_fw_health_reg(bp); + + /* In some PCIe AER scenarios, firmware may take up to + * 10 seconds to become ready in the worst case. + */ + do { + err = bnxt_try_recover_fw(bp); if (!err) - result = PCI_ERS_RESULT_RECOVERED; + break; + retry++; + } while (retry < BNXT_FW_SLOT_RESET_RETRY); - /* IRQ will be initialized later in bnxt_io_resume */ - bnxt_ulp_irq_stop(bp); - bnxt_clear_int_mode(bp); + if (err) { + dev_err(&pdev->dev, "Firmware not ready\n"); + goto reset_exit; } + err = bnxt_hwrm_func_reset(bp); + if (!err) + result = PCI_ERS_RESULT_RECOVERED; + + /* IRQ will be initialized later in bnxt_io_resume */ + bnxt_ulp_irq_stop(bp); + bnxt_clear_int_mode(bp); + reset_exit: clear_bit(BNXT_STATE_IN_FW_RESET, &bp->state); bnxt_clear_reservations(bp, true); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 4dbcd08ef7c32..a939f4ed10079 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -2470,7 +2470,6 @@ struct bnxt { #define BNXT_STATE_ABORT_ERR 5 #define BNXT_STATE_FW_FATAL_COND 6 #define BNXT_STATE_DRV_REGISTERED 7 -#define BNXT_STATE_PCI_CHANNEL_IO_FROZEN 8 #define BNXT_STATE_NAPI_DISABLED 9 #define BNXT_STATE_FW_ACTIVATE 11 #define BNXT_STATE_RECOVER 12 diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c index 96d5d4f7f51fe..984432952963a 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c @@ -253,6 +253,17 @@ int bcmgenet_wol_power_up_cfg(struct bcmgenet_priv *priv, reg = bcmgenet_umac_readl(priv, UMAC_CMD); reg &= ~CMD_CRC_FWD; bcmgenet_umac_writel(priv, reg, UMAC_CMD); + + /* + * Mirror wol_power_down_cfg(). If only UMAC_RX + * is enabled, then we must place the UMAC back + * into SW_RESET. + */ + reg = bcmgenet_umac_readl(priv, UMAC_CMD); + if ((reg & CMD_RX_EN) && !(reg & CMD_TX_EN)) { + reg |= CMD_SW_RESET; + bcmgenet_umac_writel(priv, reg, UMAC_CMD); + } spin_unlock_bh(&priv->reg_lock); /* Resume link status tracking */ diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c index f5acd4be1e69d..dbec95b497773 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c @@ -389,7 +389,7 @@ static void chcr_ktls_dev_del(struct net_device *netdev, cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, tx_info->tid, tx_info->ip_family); - xa_erase(&u_ctx->tid_list, tx_info->tid); + xa_erase_bh(&u_ctx->tid_list, tx_info->tid); } port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id]; @@ -577,7 +577,7 @@ static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, tx_info->tid, tx_info->ip_family); - xa_erase(&u_ctx->tid_list, tx_info->tid); + xa_erase_bh(&u_ctx->tid_list, tx_info->tid); put_module: /* release module refcount */ diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index e4d78ae52dafe..0dd04ee30c9a5 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -1708,17 +1708,17 @@ static int gve_xsk_pool_disable(struct net_device *dev, } napi_rx = &priv->ntfy_blocks[priv->rx[qid].ntfy_id].napi; - napi_disable(napi_rx); /* make sure current rx poll is done */ + napi_disable_locked(napi_rx); /* make sure current rx poll is done */ tx_qid = gve_xdp_tx_queue_id(priv, qid); napi_tx = &priv->ntfy_blocks[priv->tx[tx_qid].ntfy_id].napi; - napi_disable(napi_tx); /* make sure current tx poll is done */ + napi_disable_locked(napi_tx); /* make sure current tx poll is done */ gve_unreg_xsk_pool(priv, qid); smp_mb(); /* Make sure it is visible to the workers on datapath */ - napi_enable(napi_rx); - napi_enable(napi_tx); + napi_enable_locked(napi_rx); + napi_enable_locked(napi_tx); if (gve_is_gqi(priv)) { if (gve_rx_work_pending(&priv->rx[qid])) napi_schedule(napi_rx); diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c index 0ece2f6fdffb4..2e6d6abbddfc1 100644 --- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c @@ -113,6 +113,12 @@ static void gve_rx_reset_ring_dqo(struct gve_priv *priv, int idx) gve_free_to_page_pool(rx, bs, false); else gve_free_qpl_page_dqo(bs); + + if (gve_buf_state_is_allocated(rx, bs) && + bs->xsk_buff) { + xsk_buff_free(bs->xsk_buff); + bs->xsk_buff = NULL; + } } } @@ -839,7 +845,12 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, } if (unlikely(compl_desc->rx_error)) { - gve_free_buffer(rx, buf_state); + if (buf_state->xsk_buff) { + xsk_buff_free(buf_state->xsk_buff); + gve_free_buf_state(rx, buf_state); + } else { + gve_free_buffer(rx, buf_state); + } return -EINVAL; } diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c index 73e051d26b9d8..abebdb1fc2628 100644 --- a/drivers/net/ethernet/ibm/ibmveth.c +++ b/drivers/net/ethernet/ibm/ibmveth.c @@ -553,10 +553,15 @@ static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter, static void ibmveth_free_tx_ltb(struct ibmveth_adapter *adapter, int idx) { + void *ptr = adapter->tx_ltb_ptr[idx]; + + if (!ptr) + return; + + adapter->tx_ltb_ptr[idx] = NULL; dma_unmap_single(&adapter->vdev->dev, adapter->tx_ltb_dma[idx], adapter->tx_ltb_size, DMA_TO_DEVICE); - kfree(adapter->tx_ltb_ptr[idx]); - adapter->tx_ltb_ptr[idx] = NULL; + kfree(ptr); } static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter *adapter, int idx) @@ -667,7 +672,7 @@ static int ibmveth_open(struct net_device *netdev) for (i = 0; i < netdev->real_num_tx_queues; i++) { if (ibmveth_allocate_tx_ltb(adapter, i)) - goto out_free_tx_ltb; + goto out_unmap_filter_list; } adapter->rx_queue.index = 0; @@ -718,10 +723,6 @@ static int ibmveth_open(struct net_device *netdev) if (rc != 0) { netdev_err(netdev, "unable to request irq 0x%x, rc %d\n", netdev->irq, rc); - do { - lpar_rc = h_free_logical_lan(adapter->vdev->unit_address); - } while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY)); - goto out_free_buffer_pools; } @@ -737,6 +738,9 @@ static int ibmveth_open(struct net_device *netdev) return 0; out_free_buffer_pools: + do { + lpar_rc = h_free_logical_lan(adapter->vdev->unit_address); + } while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY)); while (--i >= 0) { if (adapter->rx_buff_pool[i].active) ibmveth_free_buffer_pool(adapter, @@ -746,10 +750,8 @@ static int ibmveth_open(struct net_device *netdev) dma_unmap_single(dev, adapter->filter_list_dma, 4096, DMA_BIDIRECTIONAL); -out_free_tx_ltb: - while (--i >= 0) { + for (i = netdev->real_num_tx_queues - 1; i >= 0; i--) ibmveth_free_tx_ltb(adapter, i); - } out_unmap_buffer_list: dma_unmap_single(dev, adapter->buffer_list_dma, 4096, diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 844f31ab37ad4..c282f65199e49 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -67,6 +67,13 @@ static const struct dmi_system_id disable_k1_list[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Dell Pro 16 Plus PB16250"), }, }, + { + .ident = "Lenovo ThinkPad P14s Gen 5", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad P14s Gen 5"), + }, + }, {} }; diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index 050f8241ef5e6..5fcbfa0ca8553 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -259,7 +259,6 @@ struct iavf_adapter { struct work_struct adminq_task; struct work_struct finish_config; wait_queue_head_t down_waitqueue; - wait_queue_head_t vc_waitqueue; struct iavf_q_vector *q_vectors; struct list_head vlan_filter_list; int num_vlan_filters; @@ -588,8 +587,9 @@ void iavf_configure_queues(struct iavf_adapter *adapter); void iavf_enable_queues(struct iavf_adapter *adapter); void iavf_disable_queues(struct iavf_adapter *adapter); void iavf_map_queues(struct iavf_adapter *adapter); -void iavf_add_ether_addrs(struct iavf_adapter *adapter); +int iavf_add_ether_addrs(struct iavf_adapter *adapter); void iavf_del_ether_addrs(struct iavf_adapter *adapter); +void iavf_mac_add_reject(struct iavf_adapter *adapter); void iavf_add_vlans(struct iavf_adapter *adapter); void iavf_del_vlans(struct iavf_adapter *adapter); void iavf_set_promiscuous(struct iavf_adapter *adapter); @@ -606,6 +606,13 @@ void iavf_disable_vlan_stripping(struct iavf_adapter *adapter); void iavf_virtchnl_completion(struct iavf_adapter *adapter, enum virtchnl_ops v_opcode, enum iavf_status v_retval, u8 *msg, u16 msglen); +int iavf_poll_virtchnl_response(struct iavf_adapter *adapter, + struct iavf_arq_event_info *event, + bool (*condition)(struct iavf_adapter *adapter, + const void *data, + enum virtchnl_ops v_op), + const void *cond_data, + unsigned int timeout_ms); int iavf_config_rss(struct iavf_adapter *adapter); void iavf_cfg_queues_bw(struct iavf_adapter *adapter); void iavf_cfg_queues_quanta_size(struct iavf_adapter *adapter); diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 29b8403a066bc..2f8a80a113362 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -1029,6 +1029,60 @@ static bool iavf_is_mac_set_handled(struct net_device *netdev, return ret; } +/** + * iavf_mac_change_done - Check if MAC change completed + * @adapter: board private structure + * @data: MAC address being checked (as const void *) + * @v_op: virtchnl opcode from processed message + * + * Callback for iavf_poll_virtchnl_response() to check if MAC change completed. + * + * Return: true if MAC change completed, false otherwise + */ +static bool iavf_mac_change_done(struct iavf_adapter *adapter, + const void *data, enum virtchnl_ops v_op) +{ + const u8 *addr = data; + + return iavf_is_mac_set_handled(adapter->netdev, addr); +} + +/** + * iavf_set_mac_sync - Synchronously change MAC address + * @adapter: board private structure + * @addr: MAC address to set + * + * Send MAC change request to PF and poll admin queue for response. + * Caller must hold netdev_lock. This can sleep for up to 2.5 seconds. + * Event buffer is allocated before sending to avoid state mismatch if + * allocation fails after message is sent to PF. + * + * Return: 0 on success, negative on failure + */ +static int iavf_set_mac_sync(struct iavf_adapter *adapter, const u8 *addr) +{ + struct iavf_arq_event_info event; + int ret; + + netdev_assert_locked(adapter->netdev); + + event.buf_len = IAVF_MAX_AQ_BUF_SIZE; + event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); + if (!event.msg_buf) + return -ENOMEM; + + ret = iavf_add_ether_addrs(adapter); + if (ret) + goto out; + + ret = iavf_poll_virtchnl_response(adapter, &event, + iavf_mac_change_done, addr, 2500); + +out: + kfree(event.msg_buf); + return ret; +} + /** * iavf_set_mac - NDO callback to set port MAC address * @netdev: network interface device structure @@ -1046,25 +1100,23 @@ static int iavf_set_mac(struct net_device *netdev, void *p) return -EADDRNOTAVAIL; ret = iavf_replace_primary_mac(adapter, addr->sa_data); - if (ret) return ret; - ret = wait_event_interruptible_timeout(adapter->vc_waitqueue, - iavf_is_mac_set_handled(netdev, addr->sa_data), - msecs_to_jiffies(2500)); - - /* If ret < 0 then it means wait was interrupted. - * If ret == 0 then it means we got a timeout. - * else it means we got response for set MAC from PF, - * check if netdev MAC was updated to requested MAC, - * if yes then set MAC succeeded otherwise it failed return -EACCES - */ - if (ret < 0) + ret = iavf_set_mac_sync(adapter, addr->sa_data); + if (ret) { + /* Rollback only if send failed (message never reached PF). + * Don't rollback on timeout (-EAGAIN) because the message was + * sent and PF will eventually respond. When the response arrives, + * iavf_virtchnl_completion() will handle rollback (on PF error) + * or acceptance (on PF success) automatically. + */ + if (ret != -EAGAIN) { + iavf_mac_add_reject(adapter); + ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); + } return ret; - - if (!ret) - return -EAGAIN; + } if (!ether_addr_equal(netdev->dev_addr, addr->sa_data)) return -EACCES; @@ -5394,9 +5446,6 @@ static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Setup the wait queue for indicating transition to down status */ init_waitqueue_head(&adapter->down_waitqueue); - /* Setup the wait queue for indicating virtchannel events */ - init_waitqueue_head(&adapter->vc_waitqueue); - INIT_LIST_HEAD(&adapter->ptp.aq_cmds); init_waitqueue_head(&adapter->ptp.phc_time_waitqueue); mutex_init(&adapter->ptp.aq_cmd_lock); diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index ec234cc8bd9db..e6b7e8f82c7c1 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -2,6 +2,7 @@ /* Copyright(c) 2013 - 2018 Intel Corporation. */ #include +#include #include "iavf.h" #include "iavf_ptp.h" @@ -555,20 +556,23 @@ iavf_set_mac_addr_type(struct virtchnl_ether_addr *virtchnl_ether_addr, * @adapter: adapter structure * * Request that the PF add one or more addresses to our filters. - **/ -void iavf_add_ether_addrs(struct iavf_adapter *adapter) + * + * Return: 0 on success, negative on failure + */ +int iavf_add_ether_addrs(struct iavf_adapter *adapter) { struct virtchnl_ether_addr_list *veal; struct iavf_mac_filter *f; int i = 0, count = 0; bool more = false; size_t len; + int ret; if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { /* bail because we already have a command pending */ dev_err(&adapter->pdev->dev, "Cannot add filters, command %d pending\n", adapter->current_op); - return; + return -EBUSY; } spin_lock_bh(&adapter->mac_vlan_list_lock); @@ -580,7 +584,7 @@ void iavf_add_ether_addrs(struct iavf_adapter *adapter) if (!count) { adapter->aq_required &= ~IAVF_FLAG_AQ_ADD_MAC_FILTER; spin_unlock_bh(&adapter->mac_vlan_list_lock); - return; + return 0; } adapter->current_op = VIRTCHNL_OP_ADD_ETH_ADDR; @@ -594,8 +598,9 @@ void iavf_add_ether_addrs(struct iavf_adapter *adapter) veal = kzalloc(len, GFP_ATOMIC); if (!veal) { + adapter->current_op = VIRTCHNL_OP_UNKNOWN; spin_unlock_bh(&adapter->mac_vlan_list_lock); - return; + return -ENOMEM; } veal->vsi_id = adapter->vsi_res->vsi_id; @@ -615,8 +620,15 @@ void iavf_add_ether_addrs(struct iavf_adapter *adapter) spin_unlock_bh(&adapter->mac_vlan_list_lock); - iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_ETH_ADDR, (u8 *)veal, len); + ret = iavf_send_pf_msg(adapter, VIRTCHNL_OP_ADD_ETH_ADDR, (u8 *)veal, len); kfree(veal); + if (ret) { + dev_err(&adapter->pdev->dev, + "Unable to send ADD_ETH_ADDR message to PF, error %d\n", ret); + adapter->current_op = VIRTCHNL_OP_UNKNOWN; + } + + return ret; } /** @@ -712,8 +724,8 @@ static void iavf_mac_add_ok(struct iavf_adapter *adapter) * @adapter: adapter structure * * Remove filters from list based on PF response. - **/ -static void iavf_mac_add_reject(struct iavf_adapter *adapter) + */ +void iavf_mac_add_reject(struct iavf_adapter *adapter) { struct net_device *netdev = adapter->netdev; struct iavf_mac_filter *f, *ftmp; @@ -2364,7 +2376,6 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, iavf_mac_add_reject(adapter); /* restore administratively set MAC address */ ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); - wake_up(&adapter->vc_waitqueue); break; case VIRTCHNL_OP_DEL_ETH_ADDR: dev_err(&adapter->pdev->dev, "Failed to delete MAC filter, error %s\n", @@ -2555,7 +2566,6 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, eth_hw_addr_set(netdev, adapter->hw.mac.addr); netif_addr_unlock_bh(netdev); } - wake_up(&adapter->vc_waitqueue); break; case VIRTCHNL_OP_GET_STATS: { struct iavf_eth_stats *stats = @@ -2950,3 +2960,72 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, } /* switch v_opcode */ adapter->current_op = VIRTCHNL_OP_UNKNOWN; } + +/** + * iavf_poll_virtchnl_response - Poll admin queue for virtchnl response + * @adapter: adapter structure + * @event: pre-allocated event buffer to use for polling + * @condition: callback to check if desired response received + * @cond_data: context data passed to condition callback + * @timeout_ms: maximum time to wait in milliseconds + * + * Polls the admin queue and processes all incoming virtchnl messages. + * After processing each valid message, calls the condition callback to check + * if the expected response has been received. The callback receives the opcode + * of the processed message to identify which response was received. Continues + * polling until the callback returns true or timeout expires. + * + * Caller must allocate event buffer before sending any messages to PF to avoid + * state mismatch if allocation fails after message is sent. + * + * Caller must hold netdev_lock. This can sleep for up to timeout_ms while + * polling hardware. + * + * Return: 0 on success (condition met), -EAGAIN on timeout, or error code + */ +int iavf_poll_virtchnl_response(struct iavf_adapter *adapter, + struct iavf_arq_event_info *event, + bool (*condition)(struct iavf_adapter *adapter, + const void *data, + enum virtchnl_ops v_op), + const void *cond_data, + unsigned int timeout_ms) +{ + struct iavf_hw *hw = &adapter->hw; + enum virtchnl_ops received_op; + unsigned long timeout; + int ret = -EAGAIN; + u16 pending = 0; + u32 v_retval; + + netdev_assert_locked(adapter->netdev); + + timeout = jiffies + msecs_to_jiffies(timeout_ms); + do { + if (!pending) + usleep_range(50, 75); + + if (iavf_clean_arq_element(hw, event, &pending) == IAVF_SUCCESS) { + received_op = (enum virtchnl_ops)le32_to_cpu(event->desc.cookie_high); + if (received_op != VIRTCHNL_OP_UNKNOWN) { + v_retval = le32_to_cpu(event->desc.cookie_low); + + iavf_virtchnl_completion(adapter, received_op, + (enum iavf_status)v_retval, + event->msg_buf, event->msg_len); + + if (condition(adapter, cond_data, received_op)) { + ret = 0; + break; + } + } + + memset(event->msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE); + + if (pending) + continue; + } + } while (time_before(jiffies, timeout)); + + return ret; +} diff --git a/drivers/net/ethernet/intel/ice/devlink/port.c b/drivers/net/ethernet/intel/ice/devlink/port.c index 2a2e56777f9f7..3ede246490027 100644 --- a/drivers/net/ethernet/intel/ice/devlink/port.c +++ b/drivers/net/ethernet/intel/ice/devlink/port.c @@ -590,8 +590,8 @@ static void ice_dealloc_dynamic_port(struct ice_dynamic_port *dyn_port) xa_erase(&pf->sf_nums, devlink_port->attrs.pci_sf.sf); ice_eswitch_detach_sf(pf, dyn_port); - ice_vsi_free(dyn_port->vsi); xa_erase(&pf->dyn_ports, dyn_port->vsi->idx); + ice_vsi_free(dyn_port->vsi); kfree(dyn_port); } diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.c b/drivers/net/ethernet/intel/ice/ice_eswitch.c index b069e6c514fb1..6e7bba4738989 100644 --- a/drivers/net/ethernet/intel/ice/ice_eswitch.c +++ b/drivers/net/ethernet/intel/ice/ice_eswitch.c @@ -95,7 +95,7 @@ ice_eswitch_release_repr(struct ice_pf *pf, struct ice_repr *repr) return; ice_vsi_update_security(vsi, ice_vsi_ctx_set_antispoof); - metadata_dst_free(repr->dst); + dst_release(&repr->dst->dst); repr->dst = NULL; ice_fltr_add_mac_and_broadcast(vsi, repr->parent_mac, ICE_FWD_TO_VSI); diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index c04c5856dad6e..4e26ace793ec6 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -1561,10 +1561,10 @@ ice_tx_map(struct ice_tx_ring *tx_ring, struct ice_tx_buf *first, } } tstamp_ring->next_to_use = j; - writel_relaxed(j, tstamp_ring->tail); + writel(j, tstamp_ring->tail); } else { ring_kick: - writel_relaxed(i, tx_ring->tail); + writel(i, tx_ring->tail); } return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c index 4f984f6a2cb95..55ec6387ab28d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c @@ -3,6 +3,7 @@ #include "rx.h" #include "en/xdp.h" +#include #include #include @@ -156,6 +157,7 @@ int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) xsk_buff_free(xsk_buffs[batch]); err: + bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe); rq->stats->buff_alloc_err++; return -ENOMEM; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 6fc6605d20549..c75b064b04166 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -410,8 +410,11 @@ static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq, static void mlx5e_xsk_free_rx_wqe(struct mlx5e_wqe_frag_info *wi) { - if (!(wi->flags & BIT(MLX5E_WQE_FRAG_SKIP_RELEASE))) - xsk_buff_free(*wi->xskp); + if (wi->flags & BIT(MLX5E_WQE_FRAG_SKIP_RELEASE)) + return; + + xsk_buff_free(*wi->xskp); + wi->flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE); } static void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix) @@ -905,10 +908,12 @@ int mlx5e_poll_ico_cq(struct mlx5e_cq *cq) } while (!last_wqe); } while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq))); - sq->cc = sqcc; - mlx5_cqwq_update_db_record(&cq->wq); + /* ensure cq space is freed before enabling more cqes */ + dma_wmb(); + + sq->cc = sqcc; return i; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c index 7950b82d8b8ae..3624c680a861a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c @@ -11,26 +11,6 @@ int mlx5_esw_adj_vport_modify(struct mlx5_core_dev *dev, u16 vport, lockdep_assert_held(&dev->priv.eswitch->state_lock); - if (MLX5_CAP_ESW(dev, esw_vport_state_max_tx_speed)) { - u8 op_mod = MLX5_VPORT_STATE_OP_MOD_ESW_VPORT; - struct mlx5_vport *esw_vport; - u32 speed = 0; - int err; - - err = mlx5_query_vport_max_tx_speed(dev, op_mod, vport, - true, &speed, NULL); - if (err) { - esw_vport = mlx5_eswitch_get_vport(dev->priv.eswitch, - vport); - speed = IS_ERR(esw_vport) ? 0 : - esw_vport->agg_max_tx_speed; - mlx5_core_dbg(dev, - "Failed to query vport %d max tx speed, err=%d, using cached %u\n", - vport, err, speed); - } - MLX5_SET(modify_vport_state_in, in, max_tx_speed, speed); - } - MLX5_SET(modify_vport_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VPORT_STATE); MLX5_SET(modify_vport_state_in, in, op_mod, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 3b34bec559e08..4e173b08cb37a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -1432,30 +1432,40 @@ static bool mlx5_lag_should_disable_lag(struct mlx5_lag *ldev, bool do_bond) } #ifdef CONFIG_MLX5_ESWITCH -static int -mlx5_lag_sum_devices_speed(struct mlx5_lag *ldev, u32 *sum_speed, - int (*get_speed)(struct mlx5_core_dev *, u32 *)) +static int mlx5_lag_get_devices_oper_speed(struct mlx5_lag *ldev, + u32 *sum_speed) { - struct mlx5_core_dev *pf_mdev; - struct lag_func *pf; int pf_idx; - u32 speed; - int ret; *sum_speed = 0; mlx5_ldev_for_each(pf_idx, 0, ldev) { + u8 opmod = MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT; + struct mlx5_core_dev *pf_mdev; + struct lag_func *pf; + u32 speed; + u8 state; + int ret; + pf = mlx5_lag_pf(ldev, pf_idx); if (!pf) continue; pf_mdev = pf->dev; if (!pf_mdev) continue; + ret = mlx5_query_vport_max_tx_speed(pf_mdev, opmod, 0, 0, + &speed, &state); + if (ret) { + mlx5_core_dbg(pf_mdev, "State query failed (err=%d)\n", + ret); + return ret; + } + if (state != VPORT_STATE_UP) + continue; - ret = get_speed(pf_mdev, &speed); + ret = mlx5_port_oper_linkspeed(pf_mdev, &speed); if (ret) { mlx5_core_dbg(pf_mdev, - "Failed to get device speed using %ps. Device %s speed is not available (err=%d)\n", - get_speed, dev_name(pf_mdev->device), + "Failed to get oper speed (err=%d)\n", ret); return ret; } @@ -1466,17 +1476,42 @@ mlx5_lag_sum_devices_speed(struct mlx5_lag *ldev, u32 *sum_speed, return 0; } -static int mlx5_lag_sum_devices_max_speed(struct mlx5_lag *ldev, u32 *max_speed) +static int mlx5_lag_get_devices_max_speed(struct mlx5_lag *ldev, u32 *max_speed) { - return mlx5_lag_sum_devices_speed(ldev, max_speed, - mlx5_port_max_linkspeed); -} + bool take_max; + int pf_idx; -static int mlx5_lag_sum_devices_oper_speed(struct mlx5_lag *ldev, - u32 *oper_speed) -{ - return mlx5_lag_sum_devices_speed(ldev, oper_speed, - mlx5_port_oper_linkspeed); + take_max = ldev->tracker.tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP; + if (ldev->mode == MLX5_LAG_MODE_MPESW) + take_max = false; + + *max_speed = 0; + mlx5_ldev_for_each(pf_idx, 0, ldev) { + struct mlx5_core_dev *pf_mdev; + struct lag_func *pf; + u32 speed; + int ret; + + pf = mlx5_lag_pf(ldev, pf_idx); + if (!pf) + continue; + pf_mdev = pf->dev; + if (!pf_mdev) + continue; + + ret = mlx5_port_max_linkspeed(pf_mdev, &speed); + if (ret) { + mlx5_core_dbg(pf_mdev, + "Failed to get max speed (err=%d)\n", + ret); + return ret; + } + + *max_speed = take_max ? + max(*max_speed, speed) : *max_speed + speed; + } + + return 0; } static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev, @@ -1525,7 +1560,7 @@ void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev) int pf_idx; if (ldev->mode == MLX5_LAG_MODE_MPESW) { - if (mlx5_lag_sum_devices_oper_speed(ldev, &speed)) + if (mlx5_lag_get_devices_oper_speed(ldev, &speed)) return; } else { speed = ldev->tracker.bond_speed_mbps; @@ -1533,8 +1568,8 @@ void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev) return; } - /* If speed is not set, use the sum of max speeds of all PFs */ - if (!speed && mlx5_lag_sum_devices_max_speed(ldev, &speed)) + /* If speed is not set, fall back to the max achievable speed */ + if (!speed && mlx5_lag_get_devices_max_speed(ldev, &speed)) return; speed = speed / MLX5_MAX_TX_SPEED_UNIT; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index 8aa94ec87a0e3..edac2d694e0f0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -93,30 +93,6 @@ int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, lockdep_assert_held(&mdev->priv.eswitch->state_lock); #endif - if (MLX5_CAP_ESW(mdev, esw_vport_state_max_tx_speed) && - opmod == MLX5_VPORT_STATE_OP_MOD_ESW_VPORT && - vport != MLX5_VPORT_UPLINK) { - u32 speed = 0; - int err; - - err = mlx5_query_vport_max_tx_speed(mdev, opmod, vport, - other_vport, &speed, NULL); - if (err) { -#ifdef CONFIG_MLX5_ESWITCH - struct mlx5_vport *esw_vport; - - esw_vport = mlx5_eswitch_get_vport(mdev->priv.eswitch, - vport); - speed = IS_ERR(esw_vport) ? 0 : - esw_vport->agg_max_tx_speed; -#endif - mlx5_core_dbg(mdev, - "Failed to query vport %d max tx speed, err=%d, using cached %u\n", - vport, err, speed); - } - MLX5_SET(modify_vport_state_in, in, max_tx_speed, speed); - } - MLX5_SET(modify_vport_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VPORT_STATE); MLX5_SET(modify_vport_state_in, in, op_mod, opmod); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h index b03ff9e044f9e..10af08ce72efa 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h @@ -989,6 +989,8 @@ void mlxsw_sp_acl_ruleset_prio_get(struct mlxsw_sp_acl_ruleset *ruleset, struct mlxsw_sp_acl_rule_info * mlxsw_sp_acl_rulei_create(struct mlxsw_sp_acl *acl, struct mlxsw_afa_block *afa_block); +void mlxsw_sp_acl_rulei_free(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_acl_rule_info *rulei); void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_acl_rule_info *rulei); int mlxsw_sp_acl_rulei_commit(struct mlxsw_sp_acl_rule_info *rulei); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c index cb232accb296a..f7be4f3a15184 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c @@ -340,8 +340,8 @@ mlxsw_sp_acl_rulei_create(struct mlxsw_sp_acl *acl, return ERR_PTR(err); } -void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp *mlxsw_sp, - struct mlxsw_sp_acl_rule_info *rulei) +void mlxsw_sp_acl_rulei_free(struct mlxsw_sp *mlxsw_sp, + const struct mlxsw_sp_acl_rule_info *rulei) { if (rulei->action_created) mlxsw_afa_block_destroy(rulei->act_block); @@ -351,6 +351,12 @@ void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp *mlxsw_sp, if (rulei->dst_port_range_reg_valid) mlxsw_sp_port_range_reg_put(mlxsw_sp, rulei->dst_port_range_reg_index); +} + +void mlxsw_sp_acl_rulei_destroy(struct mlxsw_sp *mlxsw_sp, + struct mlxsw_sp_acl_rule_info *rulei) +{ + mlxsw_sp_acl_rulei_free(mlxsw_sp, rulei); kfree(rulei); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c index 353fd9ca89a68..5e4632cb792d5 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_flower.c @@ -862,14 +862,19 @@ int mlxsw_sp_flower_tmplt_create(struct mlxsw_sp *mlxsw_sp, memset(&rulei, 0, sizeof(rulei)); err = mlxsw_sp_flower_parse(mlxsw_sp, block, &rulei, f); if (err) - return err; + goto out; + ruleset = mlxsw_sp_acl_ruleset_get(mlxsw_sp, block, f->common.chain_index, MLXSW_SP_ACL_PROFILE_FLOWER, &rulei.values.elusage); + err = PTR_ERR_OR_ZERO(ruleset); + +out: + mlxsw_sp_acl_rulei_free(mlxsw_sp, &rulei); /* keep the reference to the ruleset */ - return PTR_ERR_OR_ZERO(ruleset); + return err; } void mlxsw_sp_flower_tmplt_destroy(struct mlxsw_sp *mlxsw_sp, diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c index 5953e04ef14e6..2826e48af1033 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c @@ -622,18 +622,9 @@ int sparx5_ptp_init(struct sparx5 *sparx5) sparx5->ptp = 1; } - if (!sparx5->ptp) - return 0; - - for (i = 0; i < SPARX5_PHC_COUNT; ++i) { - err = sparx5_ptp_phc_init(sparx5, i, &sparx5_ptp_clock_info); - if (err) - return err; - } - - spin_lock_init(&sparx5->ptp_clock_lock); - spin_lock_init(&sparx5->ptp_ts_id_lock); - mutex_init(&sparx5->ptp_lock); + /* The base, non-PTP-capable lan969x variants need the + * first TOD counter running to forward frames. + */ /* Disable master counters */ spx5_wr(PTP_PTP_DOM_CFG_PTP_ENA_SET(0), sparx5, PTP_PTP_DOM_CFG); @@ -657,6 +648,19 @@ int sparx5_ptp_init(struct sparx5 *sparx5) /* Enable master counters */ spx5_wr(PTP_PTP_DOM_CFG_PTP_ENA_SET(0x7), sparx5, PTP_PTP_DOM_CFG); + if (!sparx5->ptp) + return 0; + + for (i = 0; i < SPARX5_PHC_COUNT; ++i) { + err = sparx5_ptp_phc_init(sparx5, i, &sparx5_ptp_clock_info); + if (err) + return err; + } + + spin_lock_init(&sparx5->ptp_clock_lock); + spin_lock_init(&sparx5->ptp_ts_id_lock); + mutex_init(&sparx5->ptp_lock); + for (i = 0; i < sparx5->data->consts->n_ports; i++) { port = sparx5->ports[i]; if (!port) diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_tc_matchall.c b/drivers/net/ethernet/microchip/sparx5/sparx5_tc_matchall.c index 702257979462d..146d7f28e8f3d 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_tc_matchall.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_tc_matchall.c @@ -160,6 +160,7 @@ static int sparx5_tc_matchall_destroy(struct net_device *ndev, } list_del(&entry->list); + kfree(entry); return err; } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 010fef79ec01e..0da3da7d0d1b8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -6222,6 +6222,13 @@ static netdev_features_t stmmac_fix_features(struct net_device *dev, if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) features &= ~NETIF_F_CSUM_MASK; + if (priv->plat->core_type == DWMAC_CORE_XGMAC) { + if (features & NETIF_F_HW_VLAN_CTAG_RX) + features |= NETIF_F_HW_VLAN_STAG_RX; + else + features &= ~NETIF_F_HW_VLAN_STAG_RX; + } + return features; } @@ -8016,14 +8023,18 @@ static int __stmmac_dvr_probe(struct device *device, ndev->watchdog_timeo = msecs_to_jiffies(watchdog); #ifdef STMMAC_VLAN_TAG_USED /* Both mac100 and gmac support receive VLAN tag detection */ - ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX; + ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; + if (priv->plat->core_type == DWMAC_CORE_XGMAC) + ndev->features |= NETIF_F_HW_VLAN_STAG_RX; + if (dwmac_is_xmac(priv->plat->core_type)) { ndev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; priv->hw->hw_vlan_en = true; } if (priv->dma_cap.vlhash) { ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; - ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; + if (priv->plat->core_type == DWMAC_CORE_XGMAC) + ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER; } if (priv->dma_cap.vlins) ndev->features |= NETIF_F_HW_VLAN_CTAG_TX; diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h index fcd3aaef27fc3..c63d99686d0ba 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet.h +++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h @@ -523,8 +523,9 @@ struct skbuf_dma_descriptor { * @stats_work: Work for reading the hardware statistics counters often enough * to catch overflows. * @dma_err_task: Work structure to process Axi DMA errors - * @stopping: Set when @dma_err_task shouldn't do anything because we are - * about to stop the device. + * @stopping: Set when we are about to stop the device: makes @dma_err_task + * a no-op (legacy DMA path) and fences RX descriptor + * resubmission in axienet_dma_rx_cb() (dmaengine path). * @tx_irq: Axidma TX IRQ number * @rx_irq: Axidma RX IRQ number * @eth_irq: Ethernet core IRQ number @@ -545,6 +546,7 @@ struct skbuf_dma_descriptor { * @tx_ring_tail: TX skb ring buffer tail index. * @rx_ring_head: RX skb ring buffer head index. * @rx_ring_tail: RX skb ring buffer tail index. + * @rx_submit_lock: Protects RX ring resubmission vs teardown in dmaengine path. */ struct axienet_local { struct net_device *ndev; @@ -626,6 +628,7 @@ struct axienet_local { int tx_ring_tail; int rx_ring_head; int rx_ring_tail; + spinlock_t rx_submit_lock; }; /** diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index fcf517069d161..1ae4fd65e5ec5 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -881,6 +881,7 @@ static void axienet_dma_tx_cb(void *data, const struct dmaengine_result *result) u64_stats_update_end(&lp->tx_stat_sync); dma_unmap_sg(lp->dev, skbuf_dma->sgl, skbuf_dma->sg_len, DMA_TO_DEVICE); dev_consume_skb_any(skbuf_dma->skb); + skbuf_dma->skb = NULL; netif_txq_completed_wake(txq, 1, len, CIRC_SPACE(lp->tx_ring_head, lp->tx_ring_tail, TX_BD_NUM_MAX), 2); @@ -1171,6 +1172,7 @@ static void axienet_dma_rx_cb(void *data, const struct dmaengine_result *result) &meta_max_len); dma_unmap_single(lp->dev, skbuf_dma->dma_address, lp->max_frm_size, DMA_FROM_DEVICE); + skbuf_dma->skb = NULL; if (IS_ERR(app_metadata)) { if (net_ratelimit()) @@ -1193,10 +1195,17 @@ static void axienet_dma_rx_cb(void *data, const struct dmaengine_result *result) u64_stats_update_end(&lp->rx_stat_sync); rx_submit: + spin_lock(&lp->rx_submit_lock); + if (lp->stopping) { + spin_unlock(&lp->rx_submit_lock); + return; + } + for (i = 0; i < CIRC_SPACE(lp->rx_ring_head, lp->rx_ring_tail, RX_BUF_NUM_DEFAULT); i++) axienet_rx_submit_desc(lp->ndev); dma_async_issue_pending(lp->rx_chan); + spin_unlock(&lp->rx_submit_lock); } /** @@ -1541,6 +1550,7 @@ static int axienet_init_dmaengine(struct net_device *ndev) lp->tx_ring_head = 0; lp->rx_ring_tail = 0; lp->rx_ring_head = 0; + lp->stopping = false; lp->tx_skb_ring = kzalloc_objs(*lp->tx_skb_ring, TX_BD_NUM_MAX); if (!lp->tx_skb_ring) { ret = -ENOMEM; @@ -1752,20 +1762,42 @@ static int axienet_stop(struct net_device *ndev) free_irq(lp->rx_irq, ndev); axienet_dma_bd_release(ndev); } else { + struct skbuf_dma_descriptor *skbuf_dma; + + spin_lock_bh(&lp->rx_submit_lock); + lp->stopping = true; + spin_unlock_bh(&lp->rx_submit_lock); + dmaengine_terminate_sync(lp->tx_chan); - dmaengine_synchronize(lp->tx_chan); dmaengine_terminate_sync(lp->rx_chan); - dmaengine_synchronize(lp->rx_chan); - - for (i = 0; i < TX_BD_NUM_MAX; i++) - kfree(lp->tx_skb_ring[i]); - kfree(lp->tx_skb_ring); - for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) - kfree(lp->rx_skb_ring[i]); - kfree(lp->rx_skb_ring); dma_release_channel(lp->rx_chan); dma_release_channel(lp->tx_chan); + + /* Unmap and free any buffer the terminate did not reclaim, so it + * is not leaked; a non-NULL skb marks such a slot. + */ + for (i = 0; i < TX_BD_NUM_MAX; i++) { + skbuf_dma = lp->tx_skb_ring[i]; + if (skbuf_dma && skbuf_dma->skb) { + dma_unmap_sg(lp->dev, skbuf_dma->sgl, + skbuf_dma->sg_len, DMA_TO_DEVICE); + dev_kfree_skb_any(skbuf_dma->skb); + } + kfree(skbuf_dma); + } + kfree(lp->tx_skb_ring); + + for (i = 0; i < RX_BUF_NUM_DEFAULT; i++) { + skbuf_dma = lp->rx_skb_ring[i]; + if (skbuf_dma && skbuf_dma->skb) { + dma_unmap_single(lp->dev, skbuf_dma->dma_address, + lp->max_frm_size, DMA_FROM_DEVICE); + dev_kfree_skb_any(skbuf_dma->skb); + } + kfree(skbuf_dma); + } + kfree(lp->rx_skb_ring); } netdev_reset_queue(ndev); @@ -3062,6 +3094,7 @@ static int axienet_probe(struct platform_device *pdev) spin_lock_init(&lp->rx_cr_lock); spin_lock_init(&lp->tx_cr_lock); + spin_lock_init(&lp->rx_submit_lock); INIT_WORK(&lp->rx_dim.work, axienet_rx_dim_work); lp->rx_dim_enabled = true; lp->rx_dim.profile_ix = 1; diff --git a/drivers/net/pfcp.c b/drivers/net/pfcp.c index 8bbaf167db759..08557c0381809 100644 --- a/drivers/net/pfcp.c +++ b/drivers/net/pfcp.c @@ -64,6 +64,13 @@ static int pfcp_encap_recv(struct sock *sk, struct sk_buff *skb) goto drop; unparsed = pfcp_hdr(skb); + if (unparsed->flags & PFCP_SEID_FLAG) { + if (unlikely(!pskb_may_pull(skb, PFCP_HLEN + + offsetofend(struct pfcphdr_session, + seid)))) + goto drop; + unparsed = pfcp_hdr(skb); + } ip_tunnel_flags_zero(flags); tun_dst = udp_tun_rx_dst(skb, sk->sk_family, flags, 0, diff --git a/drivers/net/team/team_core.c b/drivers/net/team/team_core.c index feaa75fbf8fc2..3bf7c80f1d8e5 100644 --- a/drivers/net/team/team_core.c +++ b/drivers/net/team/team_core.c @@ -820,7 +820,8 @@ static struct list_head *__team_get_qom_list(struct team *team, u16 queue_id) /* * note: already called with rcu_read_lock */ -static bool team_queue_override_transmit(struct team *team, struct sk_buff *skb) +static bool team_queue_override_transmit(struct team *team, struct sk_buff *skb, + bool *tx_success) { struct list_head *qom_list; struct team_port *port; @@ -828,11 +829,11 @@ static bool team_queue_override_transmit(struct team *team, struct sk_buff *skb) if (!team->queue_override_enabled || !skb->queue_mapping) return false; qom_list = __team_get_qom_list(team, skb->queue_mapping); - list_for_each_entry_rcu(port, qom_list, qom_list) { - if (!team_dev_queue_xmit(team, port, skb)) - return true; - } - return false; + port = list_first_or_null_rcu(qom_list, struct team_port, qom_list); + if (!port) + return false; + *tx_success = !team_dev_queue_xmit(team, port, skb); + return true; } static void __team_queue_override_port_del(struct team *team, @@ -1864,8 +1865,7 @@ static netdev_tx_t team_xmit(struct sk_buff *skb, struct net_device *dev) bool tx_success; unsigned int len = skb->len; - tx_success = team_queue_override_transmit(team, skb); - if (!tx_success) + if (!team_queue_override_transmit(team, skb, &tx_success)) tx_success = READ_ONCE(team->ops.transmit)(team, skb); if (tx_success) { struct team_pcpu_stats *pcpu_stats; diff --git a/drivers/net/team/team_mode_broadcast.c b/drivers/net/team/team_mode_broadcast.c index 61d7d79f0c363..69dc25a5847a1 100644 --- a/drivers/net/team/team_mode_broadcast.c +++ b/drivers/net/team/team_mode_broadcast.c @@ -37,6 +37,8 @@ static bool bc_transmit(struct team *team, struct sk_buff *skb) ret = !team_dev_queue_xmit(team, last, skb); if (!sum_ret) sum_ret = ret; + } else { + dev_kfree_skb_any(skb); } return sum_ret; } diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 5bbe3123979e0..dcd39815205a9 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1818,12 +1818,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, } } - if (tun_vnet_hdr_tnl_to_skb(tun->flags, features, skb, &hdr)) { - atomic_long_inc(&tun->rx_frame_errors); - err = -EINVAL; - goto free_skb; - } - + skb->dev = tun->dev; switch (tun->flags & TUN_TYPE_MASK) { case IFF_TUN: if (tun->flags & IFF_NO_PI) { @@ -1848,9 +1843,8 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, } } - skb_reset_mac_header(skb); + skb_reset_network_header(skb); skb->protocol = pi.proto; - skb->dev = tun->dev; break; case IFF_TAP: if (!pskb_may_pull(skb, ETH_HLEN)) { @@ -1858,10 +1852,19 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, drop_reason = SKB_DROP_REASON_HDR_TRUNC; goto drop; } - skb->protocol = eth_type_trans(skb, tun->dev); + skb_set_network_header(skb, ETH_HLEN); break; } + if (tun_vnet_hdr_tnl_to_skb(tun->flags, features, skb, &hdr)) { + atomic_long_inc(&tun->rx_frame_errors); + err = -EINVAL; + goto free_skb; + } + + if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) + skb->protocol = eth_type_trans(skb, tun->dev); + /* copy skb_ubuf_info for callback when skb has no error */ if (zerocopy) { skb_zcopy_init(skb, msg_control); @@ -2469,6 +2472,8 @@ static int tun_xdp_one(struct tun_struct *tun, features = tun_vnet_hdr_guest_features(READ_ONCE(tun->vnet_hdr_sz)); tnl_hdr = (struct virtio_net_hdr_v1_hash_tunnel *)gso; + skb->dev = tun->dev; + skb_set_network_header(skb, ETH_HLEN); if (tun_vnet_hdr_tnl_to_skb(tun->flags, features, skb, tnl_hdr)) { atomic_long_inc(&tun->rx_frame_errors); kfree_skb(skb); diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c index 81d8412ce8e2f..cba1dc95d3594 100644 --- a/drivers/net/usb/ax88179_178a.c +++ b/drivers/net/usb/ax88179_178a.c @@ -1457,11 +1457,11 @@ static int ax88179_rx_fixup(struct usbnet *dev, struct sk_buff *skb) return 1; } - ax_skb = netdev_alloc_skb_ip_align(dev->net, pkt_len); + ax_skb = netdev_alloc_skb_ip_align(dev->net, pkt_len - 2); if (!ax_skb) return 0; - skb_put(ax_skb, pkt_len); - memcpy(ax_skb->data, skb->data + 2, pkt_len); + skb_put(ax_skb, pkt_len - 2); + memcpy(ax_skb->data, skb->data + 2, pkt_len - 2); ax88179_rx_checksum(ax_skb, pkt_hdr); usbnet_skb_return(dev, ax_skb); diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 2780b2dd073d1..4f145c5f686ef 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -1414,6 +1414,7 @@ static int veth_open(struct net_device *dev) } veth_set_xdp_features(dev); + veth_set_xdp_features(peer); return 0; } diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index bf82ef9874abb..daab43ac92ce4 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2515,6 +2515,8 @@ static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue * goto frame_err; } + skb->dev = dev; + skb_set_network_header(skb, ETH_HLEN); if (virtio_net_hdr_tnl_to_skb(skb, &hdr->tnl_hdr, vi->rx_tnl, vi->rx_tnl_csum, virtio_is_little_endian(vi->vdev))) { diff --git a/drivers/net/wireguard/noise.c b/drivers/net/wireguard/noise.c index 9c0a09bf6c956..ec2513d6dfab3 100644 --- a/drivers/net/wireguard/noise.c +++ b/drivers/net/wireguard/noise.c @@ -784,10 +784,11 @@ wg_noise_handshake_consume_response(struct message_handshake_response *src, /* Success! Copy everything to peer */ down_write(&handshake->lock); - /* It's important to check that the state is still the same, while we - * have an exclusive lock. + /* Check that the state is the same and that this is still the + * initiation we started with, while we have an exclusive lock. */ - if (handshake->state != state) { + if (handshake->state != state || + crypto_memneq(handshake->ephemeral_private, ephemeral_private, NOISE_PUBLIC_KEY_LEN)) { up_write(&handshake->lock); goto fail; } diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h index 79b6d70de236b..663b19b783953 100644 --- a/drivers/net/wireguard/queueing.h +++ b/drivers/net/wireguard/queueing.h @@ -78,12 +78,14 @@ static inline void wg_reset_packet(struct sk_buff *skb, bool encapsulating) u8 l4_hash = skb->l4_hash; u8 sw_hash = skb->sw_hash; u32 hash = skb->hash; + u8 tstamp_type = skb->tstamp_type; skb_scrub_packet(skb, true); memset(&skb->headers, 0, sizeof(skb->headers)); if (encapsulating) { skb->l4_hash = l4_hash; skb->sw_hash = sw_hash; skb->hash = hash; + skb->tstamp_type = tstamp_type; } skb->queue_mapping = 0; skb->nohdr = 0; diff --git a/drivers/net/wireless/Kconfig b/drivers/net/wireless/Kconfig index c6599594dc995..5b84a2ffd0d3a 100644 --- a/drivers/net/wireless/Kconfig +++ b/drivers/net/wireless/Kconfig @@ -19,6 +19,7 @@ menuconfig WLAN if WLAN source "drivers/net/wireless/admtek/Kconfig" +source "drivers/net/wireless/apple/Kconfig" source "drivers/net/wireless/ath/Kconfig" source "drivers/net/wireless/atmel/Kconfig" source "drivers/net/wireless/broadcom/Kconfig" diff --git a/drivers/net/wireless/Makefile b/drivers/net/wireless/Makefile index e1c4141c60044..48d2998ca1203 100644 --- a/drivers/net/wireless/Makefile +++ b/drivers/net/wireless/Makefile @@ -4,6 +4,7 @@ # obj-$(CONFIG_WLAN_VENDOR_ADMTEK) += admtek/ +obj-$(CONFIG_WLAN_VENDOR_APPLE) += apple/ obj-$(CONFIG_WLAN_VENDOR_ATH) += ath/ obj-$(CONFIG_WLAN_VENDOR_ATMEL) += atmel/ obj-$(CONFIG_WLAN_VENDOR_BROADCOM) += broadcom/ diff --git a/drivers/net/wireless/apple/Kconfig b/drivers/net/wireless/apple/Kconfig new file mode 100644 index 0000000000000..d0e8db197ca30 --- /dev/null +++ b/drivers/net/wireless/apple/Kconfig @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0-only +config WLAN_VENDOR_APPLE + bool "Apple wireless devices" + default y + +if WLAN_VENDOR_APPLE +config APPLE_CENTAURI + tristate "Apple N1 Centauri firmware and IPC transport" + depends on ARCH_APPLE || COMPILE_TEST + depends on PCI && CFG80211 && PCIE_APPLE + select FW_LOADER + help + Boot the shared N1 firmware and establish its PCI DMA contexts. + Requires the T6050 PCIe host's root-port reset on a bootloader-trained + port, to retrain the link after the ROM boot, and a DT-supplied + personalized firmware name and platform identifier. + The endpoint must start in the stopped ROM personality. Firmware + and per-device calibration are external inputs and are not included. + +config APPLE_CENTAURI_WIFI + bool "N1 Centauri cfg80211 station interface" + depends on APPLE_CENTAURI + default y + help + Add Alpha command/data rings, scanning and WPA2-PSK/CCMP station + connections for the explicitly matched firmware protocol profile. + Radio firmware and calibration remain external inputs. +endif diff --git a/drivers/net/wireless/apple/Makefile b/drivers/net/wireless/apple/Makefile new file mode 100644 index 0000000000000..ca72cd0b4da06 --- /dev/null +++ b/drivers/net/wireless/apple/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_CENTAURI) += centauri/ diff --git a/drivers/net/wireless/apple/centauri/Makefile b/drivers/net/wireless/apple/centauri/Makefile new file mode 100644 index 0000000000000..ddd2501998887 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_APPLE_CENTAURI) += apple_centauri.o +apple_centauri-y := centauri_pci.o centauri_ipc.o +apple_centauri-$(CONFIG_APPLE_CENTAURI_WIFI) += centauri_wifi.o diff --git a/drivers/net/wireless/apple/centauri/centauri.h b/drivers/net/wireless/apple/centauri/centauri.h new file mode 100644 index 0000000000000..fb9b458887274 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri.h @@ -0,0 +1,88 @@ + +#ifndef AURORA_CENTAURI_H +#define AURORA_CENTAURI_H +#include +#include +#include +#include +#include +#include + +#define CEN_ALPHA_WINDOW_SIZE SZ_32M +#define CEN_ALPHA_SCRATCH_OFFSET SZ_8M +#define CEN_ALPHA_SCRATCH_SIZE (5 * SZ_1M) + +struct centauri_ipc; +struct centauri_wifi; +struct centauri_irqs { + struct pci_dev *pdev; + void *data; + unsigned int count; +}; + +struct cen_alpha_resources { + struct pci_dev *pdev; + void *context, *messages, *window; + dma_addr_t context_dma, messages_dma, window_dma; + void __iomem *bar; + u32 index[4], mtr, mcr; +}; +enum centauri_boot_phase { + CEN_BOOT_ROM, + CEN_BOOT_CYCLE, + CEN_BOOT_IPC, + CEN_BOOT_READY, + CEN_BOOT_FAILED, + CEN_BOOT_STOPPED, +}; + +struct centauri { + struct pci_dev *pdev; + void __iomem *bar; + resource_size_t size; + u32 platform_id, protocol; + const char *firmware_name; + struct dentry *debug; + struct mutex boot_lock; + struct delayed_work boot_work; + enum centauri_boot_phase boot_phase; + bool removing, ready; + void *firmware, *descriptor; + struct centauri_irqs irqs, alpha_irqs; + dma_addr_t firmware_dma, descriptor_dma; + size_t firmware_size; + u32 image_response; + bool boot_attempted; + unsigned int fw_wait; + bool link_ready; + struct centauri_ipc *ipc; + struct centauri_wifi *wifi; +}; + +int centauri_request_irqs(struct centauri_irqs *irqs, struct pci_dev *pdev, + unsigned int count, irq_handler_t handler, void *data); +void centauri_free_irqs(struct centauri_irqs *irqs); + +int centauri_start_ipc(struct centauri *c); +void centauri_free_ipc(struct centauri *c); +int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources); +bool centauri_stop_dma(struct pci_dev *pdev); + +#if IS_ENABLED(CONFIG_APPLE_CENTAURI_WIFI) +irqreturn_t centauri_alpha_irq(int irq, void *data); +int centauri_wifi_register(struct centauri *c); +void centauri_wifi_unregister(struct centauri *c); +#else +static inline irqreturn_t centauri_alpha_irq(int irq, void *data) +{ + return IRQ_HANDLED; +} +static inline int centauri_wifi_register(struct centauri *c) +{ + return 0; +} +static inline void centauri_wifi_unregister(struct centauri *c) +{ +} +#endif +#endif diff --git a/drivers/net/wireless/apple/centauri/centauri_ipc.c b/drivers/net/wireless/apple/centauri/centauri_ipc.c new file mode 100644 index 0000000000000..1b8b8abb00be2 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_ipc.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri.h" + +struct cen_buffer { + struct list_head list; + struct device *owner; + void *cpu; + dma_addr_t dma; + size_t size; +}; + +struct cen_context { + struct cen_buffer *buffer, *messages; + u32 index[4], mtr, mcr; + u16 tr_count, cr_count, message_head; +}; + +struct centauri_ipc { + struct list_head buffers; + struct pci_dev *function[3]; + bool enabled[3], owned[3]; + bool alpha_pm_domain; + void __iomem *bar[3]; + struct cen_context control, alpha; + struct cen_buffer *secondary, *window, *cchi_rx, *cchi_tx, *debug_rx; + u16 cchi_head, cchi_tail; +}; + +static struct cen_buffer *buffer_alloc(struct centauri *c, unsigned int function, + size_t size) +{ + struct cen_buffer *buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + + if (!buffer) + return NULL; + buffer->owner = get_device(&c->ipc->function[function]->dev); + buffer->size = PAGE_ALIGN(size); + buffer->cpu = dma_alloc_coherent(buffer->owner, buffer->size, &buffer->dma, GFP_KERNEL); + if (!buffer->cpu) { + put_device(buffer->owner); + kfree(buffer); + return NULL; + } + memset(buffer->cpu, 0, buffer->size); + list_add_tail(&buffer->list, &c->ipc->buffers); + return buffer; +} + +static int ftab_entry(const void *image, size_t size, const char tag[4], + u32 *offset, u32 *length, bool sparse) +{ + u32 count, i, off, len; + const u8 *table = image; + + if (size < 0x30 || memcmp(table + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(table + 0x28); + if (!count || count > 256 || 0x30 + count * 16 > size) + return -EINVAL; + for (i = 0; i < count; i++) { + const u8 *entry = table + 0x30 + 16 * i; + + if (memcmp(entry, tag, 4)) + continue; + off = get_unaligned_le32(entry + 4); + len = get_unaligned_le32(entry + 8); + if (off < 0x30 + count * 16 || !len || (u64)off + len > (sparse ? SZ_64M : size)) + return -EINVAL; + *offset = off; + *length = len; + return 0; + } + return -ENOENT; +} + +static int secondary_init(struct centauri *c) +{ + u32 offset, length, count, i, required; + const u8 *seed; + int ret; + + ret = ftab_entry(c->firmware, c->firmware_size, "2ftb", &offset, &length, false); + if (ret) + return ret; + seed = c->firmware + offset; + if (length < 0x30 || memcmp(seed + 0x24, "ftab", 4)) + return -EINVAL; + count = get_unaligned_le32(seed + 0x28); + if (!count || count > 256 || 0x30 + 16 * count != length) + return -EINVAL; + required = length; + for (i = 0; i < count; i++) { + u64 end = (u64)get_unaligned_le32(seed + 0x34 + i * 16) + + get_unaligned_le32(seed + 0x38 + i * 16); + + if (end > SZ_64M) + return -EINVAL; + required = max_t(u32, required, end); + } + c->ipc->secondary = buffer_alloc(c, 0, required); + if (!c->ipc->secondary) + return -ENOMEM; + memcpy(c->ipc->secondary->cpu, seed, length); + dma_wmb(); + ret = pci_write_config_dword(c->pdev, 0xf88, lower_32_bits(c->ipc->secondary->dma)); + if (!ret) + ret = pci_write_config_dword(c->pdev, 0xf8c, upper_32_bits(c->ipc->secondary->dma)); + if (!ret) + ret = pci_write_config_dword(c->pdev, 0xf90, required); + if (ret) + return pcibios_err_to_errno(ret); + + return 0; +} + +static int context_init(struct centauri *c, struct cen_context *ctx, + unsigned int function, u16 tr, u16 cr, dma_addr_t scratch, size_t scratch_size) +{ + u8 *header; + dma_addr_t dma; + int i; + + ctx->buffer = buffer_alloc(c, function, SZ_32K); + ctx->messages = buffer_alloc(c, function, SZ_16K); + if (!ctx->buffer || !ctx->messages) + return -ENOMEM; + ctx->tr_count = tr; + ctx->cr_count = cr; + ctx->index[0] = 0x78; + ctx->index[1] = ctx->index[0] + cr * 2; + ctx->index[2] = ctx->index[1] + tr * 2; + ctx->index[3] = ctx->index[2] + cr * 2; + ctx->mtr = ALIGN(ctx->index[3] + tr * 2, tr == 192 ? 16 : 2); + ctx->mcr = ALIGN(ctx->mtr + 256, 4); + header = ctx->buffer->cpu; + dma = ctx->buffer->dma; + put_unaligned_le16(768, header); + put_unaligned_le16(0x68, header + 2); + put_unaligned_le64(dma + 0x68, header + 8); + for (i = 0; i < 4; i++) + put_unaligned_le64(dma + ctx->index[i], header + 0x10 + 8 * i); + put_unaligned_le16(cr, header + 0x30); + put_unaligned_le16(tr, header + 0x32); + put_unaligned_le64(dma + ctx->mcr, header + 0x34); + put_unaligned_le64(dma + ctx->mtr, header + 0x3c); + put_unaligned_le16(16, header + 0x44); + put_unaligned_le16(16, header + 0x46); + put_unaligned_le64(scratch ?: dma + ALIGN(ctx->mcr + 256, 8), header + 0x58); + put_unaligned_le64(scratch_size, header + 0x60); + dma_wmb(); + return 0; +} + +static int handshake(void __iomem *bar, struct cen_context *ctx, + u32 status, u32 address, u32 control) +{ + u32 value; + int ret; + + writel(1, bar + control); + ret = readl_poll_timeout(bar + status, value, value == 1, 100, 2000000); + if (ret) { + pr_err("centauri: IPC init timeout status@%#x=%#x control@%#x=%#x context=%pad\n", + status, value, control, readl(bar + control), &ctx->buffer->dma); + return ret; + } + writel(lower_32_bits(ctx->buffer->dma), bar + address); + writel(upper_32_bits(ctx->buffer->dma), bar + address + 4); + dma_wmb(); + writel(2, bar + control); + ret = readl_poll_timeout(bar + status, value, value == 2, 100, 2000000); + if (ret) + pr_err("centauri: IPC run timeout status@%#x=%#x control@%#x=%#x context=%pad\n", + status, value, control, readl(bar + control), &ctx->buffer->dma); + return ret; +} + +static int message(struct cen_context *ctx, void __iomem *doorbell, + const void *payload, size_t length) +{ + u16 head = ctx->message_head; + u8 *descriptor; + u16 tail; + int ret; + + if (head >= 15 || !length || length > 512) + return -ENOSPC; + memcpy(ctx->messages->cpu + 512 * (head + 1), payload, length); + descriptor = ctx->buffer->cpu + ctx->mtr + head * 16; + put_unaligned_le32(1 | (length << 8), descriptor); + put_unaligned_le64(ctx->messages->dma + 512 * (head + 1), descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + dma_wmb(); + writel(head + 1, doorbell); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) == head + 1, + 100, 1200000, false, *(__le16 *)(ctx->buffer->cpu + ctx->index[1])); + dma_rmb(); + ctx->message_head++; + return ret; +} + +static int open_transfer(struct centauri *c, u16 id, struct cen_buffer *buffer, + u16 footer) +{ + u8 msg[0x34] = { 1 }; + u32 words = footer >> 2, shift = min_t(u32, __ffs(words), 4); + + msg[2] = words >> shift; + msg[3] = shift << 4; + put_unaligned_le16(id, msg + 4); + put_unaligned_le64(buffer->dma, msg + 8); + put_unaligned_le64(~0ULL, msg + 0x10); + put_unaligned_le16(16, msg + 0x18); + put_unaligned_le16(0xffff, msg + 0x1a); + put_unaligned_le16(id, msg + 0x1c); + put_unaligned_le16(0x40, msg + 0x1e); + return message(&c->ipc->control, c->bar + 0x9000, msg, sizeof(msg)); +} + +static int cchi_init(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + u8 config[4] = {9, 0, 0xff, 0xff}, *entry; + u16 tail; + int i, ret; + + ipc->cchi_rx = buffer_alloc(c, 0, SZ_32K); + ipc->cchi_tx = buffer_alloc(c, 0, SZ_32K); + ipc->debug_rx = buffer_alloc(c, 0, SZ_16K); + if (!ipc->cchi_rx || !ipc->cchi_tx || !ipc->debug_rx) + return -ENOMEM; + ret = message(&ipc->control, c->bar + 0x9000, config, sizeof(config)); + if (ret) + return ret; + ret = open_transfer(c, 4, ipc->cchi_rx, 1024); + if (!ret) + ret = open_transfer(c, 3, ipc->cchi_tx, 1024); + if (!ret) + ret = open_transfer(c, 5, ipc->debug_rx, 256); + if (ret) + return ret; + for (i = 0; i < 15; i++) { + entry = ipc->cchi_rx->cpu + i * 1040; + put_unaligned_le32(1 | (1024 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + entry = ipc->debug_rx->cpu + i * 272; + put_unaligned_le32(1 | (256 << 8), entry); + put_unaligned_le32(i + 1, entry + 12); + } + dma_wmb(); + writel(15, c->bar + 0x9010); + writel(15, c->bar + 0x9014); + entry = ipc->cchi_tx->cpu; + put_unaligned_le32(1 | (64 << 8), entry); + put_unaligned_le32(1, entry + 12); + put_unaligned_le16(64, entry + 16); + dma_wmb(); + writel(1, c->bar + 0x900c); + ret = read_poll_timeout(READ_ONCE, tail, le16_to_cpu(tail) != 0, + 100, 2000000, false, + *(__le16 *)(ipc->control.buffer->cpu + ipc->control.index[1] + 4 * 2)); + dma_rmb(); + if (ret) + return ret; + entry = ipc->cchi_rx->cpu; + if ((get_unaligned_le32(entry) >> 8) < 8 || entry[18] || entry[19]) + return -EPROTO; + ipc->cchi_head = ipc->cchi_tail = 1; + return 0; +} + +static int memswap_handoff(struct centauri *c, unsigned int function, const char tag[4]) +{ + struct cen_buffer *secondary = c->ipc->secondary, *working; + u32 offset, size; + int ret; + + ret = ftab_entry(secondary->cpu, secondary->size, tag, &offset, &size, false); + if (ret) + return ret; + working = buffer_alloc(c, function, size); + if (!working) + return -ENOMEM; + dma_rmb(); + memcpy(working->cpu, secondary->cpu + offset, size); + dma_wmb(); + ret = pci_write_config_dword(c->ipc->function[function], 0xf88, lower_32_bits(working->dma)); + if (!ret) + ret = pci_write_config_dword(c->ipc->function[function], 0xf8c, upper_32_bits(working->dma)); + if (!ret) + ret = pci_write_config_dword(c->ipc->function[function], 0xf90, size); + if (ret) + return pcibios_err_to_errno(ret); + return 0; +} + +static struct dev_pm_domain centauri_alpha_pm_domain; + +static int enumerate_functions(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + int i, ret; + + c->pdev->multifunction = 1; + pci_rescan_bus(c->pdev->bus); + for (i = 0; i < 3; i++) { + ipc->function[i] = pci_get_domain_bus_and_slot(pci_domain_nr(c->pdev->bus), + c->pdev->bus->number, PCI_DEVFN(PCI_SLOT(c->pdev->devfn), i)); + if (!ipc->function[i] || ipc->function[i]->vendor != PCI_VENDOR_ID_APPLE || + (i && ipc->function[i]->device != 0x1901 + i)) + return -ENODEV; + if (ipc->function[i]->cfg_size < 0xf94 || + !(pci_resource_flags(ipc->function[i], 0) & IORESOURCE_MEM) || + pci_resource_len(ipc->function[i], 0) < (i ? 0x2000 : 0x9080)) + return -ENODEV; + if (i && ipc->function[i]->dev.driver) + return -EBUSY; + ipc->owned[i] = true; + ipc->bar[i] = pci_iomap(ipc->function[i], 0, 0); + if (!ipc->bar[i]) + return -ENOMEM; + if (i) { + ret = pci_enable_device_mem(ipc->function[i]); + if (ret) + return ret; + ipc->enabled[i] = true; + } + if (i == 1 && !ipc->function[i]->dev.driver) { + dev_pm_domain_set(&ipc->function[i]->dev, ¢auri_alpha_pm_domain); + ipc->alpha_pm_domain = true; + } + ret = dma_set_mask_and_coherent(&ipc->function[i]->dev, DMA_BIT_MASK(40)); + if (ret) + return ret; + if (i == 1) { + ret = centauri_request_irqs(&c->alpha_irqs, ipc->function[i], + 16, centauri_alpha_irq, c); + if (ret) + return ret; + } + pci_set_master(ipc->function[i]); + } + return 0; +} + +static int centauri_start_ipc_locked(struct centauri *c) +{ + struct centauri_ipc *ipc; + u32 stage; + int ret; + + if (!c->firmware || !c->link_ready || c->ipc || readl(c->bar + 0x8000) != 1) + return -EBUSY; + ipc = kzalloc(sizeof(*ipc), GFP_KERNEL); + if (!ipc) + return -ENOMEM; + INIT_LIST_HEAD(&ipc->buffers); + c->ipc = ipc; + ret = enumerate_functions(c); + if (ret) + goto failed; + ret = secondary_init(c); + if (ret) + goto failed; + ret = context_init(c, &ipc->control, 0, 13, 1, 0, 0); + if (ret) + goto failed; + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + writel(0x62746632, c->bar + 0x8074); + dma_wmb(); + writel(1, c->bar + 0x9070); + ret = readl_poll_timeout(c->bar + 0x8000, stage, stage == 2 || stage == 3, 100, 3000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + ret = handshake(c->bar, &ipc->control, 0x8050, 0x8054, 0x907c); + if (ret) + goto failed; + ret = memswap_handoff(c, 0, "mswc"); + if (!ret) + ret = memswap_handoff(c, 2, "mswb"); + if (!ret) + ret = memswap_handoff(c, 1, "msww"); + if (ret) + goto failed; + ret = cchi_init(c); + if (ret) + goto failed; + ret = readl_poll_timeout(ipc->bar[1] + 8, stage, stage == 2 || stage == 3, 100, 2000000); + if (ret || stage != 2) { + ret = ret ?: -EIO; + goto failed; + } + ipc->window = buffer_alloc(c, 1, CEN_ALPHA_WINDOW_SIZE); + if (!ipc->window) { + ret = -ENOMEM; + goto failed; + } + ret = context_init(c, &ipc->alpha, 1, 192, 234, + ipc->window->dma + CEN_ALPHA_SCRATCH_OFFSET, CEN_ALPHA_SCRATCH_SIZE); + if (ret) + goto failed; + writel(lower_32_bits(ipc->window->dma), ipc->bar[1] + 0x18); + writel(upper_32_bits(ipc->window->dma), ipc->bar[1] + 0x1c); + writel(CEN_ALPHA_WINDOW_SIZE, ipc->bar[1] + 0x20); + ret = handshake(ipc->bar[1], &ipc->alpha, 0xc, 0x10, 0x1028); + if (ret) + goto failed; + ret = centauri_wifi_register(c); + if (ret) + return ret; + + /* Publish usable shared firmware before the Beta consumer probes. */ + smp_store_release(&c->ready, true); + ret = device_attach(&ipc->function[2]->dev); + if (ret < 0) + dev_warn(&c->pdev->dev, "Beta driver attachment failed: %d\n", ret); + return 0; +failed: + dev_err(&c->pdev->dev, "IPC bootstrap failed: %d (Control stage=%#x)\n", + ret, readl(c->bar + 0x8000)); + if (ipc->bar[1]) + dev_err(&c->pdev->dev, "Alpha stage=%#x IPC=%#x window=%#x/%#x/%#x\n", + readl(ipc->bar[1] + 8), readl(ipc->bar[1] + 0xc), + readl(ipc->bar[1] + 0x18), readl(ipc->bar[1] + 0x1c), readl(ipc->bar[1] + 0x20)); + if (ipc->bar[2]) + dev_err(&c->pdev->dev, "Beta stage=%#x IPC=%#x\n", + readl(ipc->bar[2] + 0x18), readl(ipc->bar[2] + 0x1c)); + + return ret; +} + +int centauri_start_ipc(struct centauri *c) +{ + int ret; + + if (!pci_trylock_rescan_remove()) + return -EAGAIN; + ret = READ_ONCE(c->removing) ? -ECANCELED : centauri_start_ipc_locked(c); + pci_unlock_rescan_remove(); + return ret; +} + +void centauri_free_ipc(struct centauri *c) +{ + struct centauri_ipc *ipc = c->ipc; + struct cen_buffer *buffer, *next; + int i; + + if (!ipc) + return; + if (ipc->owned[2]) + device_release_driver(&ipc->function[2]->dev); + centauri_wifi_unregister(c); + centauri_free_irqs(&c->alpha_irqs); + for (i = 0; i < 3; i++) { + if (!ipc->owned[i]) + continue; + pci_clear_master(ipc->function[i]); + } + for (i = 0; i < 3; i++) { + if (ipc->owned[i] && !centauri_stop_dma(ipc->function[i])) { + dev_warn(&c->pdev->dev, + "retaining IPC DMA after pending transaction timeout\n"); + return; + } + } + list_for_each_entry_safe(buffer, next, &ipc->buffers, list) { + memzero_explicit(buffer->cpu, buffer->size); + dma_free_coherent(buffer->owner, buffer->size, buffer->cpu, buffer->dma); + put_device(buffer->owner); + list_del(&buffer->list); + kfree(buffer); + } + if (ipc->alpha_pm_domain) + dev_pm_domain_set(&ipc->function[1]->dev, NULL); + for (i = 0; i < 3; i++) { + if (ipc->bar[i]) + pci_iounmap(ipc->function[i], ipc->bar[i]); + if (ipc->enabled[i]) + pci_disable_device(ipc->function[i]); + pci_dev_put(ipc->function[i]); + } + kfree(ipc); + c->ipc = NULL; +} + +int centauri_alpha_resources(struct centauri *c, struct cen_alpha_resources *resources) +{ + struct centauri_ipc *ipc = c->ipc; + + if (!ipc || !ipc->window || !ipc->alpha.buffer || !ipc->alpha.messages) + return -ENODEV; + resources->pdev = ipc->function[1]; + resources->context = ipc->alpha.buffer->cpu; + resources->context_dma = ipc->alpha.buffer->dma; + resources->messages = ipc->alpha.messages->cpu; + resources->messages_dma = ipc->alpha.messages->dma; + resources->window = ipc->window->cpu; + resources->window_dma = ipc->window->dma; + resources->bar = ipc->bar[1]; + memcpy(resources->index, ipc->alpha.index, sizeof(resources->index)); + resources->mtr = ipc->alpha.mtr; + resources->mcr = ipc->alpha.mcr; + return 0; +} diff --git a/drivers/net/wireless/apple/centauri/centauri_pci.c b/drivers/net/wireless/apple/centauri/centauri_pci.c new file mode 100644 index 0000000000000..2d0b72ecbbd04 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_pci.c @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "centauri.h" + +static struct pci_driver centauri_driver; + +bool apple_centauri_control_ready(struct pci_dev *pdev) +{ + struct centauri *c; + + if (pdev->driver != ¢auri_driver) + return false; + c = pci_get_drvdata(pdev); + return c && !READ_ONCE(c->removing) && smp_load_acquire(&c->ready); +} +EXPORT_SYMBOL_GPL(apple_centauri_control_ready); + +/* No DMA backing is released without verified bus-master shutdown. */ +bool centauri_stop_dma(struct pci_dev *pdev) +{ + u16 command; + int ret; + + pci_clear_master(pdev); + ret = pci_read_config_word(pdev, PCI_COMMAND, &command); + if (ret || (command & PCI_COMMAND_MASTER)) + return false; + return pci_wait_for_pending_transaction(pdev); +} + +void centauri_free_irqs(struct centauri_irqs *irqs) +{ + if (!irqs->pdev) + return; + while (irqs->count) + free_irq(pci_irq_vector(irqs->pdev, --irqs->count), irqs->data); + pci_free_irq_vectors(irqs->pdev); + irqs->pdev = NULL; +} + +int centauri_request_irqs(struct centauri_irqs *irqs, struct pci_dev *pdev, + unsigned int count, irq_handler_t handler, void *data) +{ + int ret; + + ret = pci_alloc_irq_vectors(pdev, count, count, PCI_IRQ_MSI); + if (ret < 0) + return ret; + irqs->pdev = pdev; + irqs->data = data; + for (irqs->count = 0; irqs->count < count; irqs->count++) { + ret = request_irq(pci_irq_vector(pdev, irqs->count), handler, 0, + "centauri", data); + if (ret) { + centauri_free_irqs(irqs); + return ret; + } + } + return 0; +} + +static irqreturn_t centauri_control_irq(int irq, void *data) +{ + return IRQ_HANDLED; +} + +/* + * This driver is built in and probes from the initramfs, before the root file system that + * holds the personalized firmware is mounted. Poll for the file instead of failing the boot. + */ +#define CEN_FW_WAIT_TRIES 240 +#define CEN_FW_WAIT_MS 500 + +static void centauri_free_boot(struct centauri *c) +{ + struct device *dev = &c->pdev->dev; + + if (!c->link_ready && c->boot_attempted) { + while (c->alpha_irqs.count) + free_irq(pci_irq_vector(c->alpha_irqs.pdev, --c->alpha_irqs.count), c); + while (c->irqs.count) + free_irq(pci_irq_vector(c->pdev, --c->irqs.count), c); + dev_warn(dev, "retaining DMA buffers after failed link transition; reboot required\n"); + return; + } + + centauri_free_ipc(c); + centauri_free_irqs(&c->irqs); + if (c->ipc || !centauri_stop_dma(c->pdev)) { + dev_warn(dev, "retaining boot DMA until outstanding transactions stop\n"); + return; + } + if (c->firmware) + dma_free_coherent(dev, c->firmware_size, c->firmware, c->firmware_dma); + if (c->descriptor) + dma_free_coherent(dev, SZ_16K, c->descriptor, c->descriptor_dma); + c->firmware = NULL; + c->descriptor = NULL; +} + +static int centauri_rom_boot(struct centauri *c) +{ + const struct firmware *fw; + struct device *dev = &c->pdev->dev; + u32 stage; + int ret; + + if (c->pdev->device != 0x1900 && c->pdev->device != 0x1901) + return -ENODEV; + if (!c->link_ready || c->boot_attempted) + return -EBUSY; + stage = readl(c->bar + 0x8000); + if (stage != 0) + return -EBUSY; + + ret = request_firmware_direct(&fw, c->firmware_name, dev); + if (ret) { + if (ret != -ENOENT || c->fw_wait >= CEN_FW_WAIT_TRIES) + dev_err(dev, "required firmware %s unavailable: %d\n", + c->firmware_name, ret); + return ret; + } + if (READ_ONCE(c->removing)) { + ret = -ECANCELED; + goto release; + } + if (fw->size < 0x30 || fw->size > SZ_64M || + memcmp(fw->data + 0x20, "rkosftab", 8)) { + ret = -EINVAL; + goto release; + } + c->firmware_size = PAGE_ALIGN(fw->size); + c->firmware = dma_alloc_coherent(dev, c->firmware_size, &c->firmware_dma, GFP_KERNEL); + c->descriptor = dma_alloc_coherent(dev, SZ_16K, &c->descriptor_dma, GFP_KERNEL); + if (!c->firmware || !c->descriptor) { + ret = -ENOMEM; + goto free_buffers; + } + memset(c->firmware, 0, c->firmware_size); + memcpy(c->firmware, fw->data, fw->size); + memset(c->descriptor, 0, SZ_16K); + + put_unaligned_le64(c->firmware_dma, c->descriptor); + put_unaligned_le32(ALIGN(fw->size, SZ_4K), c->descriptor + 8); + ret = centauri_request_irqs(&c->irqs, c->pdev, 4, + centauri_control_irq, c); + if (ret) + goto free_buffers; + pci_set_master(c->pdev); + writel(c->platform_id, c->bar + 0x807c); + writel(0, c->bar + 0x803c); + writel(lower_32_bits(c->descriptor_dma), c->bar + 0x8044); + writel(upper_32_bits(c->descriptor_dma), c->bar + 0x8048); + writel(0x20, c->bar + 0x804c); + dma_wmb(); + c->boot_attempted = true; + writel(1, c->bar + 0x9000); + ret = readl_poll_timeout(c->bar + 0x8040, c->image_response, + READ_ONCE(c->removing) || c->image_response != 0, 20000, 20000000); + if (READ_ONCE(c->removing)) { + ret = -ECANCELED; + goto release; + } + if (!ret && c->image_response != 1) + ret = -EIO; + if (!ret) + ret = readl_poll_timeout(c->bar + 0x8000, stage, + READ_ONCE(c->removing) || stage != 0, 20000, 20000000); + else + stage = readl(c->bar + 0x8000); + if (READ_ONCE(c->removing)) + ret = -ECANCELED; + dma_rmb(); + + if (!ret && stage != 4) + ret = -EIO; + if (ret) + dev_err(dev, "ROM boot: image response %#x, stage %#x, platform id %#x, image %zu bytes: %d\n", + c->image_response, stage, c->platform_id, fw->size, ret); + else + dev_info(dev, "ROM boot accepted: stage %#x, platform id %#x\n", stage, c->platform_id); + goto release; +free_buffers: + centauri_free_boot(c); +release: + release_firmware(fw); + return ret; +} + +/* + * After the ROM boot the N1 runs its firmware from host memory and must come back on a + * retrained link: the isolated function 0 is quiesced, put in D3hot with PME enabled + * (the host's PME_Turn_Off then takes the link to L2), the host resets and retrains the + * root port (PERST# asserted and released around it; the firmware keeps running), and the + * root port's and the endpoint's state are restored. + */ +static int centauri_port_cycle(struct pci_dev *endpoint) +{ + struct pci_host_bridge *host = pci_find_host_bridge(endpoint->bus); + struct pci_dev *bridge = endpoint->bus->self, *sibling; + u16 command, pmctrl; + u32 identity; + int ret, pm; + + if (!host->reset_root_port || !bridge || !pci_is_root_bus(bridge->bus)) + return -EOPNOTSUPP; + pm = endpoint->pm_cap; + if (!pm) + return -ENODEV; + if (!pci_trylock_rescan_remove()) + return -EAGAIN; + list_for_each_entry(sibling, &endpoint->bus->devices, bus_list) { + if (sibling != endpoint) { + ret = -EBUSY; + goto out; + } + } + ret = pci_save_state(bridge) ?: pci_save_state(endpoint); + if (ret) + goto out; + ret = pcibios_err_to_errno(pci_read_config_word(endpoint, PCI_COMMAND, &command)); + if (ret) + goto out; + if (!centauri_stop_dma(endpoint)) { + ret = -EBUSY; + goto stopped; + } + ret = pcibios_err_to_errno(pci_read_config_word(endpoint, pm + PCI_PM_CTRL, &pmctrl)); + if (ret) + goto stopped; + ret = pcibios_err_to_errno(pci_write_config_word(endpoint, pm + PCI_PM_CTRL, + (pmctrl & ~PCI_PM_CTRL_STATE_MASK) | PCI_D3hot | PCI_PM_CTRL_PME_ENABLE)); + if (ret) + goto stopped; + usleep_range(10000, 11000); + + ret = host->reset_root_port(host, bridge); + if (ret) + goto stopped; + pci_restore_state(bridge); + ret = pcibios_err_to_errno(pci_read_config_dword(endpoint, PCI_VENDOR_ID, &identity)); + if (!ret && identity != 0x1900106b && identity != 0x1901106b) + ret = -ENODEV; + if (ret) + goto stopped; + pci_restore_state(endpoint); + ret = pcibios_err_to_errno(pci_write_config_word(endpoint, pm + PCI_PM_CTRL, + pmctrl & ~PCI_PM_CTRL_STATE_MASK)); + if (ret) + goto stopped; + usleep_range(10000, 11000); + ret = pcibios_err_to_errno(pci_write_config_word(endpoint, PCI_COMMAND, command)); + if (!ret) + goto out; +stopped: + /* A partial transition never restores endpoint DMA or bridge decoding. */ + pci_clear_master(endpoint); + pci_write_config_word(bridge, PCI_COMMAND, 0); + pci_err(endpoint, "N1 port cycle stopped: %d\n", ret); +out: + pci_unlock_rescan_remove(); + return ret; +} + +static int centauri_cycle(struct centauri *c) +{ + int ret; + + if (!c->firmware || !c->link_ready || readl(c->bar + 0x8000) != 4) + return -EBUSY; + c->link_ready = false; + ret = centauri_port_cycle(c->pdev); + if (ret == -EAGAIN) { + c->link_ready = true; + } else if (!ret) { + pci_restore_msi_state(c->pdev); + c->link_ready = true; + } + return ret; +} + +static void centauri_boot_worker(struct work_struct *work) +{ + struct centauri *c = container_of(to_delayed_work(work), struct centauri, boot_work); + unsigned int delay_ms = 20; + bool retry = false; + int ret; + + mutex_lock(&c->boot_lock); + while (!READ_ONCE(c->removing)) { + switch (c->boot_phase) { + case CEN_BOOT_ROM: + ret = centauri_rom_boot(c); + if (!ret) + c->boot_phase = readl(c->bar + 0x8000) == 1 ? + CEN_BOOT_IPC : CEN_BOOT_CYCLE; + break; + case CEN_BOOT_CYCLE: + ret = centauri_cycle(c); + if (!ret) + c->boot_phase = CEN_BOOT_IPC; + break; + case CEN_BOOT_IPC: + ret = centauri_start_ipc(c); + if (!ret) + c->boot_phase = CEN_BOOT_READY; + break; + default: + goto out; + } + if (ret == -EAGAIN && (c->boot_phase == CEN_BOOT_CYCLE || + (c->boot_phase == CEN_BOOT_IPC && !c->ipc))) { + retry = true; + break; + } + if (ret == -ENOENT && c->boot_phase == CEN_BOOT_ROM && !c->boot_attempted && + c->fw_wait++ < CEN_FW_WAIT_TRIES) { + delay_ms = CEN_FW_WAIT_MS; + retry = true; + break; + } + if (ret) { + dev_err(&c->pdev->dev, "N1 automatic startup failed in phase%u: %d\n", + c->boot_phase, ret); + c->boot_phase = CEN_BOOT_FAILED; + break; + } + } +out: + if (READ_ONCE(c->removing)) + c->boot_phase = CEN_BOOT_STOPPED; + mutex_unlock(&c->boot_lock); + if (retry && !READ_ONCE(c->removing)) + queue_delayed_work(system_long_wq, &c->boot_work, msecs_to_jiffies(delay_ms)); +} + +static int centauri_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct centauri *c; + int ret; + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + + if (!bridge || !bridge->dev.parent || + !of_device_is_compatible(bridge->dev.parent->of_node, "apple,t6050-pcie")) + return -ENODEV; + if (PCI_FUNC(pdev->devfn) != 0) + return -ENODEV; + c = devm_kzalloc(&pdev->dev, sizeof(*c), GFP_KERNEL); + if (!c) + return -ENOMEM; + c->pdev = pdev; + c->link_ready = true; + mutex_init(&c->boot_lock); + INIT_DELAYED_WORK(&c->boot_work, centauri_boot_worker); + c->boot_phase = CEN_BOOT_ROM; + ret = device_property_read_string(&pdev->dev, "firmware-name", &c->firmware_name); + if (ret) + return dev_err_probe(&pdev->dev, ret, "missing personalized firmware name\n"); + ret = device_property_read_u32(&pdev->dev, "apple,centauri-platform-id", &c->platform_id); + if (ret) + return ret; + ret = device_property_read_u32(&pdev->dev, "apple,centauri-protocol", &c->protocol); + if (ret || c->protocol != 27) + return -EINVAL; + ret = pcim_enable_device(pdev); + if (ret) + return ret; + c->size = pci_resource_len(pdev, 0); + if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || + c->size < 0x9080) + return dev_err_probe(&pdev->dev, -ENODEV, "invalid ACIPC BAR0\n"); + ret = pcim_iomap_regions(pdev, BIT(0), "apple_centauri"); + if (ret) + return ret; + c->bar = pcim_iomap_table(pdev)[0]; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); + if (ret) + return dev_err_probe(&pdev->dev, ret, "40-bit DMA unavailable\n"); + + pci_clear_master(pdev); + pci_set_drvdata(pdev, c); + queue_delayed_work(system_long_wq, &c->boot_work, 0); + return 0; +} + +static void centauri_remove(struct pci_dev *pdev) +{ + struct centauri *c = pci_get_drvdata(pdev); + + WRITE_ONCE(c->removing, true); + cancel_delayed_work_sync(&c->boot_work); + mutex_lock(&c->boot_lock); + /* + * Nothing to stop on an N1 whose ROM boot never started (no firmware file, no + * DMA, bus mastering never enabled): on the J714c the config-space accesses of + * centauri_stop_dma hang the SoC at a restart in that state. + */ + if (!c->boot_attempted) { + dev_info(&pdev->dev, "N1 never booted: leaving it untouched\n"); + c->boot_phase = CEN_BOOT_STOPPED; + mutex_unlock(&c->boot_lock); + return; + } + centauri_wifi_unregister(c); + centauri_free_boot(c); + c->boot_phase = CEN_BOOT_STOPPED; + mutex_unlock(&c->boot_lock); +} + +static int centauri_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct centauri *c = pci_get_drvdata(pdev); + + guard(mutex)(&c->boot_lock); + if (c->boot_phase != CEN_BOOT_READY) + return -EBUSY; + return pci_save_state(pdev); +} + +static int centauri_resume(struct device *dev) +{ + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(centauri_pm_ops, centauri_suspend, centauri_resume); + +static const struct pci_device_id centauri_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1900) }, + { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1901) }, + { } +}; +MODULE_DEVICE_TABLE(pci, centauri_ids); + +static struct pci_driver centauri_driver = { + .name = "apple_centauri", + .id_table = centauri_ids, + .probe = centauri_probe, + .remove = centauri_remove, + .shutdown = centauri_remove, + .driver.pm = pm_sleep_ptr(¢auri_pm_ops), +}; + +module_pci_driver(centauri_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Apple N1 Centauri PCI firmware and ACIPC transport"); diff --git a/drivers/net/wireless/apple/centauri/centauri_wifi.c b/drivers/net/wireless/apple/centauri/centauri_wifi.c new file mode 100644 index 0000000000000..6bfdb61fba4a0 --- /dev/null +++ b/drivers/net/wireless/apple/centauri/centauri_wifi.c @@ -0,0 +1,2144 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "centauri.h" + +#define CR_BASE 0x100000 +#define TR_BASE 0x400000 +#define RING_SPAN 0x20000 +#define TX_BUFFER 0x600000 +#define BUFFER_SIZE 2048 +#define REPLY_SIZE (BUFFER_SIZE - 24) +/* The band plan: 2.4 GHz 1-14, 5 GHz 36-64, 100-144 and 149-177, 6 GHz 1-233. */ +#define CEN_CHANNELS (14 + 28 + 59) + +#define DATA_RX_BUFFER SZ_16M +#define DATA_TX_BUFFER (SZ_16M + SZ_8M) +#define DATA_RX_SLOTS 2048 +#define DATA_TX_SLOTS 2048 +#define DATA_TX_QUEUE_LIMIT DATA_TX_SLOTS + +static_assert(DATA_TX_BUFFER >= + CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_RX_BUFFER >= + CEN_ALPHA_SCRATCH_OFFSET + CEN_ALPHA_SCRATCH_SIZE); +static_assert(DATA_RX_BUFFER + DATA_RX_SLOTS * 4096 <= DATA_TX_BUFFER); +static_assert(DATA_TX_BUFFER + DATA_TX_SLOTS * BUFFER_SIZE <= + CEN_ALPHA_WINDOW_SIZE); + +struct centauri_wifi { + struct centauri *control; + struct cen_alpha_resources a; + struct wiphy *wiphy; + struct wireless_dev wdev; + struct net_device *netdev; + struct work_struct scan_work; + struct work_struct connect_work; + struct work_struct link_work; + bool irq_ready; + struct cfg80211_scan_request *scan_request; + struct mutex lock; /* Serializes firmware commands and ring consumers. */ + + u16 main_head, command_head, sequence; + u32 rx_seen[5]; + u16 rx_next[5], rx_head[5]; + u16 reply_sequence, reply_status, reply_capture; + u32 reply_command, channels_seen, scans_done, beacons_seen; + bool reply_valid, registered, configured; + u8 *reply_buffer; + size_t reply_capacity, reply_size; + bool stopping, command_failed; + bool scan_abort, connect_abort; + u8 scan_sequence; + bool connecting, connected, connect_event, disconnect_event; + bool disconnect_pending; + u16 connect_status, scan_status, disconnect_status; + struct ieee80211_channel *connect_channel; + u8 bssid[ETH_ALEN], connect_payload[1024], request_ie[256]; + u8 ssid[IEEE80211_MAX_SSID_LEN]; + size_t connect_size, request_ie_size, ssid_len; + u32 data_rx_received, data_rx_reposted, data_rx_bad; + struct sk_buff_head data_tx_queue; + u16 data_tx_length[DATA_TX_SLOTS]; + bool data_tx_busy[DATA_TX_SLOTS]; + u32 data_tx_submitted, data_tx_completed, data_tx_failed; + u32 data_tx_coalesced, data_tx_bad; + s8 station_signal; + bool station_signal_valid; + unsigned long station_signal_at, connected_at; + struct net_device_stats station_start; + u32 data_rings_open; + bool data_failed; + struct ieee80211_supported_band bands[3]; + struct ieee80211_channel channels[CEN_CHANNELS]; + struct ieee80211_sband_iftype_data iftype_6ghz; + /* The firmware's own reports, read at bring-up (system commands 19 and 8). */ + u8 hw_channels[REPLY_SIZE], system_caps[REPLY_SIZE]; + struct debugfs_blob_wrapper hw_channels_blob, system_caps_blob; + u8 link_status[REPLY_SIZE]; +}; + +static int query_link_status(struct centauri_wifi *wifi); + +static const u32 rx_buffer[5] = { 0, 0, 0x620000, 0x680000, 0x700000 }; +static const u16 ring_size[5] = { 0, 64, 192, 96, 192 }; +static const struct { + enum nl80211_band band; + u8 first, last, step; +} band_plan[] = { + { NL80211_BAND_2GHZ, 1, 14, 1 }, { NL80211_BAND_5GHZ, 36, 64, 4 }, + { NL80211_BAND_5GHZ, 100, 144, 4 }, { NL80211_BAND_5GHZ, 149, 177, 4 }, + { NL80211_BAND_6GHZ, 1, 233, 4 }, +}; +static struct ieee80211_rate rates[] = { + { .bitrate = 10 }, { .bitrate = 20 }, { .bitrate = 55 }, + { .bitrate = 110 }, { .bitrate = 60 }, { .bitrate = 90 }, + { .bitrate = 120 }, { .bitrate = 180 }, { .bitrate = 240 }, + { .bitrate = 360 }, { .bitrate = 480 }, { .bitrate = 540 }, +}; + +static const u32 akm_suites[] = { WLAN_AKM_SUITE_PSK, WLAN_AKM_SUITE_SAE }; + +static const u32 cipher_suites[] = { + WLAN_CIPHER_SUITE_CCMP, + WLAN_CIPHER_SUITE_AES_CMAC, +}; + +/* The firmware's band numbers: 0 is 2.4 GHz, 1 is 5 GHz, 2 is 6 GHz. */ +static int firmware_band(enum nl80211_band band) +{ + switch (band) { + case NL80211_BAND_2GHZ: + return 0; + case NL80211_BAND_5GHZ: + return 1; + case NL80211_BAND_6GHZ: + return 2; + default: + return -EOPNOTSUPP; + } +} + +static u16 index_read(struct centauri_wifi *wifi, unsigned int array, + unsigned int index) +{ + return le16_to_cpu(READ_ONCE(*( + __le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index))); +} + +static void index_write(struct centauri_wifi *wifi, unsigned int array, + unsigned int index, u16 value) +{ + WRITE_ONCE( + *(__le16 *)(wifi->a.context + wifi->a.index[array] + 2 * index), + cpu_to_le16(value)); +} + +static void completions(struct centauri_wifi *wifi) +{ + unsigned int vector; + + for (vector = 0; vector <= 4; vector++) + index_write(wifi, 2, vector, index_read(wifi, 0, vector)); + dma_wmb(); +} + +static int alpha_message(struct centauri_wifi *wifi, const void *message, + size_t size) +{ + u16 head = wifi->main_head; + u8 *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(2000); + + if (head >= 15 || size > 512) + return -ENOSPC; + memcpy(wifi->a.messages + (head + 1) * 512, message, size); + descriptor = wifi->a.context + wifi->a.mtr + 16 * head; + put_unaligned_le32(1 | (size << 8), descriptor); + put_unaligned_le64(wifi->a.messages_dma + (head + 1) * 512, + descriptor + 4); + put_unaligned_le32(head + 1, descriptor + 12); + index_write(wifi, 3, 0, head + 1); + if (head) + index_write(wifi, 3, 8, head + 1); + dma_wmb(); + wifi->main_head++; + writel(1, wifi->a.bar + 0x1004); + do { + if (readl(wifi->a.bar + 8) != 2) + return -EIO; + if (index_read(wifi, 1, 0) >= head + 1) { + dma_rmb(); + completions(wifi); + return 0; + } + usleep_range(100, 200); + } while (time_before(jiffies, deadline)); + return -ETIMEDOUT; +} + +static int open_pair(struct centauri_wifi *wifi, u16 id) +{ + u8 cr[0x2c] = { 2 }, tr[0x34] = { 1 }; + int ret; + + put_unaligned_le32(id | (id << 16), cr + 4); + put_unaligned_le64(wifi->a.window_dma + CR_BASE + (id - 1) * RING_SPAN, + cr + 8); + put_unaligned_le16(1024, cr + 0x10); + put_unaligned_le16(0xffff, cr + 0x12); + ret = alpha_message(wifi, cr, sizeof(cr)); + if (ret) + return ret; + put_unaligned_le32(id | (id << 16), tr + 4); + put_unaligned_le64(wifi->a.window_dma + TR_BASE + (id - 1) * RING_SPAN, + tr + 8); + put_unaligned_le64(~0ULL, tr + 0x10); + put_unaligned_le16(ring_size[id], tr + 0x18); + put_unaligned_le16(id, tr + 0x1a); + put_unaligned_le16(id, tr + 0x1c); + put_unaligned_le16(0x40, tr + 0x1e); + put_unaligned_le16(2, tr + 0x24); + return alpha_message(wifi, tr, sizeof(tr)); +} + +static void report_beacon(struct centauri_wifi *wifi, const u8 *payload, + size_t size) +{ + const u8 *frame, *ie, *end; + struct ieee80211_channel *channel; + struct cfg80211_bss *bss; + u16 frame_size; + int number = 0, frequency, signal; + enum nl80211_band band = NL80211_BAND_2GHZ; + + if (!wifi->registered || size < 0x60 + 36) + return; + + frame_size = size - 0x60; + frame = payload + 0x60; + if (!ieee80211_is_beacon(*(__le16 *)frame) && + !ieee80211_is_probe_resp(*(__le16 *)frame)) + return; + end = frame + frame_size; + for (ie = frame + 36; ie + 2 <= end && ie + 2 + ie[1] <= end; + ie += 2 + ie[1]) { + if ((ie[0] == WLAN_EID_DS_PARAMS || + ie[0] == WLAN_EID_HT_OPERATION) && + ie[1]) { + number = ie[2]; + band = number <= 14 ? NL80211_BAND_2GHZ : + NL80211_BAND_5GHZ; + } else if (ie[0] == WLAN_EID_EXTENSION && ie[1] > 1 && + ie[2] == WLAN_EID_EXT_HE_OPERATION && + ie[1] - 1 >= sizeof(struct ieee80211_he_operation) && + ie[1] >= ieee80211_he_oper_size(ie + 3)) { + /* 6 GHz beacons carry neither: the HE operation names the channel. */ + const struct ieee80211_he_6ghz_oper *oper = + ieee80211_he_6ghz_oper((const void *)(ie + 3)); + + if (oper && !number) { + number = oper->primary; + band = NL80211_BAND_6GHZ; + } + } + } + if (!number) + return; + frequency = ieee80211_channel_to_frequency(number, band); + channel = ieee80211_get_channel(wifi->wiphy, frequency); + if (!channel) + return; + signal = max_t(s8, payload[0x16], payload[0x17]); + if (signal >= 0 || signal < -127) + signal = -95; + bss = cfg80211_inform_bss_frame(wifi->wiphy, channel, + (struct ieee80211_mgmt *)frame, + frame_size, signal * 100, GFP_KERNEL); + if (bss) { + wifi->beacons_seen++; + cfg80211_put_bss(wifi->wiphy, bss); + } +} + +static void post_receive(struct centauri_wifi *wifi, unsigned int ring, + u16 slot) +{ + u8 *descriptor = + wifi->a.window + TR_BASE + (ring - 1) * RING_SPAN + slot * 16; + u32 offset = rx_buffer[ring] + slot * BUFFER_SIZE; + + memset(wifi->a.window + offset, 0, BUFFER_SIZE); + put_unaligned_le32(1 | (BUFFER_SIZE << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + offset, descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); +} + +static bool data_rx_frame(const u8 *packet, unsigned int capacity, + unsigned int length, u32 footer, unsigned int *offset, + unsigned int *frame_length) +{ + unsigned int padding = (footer >> 12) & 15; + + if (!(footer & BIT(7))) + return false; + if (length < padding + ETH_HLEN || length > capacity) + return false; + if (footer & BIT(24)) { + if (capacity < 4) + return false; + *offset = get_unaligned_le16(packet + 2); + } else { + *offset = padding; + } + *frame_length = length - padding; + return *offset <= capacity && *frame_length <= capacity - *offset; +} + +static void drain_data_rx(struct centauri_wifi *wifi) +{ + u16 head, tail, post; + unsigned int processed = 0; + + if (!(wifi->data_rings_open & BIT(4)) || readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 207); + tail = index_read(wifi, 2, 207); + post = index_read(wifi, 3, 12); + if (head >= DATA_RX_SLOTS || tail >= DATA_RX_SLOTS || + post >= DATA_RX_SLOTS) { + wifi->data_rx_bad++; + return; + } + dma_rmb(); + while (tail != head && processed < (DATA_RX_SLOTS - 1)) { + u8 *completion = wifi->a.window + 0x240000 + tail * 32; + u32 header = get_unaligned_le32(completion); + u32 info = get_unaligned_le32(completion + 4); + u16 cookie = info & 0xffff, length = info >> 16; + u16 next = (post + 1) % DATA_RX_SLOTS; + u8 *packet, *descriptor; + u32 buffer; + + if ((header >> 16) != 12 || !cookie || + cookie > (DATA_RX_SLOTS - 1) || length > 4096 || + get_unaligned_le32(completion + 8)) { + wifi->data_rx_bad++; + break; + } + + if (next == index_read(wifi, 1, 12)) + break; + buffer = DATA_RX_BUFFER + (cookie - 1) * 4096; + packet = wifi->a.window + buffer; + + wifi->data_rx_received++; + if (wifi->connected && netif_running(wifi->netdev)) { + u32 footer = get_unaligned_le32(completion + 12); + unsigned int offset, frame_length; + struct sk_buff *skb = NULL; + + if (data_rx_frame(packet, 4096, length, footer, &offset, + &frame_length)) + skb = netdev_alloc_skb_ip_align(wifi->netdev, + frame_length); + if (skb) { + skb_put_data(skb, packet + offset, + frame_length); + skb->protocol = + eth_type_trans(skb, wifi->netdev); + skb->ip_summed = CHECKSUM_NONE; + wifi->netdev->stats.rx_packets++; + wifi->netdev->stats.rx_bytes += frame_length; + netif_rx(skb); + } else { + wifi->netdev->stats.rx_dropped++; + } + } else { + wifi->netdev->stats.rx_dropped++; + } + descriptor = wifi->a.window + 0x520000 + post * 16; + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le32(cookie, descriptor + 12); + wifi->data_rx_reposted++; + tail = (tail + 1) % DATA_RX_SLOTS; + post = next; + processed++; + } + if (processed) { + dma_wmb(); + index_write(wifi, 2, 207, tail); + index_write(wifi, 3, 12, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + } +} + +static bool data_tx_completion_span(u16 cookie, u16 wire_count, u8 flags, + unsigned int *first, u16 *count) +{ + bool counted = (flags & BIT(0)) || (flags & 0x0e) == 8; + + if (!cookie || cookie > DATA_TX_SLOTS || counted != !!wire_count || + wire_count >= DATA_TX_SLOTS) + return false; + *count = counted ? wire_count : 1; + *first = (cookie + DATA_TX_SLOTS - *count) % DATA_TX_SLOTS; + return true; +} + +static void drain_data_tx(struct centauri_wifi *wifi) +{ + u16 head, tail, post; + unsigned int budget = DATA_TX_SLOTS; + bool posted = false, completed = false; + + if (wifi->command_failed || !(wifi->data_rings_open & BIT(5)) || + readl(wifi->a.bar + 8) != 2) + return; + head = index_read(wifi, 0, 192); + tail = index_read(wifi, 2, 192); + post = index_read(wifi, 3, 25); + if (head >= 2048 || tail >= 2048 || post >= DATA_TX_SLOTS) + return; + dma_rmb(); + while (tail != head && budget--) { + u8 *entry = wifi->a.window + 0x200000 + tail * 32; + u16 ring = get_unaligned_le32(entry) >> 16; + u16 cookie = get_unaligned_le16(entry + 4); + u16 count = get_unaligned_le16(entry + 9); + u8 status = entry[12]; + + if (ring == 25) { + unsigned int first, i; + + if (!data_tx_completion_span(cookie, count, entry[1], + &first, &count)) { + wifi->data_tx_bad++; + break; + } + for (i = 0; i < count; i++) + if (!wifi->data_tx_busy[(first + i) % + DATA_TX_SLOTS]) + break; + if (i != count) { + wifi->data_tx_bad++; + break; + } + if (count > 1) + wifi->data_tx_coalesced++; + for (i = 0; i < count; i++) { + unsigned int slot = (first + i) % DATA_TX_SLOTS; + + wifi->data_tx_completed++; + if (status == 0 || status == 0x83) { + wifi->netdev->stats.tx_packets++; + wifi->netdev->stats.tx_bytes += + wifi->data_tx_length[slot]; + } else { + wifi->data_tx_failed++; + wifi->netdev->stats.tx_errors++; + } + wifi->data_tx_busy[slot] = false; + } + } + tail = (tail + 1) % 2048; + completed = true; + } + if (completed) + index_write(wifi, 2, 192, tail); + budget = DATA_TX_SLOTS - 1; + while (wifi->connected && !READ_ONCE(wifi->stopping) && + netif_running(wifi->netdev) && budget--) { + u16 next = (post + 1) % DATA_TX_SLOTS; + u32 buffer = DATA_TX_BUFFER + post * BUFFER_SIZE; + u8 *descriptor = wifi->a.window + 0x540000 + post * 32; + struct sk_buff *skb; + u64 metadata = 6ULL << 27; + + if (wifi->data_tx_busy[post] || next == index_read(wifi, 1, 25)) + break; + skb = skb_dequeue(&wifi->data_tx_queue); + if (!skb) + break; + + if (skb_copy_bits(skb, 0, wifi->a.window + buffer, skb->len)) { + wifi->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + continue; + } + memset(descriptor, 0, 32); + put_unaligned_le32(3 | (skb->len << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, descriptor + 4); + put_unaligned_le16(post + 1, descriptor + 12); + + metadata |= 4ULL << 15; + put_unaligned_le64(metadata, descriptor + 16); + put_unaligned_le64(2000000, descriptor + 24); + wifi->data_tx_length[post] = skb->len; + wifi->data_tx_busy[post] = true; + wifi->data_tx_submitted++; + dev_kfree_skb_any(skb); + post = next; + posted = true; + } + if (posted || completed) { + dma_wmb(); + if (posted) + index_write(wifi, 3, 25, post); + dma_wmb(); + writel(1, wifi->a.bar + 0x1030); + } + if (wifi->connected && !READ_ONCE(wifi->stopping) && + netif_running(wifi->netdev) && + skb_queue_len(&wifi->data_tx_queue) < DATA_TX_QUEUE_LIMIT) + netif_wake_queue(wifi->netdev); +} + +/* The interface status reply: 8 fixed bytes, then TLVs. */ +static bool link_signal_read(const u8 *packet, size_t size, s8 *signal) +{ + size_t pos = 8; + + while (pos <= size && size - pos >= 4) { + u16 tag = get_unaligned_le16(packet + pos); + u16 length = get_unaligned_le16(packet + pos + 2); + + if (length > size - pos - 4) + return false; + if (tag == 268 && length >= 115) { + *signal = (s8)packet[pos + 5]; + return *signal != 0; + } + pos += 4 + length; + } + return false; +} + +static void drain(struct centauri_wifi *wifi) +{ + unsigned int ring; + bool posted = false; + + for (ring = 2; ring <= 4; ring++) { + unsigned int budget = ring_size[ring]; + + while (budget--) { + u16 slot = wifi->rx_next[ring], size, status, sequence; + u32 command, wire_command; + u8 *descriptor = wifi->a.window + TR_BASE + + (ring - 1) * RING_SPAN + slot * 16; + u8 *packet = wifi->a.window + rx_buffer[ring] + + slot * BUFFER_SIZE; + + if (!(READ_ONCE(descriptor[15]) & 4)) + break; + dma_rmb(); + size = get_unaligned_le16(packet + 0x10); + status = get_unaligned_le16(packet + 0xe); + sequence = get_unaligned_le16(packet + 0x12); + wire_command = get_unaligned_le32(packet + 0x14); + command = wire_command & ~BIT(30); + wifi->rx_seen[ring]++; + wifi->rx_next[ring] = (slot + 1) % ring_size[ring]; + if (size < 24 || size > BUFFER_SIZE) + goto refill; + if (ring == 4) { + report_beacon(wifi, packet + 24, size - 24); + } else if (command & BIT(31)) { + if (command == 0x80000030) + wifi->channels_seen++; + if (command == 0x80000005) { + wifi->scans_done++; + wifi->scan_status = status; + } + if (command == 0x80000000 && wifi->connecting) { + wifi->connect_event = true; + wifi->connect_status = status; + } + if (command == 0x80000003 || + command == 0x80000004) { + wifi->disconnect_event = true; + wifi->disconnect_status = status; + } + if (command == 0x80000004) + wifi->disconnect_pending = false; + + } else { + wifi->reply_valid = true; + wifi->reply_sequence = sequence; + wifi->reply_status = status; + + wifi->reply_command = wire_command; + if (wifi->reply_buffer && + sequence == wifi->reply_capture) { + wifi->reply_size = size - 24; + memcpy(wifi->reply_buffer, packet + 24, + min(wifi->reply_size, + wifi->reply_capacity)); + } + } +refill: + post_receive(wifi, ring, wifi->rx_head[ring]); + wifi->rx_head[ring] = + (wifi->rx_head[ring] + 1) % ring_size[ring]; + dma_wmb(); + index_write(wifi, 3, ring, wifi->rx_head[ring]); + posted = true; + } + } + completions(wifi); + if (posted) + writel(1, wifi->a.bar + 0x1004); + drain_data_rx(wifi); + drain_data_tx(wifi); +} + +/* Caller holds wifi->lock. Unknown completion keeps DMA payloads owned. */ +static void command_fault(struct centauri_wifi *wifi) +{ + wifi->command_failed = true; + wifi->configured = false; + wifi->data_failed = true; + netif_stop_queue(wifi->netdev); + if (smp_load_acquire(&wifi->irq_ready)) + schedule_work(&wifi->link_work); +} + +static bool command_payload_consumed(struct centauri_wifi *wifi, u16 slot) +{ + if (slot >= ring_size[1] || + index_read(wifi, 1, 1) != (slot + 1) % ring_size[1]) + return false; + dma_rmb(); + return true; +} + +/* + * Send one command and wait for its reply. With a reply buffer, the reply's + * payload is copied there (up to *reply_size bytes) and *reply_size becomes + * the payload's length. + */ +static int command_reply(struct centauri_wifi *wifi, u32 id, + const void *payload, size_t size, void *reply, + size_t *reply_size) +{ + u16 slot = wifi->command_head, sequence = ++wifi->sequence; + int ret; + u32 stage; + u8 *packet, *descriptor; + unsigned long deadline = jiffies + msecs_to_jiffies(1500); + + if (wifi->command_failed) + return -EIO; + if (size > BUFFER_SIZE - 24) + return -EMSGSIZE; + if ((slot + 1) % ring_size[1] == index_read(wifi, 1, 1)) + return -ENOSPC; + packet = wifi->a.window + TX_BUFFER + slot * BUFFER_SIZE; + memset(packet, 0, 24 + size); + put_unaligned_le16(24 + size, packet + 0x10); + put_unaligned_le16(sequence, packet + 0x12); + put_unaligned_le32(id, packet + 0x14); + if (size) + memcpy(packet + 24, payload, size); + descriptor = wifi->a.window + TR_BASE + slot * 16; + put_unaligned_le32(1 | ((24 + size) << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + TX_BUFFER + slot * BUFFER_SIZE, + descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + dma_wmb(); + wifi->command_head = (slot + 1) % ring_size[1]; + index_write(wifi, 3, 1, wifi->command_head); + wifi->reply_valid = false; + wifi->reply_buffer = reply; + wifi->reply_capacity = reply ? *reply_size : 0; + wifi->reply_capture = sequence; + wifi->reply_size = 0; + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + do { + if (READ_ONCE(wifi->stopping) || + READ_ONCE(wifi->control->removing)) { + command_fault(wifi); + ret = -ECANCELED; + goto out; + } + drain(wifi); + stage = readl(wifi->a.bar + 8); + if (stage != 2) { + dev_err(&wifi->control->pdev->dev, + "Wi-Fi command %#x lost transport: stage=%#x sequence=%u rx=%u\n", + id, stage, sequence, wifi->rx_seen[2]); + command_fault(wifi); + ret = -EIO; + goto out; + } + + if (wifi->reply_valid && wifi->reply_sequence == sequence && + (wifi->reply_command & ~BIT(30)) == id) { + ret = 0; + if (wifi->reply_status) { + dev_dbg(&wifi->control->pdev->dev, + "Wi-Fi command %#x: firmware status %u\n", + id, wifi->reply_status); + ret = -EREMOTEIO; + } + if (reply) + *reply_size = wifi->reply_size; + goto out; + } + usleep_range(500, 1000); + } while (time_before(jiffies, deadline)); + dev_err(&wifi->control->pdev->dev, + "Wi-Fi command %#x timed out (rx=%u)\n", id, wifi->rx_seen[2]); + command_fault(wifi); + ret = -ETIMEDOUT; +out: + wifi->reply_buffer = NULL; + return ret; +} + +static int command(struct centauri_wifi *wifi, u32 id, const void *payload, + size_t size) +{ + return command_reply(wifi, id, payload, size, NULL, NULL); +} + +/* Caller holds wifi->lock; ret is what command() returned for the request. */ +static void command_refused(struct centauri_wifi *wifi, const char *what, + int ret) +{ + if (ret == -EREMOTEIO) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi %s refused: firmware status %u\n", what, + wifi->reply_status); + else + dev_warn(&wifi->control->pdev->dev, "Wi-Fi %s failed: %d\n", + what, ret); +} + +static int station_configure(struct centauri_wifi *wifi, const u8 *address) +{ + u8 start[5 + 4 + 3 * ETH_ALEN] = { 0, 0x0b, 0, 0, 0 }; + size_t start_size = 5; + int ret; + + ret = command(wifi, 1, address, ETH_ALEN); + if (!ret) + ret = command(wifi, 2, start, start_size); + return ret; +} + +/* + * The firmware owns the radio's channel list and capabilities: system + * commands 19 (the channels the hardware supports) and 8 (the system + * capabilities) report them once the interface is configured. Kept as + * read, in debugfs. + */ +static void firmware_report(struct centauri_wifi *wifi, u32 id, u8 *buffer, + struct debugfs_blob_wrapper *blob) +{ + size_t size = REPLY_SIZE; + int ret; + + ret = command_reply(wifi, id, NULL, 0, buffer, &size); + if (ret) { + command_refused(wifi, id == 19 ? "channel list" : "capabilities", + ret); + size = 0; + } else if (size > REPLY_SIZE) { + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi command %u reply of %zu bytes truncated\n", id, + size); + size = REPLY_SIZE; + } + blob->data = buffer; + blob->size = size; +} + +/* + * The channel list: a count, then one 4-byte entry per channel the current + * country allows at each width: the channel number (the center channel for + * 40-160 MHz), then the band in bits 0-1, the width (20 << n MHz) in bits 2-3 + * and the flags in bits 4-7, then two bytes this driver does not use (a + * width mask and, on 6 GHz, power-mode bits). Scan requests take the same + * band and flag bits. + */ +#define CHANNEL_PASSIVE BIT(4) +#define CHANNEL_RADAR BIT(6) + +static const u8 *channel_entry(const u8 *list, size_t size, int band, + unsigned int width, unsigned int number) +{ + unsigned int count = size ? list[0] : 0, i, span = 2 * (1 << width) - 2; + + if (1 + 4 * count > size) + count = size ? (size - 1) / 4 : 0; + for (i = 0; i < count; i++) { + const u8 *entry = list + 1 + 4 * i; + + if ((entry[1] & 3) != band || ((entry[1] >> 2) & 3) != width) + continue; + if (number + span >= entry[0] && number <= entry[0] + span) + return entry; + } + return NULL; +} + +/* + * The firmware's list decides which channels exist and how: a band-plan + * channel it leaves out is disabled, its flags mark the rest, and a width no + * entry covers is not used. Applied over what cfg80211's regulatory domain + * set, after every change of it (the notifier) and at bring-up. + */ +static void channels_apply(struct centauri_wifi *wifi) +{ + const u8 *list = wifi->hw_channels; + size_t size = wifi->hw_channels_blob.size; + unsigned int counted[3] = {}, i; + + if (!size) + return; + for (i = 0; i < CEN_CHANNELS; i++) { + struct ieee80211_channel *channel = &wifi->channels[i]; + int band = firmware_band(channel->band); + const u8 *entry = channel_entry(list, size, band, 0, + channel->hw_value); + + if (!entry) { + channel->flags |= IEEE80211_CHAN_DISABLED; + continue; + } + if (entry[1] & CHANNEL_PASSIVE) + channel->flags |= IEEE80211_CHAN_NO_IR; + if (entry[1] & CHANNEL_RADAR) + channel->flags |= IEEE80211_CHAN_RADAR; + if (!channel_entry(list, size, band, 1, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_NO_HT40; + if (!channel_entry(list, size, band, 2, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_NO_80MHZ; + if (!channel_entry(list, size, band, 3, channel->hw_value)) + channel->flags |= IEEE80211_CHAN_NO_160MHZ; + if (!(channel->flags & IEEE80211_CHAN_DISABLED)) + counted[band]++; + } + dev_info(&wifi->control->pdev->dev, + "Wi-Fi firmware channels: 2.4 GHz %u, 5 GHz %u, 6 GHz %u (%u entries)\n", + counted[0], counted[1], counted[2], list[0]); +} + +/* + * 6 GHz needs HE: the station's HE capabilities element as the firmware + * reports it (capabilities TLV 0xff), with the 6 GHz band capabilities taken + * from its VHT element (TLV 0xbf). Without them the band is not registered. + */ +static bool he_6ghz_from_firmware(struct centauri_wifi *wifi) +{ + struct ieee80211_sband_iftype_data *data = &wifi->iftype_6ghz; + struct ieee80211_sta_he_cap *he = &data->he_cap; + const u8 *caps = wifi->system_caps, *he_elem = NULL, *vht = NULL; + size_t size = wifi->system_caps_blob.size, pos = 0, mcs; + u32 vht_cap = 0; + + while (pos + 4 <= size) { + u16 tag = get_unaligned_le16(caps + pos); + u16 length = get_unaligned_le16(caps + pos + 2); + const u8 *value = caps + pos + 4; + + if (length > size - pos - 4) + break; + if (tag == WLAN_EID_EXTENSION && length >= 3 + 6 + 11 + 4 && + value[0] == WLAN_EID_EXTENSION && value[1] == length - 2 && + value[2] == WLAN_EID_EXT_HE_CAPABILITY) + he_elem = value; + if (tag == WLAN_EID_VHT_CAPABILITY && length >= 2 + 12 && + value[0] == WLAN_EID_VHT_CAPABILITY) + vht = value; + pos += 4 + length; + } + if (!he_elem) + return false; + memcpy(&he->he_cap_elem, he_elem + 3, sizeof(he->he_cap_elem)); + mcs = he_elem[1] - 1 - sizeof(he->he_cap_elem); + memcpy(&he->he_mcs_nss_supp, he_elem + 3 + sizeof(he->he_cap_elem), + min(mcs, sizeof(he->he_mcs_nss_supp))); + if (mcs > sizeof(he->he_mcs_nss_supp)) + memcpy(he->ppe_thres, + he_elem + 3 + sizeof(he->he_cap_elem) + + sizeof(he->he_mcs_nss_supp), + min(mcs - sizeof(he->he_mcs_nss_supp), + sizeof(he->ppe_thres))); + he->has_he = true; + if (vht) + vht_cap = get_unaligned_le32(vht + 2); + data->he_6ghz_capa.capa = cpu_to_le16( + u16_encode_bits(u32_get_bits(vht_cap, + IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK), + IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP) | + u16_encode_bits(vht_cap & IEEE80211_VHT_CAP_MAX_MPDU_MASK, + IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN) | + u16_encode_bits(WLAN_HT_CAP_SM_PS_DISABLED, + IEEE80211_HE_6GHZ_CAP_SM_PS) | + (vht_cap & IEEE80211_VHT_CAP_RX_ANTENNA_PATTERN ? + IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS : 0) | + (vht_cap & IEEE80211_VHT_CAP_TX_ANTENNA_PATTERN ? + IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS : 0)); + data->types_mask = BIT(NL80211_IFTYPE_STATION); + return true; +} + +static void firmware_reports(struct centauri_wifi *wifi) +{ + firmware_report(wifi, 19, wifi->hw_channels, &wifi->hw_channels_blob); + firmware_report(wifi, 8, wifi->system_caps, &wifi->system_caps_blob); + dev_info(&wifi->control->pdev->dev, + "Wi-Fi firmware channel list: %lu bytes, capabilities: %lu bytes\n", + wifi->hw_channels_blob.size, wifi->system_caps_blob.size); +} + +/* The band plan, with the firmware's legacy rates (CCK on 2.4 GHz only). */ +static void bands_init(struct centauri_wifi *wifi) +{ + unsigned int i, n = 0, c; + + for (i = 0; i < ARRAY_SIZE(band_plan); i++) { + enum nl80211_band band = band_plan[i].band; + struct ieee80211_supported_band *sband = + &wifi->bands[firmware_band(band)]; + + if (!sband->channels) + sband->channels = wifi->channels + n; + for (c = band_plan[i].first; c <= band_plan[i].last; + c += band_plan[i].step) { + wifi->channels[n].band = band; + wifi->channels[n].hw_value = c; + wifi->channels[n].center_freq = + ieee80211_channel_to_frequency(c, band); + sband->n_channels++; + n++; + } + sband->band = band; + sband->bitrates = band == NL80211_BAND_2GHZ ? rates : rates + 4; + sband->n_bitrates = band == NL80211_BAND_2GHZ ? + ARRAY_SIZE(rates) : + ARRAY_SIZE(rates) - 4; + } + wifi->wiphy->bands[NL80211_BAND_2GHZ] = &wifi->bands[0]; + wifi->wiphy->bands[NL80211_BAND_5GHZ] = &wifi->bands[1]; + if (he_6ghz_from_firmware(wifi)) { + _ieee80211_set_sband_iftype_data(&wifi->bands[2], + &wifi->iftype_6ghz, 1); + wifi->wiphy->bands[NL80211_BAND_6GHZ] = &wifi->bands[2]; + } else { + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi: no HE capabilities from the firmware; no 6 GHz band\n"); + } +} + +static int radio_start(struct centauri_wifi *wifi) +{ + unsigned int ring, slot; + int ret; + + for (ring = 1; ring <= 4; ring++) { + ret = open_pair(wifi, ring); + if (ret) + return ret; + } + for (ring = 2; ring <= 4; ring++) { + for (slot = 0; slot < ring_size[ring] - 1; slot++) + post_receive(wifi, ring, slot); + wifi->rx_head[ring] = ring_size[ring] - 1; + dma_wmb(); + index_write(wifi, 3, ring, ring_size[ring] - 1); + } + dma_wmb(); + writel(1, wifi->a.bar + 0x1004); + msleep(400); + ret = command(wifi, 0, NULL, 0); + if (!ret) + ret = station_configure(wifi, wifi->netdev->dev_addr); + if (ret) + return ret; + msleep(1000); + firmware_reports(wifi); + wifi->configured = !wifi->command_failed; + return wifi->configured ? 0 : -EIO; +} + +#define CEN_SCAN_CHANNELS_MAX 59 +static int scan_profile27(struct centauri_wifi *wifi) +{ + struct cfg80211_scan_request *request = wifi->scan_request; + u8 scan[14 + 5 + 4 * CEN_SCAN_CHANNELS_MAX + 9 + 4 + 2 + 32]; + unsigned int next = 0; + + if (!request || !request->n_channels || request->n_ssids > 1) + return -EINVAL; + while (next < request->n_channels) { + u8 channels[1 + 4 * CEN_SCAN_CHANNELS_MAX] = {}; + u8 options[5] = { 0, 0, 1, 0, 0 }; + unsigned int count = 0; + u32 done = wifi->scans_done; + size_t size = 14; + unsigned long deadline; + int ret; + bool abort_sent = false; + + if (READ_ONCE(wifi->scan_abort)) + return -ECANCELED; + memset(scan, 0, sizeof(scan)); + scan[0] = (++wifi->scan_sequence) & 0x1f; + scan[3] = 0x80; + scan[5] = 2; + scan[7] = 0x10; + + put_unaligned_le16(40, scan + 8); + put_unaligned_le16(40, scan + 10); + put_unaligned_le16(110, scan + 12); + while (next < request->n_channels && + count < CEN_SCAN_CHANNELS_MAX) { + struct ieee80211_channel *channel = + request->channels[next++]; + unsigned int number; + int band; + + if (channel->flags & IEEE80211_CHAN_DISABLED) + continue; + band = firmware_band(channel->band); + if (band < 0) + return band; + number = ieee80211_frequency_to_channel( + channel->center_freq); + if (!number || number > 255) + return -EINVAL; + channels[1 + 4 * count] = number; + channels[2 + 4 * count] = band; + /* Flagged as in the firmware's channel list: listen only. */ + if (channel->flags & + (IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_RADAR)) + channels[2 + 4 * count] |= CHANNEL_PASSIVE; + if (channel->flags & IEEE80211_CHAN_RADAR) + channels[2 + 4 * count] |= CHANNEL_RADAR; + count++; + } + if (!count) + continue; + channels[0] = count; + put_unaligned_le16(260, scan + size); + put_unaligned_le16(1 + 4 * count, scan + size + 2); + memcpy(scan + size + 4, channels, 1 + 4 * count); + size += 5 + 4 * count; + put_unaligned_le16(357, scan + size); + put_unaligned_le16(sizeof(options), scan + size + 2); + memcpy(scan + size + 4, options, sizeof(options)); + size += 4 + sizeof(options); + if (request->n_ssids) { + const struct cfg80211_ssid *ssid = &request->ssids[0]; + + if (ssid->ssid_len > 32) + return -EINVAL; + + scan[2] = !!ssid->ssid_len; + put_unaligned_le16(258, scan + size); + put_unaligned_le16(2 + ssid->ssid_len, scan + size + 2); + scan[size + 4] = ssid->ssid_len ? 5 : 4; + scan[size + 5] = ssid->ssid_len; + memcpy(scan + size + 6, ssid->ssid, ssid->ssid_len); + size += 6 + ssid->ssid_len; + } + ret = command(wifi, 65551, scan, size); + if (ret) { + command_refused(wifi, "scan", ret); + return ret; + } + deadline = jiffies + + msecs_to_jiffies(max(8000U, count * 220 + 2000)); + do { + drain(wifi); + if (READ_ONCE(wifi->stopping) || + readl(wifi->a.bar + 8) != 2) + return -EIO; + if (wifi->scans_done != done) + break; + if (READ_ONCE(wifi->scan_abort) && !abort_sent) { + ret = command(wifi, 65537, NULL, 0); + if (ret) + return ret; + abort_sent = true; + } + usleep_range(1000, 2000); + } while (time_before(jiffies, deadline)); + if (wifi->scans_done == done) { + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi scan of %u channels: no completion event\n", + count); + command(wifi, 65537, NULL, 0); + return -ETIMEDOUT; + } + if (wifi->scan_status && !abort_sent) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi scan of %u channels ended with firmware status %u\n", + count, wifi->scan_status); + if (READ_ONCE(wifi->scan_abort)) + return -ECANCELED; + } + return 0; +} + +static void scan_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = + container_of(work, struct centauri_wifi, scan_work); + struct cfg80211_scan_info info = {}; + struct cfg80211_scan_request *request; + int ret = 0; + + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->scan_abort) || READ_ONCE(wifi->stopping)) { + ret = -ECANCELED; + goto completed; + } + { + ret = scan_profile27(wifi); + goto completed; + } +completed: + info.aborted = !!ret || READ_ONCE(wifi->scan_abort) || + READ_ONCE(wifi->stopping); + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + if (request) + cfg80211_scan_done(request, &info); +} + +static int wifi_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) + ret = -ENODEV; + else if (wifi->command_failed) + ret = -EIO; + else if (!wifi->configured) + ret = -ENETDOWN; + else if (wifi->scan_request || wifi->connecting || + wifi->disconnect_pending) + ret = -EBUSY; + else { + WRITE_ONCE(wifi->scan_abort, false); + wifi->scan_request = request; + schedule_work(&wifi->scan_work); + } + mutex_unlock(&wifi->lock); + return ret; +} + +static void wifi_abort_scan(struct wiphy *wiphy, struct wireless_dev *wdev) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)wiphy_priv(wiphy); + + if (wdev == &wifi->wdev) + WRITE_ONCE(wifi->scan_abort, true); +} + +static void cancel_scan(struct centauri_wifi *wifi) +{ + struct cfg80211_scan_info info = { .aborted = true }; + struct cfg80211_scan_request *request; + + WRITE_ONCE(wifi->scan_abort, true); + cancel_work_sync(&wifi->scan_work); + mutex_lock(&wifi->lock); + request = wifi->scan_request; + wifi->scan_request = NULL; + mutex_unlock(&wifi->lock); + + if (request) + cfg80211_scan_done(request, &info); +} + +static void cancel_connect_locked(struct centauri_wifi *wifi) +{ + if (!wifi->connecting) + return; + wifi->connecting = false; + wifi->connected = false; + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + cfg80211_connect_timeout(wifi->netdev, wifi->bssid, wifi->request_ie, + wifi->request_ie_size, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); +} + +static int rsn_caps_offset(const u8 *body, size_t size) +{ + size_t offset = 6; + u16 count; + unsigned int i; + + for (i = 0; i < 2; i++) { + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + offset += 2 + 4 * count; + if (offset > size) + return -EINVAL; + } + return offset + 2 <= size ? offset : -EINVAL; +} + +/* The offset of the RSN element's AKM list when it offers akm. */ +static int rsn_akm_offset(const u8 *body, size_t size, u32 akm) +{ + size_t offset, end; + u16 count; + unsigned int i; + + if (size < 8 || get_unaligned_le16(body) != 1) + return -EINVAL; + count = get_unaligned_le16(body + 6); + if (!count) + return -EINVAL; + offset = 8 + 4 * count; + if (offset + 2 > size) + return -EINVAL; + count = get_unaligned_le16(body + offset); + end = offset + 2 + 4 * count; + if (!count || end + 2 > size) + return -EINVAL; + for (i = 0; i < count; i++) + if (get_unaligned_be32(body + offset + 2 + 4 * i) == akm) + return offset; + return -EOPNOTSUPP; +} + +/* The station's RSN element with akm as its only AKM. */ +static int select_rsn(const struct cfg80211_crypto_settings *crypto, u32 akm, + const u8 *body, size_t size, u8 *selected) +{ + size_t tail; + unsigned int i; + int offset; + + for (i = 0; i < crypto->n_akm_suites; i++) + if (crypto->akm_suites[i] == akm) + break; + if (i == crypto->n_akm_suites) + return -EOPNOTSUPP; + offset = rsn_akm_offset(body, size, akm); + if (offset < 0) + return offset; + + if (get_unaligned_be32(body + 2) != WLAN_CIPHER_SUITE_CCMP || + get_unaligned_le16(body + 6) != 1 || + get_unaligned_be32(body + 8) != WLAN_CIPHER_SUITE_CCMP) + return -EOPNOTSUPP; + tail = offset + 2 + 4 * get_unaligned_le16(body + offset); + memcpy(selected, body, offset); + put_unaligned_le16(1, selected + offset); + put_unaligned_be32(akm, selected + offset + 2); + memcpy(selected + offset + 6, body + tail, size - tail); + return offset + 6 + size - tail; +} + +/* + * The connect command carries one authentication and AKM pair, in the bytes + * at 0x46 and 0x47: WPA2-Personal is open authentication with AKM 4 and the + * PMK at 0x4c, WPA3-Personal is SAE authentication with AKM 8 and the + * password in TLV 313, which the firmware runs SAE from. + */ +#define CONNECT_AUTH_OPEN 0 +#define CONNECT_AUTH_SAE 3 +#define CONNECT_AKM_PSK 4 +#define CONNECT_AKM_SAE 8 + +/* The AKM of the client's request whose credential it supplies, SAE first. */ +static u32 connect_akm(const struct cfg80211_crypto_settings *crypto) +{ + bool psk = false; + unsigned int i; + + for (i = 0; i < crypto->n_akm_suites; i++) { + if (crypto->akm_suites[i] == WLAN_AKM_SUITE_SAE && + crypto->sae_pwd_len) + return WLAN_AKM_SUITE_SAE; + if (crypto->akm_suites[i] == WLAN_AKM_SUITE_PSK && crypto->psk) + psk = true; + } + return psk ? WLAN_AKM_SUITE_PSK : 0; +} + +static void append_tlv(u8 *payload, size_t *size, u16 tag, const void *data, + u16 length) +{ + put_unaligned_le16(tag, payload + *size); + put_unaligned_le16(length, payload + *size + 2); + memcpy(payload + *size + 4, data, length); + *size += 4 + length; +} + +/* + * The connect request names its target: nl80211 clients pass the BSSID and + * channel they chose from their own scan results. The kernel's scan entry for + * it may already be gone (a client's flushed scan expires every BSS its last + * channel subset did not see), so the entry is looked up only when the + * request names no target. + */ +static int wifi_connect(struct wiphy *wiphy, struct net_device *netdev, + struct cfg80211_connect_params *params) +{ + static const u8 mld_defaults[14] = { + 1, 0, 1, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 3, 0, 0, + }; + + /* TLV 304: the join mode, 5 for every join. */ + static const u8 join_mode = 5; + /* TLV 316: no security roaming groups. */ + static const u8 roaming; + static const __le32 connection_timeouts[] = { + cpu_to_le32(260), cpu_to_le32(6000), cpu_to_le32(2000), + cpu_to_le32(15000), cpu_to_le32(300), + }; + struct centauri_wifi *wifi = + *(struct centauri_wifi **)wiphy_priv(wiphy); + const u8 *bssid = params->bssid ?: params->bssid_hint; + struct ieee80211_channel *channel = + params->channel ?: params->channel_hint; + struct cfg80211_bss *bss; + const u8 *rsn; + u8 rsn_body[255], ssid[35]; + u8 *payload = wifi->connect_payload; + size_t size = 130, rsn_size, rsn_prefix, rsn_tail; + int ret = 0, offset, band; + u32 akm = connect_akm(¶ms->crypto); + bool sae = akm == WLAN_AKM_SUITE_SAE, mfpc; + u8 password[2 + SAE_PASSWORD_MAX_LEN]; + + if (!params->ssid_len || params->ssid_len > 32 || !akm || + params->crypto.sae_pwd_len > SAE_PASSWORD_MAX_LEN || + params->crypto.n_ciphers_pairwise != 1 || + params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || + params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || + params->ie_len > sizeof(wifi->request_ie)) + return -EOPNOTSUPP; + rsn = cfg80211_find_ie(WLAN_EID_RSN, params->ie, params->ie_len); + if (!rsn) + return -EINVAL; + ret = select_rsn(¶ms->crypto, akm, rsn + 2, rsn[1], rsn_body); + if (ret < 0) + return ret; + rsn_size = ret; + /* + * The station's own RSN element carries its management frame + * protection to the firmware; bit 7 of its capabilities is MFPC. + */ + offset = rsn_caps_offset(rsn_body, rsn_size); + if (offset < 0) + return -EINVAL; + mfpc = get_unaligned_le16(rsn_body + offset) & BIT(7); + ret = 0; + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) { + ret = -ENODEV; + goto out; + } + if (!wifi->configured || wifi->connecting || wifi->connected || + wifi->scan_request || wifi->disconnect_pending) { + ret = -EBUSY; + goto out; + } + + if (bssid && channel) { + memcpy(wifi->bssid, bssid, ETH_ALEN); + } else { + bss = cfg80211_get_bss(wiphy, channel, bssid, params->ssid, + params->ssid_len, IEEE80211_BSS_TYPE_ESS, + IEEE80211_PRIVACY_ON); + if (!bss) { + ret = -ENOENT; + goto out; + } + memcpy(wifi->bssid, bss->bssid, ETH_ALEN); + channel = bss->channel; + cfg80211_put_bss(wiphy, bss); + } + ret = firmware_band(channel->band); + if (ret < 0) + goto out; + band = ret; + ret = 0; + /* 6 GHz admits WPA3-Personal only with management frame protection. */ + if (channel->band == NL80211_BAND_6GHZ && (!sae || !mfpc)) { + ret = -EOPNOTSUPP; + goto out; + } + wifi->connect_channel = channel; + memcpy(wifi->ssid, params->ssid, params->ssid_len); + wifi->ssid_len = params->ssid_len; + memset(payload, 0, sizeof(wifi->connect_payload)); + payload[0] = 1; + memcpy(payload + 1, wifi->bssid, ETH_ALEN); + payload[7] = ieee80211_frequency_to_channel(channel->center_freq); + + payload[8] = band; + payload[0x46] = sae ? CONNECT_AUTH_SAE : CONNECT_AUTH_OPEN; + payload[0x47] = sae ? CONNECT_AKM_SAE : CONNECT_AKM_PSK; + + put_unaligned_le16(4, payload + 0x48); + if (!sae) { + put_unaligned_le16(32, payload + 0x4a); + memcpy(payload + 0x4c, params->crypto.psk, 32); + } + + append_tlv(payload, &size, 304, &join_mode, 1); + append_tlv(payload, &size, 63763, connection_timeouts, + sizeof(connection_timeouts)); + ssid[0] = 1; + ssid[1] = 4; + ssid[2] = params->ssid_len; + memcpy(ssid + 3, params->ssid, params->ssid_len); + append_tlv(payload, &size, 342, ssid, 3 + params->ssid_len); + append_tlv(payload, &size, WLAN_EID_RSN, rsn_body, rsn_size); + if (sae) { + put_unaligned_le16(params->crypto.sae_pwd_len, password); + memcpy(password + 2, params->crypto.sae_pwd, + params->crypto.sae_pwd_len); + append_tlv(payload, &size, 313, password, + 2 + params->crypto.sae_pwd_len); + memzero_explicit(password, sizeof(password)); + } + + append_tlv(payload, &size, 316, &roaming, 1); + append_tlv(payload, &size, 376, mld_defaults, sizeof(mld_defaults)); + wifi->connect_size = size; + + rsn_prefix = rsn - params->ie; + rsn_tail = rsn_prefix + 2 + rsn[1]; + memcpy(wifi->request_ie, params->ie, rsn_prefix); + wifi->request_ie[rsn_prefix] = WLAN_EID_RSN; + wifi->request_ie[rsn_prefix + 1] = rsn_size; + memcpy(wifi->request_ie + rsn_prefix + 2, rsn_body, rsn_size); + memcpy(wifi->request_ie + rsn_prefix + 2 + rsn_size, + params->ie + rsn_tail, params->ie_len - rsn_tail); + wifi->request_ie_size = params->ie_len - rsn[1] + rsn_size; + WRITE_ONCE(wifi->connect_abort, false); + wifi->connecting = true; + wifi->connect_status = 0; + wifi->connect_event = wifi->disconnect_event = false; + schedule_work(&wifi->connect_work); +out: + mutex_unlock(&wifi->lock); + return ret; +} + +/* + * cfg80211 records a connection against a BSS entry. When the client's scans + * have expired the target's, inform one from the request until its beacons + * (reported on ring 4 once associated) replace the elements. + */ +static struct cfg80211_bss *connected_bss(struct centauri_wifi *wifi) +{ + struct cfg80211_bss *bss; + u8 ie[2 + IEEE80211_MAX_SSID_LEN]; + + bss = cfg80211_get_bss(wifi->wiphy, wifi->connect_channel, wifi->bssid, + NULL, 0, IEEE80211_BSS_TYPE_ESS, + IEEE80211_PRIVACY_ANY); + if (bss) + return bss; + ie[0] = WLAN_EID_SSID; + ie[1] = wifi->ssid_len; + memcpy(ie + 2, wifi->ssid, wifi->ssid_len); + return cfg80211_inform_bss(wifi->wiphy, wifi->connect_channel, + CFG80211_BSS_FTYPE_UNKNOWN, wifi->bssid, 0, + WLAN_CAPABILITY_ESS | WLAN_CAPABILITY_PRIVACY, + 100, ie, 2 + wifi->ssid_len, -95 * 100, + GFP_KERNEL); +} + +static void connect_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = + container_of(work, struct centauri_wifi, connect_work); + unsigned long deadline; + u16 slot, status; + int ret; + + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->connect_abort) || READ_ONCE(wifi->stopping)) { + cancel_connect_locked(wifi); + mutex_unlock(&wifi->lock); + return; + } + slot = wifi->command_head; + ret = command(wifi, 131072, wifi->connect_payload, wifi->connect_size); + memzero_explicit(wifi->connect_payload, sizeof(wifi->connect_payload)); + deadline = jiffies + msecs_to_jiffies(15000); + while (!ret && !wifi->connect_event && + !READ_ONCE(wifi->connect_abort) && !READ_ONCE(wifi->stopping) && + time_before(jiffies, deadline)) { + drain(wifi); + + if (wifi->disconnect_event) + ret = -ENOTCONN; + if (readl(wifi->a.bar + 8) != 2) + ret = -EIO; + usleep_range(1000, 2000); + } + if (READ_ONCE(wifi->connect_abort) || READ_ONCE(wifi->stopping)) + ret = -ECANCELED; + else if (!ret && !wifi->connect_event) + ret = -ETIMEDOUT; + if (ret == -ENOTCONN) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi connect to %pM: disconnect event, firmware status %u\n", + wifi->bssid, wifi->disconnect_status); + else if (ret == -ETIMEDOUT) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi connect to %pM: no connect event\n", + wifi->bssid); + else if (ret && ret != -ECANCELED) + command_refused(wifi, "connect", ret); + else if (!ret && wifi->connect_status) + dev_warn(&wifi->control->pdev->dev, + "Wi-Fi connect to %pM failed: firmware status %u\n", + wifi->bssid, wifi->connect_status); + status = !ret && wifi->connect_status == 0 ? + WLAN_STATUS_SUCCESS : + WLAN_STATUS_UNSPECIFIED_FAILURE; + wifi->connecting = false; + wifi->connected = status == WLAN_STATUS_SUCCESS; + wifi->station_signal_valid = false; + if (wifi->connected) { + wifi->connected_at = jiffies; + wifi->station_signal_at = jiffies - HZ; + wifi->station_start = wifi->netdev->stats; + } + + if (command_payload_consumed(wifi, slot)) + memzero_explicit(wifi->a.window + TX_BUFFER + + slot * BUFFER_SIZE, + BUFFER_SIZE); + + if (ret == -ECANCELED || ret == -ETIMEDOUT) + cfg80211_connect_timeout(wifi->netdev, wifi->bssid, + wifi->request_ie, + wifi->request_ie_size, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); + else + cfg80211_connect_bss(wifi->netdev, wifi->bssid, + status == WLAN_STATUS_SUCCESS ? + connected_bss(wifi) : + NULL, + wifi->request_ie, wifi->request_ie_size, + NULL, 0, status, GFP_KERNEL, + NL80211_TIMEOUT_UNSPECIFIED); + if (status == WLAN_STATUS_SUCCESS && !READ_ONCE(wifi->stopping)) { + cfg80211_port_authorized(wifi->netdev, wifi->bssid, NULL, 0, + GFP_KERNEL); + netif_carrier_on(wifi->netdev); + if (wifi->data_rings_open & BIT(5)) + netif_wake_queue(wifi->netdev); + schedule_work(&wifi->link_work); + } + mutex_unlock(&wifi->lock); +} + +irqreturn_t centauri_alpha_irq(int irq, void *data) +{ + struct centauri *c = data; + struct centauri_wifi *wifi = smp_load_acquire(&c->wifi); + + if (wifi && smp_load_acquire(&wifi->irq_ready) && + !READ_ONCE(wifi->stopping)) + schedule_work(&wifi->link_work); + return IRQ_HANDLED; +} + +static void link_worker(struct work_struct *work) +{ + struct centauri_wifi *wifi = + container_of(work, struct centauri_wifi, link_work); + bool disconnected, was_connected; + + mutex_lock(&wifi->lock); + was_connected = wifi->connected; + drain(wifi); + disconnected = wifi->command_failed || wifi->disconnect_event || + READ_ONCE(wifi->stopping) || readl(wifi->a.bar + 8) != 2; + if (disconnected) + wifi->connected = false; + if (disconnected && was_connected) { + netif_stop_queue(wifi->netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(wifi->netdev); + cfg80211_disconnected(wifi->netdev, WLAN_REASON_UNSPECIFIED, + NULL, 0, false, GFP_KERNEL); + } + mutex_unlock(&wifi->lock); +} + +static int wait_disconnect(struct centauri_wifi *wifi) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(5000); + + while (wifi->disconnect_pending) { + drain(wifi); + if (READ_ONCE(wifi->stopping) || readl(wifi->a.bar + 8) != 2) + return -EIO; + if (!wifi->disconnect_pending) + break; + if (time_after_eq(jiffies, deadline)) + return -ETIMEDOUT; + usleep_range(500, 1000); + } + return 0; +} + +static int wifi_disconnect(struct wiphy *wiphy, struct net_device *netdev, + u16 reason) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 payload[16] = {}; + bool connected; + u16 reply_status; + int ret; + + WRITE_ONCE(wifi->connect_abort, true); + cancel_scan(wifi); + cancel_work_sync(&wifi->connect_work); + mutex_lock(&wifi->lock); + cancel_connect_locked(wifi); + put_unaligned_le16(reason, payload); + put_unaligned_le16(328, payload + 8); + put_unaligned_le16(4, payload + 10); + + put_unaligned_le32(10, payload + 12); + wifi->disconnect_pending = !READ_ONCE(wifi->stopping); + ret = READ_ONCE(wifi->stopping) ? + 0 : + command(wifi, 131073, payload, sizeof(payload)); + reply_status = wifi->reply_status; + if (!ret && wifi->disconnect_pending && (wifi->reply_command & BIT(30))) + ret = wait_disconnect(wifi); + else if (ret != -EIO && ret != -ETIMEDOUT) + wifi->disconnect_pending = false; + + connected = wifi->connected; + wifi->connected = false; + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + netif_carrier_off(netdev); + if (connected) + cfg80211_disconnected(netdev, reason, NULL, 0, true, + GFP_KERNEL); + mutex_unlock(&wifi->lock); + cancel_work_sync(&wifi->link_work); + return ret == -EREMOTEIO && reply_status == 326 ? 0 : ret; +} + +static int wifi_power_mgmt(struct wiphy *wiphy, struct net_device *netdev, + bool enabled, int timeout) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 mode = enabled ? 0 : 3; + int ret; + + mutex_lock(&wifi->lock); + ret = wifi->configured ? command(wifi, 7, &mode, sizeof(mode)) : + -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; +} + +static void wifi_regulatory(struct wiphy *wiphy, + struct regulatory_request *request) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)wiphy_priv(wiphy); + u8 country[3] = { request->alpha2[0], request->alpha2[1], 1 }; + bool named = country[0] >= 'A' && country[0] <= 'Z' && + country[1] >= 'A' && country[1] <= 'Z'; + int ret; + + mutex_lock(&wifi->lock); + if (wifi->configured && named) { + ret = command(wifi, 9, country, sizeof(country)); + if (ret) + dev_err(&wifi->control->pdev->dev, + "firmware country update failed: %d\n", ret); + else + firmware_report(wifi, 19, wifi->hw_channels, + &wifi->hw_channels_blob); + } + /* cfg80211 has just recomputed the channel flags from its own rules. */ + channels_apply(wifi); + mutex_unlock(&wifi->lock); +} + +static void fill_station(struct centauri_wifi *wifi, struct station_info *sinfo) +{ + const struct net_device_stats *stats = &wifi->netdev->stats; + + if (time_after_eq(jiffies, wifi->station_signal_at + HZ)) { + wifi->station_signal_valid = false; + wifi->station_signal_at = jiffies; + query_link_status(wifi); + } + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_CONNECTED_TIME) | + BIT_ULL(NL80211_STA_INFO_RX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_TX_BYTES64) | + BIT_ULL(NL80211_STA_INFO_RX_PACKETS) | + BIT_ULL(NL80211_STA_INFO_TX_PACKETS); + sinfo->connected_time = (jiffies - wifi->connected_at) / HZ; + sinfo->rx_bytes = stats->rx_bytes - wifi->station_start.rx_bytes; + sinfo->tx_bytes = stats->tx_bytes - wifi->station_start.tx_bytes; + sinfo->rx_packets = stats->rx_packets - wifi->station_start.rx_packets; + sinfo->tx_packets = stats->tx_packets - wifi->station_start.tx_packets; + if (wifi->station_signal_valid) { + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_SIGNAL); + sinfo->signal = wifi->station_signal; + } +} + +static int wifi_get_station(struct wiphy *wiphy, struct wireless_dev *wdev, + const u8 *mac, struct station_info *sinfo) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (wdev != &wifi->wdev || !wifi->connected || + READ_ONCE(wifi->stopping) || !ether_addr_equal(mac, wifi->bssid)) + ret = -ENOENT; + else + fill_station(wifi, sinfo); + mutex_unlock(&wifi->lock); + return ret; +} + +static int wifi_dump_station(struct wiphy *wiphy, struct wireless_dev *wdev, + int index, u8 *mac, struct station_info *sinfo) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)wiphy_priv(wiphy); + int ret = 0; + + mutex_lock(&wifi->lock); + if (index || wdev != &wifi->wdev || !wifi->connected || + READ_ONCE(wifi->stopping)) + ret = -ENOENT; + else { + ether_addr_copy(mac, wifi->bssid); + fill_station(wifi, sinfo); + } + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct cfg80211_ops wifi_ops = { + .scan = wifi_scan, + .abort_scan = wifi_abort_scan, + .connect = wifi_connect, + .disconnect = wifi_disconnect, + .set_power_mgmt = wifi_power_mgmt, + .get_station = wifi_get_station, + .dump_station = wifi_dump_station, +}; + +static int data_start(struct centauri_wifi *wifi); + +static int net_open(struct net_device *netdev) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)netdev_priv(netdev); + int ret; + + netif_carrier_off(netdev); + netif_stop_queue(netdev); + mutex_lock(&wifi->lock); + ret = wifi->configured ? data_start(wifi) : -ENETDOWN; + mutex_unlock(&wifi->lock); + return ret; +} + +static int net_stop(struct net_device *netdev) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)netdev_priv(netdev); + + cancel_scan(wifi); + netif_stop_queue(netdev); + skb_queue_purge(&wifi->data_tx_queue); + return 0; +} + +static netdev_tx_t net_transmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)netdev_priv(netdev); + + if (READ_ONCE(wifi->stopping) || !READ_ONCE(wifi->connected) || + skb->len < ETH_HLEN || skb->len > BUFFER_SIZE) { + netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) { + netif_stop_queue(netdev); + return NETDEV_TX_BUSY; + } + skb_queue_tail(&wifi->data_tx_queue, skb); + if (skb_queue_len(&wifi->data_tx_queue) >= DATA_TX_QUEUE_LIMIT) + netif_stop_queue(netdev); + schedule_work(&wifi->link_work); + return NETDEV_TX_OK; +} + +static int station_reconfigure(struct centauri_wifi *wifi, const u8 *address) +{ + const u8 stop[5] = {}; + int ret; + + wifi->configured = false; + ret = command(wifi, 3, stop, sizeof(stop)); + if (!ret) + ret = station_configure(wifi, address); + if (!ret) + wifi->configured = true; + return ret; +} + +static int net_set_mac_address(struct net_device *netdev, void *data) +{ + struct centauri_wifi *wifi = + *(struct centauri_wifi **)netdev_priv(netdev); + struct sockaddr *address = data; + int ret, restore; + + ret = eth_prepare_mac_addr_change(netdev, data); + if (ret) + return ret; + mutex_lock(&wifi->lock); + if (READ_ONCE(wifi->stopping)) { + ret = -ENODEV; + goto out; + } + if (!wifi->configured) { + ret = -ENETDOWN; + goto out; + } + if (wifi->connecting || wifi->connected || wifi->scan_request || + wifi->disconnect_pending) { + ret = -EBUSY; + goto out; + } + if (ether_addr_equal(netdev->dev_addr, address->sa_data)) + goto out; + + ret = station_reconfigure(wifi, (const u8 *)address->sa_data); + if (!ret) { + eth_commit_mac_addr_change(netdev, data); + } else { + restore = station_reconfigure(wifi, netdev->dev_addr); + if (restore) + dev_err(&wifi->control->pdev->dev, + "Wi-Fi MAC restore failed: %d (change: %d)\n", + restore, ret); + } +out: + mutex_unlock(&wifi->lock); + return ret; +} + +static const struct net_device_ops net_ops = { + .ndo_open = net_open, + .ndo_stop = net_stop, + .ndo_start_xmit = net_transmit, + .ndo_validate_addr = eth_validate_addr, + .ndo_set_mac_address = net_set_mac_address, +}; + +static int data_ring_open(struct centauri_wifi *wifi, unsigned int id) +{ + u8 message[0x34 + 10] = { 0 }; + unsigned int vector, offset, size, bit, doorbell, irq; + bool transfer = false; + int ret; + + lockdep_assert_held(&wifi->lock); + if (wifi->data_failed) + return -EIO; + if (id == 76) { + vector = 192; + offset = 0x200000; + size = 2048; + bit = 0; + doorbell = 18; + irq = 3; + } else if (id == 14) { + vector = 206; + offset = 0x220000; + size = 256; + bit = 1; + doorbell = 11; + irq = 8; + } else if (id == 15) { + vector = 207; + offset = 0x240000; + size = DATA_RX_SLOTS; + bit = 2; + doorbell = 11; + irq = 8; + } else if (id == 24 || id == 25 || id == 12) { + transfer = true; + vector = id; + offset = id == 12 ? 0x520000 : id == 24 ? 0x500000 : 0x540000; + size = id == 12 ? DATA_RX_SLOTS : DATA_TX_SLOTS; + bit = id == 12 ? 4 : id == 24 ? 3 : 5; + doorbell = id == 12 ? 11 : 18; + irq = id == 12 ? 8 : 3; + } else { + return -EINVAL; + } + if (readl(wifi->a.bar + 8) != 2) { + ret = -EIO; + goto out; + } + if (wifi->data_rings_open & BIT(bit)) + return 0; + if (wifi->connecting || wifi->scan_request) { + ret = -EBUSY; + goto out; + } + memset(wifi->a.window + offset, 0, size * 32); + index_write(wifi, transfer ? 1 : 0, vector, 0); + index_write(wifi, transfer ? 3 : 2, vector, 0); + message[0] = transfer ? 1 : 2; + message[2] = transfer && id == 12 ? 0 : 4; + put_unaligned_le16(id, message + 4); + put_unaligned_le16(vector, message + 6); + put_unaligned_le64(wifi->a.window_dma + offset, message + 8); + if (transfer) { + put_unaligned_le64(~0ULL, message + 0x10); + put_unaligned_le16(size, message + 0x18); + put_unaligned_le16(id == 12 ? 12 : 76, message + 0x1a); + put_unaligned_le16(doorbell, message + 0x1c); + put_unaligned_le16(id == 12 ? 0x220 : 0, message + 0x1e); + put_unaligned_le32(16, message + 0x20); + put_unaligned_le16(irq, message + 0x24); + } else { + put_unaligned_le16(size, message + 0x10); + put_unaligned_le16(id == 15 ? 12 : 0xffff, message + 0x12); + put_unaligned_le16(doorbell, message + 0x14); + put_unaligned_le32(1, message + 0x18); + put_unaligned_le16(irq, message + 0x1c); + } + + ret = alpha_message(wifi, message, (transfer ? 0x34 : 0x2c) + 10); + if (!ret && readl(wifi->a.bar + 8) != 2) + ret = -EIO; + if (ret) { + wifi->data_failed = true; + goto out; + } + wifi->data_rings_open |= BIT(bit); + if (transfer && id == 12) { + unsigned int slot; + + for (slot = 0; slot < size - 1; slot++) { + u8 *descriptor = wifi->a.window + offset + slot * 16; + u32 buffer = DATA_RX_BUFFER + slot * 4096; + + memset(wifi->a.window + buffer, 0, 4096); + put_unaligned_le32(1 | (4096 << 8), descriptor); + put_unaligned_le64(wifi->a.window_dma + buffer, + descriptor + 4); + put_unaligned_le32(slot + 1, descriptor + 12); + } + dma_wmb(); + index_write(wifi, 3, vector, size - 1); + dma_wmb(); + writel(1, wifi->a.bar + 0x1050); + } +out: + return ret; +} + +static int data_start(struct centauri_wifi *wifi) +{ + static const unsigned int rings[] = { 76, 15, 12, 25 }; + int ret; + + for (unsigned int i = 0; i < ARRAY_SIZE(rings); i++) { + ret = data_ring_open(wifi, rings[i]); + if (ret) + return ret; + } + return 0; +} + +static int query_link_status(struct centauri_wifi *wifi) +{ + size_t size = sizeof(wifi->link_status); + int ret; + + if (!wifi->configured) + return -ENETDOWN; + ret = command_reply(wifi, 20, NULL, 0, wifi->link_status, &size); + if (!ret) + wifi->station_signal_valid = + link_signal_read(wifi->link_status, + min(size, sizeof(wifi->link_status)), + &wifi->station_signal); + wifi->station_signal_at = jiffies; + return ret; +} + +int centauri_wifi_register(struct centauri *c) +{ + struct centauri_wifi *wifi; + u8 address[ETH_ALEN]; + int ret; + + wifi = kzalloc_obj(*wifi); + if (!wifi) + return -ENOMEM; + wifi->control = c; + mutex_init(&wifi->lock); + skb_queue_head_init(&wifi->data_tx_queue); + INIT_WORK(&wifi->scan_work, scan_worker); + INIT_WORK(&wifi->connect_work, connect_worker); + INIT_WORK(&wifi->link_work, link_worker); + smp_store_release(&c->wifi, wifi); + ret = centauri_alpha_resources(c, &wifi->a); + if (ret) + goto failed; + wifi->wiphy = wiphy_new(&wifi_ops, sizeof(struct centauri_wifi *)); + wifi->netdev = alloc_etherdev(sizeof(struct centauri_wifi *)); + if (!wifi->wiphy || !wifi->netdev) { + ret = -ENOMEM; + goto failed; + } + *(struct centauri_wifi **)wiphy_priv(wifi->wiphy) = wifi; + *(struct centauri_wifi **)netdev_priv(wifi->netdev) = wifi; + set_wiphy_dev(wifi->wiphy, &c->pdev->dev); + wifi->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION); + wifi->wiphy->max_scan_ssids = 1; + wifi->wiphy->signal_type = CFG80211_SIGNAL_TYPE_MBM; + wifi->wiphy->reg_notifier = wifi_regulatory; + wifi->wiphy->cipher_suites = cipher_suites; + wifi->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); + wifi->wiphy->akm_suites = akm_suites; + wifi->wiphy->n_akm_suites = ARRAY_SIZE(akm_suites); + wiphy_ext_feature_set(wifi->wiphy, + NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); + wiphy_ext_feature_set(wifi->wiphy, NL80211_EXT_FEATURE_SAE_OFFLOAD); + + ret = eth_platform_get_mac_address(&wifi->a.pdev->dev, address); + if (!ret) { + eth_hw_addr_set(wifi->netdev, address); + ether_addr_copy(wifi->wiphy->perm_addr, address); + } else if (ret == -EPROBE_DEFER) { + goto failed; + } else { + eth_hw_addr_random(wifi->netdev); + dev_warn( + &wifi->a.pdev->dev, + "No valid factory MAC property; using a random address\n"); + } + /* The bands come from the firmware's reports: start the radio first. */ + ret = radio_start(wifi); + if (ret) { + dev_err(&c->pdev->dev, + "Wi-Fi ring/interface setup failed: %d\n", ret); + goto failed; + } + bands_init(wifi); + ret = wiphy_register(wifi->wiphy); + if (ret) + goto failed; + wifi->registered = true; + debugfs_create_blob("hw_channels", 0400, wifi->wiphy->debugfsdir, + &wifi->hw_channels_blob); + debugfs_create_blob("system_caps", 0400, wifi->wiphy->debugfsdir, + &wifi->system_caps_blob); + rtnl_lock(); + channels_apply(wifi); + rtnl_unlock(); + wifi->wdev.wiphy = wifi->wiphy; + wifi->wdev.iftype = NL80211_IFTYPE_STATION; + wifi->wdev.netdev = wifi->netdev; + wifi->netdev->ieee80211_ptr = &wifi->wdev; + wifi->netdev->netdev_ops = &net_ops; + SET_NETDEV_DEV(wifi->netdev, &c->pdev->dev); + strscpy(wifi->netdev->name, "wlan%d", IFNAMSIZ); + wifi->netdev->max_mtu = ETH_DATA_LEN; + + ret = register_netdev(wifi->netdev); + if (ret) + goto failed; + smp_store_release(&wifi->irq_ready, true); + schedule_work(&wifi->link_work); + + return 0; +failed: + centauri_wifi_unregister(c); + return ret; +} + +void centauri_wifi_unregister(struct centauri *c) +{ + struct centauri_wifi *wifi = c->wifi; + + if (!wifi) + return; + WRITE_ONCE(wifi->stopping, true); + WRITE_ONCE(wifi->connect_abort, true); + WRITE_ONCE(wifi->scan_abort, true); + smp_store_release(&c->wifi, NULL); + + for (unsigned int i = 0; i < c->alpha_irqs.count; i++) + synchronize_irq(pci_irq_vector(c->alpha_irqs.pdev, i)); + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + netif_tx_disable(wifi->netdev); + + cancel_scan(wifi); + cancel_work_sync(&wifi->connect_work); + cancel_work_sync(&wifi->link_work); + mutex_lock(&wifi->lock); + cancel_connect_locked(wifi); + mutex_unlock(&wifi->lock); + skb_queue_purge(&wifi->data_tx_queue); + if (wifi->netdev && wifi->netdev->reg_state == NETREG_REGISTERED) + unregister_netdev(wifi->netdev); + if (wifi->registered) + wiphy_unregister(wifi->wiphy); + if (wifi->wiphy) + wiphy_free(wifi->wiphy); + if (wifi->netdev) + free_netdev(wifi->netdev); + memzero_explicit(wifi, sizeof(*wifi)); + kfree(wifi); +} diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c index 5cdf4973d986a..992602965fb49 100644 --- a/drivers/net/wireless/ath/ath12k/core.c +++ b/drivers/net/wireless/ath/ath12k/core.c @@ -1770,29 +1770,6 @@ int ath12k_core_pre_init(struct ath12k_base *ab) return 0; } -static int ath12k_core_panic_handler(struct notifier_block *nb, - unsigned long action, void *data) -{ - struct ath12k_base *ab = container_of(nb, struct ath12k_base, - panic_nb); - - return ath12k_hif_panic_handler(ab); -} - -static int ath12k_core_panic_notifier_register(struct ath12k_base *ab) -{ - ab->panic_nb.notifier_call = ath12k_core_panic_handler; - - return atomic_notifier_chain_register(&panic_notifier_list, - &ab->panic_nb); -} - -static void ath12k_core_panic_notifier_unregister(struct ath12k_base *ab) -{ - atomic_notifier_chain_unregister(&panic_notifier_list, - &ab->panic_nb); -} - static inline bool ath12k_core_hw_group_create_ready(struct ath12k_hw_group *ag) { @@ -2208,18 +2185,13 @@ int ath12k_core_init(struct ath12k_base *ab) struct ath12k_hw_group *ag; int ret; - ret = ath12k_core_panic_notifier_register(ab); - if (ret) - ath12k_warn(ab, "failed to register panic handler: %d\n", ret); - mutex_lock(&ath12k_hw_group_mutex); ag = ath12k_core_hw_group_assign(ab); if (!ag) { mutex_unlock(&ath12k_hw_group_mutex); ath12k_warn(ab, "unable to get hw group\n"); - ret = -ENODEV; - goto err_unregister_notifier; + return -ENODEV; } mutex_unlock(&ath12k_hw_group_mutex); @@ -2244,8 +2216,6 @@ int ath12k_core_init(struct ath12k_base *ab) err_unassign_hw_group: ath12k_core_hw_group_unassign(ab); -err_unregister_notifier: - ath12k_core_panic_notifier_unregister(ab); return ret; } @@ -2254,7 +2224,6 @@ void ath12k_core_deinit(struct ath12k_base *ab) { ath12k_core_hw_group_destroy(ab->ag); ath12k_core_hw_group_unassign(ab); - ath12k_core_panic_notifier_unregister(ab); } void ath12k_core_free(struct ath12k_base *ab) diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h index f28d2e90b67b2..708479df4b89e 100644 --- a/drivers/net/wireless/ath/ath12k/core.h +++ b/drivers/net/wireless/ath/ath12k/core.h @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -1146,8 +1145,6 @@ struct ath12k_base { #endif /* CONFIG_ACPI */ - struct notifier_block panic_nb; - struct ath12k_hw_group *ag; struct ath12k_wsi_info wsi_info; enum ath12k_firmware_mode fw_mode; diff --git a/drivers/net/wireless/ath/ath12k/hif.h b/drivers/net/wireless/ath/ath12k/hif.h index 4de8667690e91..aecdb0c5f2858 100644 --- a/drivers/net/wireless/ath/ath12k/hif.h +++ b/drivers/net/wireless/ath/ath12k/hif.h @@ -30,7 +30,6 @@ struct ath12k_hif_ops { void (*ce_irq_enable)(struct ath12k_base *ab); void (*ce_irq_disable)(struct ath12k_base *ab); void (*get_ce_msi_idx)(struct ath12k_base *ab, u32 ce_id, u32 *msi_idx); - int (*panic_handler)(struct ath12k_base *ab); void (*coredump_download)(struct ath12k_base *ab); }; @@ -149,14 +148,6 @@ static inline void ath12k_hif_power_down(struct ath12k_base *ab, bool is_suspend ab->hif.ops->power_down(ab, is_suspend); } -static inline int ath12k_hif_panic_handler(struct ath12k_base *ab) -{ - if (!ab->hif.ops->panic_handler) - return NOTIFY_DONE; - - return ab->hif.ops->panic_handler(ab); -} - static inline void ath12k_hif_coredump_download(struct ath12k_base *ab) { if (ab->hif.ops->coredump_download) diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c index fee4129ea4055..a41544665080a 100644 --- a/drivers/net/wireless/ath/ath12k/pci.c +++ b/drivers/net/wireless/ath/ath12k/pci.c @@ -1473,13 +1473,6 @@ void ath12k_pci_power_down(struct ath12k_base *ab, bool is_suspend) ath12k_pci_sw_reset(ab_pci->ab, false); } -static int ath12k_pci_panic_handler(struct ath12k_base *ab) -{ - ath12k_pci_sw_reset(ab, false); - - return NOTIFY_OK; -} - static const struct ath12k_hif_ops ath12k_pci_hif_ops = { .start = ath12k_pci_start, .stop = ath12k_pci_stop, @@ -1497,7 +1490,6 @@ static const struct ath12k_hif_ops ath12k_pci_hif_ops = { .ce_irq_enable = ath12k_pci_hif_ce_irq_enable, .ce_irq_disable = ath12k_pci_hif_ce_irq_disable, .get_ce_msi_idx = ath12k_pci_get_ce_msi_idx, - .panic_handler = ath12k_pci_panic_handler, #ifdef CONFIG_ATH12K_COREDUMP .coredump_download = ath12k_pci_coredump_download, #endif diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index cc660583f7cb4..891439493a5c0 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1634,8 +1634,31 @@ void brcmf_detach(struct device *dev) brcmf_cfg80211_awdl_stop(drvr->config); /* make sure primary interface removed last */ for (i = BRCMF_MAX_IFS - 1; i > -1; i--) { - if (drvr->iflist[i]) - brcmf_remove_interface(drvr->iflist[i], false); + struct brcmf_if *ifp = drvr->iflist[i]; + + if (!ifp) + continue; + + if (ifp->ndev) { + brcmf_remove_interface(ifp, false); + continue; + } + + /* The P2P device interface has no netdev. Its removal can + * race with NL80211_CMD_DEL_INTERFACE issued by a user space + * process that is exiting (e.g. wpa_supplicant), which ends + * in brcmf_p2p_del_vif() under RTNL and the wiphy mutex. If + * both paths remove the interface, the wdev is unregistered + * twice and the vif is used after being freed. Take the same + * locks and re-read iflist, so only one path removes it. + */ + rtnl_lock(); + wiphy_lock(drvr->wiphy); + ifp = drvr->iflist[i]; + if (ifp) + brcmf_remove_interface(ifp, true); + wiphy_unlock(drvr->wiphy); + rtnl_unlock(); } brcmf_proto_detach(drvr); diff --git a/drivers/net/wireless/marvell/libertas/tx.c b/drivers/net/wireless/marvell/libertas/tx.c index 13d08022e4141..27304a98787d6 100644 --- a/drivers/net/wireless/marvell/libertas/tx.c +++ b/drivers/net/wireless/marvell/libertas/tx.c @@ -117,13 +117,6 @@ netdev_tx_t lbs_hard_start_xmit(struct sk_buff *skb, struct net_device *dev) if (priv->wdev->iftype == NL80211_IFTYPE_MONITOR) { struct tx_radiotap_hdr *rtap_hdr = (void *)skb->data; - if (skb->len < sizeof(*rtap_hdr) + 4 + ETH_ALEN) { - lbs_deb_tx("tx err: short monitor frame %u\n", skb->len); - dev->stats.tx_dropped++; - dev->stats.tx_errors++; - goto free; - } - /* set txpd fields from the radiotap header */ txpd->tx_control = cpu_to_le32(convert_radiotap_rate_to_mv(rtap_hdr->rate)); diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index 476578e2fbf0c..9188a27e78652 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -539,6 +539,7 @@ struct mt76_hw_cap { #define MT_DRV_HW_MGMT_TXQ BIT(4) #define MT_DRV_AMSDU_OFFLOAD BIT(5) #define MT_DRV_IGNORE_TXS_FAILED BIT(6) +#define MT_DRV_HW_PS_BUFFERING BIT(7) struct mt76_driver_ops { u32 drv_flags; @@ -1424,6 +1425,13 @@ mtxq_to_txq(struct mt76_txq *mtxq) return container_of(ptr, struct ieee80211_txq, drv_priv); } +/* peer-wide state uses the wcid of the primary link */ +static inline struct mt76_wcid * +mt76_wcid_primary(struct mt76_wcid *wcid) +{ + return wcid->def_wcid ? wcid->def_wcid : wcid; +} + static inline struct ieee80211_sta * wcid_to_sta(struct mt76_wcid *wcid) { @@ -1541,6 +1549,8 @@ void mt76_release_buffered_frames(struct ieee80211_hw *hw, u16 tids, int nframes, enum ieee80211_frame_release_type reason, bool more_data); +void mt76_sta_ps_transition(struct mt76_dev *dev, struct mt76_wcid *wcid, + bool ps); bool mt76_has_tx_pending(struct mt76_phy *phy); int mt76_update_channel(struct mt76_phy *phy); void mt76_update_survey(struct mt76_phy *phy); diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/init.c b/drivers/net/wireless/mediatek/mt76/mt7603/init.c index 10f2ec70c792b..218ec406d2a87 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7603/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7603/init.c @@ -230,6 +230,7 @@ mt7603_mac_init(struct mt7603_dev *dev) eth_broadcast_addr(bc_addr); mt7603_wtbl_init(dev, MT7603_WTBL_RESERVED, -1, bc_addr); + mt76_wcid_init(&dev->global_sta.wcid, 0); dev->global_sta.wcid.idx = MT7603_WTBL_RESERVED; rcu_assign_pointer(dev->mt76.wcid[MT7603_WTBL_RESERVED], &dev->global_sta.wcid); diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h index a9a4a87ae0a76..fe03bc70e63a4 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h @@ -1068,6 +1068,7 @@ enum { MCU_UNI_EVENT_IE_COUNTDOWN = 0x09, MCU_UNI_EVENT_COREDUMP = 0x0a, MCU_UNI_EVENT_BSS_BEACON_LOSS = 0x0c, + MCU_UNI_EVENT_PS_SYNC = 0x0d, MCU_UNI_EVENT_SCAN_DONE = 0x0e, MCU_UNI_EVENT_RDD_REPORT = 0x11, MCU_UNI_EVENT_ROC = 0x27, diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c index 2ab38f9e52581..e768f4f72ed8e 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c @@ -1208,6 +1208,12 @@ static void mt7915_unregister_ext_phy(struct mt7915_dev *dev) static void mt7915_stop_hardware(struct mt7915_dev *dev) { + int i; + + mt76_for_each_q_rx(&dev->mt76, i) { + if (!mt76_queue_is_wed_rro(&dev->mt76.q_rx[i])) + napi_disable(&dev->mt76.napi[i]); + } mt7915_mcu_exit(dev); mt76_connac2_tx_token_put(&dev->mt76); mt7915_dma_cleanup(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index 4783e5f52d229..1830398ff719f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -805,6 +805,7 @@ __mt7915_drop_sta(void *ptr, u8 *mac, struct ieee80211_vif *vif) msta = (struct mt7915_sta *)sta->drv_priv; mt7915_mcu_add_sta(data->dev, vif, sta, CONN_STATE_DISCONNECT, false); + msta->wcid.tx_info &= ~MT_WCID_TX_INFO_SET; msta->wcid.sta_disabled = 1; msta->wcid.sta = 0; } @@ -871,6 +872,7 @@ int mt7915_mac_sta_event(struct mt76_dev *mdev, struct ieee80211_vif *vif, mutex_unlock(&dev->mt76.mutex); mt7915_mcu_add_sta(dev, vif, sta, CONN_STATE_DISCONNECT, false); + msta->wcid.tx_info &= ~MT_WCID_TX_INFO_SET; msta->wcid.sta_disabled = 1; msta->wcid.sta = 0; return 0; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index b52b678330604..8603915ddc2ef 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -781,7 +781,8 @@ mt7925_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi, val = FIELD_PREP(MT_TXD1_WLAN_IDX, wcid->idx) | FIELD_PREP(MT_TXD1_OWN_MAC, omac_idx); - if (band_idx) + /* 0xff means BAND_AUTO to the firmware, and TGID cannot encode it */ + if (band_idx && band_idx != 0xff) val |= FIELD_PREP(MT_TXD1_TGID, band_idx); txwi[1] = cpu_to_le32(val); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 9f080da13341b..e6e5c26307a11 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -2220,6 +2220,18 @@ mt7925_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif, mvif->valid_links = new_links; + /* Restore the legacy BSS after disabling MLO */ + if (old_links && !new_links) { + mt792x_mac_link_bss_remove(dev, &mvif->bss_conf, + &mvif->sta.deflink); + err = mt7925_mac_link_bss_add(dev, &vif->bss_conf, + &mvif->sta.deflink); + if (err < 0) { + mt792x_mutex_release(dev); + return err; + } + } + mt792x_mutex_release(dev); return 0; diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.c b/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.c index 946dd7956e4a0..ce5d5909d838d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.c +++ b/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.c @@ -108,68 +108,98 @@ mt792x_asar_acpi_read_mtcl(struct mt792x_dev *dev, u8 **table, u8 *version) return ret; } +/* A table's layout is decided by the version byte in MTCL, but some firmware + * labels a version 2 table as version 1. Accept whichever layout the table + * actually has: for every accepted entry count the two do not overlap, since + * one is 6 + 5n bytes and the other 7 + 7n. + */ +static bool +mt792x_asar_fits(u8 *table, int len, int prelen, int sarlen, int nr_off, + int min, int max) +{ + int tblcnt; + + if (len < prelen || (len - prelen) % sarlen) + return false; + + tblcnt = (len - prelen) / sarlen; + if (tblcnt < min || tblcnt > max) + return false; + + return tblcnt == table[nr_off]; +} + /* MTDS : Dynamic SAR Power Table */ static int -mt792x_asar_acpi_read_mtds(struct mt792x_dev *dev, u8 **table, u8 version) +mt792x_asar_acpi_read_mtds(struct mt792x_dev *dev, u8 **table, u8 version, + u8 *used) { - int len, ret, sarlen, prelen, tblcnt; - bool enable; + int len, ret; + u8 *t; ret = mt792x_acpi_read(dev, MT792x_ACPI_MTDS, table, &len); if (ret) return ret; + t = *table; + /* Table content validation */ - switch (version) { - case 1: - enable = ((struct mt792x_asar_dyn *)*table)->enable; - sarlen = sizeof(struct mt792x_asar_dyn_limit); - prelen = sizeof(struct mt792x_asar_dyn); - break; - case 2: - enable = ((struct mt792x_asar_dyn_v2 *)*table)->enable; - sarlen = sizeof(struct mt792x_asar_dyn_limit_v2); - prelen = sizeof(struct mt792x_asar_dyn_v2); - break; - default: + if (mt792x_asar_fits(t, len, sizeof(struct mt792x_asar_dyn), + sizeof(struct mt792x_asar_dyn_limit), + offsetof(struct mt792x_asar_dyn, nr_tbl), + MT792x_ASAR_MIN_DYN, MT792x_ASAR_MAX_DYN)) + *used = 1; + else if (mt792x_asar_fits(t, len, sizeof(struct mt792x_asar_dyn_v2), + sizeof(struct mt792x_asar_dyn_limit_v2), + offsetof(struct mt792x_asar_dyn_v2, nr_tbl), + MT792x_ASAR_MIN_DYN, MT792x_ASAR_MAX_DYN)) + *used = 2; + else return -EINVAL; - } - tblcnt = (len - prelen) / sarlen; - if (!enable || - tblcnt > MT792x_ASAR_MAX_DYN || tblcnt < MT792x_ASAR_MIN_DYN) + if (!t[offsetof(struct mt792x_asar_dyn, enable)]) return -EINVAL; + if (version && *used != version) + dev_info(dev->mt76.dev, + "MTDS is v%u, MTCL says v%u; using v%u\n", + *used, version, *used); + return 0; } /* MTGS : Geo SAR Power Table */ static int -mt792x_asar_acpi_read_mtgs(struct mt792x_dev *dev, u8 **table, u8 version) +mt792x_asar_acpi_read_mtgs(struct mt792x_dev *dev, u8 **table, u8 version, + u8 *used) { - int len, ret, sarlen, prelen, tblcnt; + int len, ret; + u8 *t; ret = mt792x_acpi_read(dev, MT792x_ACPI_MTGS, table, &len); if (ret) return ret; + t = *table; + /* Table content validation */ - switch (version) { - case 1: - sarlen = sizeof(struct mt792x_asar_geo_limit); - prelen = sizeof(struct mt792x_asar_geo); - break; - case 2: - sarlen = sizeof(struct mt792x_asar_geo_limit_v2); - prelen = sizeof(struct mt792x_asar_geo_v2); - break; - default: + if (mt792x_asar_fits(t, len, sizeof(struct mt792x_asar_geo), + sizeof(struct mt792x_asar_geo_limit), + offsetof(struct mt792x_asar_geo, nr_tbl), + MT792x_ASAR_MIN_GEO, MT792x_ASAR_MAX_GEO)) + *used = 1; + else if (mt792x_asar_fits(t, len, sizeof(struct mt792x_asar_geo_v2), + sizeof(struct mt792x_asar_geo_limit_v2), + offsetof(struct mt792x_asar_geo_v2, nr_tbl), + MT792x_ASAR_MIN_GEO, MT792x_ASAR_MAX_GEO)) + *used = 2; + else return -EINVAL; - } - tblcnt = (len - prelen) / sarlen; - if (tblcnt > MT792x_ASAR_MAX_GEO || tblcnt < MT792x_ASAR_MIN_GEO) - return -EINVAL; + if (version && *used != version) + dev_info(dev->mt76.dev, + "MTGS is v%u, MTCL says v%u; using v%u\n", + *used, version, *used); return 0; } @@ -205,14 +235,16 @@ int mt792x_init_acpi_sar(struct mt792x_dev *dev) asar->countrylist = NULL; } - ret = mt792x_asar_acpi_read_mtds(dev, (u8 **)&asar->dyn, asar->ver); + ret = mt792x_asar_acpi_read_mtds(dev, (u8 **)&asar->dyn, asar->ver, + &asar->dyn_ver); if (ret) { devm_kfree(dev->mt76.dev, asar->dyn); asar->dyn = NULL; } /* MTGS is optional */ - ret = mt792x_asar_acpi_read_mtgs(dev, (u8 **)&asar->geo, asar->ver); + ret = mt792x_asar_acpi_read_mtgs(dev, (u8 **)&asar->geo, asar->ver, + &asar->geo_ver); if (ret) { devm_kfree(dev->mt76.dev, asar->geo); asar->geo = NULL; @@ -254,7 +286,7 @@ mt792x_asar_get_geo_pwr(struct mt792x_phy *phy, break; } - if (asar->ver == 1) { + if (asar->geo_ver == 1) { band_pwr = &asar->geo->tbl[idx].band[0]; max = ARRAY_SIZE(asar->geo->tbl[idx].band); } else { @@ -279,10 +311,18 @@ mt792x_asar_get_geo_pwr(struct mt792x_phy *phy, if (idx >= max) return dyn_power; - geo_power = (band_pwr + idx)->pwr; - dyn_power += (band_pwr + idx)->offset; + band_pwr += idx; + + /* Some OEM tables carry 0xff here; narrowed to s8 that is -1 and + * wins the min() below. Take it as no limit. + */ + if (band_pwr->pwr == 0xff) + return dyn_power; + + geo_power = band_pwr->pwr; - return min(geo_power, dyn_power); + return min_t(int, geo_power, + min_t(int, dyn_power + band_pwr->offset, 127)); } static s8 @@ -297,7 +337,7 @@ mt792x_asar_range_pwr(struct mt792x_phy *phy, if (!capa) return 127; - if (asar->ver == 1) { + if (asar->dyn_ver == 1) { limit = &asar->dyn->tbl[0].frp[0]; max = ARRAY_SIZE(asar->dyn->tbl[0].frp); } else { @@ -315,7 +355,9 @@ mt792x_asar_range_pwr(struct mt792x_phy *phy, else band = NL80211_BAND_2GHZ; - return mt792x_asar_get_geo_pwr(phy, band, limit[idx]); + /* 0xff: no dynamic limit, see mt792x_asar_get_geo_pwr() */ + return mt792x_asar_get_geo_pwr(phy, band, + limit[idx] == 0xff ? 127 : limit[idx]); } int mt792x_init_acpi_sar_power(struct mt792x_phy *phy, bool set_default) diff --git a/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.h b/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.h index 4740330738315..1d42f24b05f66 100644 --- a/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.h +++ b/drivers/net/wireless/mediatek/mt76/mt792x_acpi_sar.h @@ -105,6 +105,8 @@ struct mt792x_asar_fg { struct mt792x_acpi_sar { u8 ver; + u8 dyn_ver; + u8 geo_ver; union { struct mt792x_asar_dyn *dyn; struct mt792x_asar_dyn_v2 *dyn_v2; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c index fa74aba426903..95fb30404b857 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c @@ -536,6 +536,7 @@ mt7996_init_wiphy(struct ieee80211_hw *hw, struct mtk_wed_device *wed) ieee80211_hw_set(hw, NO_VIRTUAL_MONITOR); ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID); ieee80211_hw_set(hw, CHANCTX_STA_CSA); + ieee80211_hw_set(hw, CONNECTION_MONITOR); hw->max_tx_fragments = 4; wiphy->txq_memory_limit = 32 << 20; /* 32 MiB */ @@ -1237,7 +1238,7 @@ static int mt7996_variant_fem_init(struct mt7996_dev *dev) adie_ver = u32_get_bits(regval, MT_ADIE_VERSION_MASK); idx = MT7976C_EFUSE_OFFSET % MT7996_EEPROM_BLOCK_SIZE; is_7976c = adie_ver == 0x8a10 || adie_ver == 0x8b00 || - adie_ver == 0x8c10 || buf[idx] == 0xc; + adie_ver == 0x8c10 || (!ret && buf[idx] == 0xc); adie_id = u32_get_bits(regval, MT_ADIE_CHIP_ID_MASK); if (adie_id == 0x7975 || adie_id == 0x7979 || diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c index c6140412ed322..ae006f29091ff 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c @@ -310,7 +310,14 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, if (vif->type == NL80211_IFTYPE_AP) return mt7996_mcu_mld_link_oper(dev, link_conf, link, true); - return 0; + + /* update the link address */ + ret = mt7996_mcu_add_dev_info(phy, vif, link_conf, mlink, true); + if (ret) + return ret; + + return mt7996_mcu_add_bss_info(phy, vif, link_conf, mlink, + msta_link, true); } mlink->idx = __ffs64(~dev->mt76.vif_mask); @@ -1422,6 +1429,7 @@ mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif, else if (sta->mlo && links == BIT(link_id)) /* last link */ mt7996_mcu_teardown_mld_sta(dev, link, msta_link); + msta_link->wcid.tx_info &= ~MT_WCID_TX_INFO_SET; msta_link->wcid.sta_disabled = 1; msta_link->wcid.sta = 0; links = links & ~BIT(link_id); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h index c673e986ecb52..288b923750c0d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h @@ -284,6 +284,9 @@ struct mt7996_mcu_wed_rro_ba_delete_event { __le16 len; __le16 session_id; + __le16 wlan_id; + u8 tid; + u8 __rsv1; u8 __rsv2[2]; } __packed; diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c index b03be0eb47128..65703dbb185c9 100644 --- a/drivers/net/wireless/mediatek/mt76/tx.c +++ b/drivers/net/wireless/mediatek/mt76/tx.c @@ -234,6 +234,8 @@ mt76_tx_check_non_aql(struct mt76_dev *dev, struct mt76_wcid *wcid, if (!wcid || info->tx_time_est) return; + wcid = mt76_wcid_primary(wcid); + pending = atomic_dec_return(&wcid->non_aql_packets); if (pending < 0) atomic_cmpxchg(&wcid->non_aql_packets, pending, 0); @@ -267,6 +269,21 @@ void __mt76_tx_complete_skb(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff * wcid = __mt76_wcid_ptr(dev, wcid_idx); mt76_tx_check_non_aql(dev, wcid, skb); + if (wcid && (dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING) && + test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { + struct ieee80211_sta *sta = wcid_to_sta(wcid); + + if (sta) { + struct ieee80211_hw *hw = mt76_phy_hw(dev, wcid->phy_idx); + int i; + + for (i = 0; i < ARRAY_SIZE(sta->txq); i++) + if (sta->txq[i]) + ieee80211_schedule_txq(hw, sta->txq[i]); + mt76_worker_schedule(&dev->tx_worker); + } + } + #ifdef CONFIG_NL80211_TESTMODE if (mt76_is_testmode_skb(dev, skb, &hw)) { struct mt76_phy *phy = hw->priv; @@ -329,7 +346,8 @@ __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb, if (!non_aql) return idx; - pending = atomic_inc_return(&wcid->non_aql_packets); + /* the hardware can report the completion on a different link */ + pending = atomic_inc_return(&mt76_wcid_primary(wcid)->non_aql_packets); if (stop && pending >= MT_MAX_NON_AQL_PKT) *stop = true; @@ -476,10 +494,15 @@ mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_queue *q, bool stop = false; int idx; - if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) - return 0; + if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { + if (!(dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING)) + return 0; + if (ieee80211_txq_aql_pending(phy->hw, txq)) + return 0; + } - if (atomic_read(&wcid->non_aql_packets) >= MT_MAX_NON_AQL_PKT) + if (atomic_read(&mt76_wcid_primary(wcid)->non_aql_packets) >= + MT_MAX_NON_AQL_PKT) return 0; skb = mt76_txq_dequeue(phy, mtxq); @@ -497,6 +520,9 @@ mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_queue *q, if (idx < 0) return idx; + if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) + goto out; + do { if (test_bit(MT76_RESET, &phy->state) || phy->offchannel) break; @@ -522,6 +548,7 @@ mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_queue *q, n_frames++; } while (1); +out: spin_lock(&q->lock); dev->queue_ops->kick(dev, q); spin_unlock(&q->lock); @@ -548,10 +575,7 @@ mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) mtxq = (struct mt76_txq *)txq->drv_priv; wcid = __mt76_wcid_ptr(dev, mtxq->wcid); - if (!wcid || test_bit(MT_WCID_FLAG_PS, &wcid->flags)) - continue; - - if (atomic_read(&wcid->non_aql_packets) >= MT_MAX_NON_AQL_PKT) + if (!wcid) continue; phy = mt76_dev_phy(dev, wcid->phy_idx); @@ -559,6 +583,25 @@ mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) continue; q = phy->q_tx[qid]; + + if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) { + if (!(dev->drv->drv_flags & MT_DRV_HW_PS_BUFFERING)) + continue; + + if (!mt76_txq_stopped(q)) + n_frames = mt76_txq_send_burst(phy, q, mtxq, wcid); + + ieee80211_return_txq(phy->hw, txq, false); + + if (unlikely(n_frames < 0)) + return n_frames; + ret += n_frames; + continue; + } + + if (atomic_read(&mt76_wcid_primary(wcid)->non_aql_packets) >= + MT_MAX_NON_AQL_PKT) + continue; if (dev->queue_ops->tx_cleanup && q->queued + 2 * MT_TXQ_FREE_THR >= q->ndesc) { dev->queue_ops->tx_cleanup(dev, q, false); @@ -629,7 +672,8 @@ mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid, if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) && !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && !ieee80211_is_data_present(hdr->frame_control) && - (!ieee80211_is_bufferable_mmpdu(skb) || + (!(wcid->tx_info & MT_WCID_TX_INFO_SET) || + !ieee80211_is_bufferable_mmpdu(skb) || ieee80211_is_deauth(hdr->frame_control) || ieee80211_is_disassoc(hdr->frame_control) || head == &wcid->tx_offchannel)) @@ -765,6 +809,33 @@ void mt76_stop_tx_queues(struct mt76_phy *phy, struct ieee80211_sta *sta, } EXPORT_SYMBOL_GPL(mt76_stop_tx_queues); +void mt76_sta_ps_transition(struct mt76_dev *dev, struct mt76_wcid *wcid, + bool ps) +{ + struct ieee80211_sta *sta; + struct ieee80211_hw *hw; + int i; + + if (ps) { + set_bit(MT_WCID_FLAG_PS, &wcid->flags); + mt76_worker_schedule(&dev->tx_worker); + return; + } + + clear_bit(MT_WCID_FLAG_PS, &wcid->flags); + + sta = wcid_to_sta(wcid); + if (!sta) + return; + + hw = mt76_phy_hw(dev, wcid->phy_idx); + for (i = 0; i < ARRAY_SIZE(sta->txq); i++) + if (sta->txq[i]) + ieee80211_schedule_txq(hw, sta->txq[i]); + mt76_worker_schedule(&dev->tx_worker); +} +EXPORT_SYMBOL_GPL(mt76_sta_ps_transition); + void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq) { struct mt76_phy *phy = hw->priv; diff --git a/drivers/net/wireless/mediatek/mt7932/Kconfig b/drivers/net/wireless/mediatek/mt7932/Kconfig index 49a6fbad0e67e..7f0efefffee3f 100644 --- a/drivers/net/wireless/mediatek/mt7932/Kconfig +++ b/drivers/net/wireless/mediatek/mt7932/Kconfig @@ -4,11 +4,15 @@ config MT7932_FULLMAC depends on PCI && OF && ARM64 && IOMMU_DMA depends on PCIE_APPLE_PIODMA_DIAG && CFG80211 select FW_LOADER + select CRYPTO + select CRYPTO_HMAC + select CRYPTO_SHA1 + select CRYPTO_LIB_SHA256 help Support for the Wi-Fi function of the MediaTek MT7932 radio in the Apple MacBook Neo (J700). The firmware runs association and the - WPA2-PSK handshake; the driver is a cfg80211 station on 2.4 GHz and - on the 5 GHz UNII-1, -2, -2e and -3 channels that the regulatory + WPA2-PSK or WPA3-SAE handshake; the driver is a cfg80211 station on + 2.4 GHz and the 5 GHz UNII-1, -2, -2e and -3 channels that the regulatory domain permits. Radar (DFS) channels are scanned passively but not joined. Other MediaTek devices and other machines are not matched. @@ -41,4 +45,5 @@ config MT7932_KUNIT_TEST to the driver, with synthetic, truncated and corrupted input and without radio hardware: receive descriptors, including frames the hardware did not decrypt, command replies, regulatory snapshots - and country power packages. + and country power packages. Also check connection admission, SAE + request encoding and WPA passphrase derivation. diff --git a/drivers/net/wireless/mediatek/mt7932/Makefile b/drivers/net/wireless/mediatek/mt7932/Makefile index 781d4530fbee1..c319e93d49277 100644 --- a/drivers/net/wireless/mediatek/mt7932/Makefile +++ b/drivers/net/wireless/mediatek/mt7932/Makefile @@ -4,3 +4,4 @@ mt7932-fullmac-y := mt7932.o recovery.o calibration_transport.o data.o frontend. mt7932-fullmac-$(CONFIG_MT7932_RF_KUNIT_TEST) += rf-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += input-test.o mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += regulatory-test.o +mt7932-fullmac-$(CONFIG_MT7932_KUNIT_TEST) += association-test.o diff --git a/drivers/net/wireless/mediatek/mt7932/association-test.c b/drivers/net/wireless/mediatek/mt7932/association-test.c new file mode 100644 index 0000000000000..7ee7dad7fb206 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt7932/association-test.c @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include + +#include "association.h" + +static void mt_sae_pmk_vectors_test(struct kunit *test) +{ + /* IEEE 802.11i-2004, Annex H.4. These are public test credentials. */ + static const struct { + const char *password, *ssid, *pmk; + } vectors[] = { + { + "password", "IEEE", + "f42c6fc52df0ebef9ebb4b90b38a5f902e83fe1b135a70e23aed762e9710a12e", + }, { + "ThisIsAPassword", "ThisIsASSID", + "0dc0d6eb90555ed6419756b9a15ec3e3209b63df707dd508d14581f8982721af", + }, { + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ", + "becb93866bb8c3832cb777c2f559807c8c59afcb6eae734885001300a981cc62", + }, + }; + u8 pmk[32], expected[32]; + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(vectors); i++) { + KUNIT_ASSERT_EQ(test, hex2bin(expected, vectors[i].pmk, 32), 0); + ret = mt7932_sae_pmk(pmk, vectors[i].password, + strlen(vectors[i].password), vectors[i].ssid, + strlen(vectors[i].ssid)); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_MEMEQ(test, pmk, expected, sizeof(pmk)); + } + memzero_explicit(pmk, sizeof(pmk)); +} + +static void mt_sae_pmk_bounds_test(struct kunit *test) +{ + u8 password[64], ssid[33], pmk[32], zero[32] = {}; + static const struct { + size_t password_length, ssid_length; + } invalid[] = { { 7, 4 }, { 64, 4 }, { 8, 0 }, { 8, 33 } }; + unsigned int i; + + memset(password, 'p', sizeof(password)); + memset(ssid, 's', sizeof(ssid)); + for (i = 0; i < ARRAY_SIZE(invalid); i++) { + int ret; + + memset(pmk, 0xa5, sizeof(pmk)); + ret = mt7932_sae_pmk(pmk, password, invalid[i].password_length, + ssid, invalid[i].ssid_length); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_MEMEQ(test, pmk, zero, sizeof(pmk)); + } + KUNIT_EXPECT_EQ(test, mt7932_sae_pmk(pmk, NULL, 8, ssid, 4), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_sae_pmk(pmk, password, 8, NULL, 4), -EINVAL); + KUNIT_EXPECT_EQ(test, mt7932_sae_pmk(pmk, password, 63, ssid, 32), 0); + memzero_explicit(pmk, sizeof(pmk)); +} + +static void mt_connect_body_test(struct kunit *test) +{ + static const struct mt7932_peer_ids ids = { { 1, 2 }, { 3, 4, 5, 6 } }; + static const u8 bssid[6] = { 2, 0, 0, 0, 0, 1 }; + static const u8 channels[] = { 11, 157 }; + u8 pmk[32], password[63], body[856], expected[856]; + unsigned int mode, band; + + memset(pmk, 0xa5, sizeof(pmk)); + memset(password, 'p', sizeof(password)); + for (mode = 0; mode < 4; mode++) { + bool open = mode == 0, sae = mode == 2, owe = mode == 3; + + for (band = 0; band < ARRAY_SIZE(channels); band++) { + int ret; + + memset(body, 0xff, sizeof(body)); + ret = mt7932_connect_body(body, sizeof(body), "IEEE", 4, + open || owe ? NULL : pmk, bssid, channels[band], + 40, 2, &ids, open, + sae ? password : NULL, sae ? 63 : 0, owe); + KUNIT_ASSERT_EQ(test, ret, 856); + /* Complete golden requests also catch writes to reserved + * fields and keep the established PSK and OPEN encodings. + */ + memset(expected, 0, sizeof(expected)); + expected[1] = open || owe ? 1 : sae ? 8 : 4; + expected[4] = open || owe ? 0 : sae ? 6 : 0x10; + expected[8] = 0x78; + expected[0x12] = sae ? 5 : 1; + if (owe) + expected[0x20e] = 4; + if (!open && !owe) { + put_unaligned_le16(sae ? 0x1000 : 8, expected + 0x14); + expected[0x40] = sae ? 63 : 32; + if (sae) { + expected[0x44] = 10; + memcpy(expected + 0x4c, password, 63); + expected[0x1c8] = 32; + memcpy(expected + 0x1cc, pmk, 32); + expected[0x20d] = 1; + } else { + expected[0x44] = 6; + memcpy(expected + 0x4c, pmk, 32); + } + expected[0x220] = 5; + } + expected[0x18] = 4; + memcpy(expected + 0x1c, "IEEE", 4); + expected[0x1c0] = 2; + expected[0x228] = 20; + memcpy(expected + 0x32c, bssid, sizeof(bssid)); + expected[0x338] = band + 1; + expected[0x339] = channels[band]; + expected[0x33c] = 1; + expected[0x33d] = 2; + expected[0x33e] = 4; + memcpy(expected + 0x340, ids.station, sizeof(ids.station)); + memcpy(expected + 0x348, ids.wtbl, sizeof(ids.wtbl)); + expected[0x354] = 1; + KUNIT_EXPECT_MEMEQ(test, body, expected, sizeof(body)); + } + } + memzero_explicit(body, sizeof(body)); + memzero_explicit(expected, sizeof(expected)); + memzero_explicit(pmk, sizeof(pmk)); +} + +static void mt_sae_body_bounds_test(struct kunit *test) +{ + static const struct mt7932_peer_ids ids = { { 1, 2 }, { 3, 4, 5, 6 } }; + static const u8 bssid[6] = { 2, 0, 0, 0, 0, 1 }; + static const struct { + unsigned int capacity, password_length; + bool open, password, pmk; + int expected; + } requests[] = { + { 856, 8, false, true, true, 856 }, + { 856, 63, false, true, true, 856 }, + { 855, 8, false, true, true, -EINVAL }, + { 856, 7, false, true, true, -EINVAL }, + { 856, 64, false, true, true, -EINVAL }, + { 856, 8, false, false, true, -EINVAL }, + { 856, 0, false, true, true, -EINVAL }, + { 856, 8, true, true, false, -EINVAL }, + { 856, 8, false, true, false, -EINVAL }, + }; + u8 body[857], password[64] = {}, pmk[32] = {}; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(requests); i++) { + int ret; + + memset(body, 0xa5, sizeof(body)); + ret = mt7932_connect_body(body, requests[i].capacity, "IEEE", 4, + requests[i].pmk ? pmk : NULL, bssid, 11, 20, 2, &ids, + requests[i].open, requests[i].password ? password : NULL, + requests[i].password_length, false); + KUNIT_EXPECT_EQ(test, ret, requests[i].expected); + KUNIT_EXPECT_EQ(test, body[856], (u8)0xa5); + if (requests[i].expected < 0) + KUNIT_EXPECT_EQ(test, body[0], (u8)0xa5); + } + memzero_explicit(body, sizeof(body)); +} + +enum mt_auth_change { + MT_AUTH_NONE, MT_AUTH_TYPE, MT_AUTH_MFP, MT_AUTH_PWE, MT_AUTH_PASSWORD_LEN, + MT_AUTH_NO_PASSWORD, MT_AUTH_WPA, MT_AUTH_PAIR_COUNT, + MT_AUTH_PAIR, MT_AUTH_GROUP, MT_AUTH_NO_AKM, MT_AUTH_PSK_FIRST, + MT_AUTH_SAE_FIRST, MT_AUTH_OTHER_FIRST, MT_AUTH_OTHER_AKM, MT_AUTH_PSK, +}; + +static void mt_connect_auth_test(struct kunit *test) +{ + static const struct { + const char *name; + enum mt_auth_change change; + unsigned int value; + int expected; + } cases[] = { + { "pure PSK", MT_AUTH_PSK, 0, WLAN_AKM_SUITE_PSK }, + { "PSK missing PMK", MT_AUTH_PSK, 1, -EOPNOTSUPP }, + { "PSK required MFP", MT_AUTH_PSK, 2, -EOPNOTSUPP }, + { "pure SAE", MT_AUTH_NONE, 0, WLAN_AKM_SUITE_SAE }, + { "optional MFP", MT_AUTH_MFP, NL80211_MFP_OPTIONAL, WLAN_AKM_SUITE_SAE }, + { "no MFP", MT_AUTH_MFP, NL80211_MFP_NO, -EOPNOTSUPP }, + { "wrong auth", MT_AUTH_TYPE, NL80211_AUTHTYPE_OPEN_SYSTEM, -EOPNOTSUPP }, + { "automatic auth", MT_AUTH_TYPE, NL80211_AUTHTYPE_AUTOMATIC, -EOPNOTSUPP }, + { "both PWE", MT_AUTH_PWE, NL80211_SAE_PWE_BOTH, WLAN_AKM_SUITE_SAE }, + { "H2E", MT_AUTH_PWE, NL80211_SAE_PWE_HASH_TO_ELEMENT, WLAN_AKM_SUITE_SAE }, + { "HnP", MT_AUTH_PWE, NL80211_SAE_PWE_HUNT_AND_PECK, WLAN_AKM_SUITE_SAE }, + { "unknown PWE", MT_AUTH_PWE, 255, -EOPNOTSUPP }, + { "short password", MT_AUTH_PASSWORD_LEN, 7, -EOPNOTSUPP }, + { "long password", MT_AUTH_PASSWORD_LEN, 64, -EOPNOTSUPP }, + { "maximum password", MT_AUTH_PASSWORD_LEN, 63, WLAN_AKM_SUITE_SAE }, + { "no password", MT_AUTH_NO_PASSWORD, 0, -EOPNOTSUPP }, + { "SAE VERSION_3", MT_AUTH_WPA, NL80211_WPA_VERSION_3, WLAN_AKM_SUITE_SAE }, + { "SAE legacy VERSION_2", MT_AUTH_WPA, NL80211_WPA_VERSION_2, WLAN_AKM_SUITE_SAE }, + { "SAE mixed RSN versions", MT_AUTH_WPA, + NL80211_WPA_VERSION_2 | NL80211_WPA_VERSION_3, WLAN_AKM_SUITE_SAE }, + { "no WPA version", MT_AUTH_WPA, 0, -EOPNOTSUPP }, + { "unknown WPA version", MT_AUTH_WPA, 8, -EOPNOTSUPP }, + { "WPA1", MT_AUTH_WPA, NL80211_WPA_VERSION_1, -EOPNOTSUPP }, + { "mixed WPA", MT_AUTH_WPA, 3, -EOPNOTSUPP }, + { "no pairwise", MT_AUTH_PAIR_COUNT, 0, -EOPNOTSUPP }, + { "multiple pairwise", MT_AUTH_PAIR_COUNT, 2, -EOPNOTSUPP }, + { "TKIP pairwise", MT_AUTH_PAIR, WLAN_CIPHER_SUITE_TKIP, -EOPNOTSUPP }, + { "TKIP group", MT_AUTH_GROUP, WLAN_CIPHER_SUITE_TKIP, -EOPNOTSUPP }, + { "no AKM", MT_AUTH_NO_AKM, 0, -EOPNOTSUPP }, + { "SAE EXT KEY", MT_AUTH_OTHER_AKM, SUITE(0x000fac, 24), -EOPNOTSUPP }, + { "PSK SHA256", MT_AUTH_OTHER_AKM, WLAN_AKM_SUITE_PSK_SHA256, -EOPNOTSUPP }, + { "SAE first with both credentials", MT_AUTH_SAE_FIRST, 0, WLAN_AKM_SUITE_SAE }, + { "SAE first missing password", MT_AUTH_SAE_FIRST, 1, -EOPNOTSUPP }, + { "PSK first optional MFP", MT_AUTH_PSK_FIRST, 0, WLAN_AKM_SUITE_PSK }, + { "PSK first missing PMK", MT_AUTH_PSK_FIRST, 1, -EOPNOTSUPP }, + { "PSK first required MFP", MT_AUTH_PSK_FIRST, 2, -EOPNOTSUPP }, + { "PSK first no MFP", MT_AUTH_PSK_FIRST, 3, WLAN_AKM_SUITE_PSK }, + { "PSK first mixed versions", MT_AUTH_PSK_FIRST, 4, WLAN_AKM_SUITE_PSK }, + { "PSK first VERSION_3 only", MT_AUTH_PSK_FIRST, 5, -EOPNOTSUPP }, + { "SAE first mixed versions", MT_AUTH_SAE_FIRST, 2, WLAN_AKM_SUITE_SAE }, + { "unsupported AKM before SAE", MT_AUTH_OTHER_FIRST, 0, -EOPNOTSUPP }, + }; + u8 password[64] = {}, pmk[32] = {}; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cases); i++) { + struct cfg80211_connect_params p = { + .auth_type = NL80211_AUTHTYPE_SAE, + .mfp = NL80211_MFP_REQUIRED, + .crypto = { + .wpa_versions = NL80211_WPA_VERSION_3, + .n_ciphers_pairwise = 1, + .ciphers_pairwise = { WLAN_CIPHER_SUITE_CCMP }, + .cipher_group = WLAN_CIPHER_SUITE_CCMP, + .n_akm_suites = 1, + .akm_suites = { WLAN_AKM_SUITE_SAE }, + .sae_pwd = password, + .sae_pwd_len = 8, + }, + }; + unsigned int value = cases[i].value; + + switch (cases[i].change) { + case MT_AUTH_NONE: + break; + case MT_AUTH_TYPE: + p.auth_type = value; + break; + case MT_AUTH_MFP: + p.mfp = value; + break; + case MT_AUTH_PWE: + p.crypto.sae_pwe = value; + break; + case MT_AUTH_PASSWORD_LEN: + p.crypto.sae_pwd_len = value; + break; + case MT_AUTH_NO_PASSWORD: + p.crypto.sae_pwd = NULL; + break; + case MT_AUTH_WPA: + p.crypto.wpa_versions = value; + break; + case MT_AUTH_PAIR_COUNT: + p.crypto.n_ciphers_pairwise = value; + break; + case MT_AUTH_PAIR: + p.crypto.ciphers_pairwise[0] = value; + break; + case MT_AUTH_GROUP: + p.crypto.cipher_group = value; + break; + case MT_AUTH_NO_AKM: + p.crypto.n_akm_suites = 0; + break; + case MT_AUTH_OTHER_AKM: + p.crypto.akm_suites[0] = value; + break; + case MT_AUTH_PSK: + p.crypto.wpa_versions = NL80211_WPA_VERSION_2; + p.crypto.akm_suites[0] = WLAN_AKM_SUITE_PSK; + p.crypto.psk = value == 1 ? NULL : pmk; + p.crypto.sae_pwd = NULL; + p.crypto.sae_pwd_len = 0; + p.auth_type = NL80211_AUTHTYPE_AUTOMATIC; + p.mfp = value == 2 ? NL80211_MFP_REQUIRED : NL80211_MFP_NO; + break; + case MT_AUTH_PSK_FIRST: + p.crypto.wpa_versions = value == 5 ? NL80211_WPA_VERSION_3 : + NL80211_WPA_VERSION_2; + if (value == 4) + p.crypto.wpa_versions |= NL80211_WPA_VERSION_3; + p.crypto.n_akm_suites = 2; + p.crypto.akm_suites[0] = WLAN_AKM_SUITE_PSK; + p.crypto.akm_suites[1] = WLAN_AKM_SUITE_SAE; + p.crypto.psk = value == 1 ? NULL : pmk; + p.auth_type = NL80211_AUTHTYPE_OPEN_SYSTEM; + p.mfp = value == 2 ? NL80211_MFP_REQUIRED : + value == 3 ? NL80211_MFP_NO : NL80211_MFP_OPTIONAL; + break; + case MT_AUTH_SAE_FIRST: + p.crypto.n_akm_suites = 2; + p.crypto.akm_suites[1] = WLAN_AKM_SUITE_PSK; + p.crypto.psk = pmk; + if (value == 1) + p.crypto.sae_pwd = NULL; + if (value == 2) + p.crypto.wpa_versions |= NL80211_WPA_VERSION_2; + p.mfp = NL80211_MFP_OPTIONAL; + break; + case MT_AUTH_OTHER_FIRST: + p.crypto.n_akm_suites = 2; + p.crypto.akm_suites[0] = WLAN_AKM_SUITE_PSK_SHA256; + p.crypto.akm_suites[1] = WLAN_AKM_SUITE_SAE; + break; + } + KUNIT_EXPECT_EQ_MSG(test, mt7932_connect_auth(&p), cases[i].expected, + "%s", cases[i].name); + } +} + +static void mt_owe_body_bounds_test(struct kunit *test) +{ + static const struct mt7932_peer_ids ids = { { 1, 2 }, { 3, 4, 5, 6 } }; + static const u8 bssid[6] = { 2, 0, 0, 0, 0, 1 }; + u8 body[857], pmk[32] = {}, password[8] = {}; + unsigned int i; + + for (i = 0; i < 6; i++) { + int ret; + + memset(body, 0xa5, sizeof(body)); + ret = mt7932_connect_body(body, i == 0 ? 855 : 856, "IEEE", 4, + i == 1 ? pmk : NULL, bssid, 11, 20, 2, &ids, + i == 2, i == 3 || i == 4 ? password : NULL, + i == 3 || i == 5 ? 8 : 0, true); + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, body[0], (u8)0xa5); + KUNIT_EXPECT_EQ(test, body[856], (u8)0xa5); + } +} + +static void mt_owe_connect_auth_test(struct kunit *test) +{ + u8 credential[32] = {}; + unsigned int i; + + for (i = 0; i < 20; i++) { + struct cfg80211_connect_params p = { + .auth_type = NL80211_AUTHTYPE_OPEN_SYSTEM, + .mfp = NL80211_MFP_REQUIRED, + .crypto = { + .wpa_versions = NL80211_WPA_VERSION_2, + .n_ciphers_pairwise = 1, + .ciphers_pairwise = { WLAN_CIPHER_SUITE_CCMP }, + .cipher_group = WLAN_CIPHER_SUITE_CCMP, + .n_akm_suites = 1, + .akm_suites = { WLAN_AKM_SUITE_OWE }, + }, + }; + int expected = -EOPNOTSUPP; + + switch (i) { + case 0: + expected = WLAN_AKM_SUITE_OWE; + break; + case 1: + p.crypto.psk = credential; + break; + case 2: + p.crypto.sae_pwd = credential; + break; + case 3: + p.crypto.sae_pwd_len = 8; + break; + case 4: + p.auth_type = NL80211_AUTHTYPE_AUTOMATIC; + break; + case 5: + p.auth_type = NL80211_AUTHTYPE_SAE; + break; + case 6: + p.mfp = NL80211_MFP_OPTIONAL; + break; + case 7: + p.mfp = NL80211_MFP_NO; + break; + case 8: + p.crypto.wpa_versions = NL80211_WPA_VERSION_1; + break; + case 9: + p.crypto.n_ciphers_pairwise = 0; + break; + case 10: + p.crypto.n_ciphers_pairwise = 2; + break; + case 11: + p.crypto.ciphers_pairwise[0] = WLAN_CIPHER_SUITE_TKIP; + break; + case 12: + p.crypto.cipher_group = WLAN_CIPHER_SUITE_TKIP; + break; + case 13: + p.crypto.n_akm_suites = 0; + break; + case 14: + p.key = credential; + break; + case 15: + p.key_len = 1; + break; + case 16: + p.want_1x = true; + break; + case 17: + p.crypto.n_akm_suites = 2; + p.crypto.akm_suites[1] = WLAN_AKM_SUITE_PSK; + expected = WLAN_AKM_SUITE_OWE; + break; + case 18: + p.crypto.n_akm_suites = 2; + p.crypto.akm_suites[0] = WLAN_AKM_SUITE_PSK; + p.crypto.akm_suites[1] = WLAN_AKM_SUITE_OWE; + break; + case 19: + p.crypto.wpa_versions |= NL80211_WPA_VERSION_1; + break; + } + KUNIT_EXPECT_EQ_MSG(test, mt7932_connect_auth(&p), expected, "case %u", i); + } +} + +static struct kunit_case mt_association_test_cases[] = { + KUNIT_CASE(mt_owe_body_bounds_test), + KUNIT_CASE(mt_owe_connect_auth_test), + KUNIT_CASE(mt_sae_pmk_vectors_test), + KUNIT_CASE(mt_sae_pmk_bounds_test), + KUNIT_CASE(mt_connect_body_test), + KUNIT_CASE(mt_sae_body_bounds_test), + KUNIT_CASE(mt_connect_auth_test), + {} +}; + +static struct kunit_suite mt_association_test_suite = { + .name = "mt7932-association", + .test_cases = mt_association_test_cases, +}; + +kunit_test_suite(mt_association_test_suite); diff --git a/drivers/net/wireless/mediatek/mt7932/association.h b/drivers/net/wireless/mediatek/mt7932/association.h index ba733a885e261..5963eeac25ee1 100644 --- a/drivers/net/wireless/mediatek/mt7932/association.h +++ b/drivers/net/wireless/mediatek/mt7932/association.h @@ -2,9 +2,136 @@ #ifndef MT7932_ASSOCIATION_H #define MT7932_ASSOCIATION_H +#include +#include +#include + #include "protocol.h" #include "channels.h" +#define MT7932_SAE_PASSWORD_MIN 8 +#define MT7932_SAE_PASSWORD_MAX 63 + +/* The first AKM is selected by userspace. Credentials for an alternative + * must never change the selected security arm on a transition BSS. + */ +static inline int +mt7932_connect_auth(const struct cfg80211_connect_params *p) +{ + u32 versions = p->crypto.wpa_versions; + + /* SAE offload clients use VERSION_3; older clients use VERSION_2. + * A transition request can carry both. The selected AKM still owns + * the firmware arm, so these version bits never enable a fallback. + */ + if (!versions || + (versions & ~(NL80211_WPA_VERSION_2 | NL80211_WPA_VERSION_3)) || + p->crypto.n_ciphers_pairwise != 1 || + p->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || + p->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || + p->crypto.n_akm_suites < 1) + return -EOPNOTSUPP; + + switch (p->crypto.akm_suites[0]) { + case WLAN_AKM_SUITE_PSK: + if ((versions & NL80211_WPA_VERSION_2) && p->crypto.psk && + (p->mfp == NL80211_MFP_NO || p->mfp == NL80211_MFP_OPTIONAL)) + return WLAN_AKM_SUITE_PSK; + break; + case WLAN_AKM_SUITE_OWE: + if (p->auth_type == NL80211_AUTHTYPE_OPEN_SYSTEM && + p->mfp == NL80211_MFP_REQUIRED && !p->crypto.psk && + !p->crypto.sae_pwd && !p->crypto.sae_pwd_len && + !p->key && !p->key_len && !p->want_1x) + return WLAN_AKM_SUITE_OWE; + break; + case WLAN_AKM_SUITE_SAE: + if (p->auth_type != NL80211_AUTHTYPE_SAE || + !p->crypto.sae_pwd || + p->crypto.sae_pwd_len < MT7932_SAE_PASSWORD_MIN || + p->crypto.sae_pwd_len > MT7932_SAE_PASSWORD_MAX || + (p->mfp != NL80211_MFP_REQUIRED && + p->mfp != NL80211_MFP_OPTIONAL)) + break; + /* Firmware selects PWE itself; there is no request field for it. */ + if (p->crypto.sae_pwe == NL80211_SAE_PWE_UNSPECIFIED || + p->crypto.sae_pwe == NL80211_SAE_PWE_HUNT_AND_PECK || + p->crypto.sae_pwe == NL80211_SAE_PWE_BOTH || + p->crypto.sae_pwe == NL80211_SAE_PWE_HASH_TO_ELEMENT) + return WLAN_AKM_SUITE_SAE; + break; + } + return -EOPNOTSUPP; +} + +/* The SAE offload needs both the password and a WPA passphrase-derived PMK. + * This is PBKDF2-HMAC-SHA1(password, SSID, 4096, 32), not host SAE crypto. + * Run only in sleepable context and erase every keyed intermediate. + */ +static inline int mt7932_sae_pmk(u8 pmk[32], const u8 *password, + size_t password_length, const u8 *ssid, + size_t ssid_length) +{ + u8 salt[32 + sizeof(__be32)]; + u8 u[SHA1_DIGEST_SIZE], next[SHA1_DIGEST_SIZE]; + u8 block_value[SHA1_DIGEST_SIZE]; + struct crypto_shash *tfm; + struct shash_desc *desc; + unsigned int block, iteration, i, copied = 0; + int ret; + + memzero_explicit(pmk, 32); + if (!password || password_length < MT7932_SAE_PASSWORD_MIN || + password_length > MT7932_SAE_PASSWORD_MAX || + !ssid || !ssid_length || ssid_length > 32) + return -EINVAL; + tfm = crypto_alloc_shash("hmac(sha1)", 0, 0); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + desc = kzalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL); + if (!desc) { + ret = -ENOMEM; + goto free_tfm; + } + desc->tfm = tfm; + ret = crypto_shash_setkey(tfm, password, password_length); + if (ret) + goto clear; + memcpy(salt, ssid, ssid_length); + for (block = 1; copied < 32; block++) { + size_t length = min_t(size_t, SHA1_DIGEST_SIZE, 32 - copied); + + put_unaligned_be32(block, salt + ssid_length); + ret = crypto_shash_digest(desc, salt, + ssid_length + sizeof(__be32), u); + if (ret) + goto clear; + memcpy(block_value, u, sizeof(block_value)); + for (iteration = 1; iteration < 4096; iteration++) { + ret = crypto_shash_digest(desc, u, sizeof(u), next); + if (ret) + goto clear; + memcpy(u, next, sizeof(u)); + for (i = 0; i < sizeof(block_value); i++) + block_value[i] ^= u[i]; + } + memcpy(pmk + copied, block_value, length); + copied += length; + cond_resched(); + } +clear: + memzero_explicit(salt, sizeof(salt)); + memzero_explicit(u, sizeof(u)); + memzero_explicit(next, sizeof(next)); + memzero_explicit(block_value, sizeof(block_value)); + if (ret) + memzero_explicit(pmk, 32); + kfree_sensitive(desc); +free_tfm: + crypto_free_shash(tfm); + return ret; +} + struct mt7932_peer_ids { u8 station[2], wtbl[4]; }; @@ -110,20 +237,48 @@ static inline int mt7932_connect_binary_pmk(u8 *body, size_t capacity, return 0; } -/* Explicit OPEN or PMK-only WPA2/CCMP; never an automatic security fallback. +/* SAE offload owns PMF and the full handshake; no host key installation. */ +static inline void mt7932_connect_sae_body(u8 *body, const u8 *password, + size_t password_length, const u8 pmk[32]) +{ + body[1] = 8; + put_unaligned_le32(6, body + 4); + put_unaligned_le32(0x78, body + 8); + put_unaligned_le16(5, body + 0x12); + put_unaligned_le16(0x1000, body + 0x14); + put_unaligned_le32(password_length, body + 0x40); + /* Key cipher shares +0x44 with the PSK arm's binary-PMK key type; + * +0x48 holds 16-bit key flags and must stay zero. + */ + put_unaligned_le32(10, body + 0x44); + memcpy(body + 0x4c, password, password_length); + put_unaligned_le32(32, body + 0x1c8); + memcpy(body + 0x1cc, pmk, 32); + /* SAE implies required PMF; BIT(30) would exclude pure SAE. */ + put_unaligned_le32(BIT(8), body + 0x20c); +} + +/* Explicit OPEN, OWE, PMK-only WPA2/CCMP or SAE; no automatic security fallback. * Caller owns identities/resources and validated BSS state. Never log this * body: it carries the PMK. */ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, size_t ssid_length, const u8 pmk[32], const u8 bssid[6], u8 channel, unsigned int listen_interval, u8 connection_flags, - const struct mt7932_peer_ids *ids, bool open) + const struct mt7932_peer_ids *ids, bool open, + const u8 *sae_pwd, size_t sae_pwd_length, bool owe) { u8 resources[6]; + bool sae = sae_pwd_length != 0; unsigned int i, j; int ret; - if ((open ? pmk != NULL : pmk == NULL) || + if (((open || owe) ? !!pmk : !pmk) || + (open && owe) || + (sae && (open || owe || !sae_pwd || + sae_pwd_length < MT7932_SAE_PASSWORD_MIN || + sae_pwd_length > MT7932_SAE_PASSWORD_MAX)) || + (sae_pwd && !sae) || capacity < 856 || !ssid_length || ssid_length > 32 || !mt7932_channel_band(channel) || connection_flags > 63 || (bssid[0] & 1)) return -EINVAL; @@ -142,10 +297,17 @@ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, * automatic-authentication policy. Keep this coherent version/auth/ * policy tuple together; no dummy keys or stale security state. */ - if (open) { + if (open || owe) { body[1] = 1; put_unaligned_le32(0x78, body + 8); put_unaligned_le16(1, body + 0x12); + /* Firmware owns OWE DH, keys and RSN generation. No host key + * or cipher encoding belongs in this request arm. + */ + if (owe) + put_unaligned_le32(BIT(18), body + 0x20c); + } else if (sae) { + mt7932_connect_sae_body(body, sae_pwd, sae_pwd_length, pmk); } else { body[1] = 4; put_unaligned_le32(0x10, body + 4); @@ -155,13 +317,13 @@ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, } put_unaligned_le32(ssid_length, body + 0x18); memcpy(body + 0x1c, ssid, ssid_length); - /* WCLJoinRequest projects six flags here, NOT an environmental count. */ + /* Retain the established policy byte for the qualified request. */ body[0x1c0] = connection_flags; /* Fresh HOST AIS media state1 selects CCMP type5 here. This is not * structured-key type0x44 or the association event's RSN suite encoding. - * This builder is only used for the first, unassociated attempt. + * This builder is only used for an unassociated attempt. */ - put_unaligned_le32(open ? 0 : 5, body + 0x220); + put_unaligned_le32(open || owe ? 0 : 5, body + 0x220); put_unaligned_le16(listen_interval > 20 ? 20 : listen_interval, body + 0x228); memcpy(body + 0x32c, bssid, 6); body[0x338] = mt7932_channel_band(channel); @@ -172,7 +334,8 @@ static inline int mt7932_connect_body(u8 *body, size_t capacity, const u8 *ssid, memcpy(body + 0x340, ids->station, 2); memcpy(body + 0x348, ids->wtbl, 4); body[0x354] = 1; - ret = open ? 0 : mt7932_connect_binary_pmk(body, capacity, pmk, 32); + ret = open || owe || sae ? 0 : + mt7932_connect_binary_pmk(body, capacity, pmk, 32); return ret ? ret : 856; } diff --git a/drivers/net/wireless/mediatek/mt7932/data.c b/drivers/net/wireless/mediatek/mt7932/data.c index ba06db134414e..d3eb03db997f2 100644 --- a/drivers/net/wireless/mediatek/mt7932/data.c +++ b/drivers/net/wireless/mediatek/mt7932/data.c @@ -219,6 +219,7 @@ static int mt_rate_width(enum rate_info_bw bw) void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) { struct sk_buff *skb; + enum mt7932_rx_verdict verdict; unsigned long flags; int signal; @@ -229,12 +230,25 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) if (!m->connected || !m->peer_valid || m->disconnecting || m->stopping) goto unlock; if ((frame->unicast && frame->wtbl != m->peer_wtbl) || - (!frame->unicast && frame->wtbl != m->peer_ids.wtbl[0] && frame->wtbl != m->peer_wtbl) || - mt7932_rx_duplicate(m->duplicates, frame)) { + (!frame->unicast && frame->wtbl != m->peer_ids.wtbl[0] && + frame->wtbl != m->peer_wtbl)) { + m->rx_wrong_peer++; m->netdev->stats.rx_dropped++; goto unlock; } - if (!mt7932_rx_protection_ok(frame, m->connect_open)) { + /* These count MSDU records, including split A-MSDU subframes. */ + if (frame->group4 && (frame->frame_control & IEEE80211_FCTL_RETRY)) + m->rx_retry++; + if (frame->amsdu) + m->rx_amsdu++; + verdict = mt7932_rx_filter(m->duplicates, frame, m->connect_open); + if (verdict == MT7932_RX_DUPLICATE) { + m->rx_duplicate++; + m->netdev->stats.rx_dropped++; + goto unlock; + } + if (verdict == MT7932_RX_UNPROTECTED) { + m->rx_unprotected++; m->netdev->stats.rx_dropped++; dev_dbg_ratelimited(&m->pdev->dev, "RX_UNPROTECTED_DROPPED: cipher=%u mismatch=%u\n", @@ -243,6 +257,7 @@ void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame) } skb = netdev_alloc_skb_ip_align(m->netdev, frame->length); if (!skb) { + m->rx_no_memory++; m->netdev->stats.rx_dropped++; goto unlock; } diff --git a/drivers/net/wireless/mediatek/mt7932/frontend.c b/drivers/net/wireless/mediatek/mt7932/frontend.c index 84170c8f61326..55c7bb4e5babd 100644 --- a/drivers/net/wireless/mediatek/mt7932/frontend.c +++ b/drivers/net/wireless/mediatek/mt7932/frontend.c @@ -162,7 +162,6 @@ int mt_scan_quiesce(struct mt7932 *m) static int mt_scan_submit_next(struct mt7932 *m) { struct cfg80211_scan_request *request = m->scan_request; - const struct cfg80211_ssid *ssid = NULL; u8 body[0x4d4], channels[MT7932_CHANNELS_5G]; unsigned int count = 0, i; unsigned long flags; @@ -195,8 +194,6 @@ static int mt_scan_submit_next(struct mt7932 *m) } if (!count) return 1; - if (!passive && request->n_ssids) - ssid = &request->ssids[0]; spin_lock_irqsave(&m->response_lock, flags); if (m->scan_aborted || !mt_rf_allowed(m) || m->disconnecting || (m->scan_home_channel && !m->connected)) { @@ -217,9 +214,10 @@ static int mt_scan_submit_next(struct mt7932 *m) if (ret >= 0) ret = mt7932_scan_home(body, m->scan_home_channel); if (ret >= 0) - ret = mt7932_scan_options(body, ssid ? ssid->ssid : NULL, - ssid ? ssid->ssid_len : 0, - request->ie, request->ie_len); + ret = mt7932_scan_options(body, NULL, 0, request->ie, request->ie_len); + for (i = 0; ret >= 0 && !passive && i < request->n_ssids; i++) + ret = mt7932_scan_ssid_slot(body, request->ssids[i].ssid, + request->ssids[i].ssid_len); if (ret >= 0) { schedule_delayed_work(&m->scan_timeout_work, msecs_to_jiffies(60000)); ret = mt_request(m, 3, true, true, false, body, sizeof(body)); @@ -358,9 +356,12 @@ int mt_scan(struct wiphy *wiphy, struct cfg80211_scan_request *request) return -EOPNOTSUPP; if (!request->n_channels || request->ie_len > 600 || request->duration_mandatory || - (request->flags & ~(NL80211_SCAN_FLAG_FLUSH | NL80211_SCAN_FLAG_COLOCATED_6GHZ)) || request->n_ssids > 1 || - (request->n_ssids && request->ssids[0].ssid_len > 32)) + (request->flags & ~(NL80211_SCAN_FLAG_FLUSH | NL80211_SCAN_FLAG_COLOCATED_6GHZ)) || + request->n_ssids > MT7932_SCAN_SSIDS) return -EOPNOTSUPP; + for (i = 0; i < request->n_ssids; i++) + if (request->ssids[i].ssid_len > 32) + return -EOPNOTSUPP; for (i = 0; i < request->n_channels; i++) { struct ieee80211_channel *channel = request->channels[i]; @@ -576,6 +577,7 @@ int mt_register_interface(struct mt7932 *m) m->wdev.netdev = netdev; netdev->ieee80211_ptr = &m->wdev; netdev->netdev_ops = &mt_net_ops; + netdev->ethtool_ops = &mt_ethtool_ops; netdev->min_mtu = ETH_MIN_MTU; netdev->max_mtu = ETH_DATA_LEN; SET_NETDEV_DEV(netdev, &m->pdev->dev); @@ -654,7 +656,20 @@ void mt_packet_receive(struct mt7932 *m, const u8 *packet, size_t length) unsigned int band; int ret = mt7932_rx_frame(packet, length, &frame); - if (ret || !READ_ONCE(m->rf_ready)) + if (ret) { + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + if (ret == -EBADMSG) + m->rx_integrity++; + else if (ret == -EOPNOTSUPP) + m->rx_unsupported++; + else if (ret != -ENOMSG) + m->rx_malformed++; + spin_unlock_irqrestore(&m->response_lock, flags); + return; + } + if (!READ_ONCE(m->rf_ready)) return; band = mt7932_channel_band(frame.channel); channel = ieee80211_get_channel(m->wiphy, band == 1 ? 2407 + 5 * frame.channel : 5000 + 5 * frame.channel); diff --git a/drivers/net/wireless/mediatek/mt7932/input-test.c b/drivers/net/wireless/mediatek/mt7932/input-test.c index 9ee9253f34a30..b5c9f22912ceb 100644 --- a/drivers/net/wireless/mediatek/mt7932/input-test.c +++ b/drivers/net/wireless/mediatek/mt7932/input-test.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-only /* * KUnit tests for the validation of firmware input: RX descriptors and - * command replies. The firmware and the device are treated as untrusted. + * command replies, plus the directed-SSID slots of scan requests. The + * firmware and the device are treated as untrusted. */ #include @@ -171,6 +172,136 @@ static void mt7932_event_reply_eid_test(struct kunit *test) KUNIT_EXPECT_EQ(test, mt7932_reply_eid(0x50, false), -1); } +static struct mt7932_rx_frame rx_qos_frame(void) +{ + return (struct mt7932_rx_frame) { + .translated = true, + .group4 = true, + .unicast = true, + .frame_control = IEEE80211_FTYPE_DATA | IEEE80211_STYPE_QOS_DATA, + .sequence_control = 0x1230, + .qos_control = 3, + .sec_mode = 4, + }; +} + +static void mt7932_rx_reject_preserves_sequence_test(struct kunit *test) +{ + struct mt7932_duplicate cache[17] = {}, before[17]; + struct mt7932_rx_frame frame = rx_qos_frame(); + + KUNIT_ASSERT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); + memcpy(before, cache, sizeof(cache)); + frame.sequence_control += 0x10; + frame.sec_mode = 0; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_UNPROTECTED); + KUNIT_EXPECT_MEMEQ(test, cache, before, sizeof(cache)); + frame.sec_mode = 4; + frame.cipher_mismatch = true; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_UNPROTECTED); + KUNIT_EXPECT_MEMEQ(test, cache, before, sizeof(cache)); +} + +static void mt7932_rx_decrypted_retry_test(struct kunit *test) +{ + struct mt7932_duplicate cache[17] = {}; + struct mt7932_rx_frame frame = rx_qos_frame(); + + frame.sec_mode = 0; + KUNIT_ASSERT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_UNPROTECTED); + frame.sec_mode = 4; + frame.frame_control |= IEEE80211_FCTL_RETRY; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_DUPLICATE); +} + +static void mt7932_rx_retry_amsdu_test(struct kunit *test) +{ + struct mt7932_duplicate cache[17] = {}; + struct mt7932_rx_frame frame = rx_qos_frame(); + + /* The rejected first copy must not suppress the decrypted aggregate. */ + frame.amsdu = 3; + frame.sec_mode = 0; + KUNIT_ASSERT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_UNPROTECTED); + frame.sec_mode = 4; + frame.frame_control |= IEEE80211_FCTL_RETRY; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); + frame.amsdu = 2; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); + frame.amsdu = 1; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); + /* A second retry of the completed aggregate is still suppressed. */ + frame.amsdu = 3; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_DUPLICATE); + frame.amsdu = 2; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_DUPLICATE); + frame.amsdu = 1; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_DUPLICATE); +} + +static void mt7932_rx_sequence_domains_test(struct kunit *test) +{ + struct mt7932_duplicate cache[17] = {}; + struct mt7932_rx_frame frame = rx_qos_frame(); + + KUNIT_ASSERT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); + /* Same sequence without Retry is a new frame, including after wrap. */ + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); + frame.frame_control |= IEEE80211_FCTL_RETRY; + frame.qos_control = 4; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); + frame.qos_control = 3; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_DUPLICATE); + frame.sequence_control = 0xfff0; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); + frame.sequence_control = 0; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, false), MT7932_RX_ACCEPT); +} + +static void mt7932_rx_open_duplicate_test(struct kunit *test) +{ + struct mt7932_duplicate cache[17] = {}; + struct mt7932_rx_frame frame = rx_qos_frame(); + + frame.sec_mode = 0; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, true), MT7932_RX_ACCEPT); + frame.frame_control |= IEEE80211_FCTL_RETRY; + KUNIT_EXPECT_EQ(test, mt7932_rx_filter(cache, &frame, true), MT7932_RX_DUPLICATE); +} + +static void mt7932_scan_directed_ssids_test(struct kunit *test) +{ + static const u8 channels[] = { 36 }; + u8 body[0x4d4]; + + KUNIT_ASSERT_EQ(test, mt7932_scan_body(body, sizeof(body), 1, channels, 1), 0x4d4); + KUNIT_EXPECT_EQ(test, mt7932_scan_options(body, NULL, 0, NULL, 0), 0); + KUNIT_EXPECT_EQ(test, body[3], 1); + KUNIT_EXPECT_EQ(test, body[4], 0); + KUNIT_EXPECT_EQ(test, mt7932_scan_options(body, (const u8 *)"hidden", 6, NULL, 0), 0); + KUNIT_EXPECT_EQ(test, body[3], 8); + KUNIT_EXPECT_EQ(test, body[4], 1); + /* Further directed SSIDs fill the next slots; wildcards take none. */ + KUNIT_EXPECT_EQ(test, mt7932_scan_ssid_slot(body, NULL, 0), 0); + KUNIT_EXPECT_EQ(test, mt7932_scan_ssid_slot(body, (const u8 *)"second", 6), 0); + KUNIT_EXPECT_EQ(test, body[4], 2); + KUNIT_EXPECT_EQ(test, get_unaligned_le32(body + 8 + 36), 6); + KUNIT_EXPECT_EQ(test, memcmp(body + 12 + 36, "second", 6), 0); + KUNIT_EXPECT_EQ(test, memcmp(body + 12, "hidden", 6), 0); + KUNIT_EXPECT_EQ(test, mt7932_scan_ssid_slot(body, (const u8 *)"c", 1), 0); + KUNIT_EXPECT_EQ(test, mt7932_scan_ssid_slot(body, (const u8 *)"d", 1), 0); + KUNIT_EXPECT_EQ(test, body[4], 4); + KUNIT_EXPECT_EQ(test, mt7932_scan_ssid_slot(body, (const u8 *)"e", 1), -ENOSPC); + KUNIT_EXPECT_EQ(test, get_unaligned_le16(body + 0xe0), 0); + KUNIT_EXPECT_EQ(test, body[0xa0], 2); + KUNIT_EXPECT_EQ(test, body[0xa1], 36); + /* A passive request carries no directed SSID. */ + mt7932_scan_passive(body); + KUNIT_EXPECT_EQ(test, body[4], 0); + KUNIT_EXPECT_LT(test, mt7932_scan_ssid_slot(body, (const u8 *)"hidden", 6), 0); +} + static struct kunit_case mt7932_input_test_cases[] = { KUNIT_CASE(mt7932_rx_decrypted_test), KUNIT_CASE(mt7932_rx_plaintext_test), @@ -179,6 +310,12 @@ static struct kunit_case mt7932_input_test_cases[] = { KUNIT_CASE(mt7932_rx_short_test), KUNIT_CASE(mt7932_event_reply_test), KUNIT_CASE(mt7932_event_reply_eid_test), + KUNIT_CASE(mt7932_rx_reject_preserves_sequence_test), + KUNIT_CASE(mt7932_rx_decrypted_retry_test), + KUNIT_CASE(mt7932_rx_retry_amsdu_test), + KUNIT_CASE(mt7932_rx_sequence_domains_test), + KUNIT_CASE(mt7932_rx_open_duplicate_test), + KUNIT_CASE(mt7932_scan_directed_ssids_test), {} }; diff --git a/drivers/net/wireless/mediatek/mt7932/lifecycle.c b/drivers/net/wireless/mediatek/mt7932/lifecycle.c index d05d05cd7d7a0..9a05219172bde 100644 --- a/drivers/net/wireless/mediatek/mt7932/lifecycle.c +++ b/drivers/net/wireless/mediatek/mt7932/lifecycle.c @@ -66,4 +66,7 @@ void mt_stop_host(struct mt7932 *m) NULL, 0, true, GFP_KERNEL); } memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); + memzero_explicit(m->connect_sae_pwd, sizeof(m->connect_sae_pwd)); + m->connect_sae_pwd_length = 0; + m->connect_pmk_valid = false; } diff --git a/drivers/net/wireless/mediatek/mt7932/link.c b/drivers/net/wireless/mediatek/mt7932/link.c index 1d6ef0596de36..eb219601862f5 100644 --- a/drivers/net/wireless/mediatek/mt7932/link.c +++ b/drivers/net/wireless/mediatek/mt7932/link.c @@ -89,7 +89,7 @@ void mt_rf_fail_locked(struct mt7932 *m, int error) } } -/* One owned station, firmware WPA2-PSK handshake; no host key fabrication. */ +/* One owned station and firmware handshake; no host key fabrication. */ static int mt_peer_reserve(struct mt7932 *m) { u8 *ids = (u8 *)&m->peer_ids; @@ -176,11 +176,19 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) complete(&m->assoc_start); complete(&m->assoc_done); } else { + const struct element *ht; const struct element *he = cfg80211_find_ext_elem( WLAN_EID_EXT_HE_CAPABILITY, m->assoc_request_ies, ret); m->assoc_request_ie_len = ret; m->assoc_request_seen = true; + ht = cfg80211_find_elem(WLAN_EID_HT_CAPABILITY, + m->assoc_request_ies, ret); + if (ht) + dev_dbg(&m->pdev->dev, + "ASSOC_HT_CAPABILITIES: epoch=%u cap=%04x ampdu=%02x mcs=%02x/%02x\n", + m->connection_generation, get_unaligned_le16(ht->data), + ht->data[2], ht->data[3], ht->data[4]); /* Generated station request, not the AP's capability or a * rate-control setting. Bounds were checked by the exporter. */ @@ -212,6 +220,12 @@ void mt_link_event(struct mt7932 *m, const struct mt7932_event *event) /* OPEN reports retain cipher as numeric metadata only; it neither * installs a key nor changes the immutable requested security mode. */ + if (length >= 52 && get_unaligned_le16(body)) { + m->connect_error = -ECONNREFUSED; + complete(&m->assoc_start); + complete(&m->assoc_done); + break; + } if (!mt7932_assoc_resources(body, length, &m->peer_ids, m->connect_channel, m->connect_open)) { m->connect_error = -EPROTO; complete(&m->assoc_start); @@ -271,6 +285,9 @@ static void mt_peer_release(struct mt7932 *m) memset(&m->peer_ids, 0, sizeof(m->peer_ids)); memset(m->duplicates, 0, sizeof(m->duplicates)); memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); + memzero_explicit(m->connect_sae_pwd, sizeof(m->connect_sae_pwd)); + m->connect_sae_pwd_length = 0; + m->connect_pmk_valid = false; m->peer_valid = false; m->peer_wtbl = 0; m->event_cipher = 0; @@ -390,58 +407,119 @@ struct cfg80211_bss *mt_connect_find_bss(struct mt7932 *m) return bss; } -void mt_connect_work(struct work_struct *work) +/* command_mutex held. Select an admitted scan candidate and own six IDs. */ +static int mt_connect_prepare(struct mt7932 *m, bool *peer_owned) { - struct mt7932 *m = container_of(work, struct mt7932, connect_work); - struct cfg80211_connect_resp_params response = {}; - u8 *body, mac[8] = {}, queue[32]; unsigned long flags; - bool submitted = false, peer_owned = false, cancelled; - u16 reason; int ret; - mutex_lock(&m->command_mutex); - body = kzalloc(856, GFP_KERNEL); - ret = body ? 0 : -ENOMEM; - if (ret) - goto report; - if (!mt_rf_allowed(m) || READ_ONCE(m->connect_error)) { - ret = -ECANCELED; - goto free; - } + if (!mt_rf_allowed(m) || READ_ONCE(m->connect_error)) + return -ECANCELED; if (!m->connect_bss) { dev_dbg(&m->pdev->dev, "CONNECT_DISCOVERY: refreshing missing scan candidate\n"); ret = mt_connect_discover(m); if (ret) - goto free; + return ret; } if (m->connect_bss->channel->flags & (IEEE80211_CHAN_DISABLED | IEEE80211_CHAN_NO_IR | IEEE80211_CHAN_NO_20MHZ | IEEE80211_CHAN_RADAR)) { - ret = -EINVAL; - goto free; + return -EINVAL; } spin_lock_irqsave(&m->response_lock, flags); ret = m->connect_error ?: mt_peer_reserve(m); if (!ret) { - peer_owned = true; + *peer_owned = true; m->connect_channel = m->connect_bss->channel->hw_value; memcpy(m->connect_bssid, m->connect_bss->bssid, ETH_ALEN); } spin_unlock_irqrestore(&m->response_lock, flags); + return ret; +} + +/* command_mutex held on entry and return. Use the ordinary teardown fences + * for both a cold OWE session and a failed authentication attempt. + */ +static int mt_owe_retire_attempt(struct mt7932 *m, bool *submitted, + bool *peer_owned, bool *retirement_attempted) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&m->response_lock, flags); + m->disconnecting = true; + spin_unlock_irqrestore(&m->response_lock, flags); + *submitted = true; + mutex_unlock(&m->command_mutex); + *retirement_attempted = true; + ret = mt_retire_connection(m); + mutex_lock(&m->command_mutex); if (ret) - goto free; - /* Initial station policy: framework M=0 branch and no optional flags. - * This yields flag2 independently of credentials; no scan-result count - * belongs in this field. The original driver uses listen interval 20. - * Other platform flag policies remain unsupported, not guessed mappings. + return ret; + + *submitted = false; + *peer_owned = false; + *retirement_attempted = false; + spin_lock_irqsave(&m->response_lock, flags); + if (m->connect_cancelled) { + ret = -ECANCELED; + } else if (m->link_failed || !mt_rf_allowed(m)) { + ret = -EIO; + } else { + m->owe_cold_prepared = true; + m->connect_error = 0; + m->disconnecting = false; + m->disconnect_seen = false; + m->disconnect_event_received = false; + reinit_completion(&m->assoc_start); + reinit_completion(&m->assoc_done); + reinit_completion(&m->disconnect_done); + m->assoc_request_ie_len = 0; + m->assoc_request_seen = false; + m->connect_center = 0; + memset(&m->connect_chandef, 0, sizeof(m->connect_chandef)); + m->connection_generation++; + } + spin_unlock_irqrestore(&m->response_lock, flags); + return ret; +} + +/* command_mutex held on entry and return. D7 calibration remains serviceable + * while waiting for association events. A retry has one ten-second deadline. + */ +static int mt_connect_attempt(struct mt7932 *m, bool *submitted, + bool *peer_owned, bool *auth_submitted, bool retry) +{ + u8 *body, mac[8] = {}, queue[32]; + unsigned long deadline, timeout; + int ret; + + ret = mt_connect_prepare(m, peer_owned); + if (ret) + return ret; + body = kzalloc(856, GFP_KERNEL); + ret = body ? 0 : -ENOMEM; + if (ret) + goto out; + /* Keep policy value 2 and listen interval 20 for all security arms. + * SAE is qualified with the same policy after correcting key_cipher. */ + if (m->connect_sae_pwd_length && !m->connect_pmk_valid) { + ret = mt7932_sae_pmk(m->connect_pmk, m->connect_sae_pwd, + m->connect_sae_pwd_length, m->connect_ssid, + m->connect_ssid_length); + if (ret) + goto free; + } ret = mt7932_connect_body(body, 856, m->connect_ssid, m->connect_ssid_length, - m->connect_open ? NULL : m->connect_pmk, m->connect_bssid, - m->connect_channel, 20, 2, &m->peer_ids, m->connect_open); + m->connect_open || m->connect_owe ? NULL : m->connect_pmk, m->connect_bssid, + m->connect_channel, 20, 2, &m->peer_ids, m->connect_open, + m->connect_sae_pwd_length ? m->connect_sae_pwd : NULL, + m->connect_sae_pwd_length, m->connect_owe); if (ret < 0) goto free; + /* Reprogram the same on-air station identity for either attempt. */ memcpy(mac + 2, m->mac, ETH_ALEN); - submitted = true; + *submitted = true; ret = mt_request(m, 0xb8, true, true, false, mac, sizeof(mac)); ret = ret ?: READ_ONCE(m->connect_error); if (!ret) @@ -454,37 +532,104 @@ void mt_connect_work(struct work_struct *work) free: kfree_sensitive(body); memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); + memzero_explicit(m->connect_sae_pwd, sizeof(m->connect_sae_pwd)); + m->connect_sae_pwd_length = 0; + m->connect_pmk_valid = false; if (ret) - goto report; + goto out; dev_dbg(&m->pdev->dev, "CONNECT_SUBMITTED: BSSID=%pM channel=%u\n", m->connect_bssid, m->connect_channel); + *auth_submitted = true; + deadline = jiffies + msecs_to_jiffies(10000); /* Genuine D7 requests must remain serviceable during asynchronous waits. */ mutex_unlock(&m->command_mutex); - ret = wait_for_completion_timeout(&m->assoc_start, msecs_to_jiffies(5000)) ? 0 : -ETIMEDOUT; + timeout = msecs_to_jiffies(retry ? 10000 : 5000); + ret = wait_for_completion_timeout(&m->assoc_start, timeout) ? 0 : -ETIMEDOUT; mutex_lock(&m->command_mutex); if (ret) { ret = -ETIMEDOUT; - goto report; + goto out; } ret = READ_ONCE(m->connect_error); if (ret) - goto report; + goto out; ret = mt_association_calibration(m); ret = ret ?: READ_ONCE(m->connect_error); if (ret) - goto report; + goto out; dev_dbg(&m->pdev->dev, "ASSOCIATION_CALIBRATED: primary=%u center=%u\n", m->connect_channel, m->connect_center); + if (retry && time_after_eq(jiffies, deadline)) { + ret = -ETIMEDOUT; + goto out; + } mt7932_peer_queue(queue, m->peer_ids.station[0], true); ret = mt_request(m, 0x6b, true, true, false, queue, sizeof(queue)); if (ret) - goto report; + goto out; + if (retry && time_after_eq(jiffies, deadline)) { + ret = -ETIMEDOUT; + goto out; + } mutex_unlock(&m->command_mutex); - if (!wait_for_completion_timeout(&m->assoc_done, msecs_to_jiffies(10000))) + timeout = retry ? max_t(long, 1, deadline - jiffies) : msecs_to_jiffies(10000); + if (!wait_for_completion_timeout(&m->assoc_done, timeout)) ret = -ETIMEDOUT; else ret = READ_ONCE(m->connect_error); mutex_lock(&m->command_mutex); +out: + return ret ?: READ_ONCE(m->connect_error); +} + +void mt_connect_work(struct work_struct *work) +{ + struct mt7932 *m = container_of(work, struct mt7932, connect_work); + struct cfg80211_connect_resp_params response = {}; + unsigned long flags; + bool submitted = false, peer_owned = false, cancelled, retry = false; + bool retirement_attempted = false, auth_submitted = false; + u16 reason; + int ret; + + mutex_lock(&m->command_mutex); + /* A fresh firmware session needs the same typed cleanup as a retired + * OWE connection. Reserve temporary live IDs rather than inventing a + * peer identity, and perform this once before the first OWE request. + */ + if (m->connect_owe && !m->owe_cold_prepared) { + ret = mt_connect_prepare(m, &peer_owned); + if (ret) + goto report; + ret = mt_owe_retire_attempt(m, &submitted, &peer_owned, + &retirement_attempted); + if (ret) + goto report; + } + ret = mt_connect_attempt(m, &submitted, &peer_owned, &auth_submitted, false); + /* The first OWE authentication can stall after a firmware load. Retry + * once, only after the normal typed disconnect, queue retirement and + * DMA/reader fences have released the old peer. Cancellation and an + * uncertain firmware owner must never be turned into another attempt. + */ + spin_lock_irqsave(&m->response_lock, flags); + if (m->connect_owe && auth_submitted && + (ret == -ETIMEDOUT || ret == -ECONNREFUSED || ret == -ECONNRESET) && + !m->connect_cancelled && + !m->link_failed && mt_rf_allowed(m)) { + m->disconnecting = true; + retry = true; + } + spin_unlock_irqrestore(&m->response_lock, flags); + if (retry) { + dev_dbg(&m->pdev->dev, "OWE_RETRY: first attempt failed: %d\n", ret); + mt_transport_snapshot(m); + ret = mt_owe_retire_attempt(m, &submitted, &peer_owned, + &retirement_attempted); + if (!ret) + ret = mt_connect_attempt(m, &submitted, &peer_owned, + &auth_submitted, true); + } report: ret = ret ?: READ_ONCE(m->connect_error); if (!ret && !mt_rf_allowed(m)) @@ -495,14 +640,17 @@ void mt_connect_work(struct work_struct *work) if (ret && !READ_ONCE(m->stopping)) mt_transport_snapshot(m); memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); + memzero_explicit(m->connect_sae_pwd, sizeof(m->connect_sae_pwd)); + m->connect_sae_pwd_length = 0; + m->connect_pmk_valid = false; if (ret) { spin_lock_irqsave(&m->response_lock, flags); m->disconnecting = true; spin_unlock_irqrestore(&m->response_lock, flags); mutex_unlock(&m->command_mutex); - if (submitted && !READ_ONCE(m->stopping)) { + if (submitted && !retirement_attempted && !READ_ONCE(m->stopping)) { mt_retire_connection(m); - } else if (peer_owned && !submitted) { + } else if (peer_owned && !submitted && !retirement_attempted) { mutex_lock(&m->command_mutex); spin_lock_irqsave(&m->response_lock, flags); mt_peer_release(m); @@ -592,7 +740,7 @@ void mt_connect_work(struct work_struct *work) READ_ONCE(m->data_ready)); else if (!ret) dev_dbg(&m->pdev->dev, - "WPA2_AUTHORIZED: firmware handshake complete; Ethernet=%u\n", + "FIRMWARE_AUTHORIZED: handshake complete; Ethernet=%u\n", READ_ONCE(m->data_ready)); else if (cancelled) dev_dbg(&m->pdev->dev, "CONNECT_CANCELLED: %d\n", ret); @@ -629,7 +777,7 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, struct mt7932 *m = mt_from_wiphy(wiphy); unsigned long flags; bool open = mt_open_request(params); - int ret = 0; + int auth, ret = 0; if (netdev == m->netdev) mt_retry_missing_policy(m); @@ -642,13 +790,9 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, !!params->crypto.psk, params->mfp, params->ie_len); if (!params->ssid_len || params->ssid_len > 32) return -EINVAL; - if (!open && (params->crypto.wpa_versions != NL80211_WPA_VERSION_2 || - params->crypto.n_ciphers_pairwise != 1 || params->crypto.ciphers_pairwise[0] != WLAN_CIPHER_SUITE_CCMP || - params->crypto.cipher_group != WLAN_CIPHER_SUITE_CCMP || - !params->crypto.n_akm_suites || params->crypto.akm_suites[0] != WLAN_AKM_SUITE_PSK || - !params->crypto.psk || params->mfp == NL80211_MFP_REQUIRED || - !params->ssid_len || params->ssid_len > 32)) - return -EOPNOTSUPP; + auth = open ? 0 : mt7932_connect_auth(params); + if (auth < 0) + return auth; if (params->bssid && !is_valid_ether_addr(params->bssid)) return -EINVAL; mutex_lock(&m->command_mutex); @@ -674,9 +818,21 @@ int mt_connect(struct wiphy *wiphy, struct net_device *netdev, memcpy(m->connect_ssid, params->ssid, params->ssid_len); m->connect_ssid_length = params->ssid_len; m->connect_open = open; + m->connect_owe = auth == WLAN_AKM_SUITE_OWE; memzero_explicit(m->connect_pmk, sizeof(m->connect_pmk)); - if (!open) + memzero_explicit(m->connect_sae_pwd, sizeof(m->connect_sae_pwd)); + m->connect_sae_pwd_length = 0; + m->connect_pmk_valid = false; + if (auth == WLAN_AKM_SUITE_SAE) { + m->connect_sae_pwd_length = params->crypto.sae_pwd_len; + memcpy(m->connect_sae_pwd, params->crypto.sae_pwd, + m->connect_sae_pwd_length); + } + /* Transition requests may already carry a PMK; pure SAE needs derivation. */ + if (!open && params->crypto.psk) { memcpy(m->connect_pmk, params->crypto.psk, 32); + m->connect_pmk_valid = true; + } m->connect_error = 0; m->assoc_request_ie_len = 0; m->assoc_request_seen = false; diff --git a/drivers/net/wireless/mediatek/mt7932/mt7932.h b/drivers/net/wireless/mediatek/mt7932/mt7932.h index aae173b97cc13..621d1bda84082 100644 --- a/drivers/net/wireless/mediatek/mt7932/mt7932.h +++ b/drivers/net/wireless/mediatek/mt7932/mt7932.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -121,6 +122,8 @@ struct mt7932 { bool reg_deferral; /* Channels the applied country package leaves out; under RTNL. */ DECLARE_BITMAP(policy_disabled, MT7932_CHANNELS); + /* Last package pass for the current request; under response_lock. */ + struct mt7932_policy_choice policy_choice; struct net_device *netdev; struct wireless_dev wdev; struct cfg80211_scan_request *scan_request; @@ -145,12 +148,15 @@ struct mt7932 { u8 connect_bssid_req[ETH_ALEN], connect_bssid_hint[ETH_ALEN]; bool discovering, discovery_finished, connect_cancelled; u8 connect_ssid[32], connect_pmk[32], connect_bssid[6]; + u8 connect_sae_pwd[MT7932_SAE_PASSWORD_MAX], connect_sae_pwd_length; + bool connect_pmk_valid; u8 assoc_request_ies[512]; u16 assoc_request_ie_len; bool assoc_request_seen; u32 connection_generation; u8 connect_ssid_length, connect_channel, peer_wtbl; - bool connecting, connected, peer_valid, link_failed, connect_open; + bool connecting, connected, peer_valid, link_failed, connect_open, connect_owe; + bool owe_cold_prepared; bool disconnecting, disconnect_seen, disconnect_event_received, disconnect_local; /* The connect worker is reporting its outcome; an RF failure meanwhile * sets connect_lost for the worker to report. Under response_lock. @@ -180,6 +186,10 @@ struct mt7932 { bool data_live[512], data_token[512], data_retired[512]; struct mt7932_duplicate duplicates[17]; u64 tx_released, rx_ethernet; + /* Lifetime host RX observations, protected by response_lock. */ + u64 rx_integrity, rx_malformed, rx_unsupported; + u64 rx_wrong_peer, rx_unprotected, rx_duplicate, rx_no_memory; + u64 rx_retry, rx_amsdu; /* TX0 descriptors published since the last doorbell; data_lock. */ unsigned int data_unkicked; bool data_ready; @@ -193,6 +203,7 @@ struct mt7932 { bool mt_rf_allowed(struct mt7932 *m); void mt_retry_missing_policy(struct mt7932 *m); void mt_reg_apply_deferred_locked(struct mt7932 *m); +void mt_policy_debugfs_init(struct mt7932 *m); int mt_net_open(struct net_device *netdev); int mt_request_input(struct mt7932 *m, const struct firmware **fw, const char *name); /* response_lock held; closes RF admission and reports terminal host state. */ @@ -221,6 +232,7 @@ void mt_bss_presence(struct mt7932 *m, const struct mt7932_event *event); void mt_data_complete(struct mt7932 *m, const u8 *packet, size_t length); netdev_tx_t mt_net_xmit(struct sk_buff *skb, struct net_device *netdev); void mt_data_receive(struct mt7932 *m, const struct mt7932_rx_frame *frame); +extern const struct ethtool_ops mt_ethtool_ops; int mt_data_prepare(struct mt7932 *m); struct mt7932 *mt_from_wiphy(struct wiphy *wiphy); void mt_scan_event(struct mt7932 *m, const struct mt7932_event *event); diff --git a/drivers/net/wireless/mediatek/mt7932/packet.h b/drivers/net/wireless/mediatek/mt7932/packet.h index d7480c81872b2..724cba42dc171 100644 --- a/drivers/net/wireless/mediatek/mt7932/packet.h +++ b/drivers/net/wireless/mediatek/mt7932/packet.h @@ -220,6 +220,27 @@ static inline bool mt7932_rx_duplicate(struct mt7932_duplicate cache[17], return false; } +enum mt7932_rx_verdict { + MT7932_RX_ACCEPT, + MT7932_RX_UNPROTECTED, + MT7932_RX_DUPLICATE, +}; + +/* Rejected protection must not record a sequence as successfully received. + * Otherwise a later decrypted retry can be discarded as a duplicate even + * though no copy of its Ethernet payload reached the network stack. + */ +static inline enum mt7932_rx_verdict +mt7932_rx_filter(struct mt7932_duplicate cache[17], + const struct mt7932_rx_frame *frame, bool open) +{ + if (!mt7932_rx_protection_ok(frame, open)) + return MT7932_RX_UNPROTECTED; + if (mt7932_rx_duplicate(cache, frame)) + return MT7932_RX_DUPLICATE; + return MT7932_RX_ACCEPT; +} + /* Validate the complete variable walk before returning any completion IDs. */ static inline int mt7932_tx_free(const u8 *packet, size_t size, u16 *tokens, unsigned int capacity) diff --git a/drivers/net/wireless/mediatek/mt7932/power.c b/drivers/net/wireless/mediatek/mt7932/power.c index eeb5071c4fe87..62537ae867d90 100644 --- a/drivers/net/wireless/mediatek/mt7932/power.c +++ b/drivers/net/wireless/mediatek/mt7932/power.c @@ -14,6 +14,7 @@ static int mt_power_submit(struct mt7932 *m, u8 mode) m->power_applied_valid = !ret; if (!ret) m->power_applied = mode; + dev_dbg(&m->pdev->dev, "POWER_SAVE_SUBMITTED: mode=%u status=%d\n", mode, ret); return ret; } diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c index 225f3b8f35a0c..ecb79adfb73fd 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory-test.c +++ b/drivers/net/wireless/mediatek/mt7932/regulatory-test.c @@ -1,8 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * KUnit tests for the regulatory snapshot logic and the power package - * parser and channel filter. The packages are local files, so - * the parser treats them as untrusted. + * KUnit tests for the regulatory snapshot logic, the power package + * parser and channel filter, and the country package lookup with its + * world fallback. The packages are local files, so the parser treats + * them as untrusted. */ #include #include @@ -407,6 +408,322 @@ static void mt7932_policy_corruption_test(struct kunit *test) } } +#define FAKE_FILES 8 +#define FAKE_LOG 16 + +/* An in-memory firmware directory that records every lookup. */ +struct fake_fs { + struct { + const char *path; + const u8 *data; + size_t size; + } file[FAKE_FILES]; + unsigned int files, held; + char log[FAKE_LOG][MT7932_POLICY_PATH_MAX]; + unsigned int requests; +}; + +static int fake_get(void *ctx, const char *path, struct mt7932_policy_file *file) +{ + struct fake_fs *fs = ctx; + unsigned int i; + + if (fs->requests < FAKE_LOG) + strscpy(fs->log[fs->requests], path, MT7932_POLICY_PATH_MAX); + fs->requests++; + for (i = 0; i < fs->files; i++) { + if (strcmp(fs->file[i].path, path)) + continue; + file->handle = fs; + file->data = fs->file[i].data; + file->size = fs->file[i].size; + fs->held++; + return 0; + } + return -ENOENT; +} + +static void fake_put(void *ctx, struct mt7932_policy_file *file) +{ + struct fake_fs *fs = ctx; + + fs->held--; + file->handle = NULL; +} + +static void fake_add(struct fake_fs *fs, const char *path, const u8 *data, size_t size) +{ + fs->file[fs->files].path = path; + fs->file[fs->files].data = data; + fs->file[fs->files].size = size; + fs->files++; +} + +/* A country snapshot as the notifier builds it: 1, 6, 13, 36 and 149. */ +static void reg_build_country(struct mt7932_reg_snapshot *reg, const char *alpha2) +{ + static const u16 channels[] = {1, 6, 13, 36, 149}; + unsigned int i; + + memset(reg, 0, sizeof(*reg)); + memcpy(reg->alpha2, alpha2, 2); + memcpy(reg->domain, alpha2, 2); + if (!memcmp(alpha2, "00", 2)) + memcpy(reg->domain, "XZ", 2); + reg->domain[4] = 1; + reg->domain[5] = 2; + reg->domain[8] = 3; + reg->domain[9] = 2; + for (i = 0; i < ARRAY_SIZE(channels); i++) { + put_unaligned_le16(channels[i], reg->domain + 12 + i * 8); + reg->power[i] = 20; + } + reg->length = 12 + ARRAY_SIZE(channels) * 8; +} + +struct select_env { + struct fake_fs fs; + struct mt7932_policy policy; + struct mt7932_policy_file file; + struct mt7932_policy_choice choice; + struct mt7932_reg_snapshot reg; + struct mt7932_policy_loader loader; +}; + +static struct select_env *select_env(struct kunit *test) +{ + struct select_env *env = kunit_kzalloc(test, sizeof(*env), GFP_KERNEL); + + KUNIT_ASSERT_NOT_NULL(test, env); + env->loader.get = fake_get; + env->loader.put = fake_put; + env->loader.ctx = &env->fs; + return env; +} + +static int select_run(struct select_env *env, const char *alpha2) +{ + if (env->file.handle) + fake_put(&env->fs, &env->file); + env->fs.requests = 0; + reg_build_country(&env->reg, alpha2); + return mt7932_policy_select(&env->loader, &env->reg, &env->policy, &env->file, + &env->choice); +} + +/* Lookup order: policy/CC, then world; 00 only the world package. */ +static void mt7932_policy_order_test(struct kunit *test) +{ + enum mt7932_policy_source sources[MT7932_POLICY_CANDIDATES]; + char path[MT7932_POLICY_PATH_MAX]; + struct select_env *env = select_env(test); + + KUNIT_ASSERT_EQ(test, mt7932_policy_candidates("AU", sources), 2); + KUNIT_EXPECT_EQ(test, sources[0], MT7932_POLICY_V1); + KUNIT_EXPECT_EQ(test, sources[1], MT7932_POLICY_WORLD); + KUNIT_ASSERT_EQ(test, mt7932_policy_candidates("00", sources), 1); + KUNIT_EXPECT_EQ(test, sources[0], MT7932_POLICY_WORLD); + mt7932_policy_path(path, sizeof(path), MT7932_POLICY_V1, "AU"); + KUNIT_EXPECT_STREQ(test, path, "mediatek/mt7932/policy/AU.bin"); + mt7932_policy_path(path, sizeof(path), MT7932_POLICY_WORLD, "AU"); + KUNIT_EXPECT_STREQ(test, path, "mediatek/mt7932/policy/world-XZ.bin"); + + /* Nothing installed: every candidate is asked for, in order. */ + KUNIT_EXPECT_EQ(test, select_run(env, "NZ"), -ENOENT); + KUNIT_ASSERT_EQ(test, env->fs.requests, 2); + KUNIT_EXPECT_STREQ(test, env->fs.log[0], "mediatek/mt7932/policy/NZ.bin"); + KUNIT_EXPECT_STREQ(test, env->fs.log[1], "mediatek/mt7932/policy/world-XZ.bin"); + KUNIT_EXPECT_PTR_EQ(test, env->file.handle, NULL); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_NONE); + KUNIT_EXPECT_EQ(test, env->choice.n_tried, 2); + + /* cfg80211's numeric codes have no package of their own. */ + KUNIT_EXPECT_EQ(test, mt7932_policy_candidates("98", sources), 1); + KUNIT_EXPECT_EQ(test, mt7932_policy_candidates("99", sources), 1); + KUNIT_EXPECT_FALSE(test, mt7932_policy_country("00")); + KUNIT_EXPECT_FALSE(test, mt7932_policy_country("aU")); + KUNIT_EXPECT_TRUE(test, mt7932_policy_country("AU")); + + /* 00 never opens a country file. */ + KUNIT_EXPECT_EQ(test, select_run(env, "00"), -ENOENT); + KUNIT_ASSERT_EQ(test, env->fs.requests, 1); + KUNIT_EXPECT_STREQ(test, env->fs.log[0], "mediatek/mt7932/policy/world-XZ.bin"); + KUNIT_EXPECT_EQ(test, env->fs.held, 0); +} + +/* The country's own package wins; an unusable one gives way to the world. */ +static void mt7932_policy_version_test(struct kunit *test) +{ + struct select_env *env = select_env(test); + size_t v1_size, xz_size, big_size = 16385; + u8 *v1, *other, *xz, *big; + + v1 = pkg_build(test, "AU", &v1_size); + xz = pkg_build(test, "XZ", &xz_size); + other = kunit_kzalloc(test, v1_size, GFP_KERNEL); + big = kunit_kzalloc(test, big_size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, other); + KUNIT_ASSERT_NOT_NULL(test, big); + + fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", v1, v1_size); + fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); + KUNIT_ASSERT_EQ(test, select_run(env, "AU"), 0); + KUNIT_EXPECT_EQ(test, env->fs.requests, 1); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V1); + KUNIT_EXPECT_EQ(test, env->choice.version, 1); + KUNIT_EXPECT_EQ(test, env->choice.size, v1_size); + KUNIT_EXPECT_EQ(test, env->choice.fallback, 0); + KUNIT_EXPECT_MEMEQ(test, env->choice.domain, "AU", 2); + KUNIT_EXPECT_PTR_EQ(test, env->file.data, (const u8 *)v1); + KUNIT_EXPECT_EQ(test, env->fs.held, 1); + + /* A package of another J7RP version is not used. */ + memcpy(other, v1, v1_size); + put_unaligned_le16(2, other + 4); + env->fs.files = 0; + fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", other, v1_size); + fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); + KUNIT_ASSERT_EQ(test, select_run(env, "AU"), 0); + KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); + KUNIT_EXPECT_EQ(test, env->choice.fallback, -EINVAL); + + /* Over 16 KiB. */ + memcpy(big, v1, v1_size); + put_unaligned_le32(big_size, big + 12); + env->fs.files = 0; + fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", big, big_size); + fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); + KUNIT_ASSERT_EQ(test, select_run(env, "AU"), 0); + KUNIT_EXPECT_EQ(test, env->choice.tried[0].error, -EINVAL); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); + + /* Another country's package under this country's name. */ + env->fs.files = 0; + fake_add(&env->fs, "mediatek/mt7932/policy/NZ.bin", v1, v1_size); + fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); + KUNIT_ASSERT_EQ(test, select_run(env, "NZ"), 0); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); + KUNIT_EXPECT_EQ(test, env->choice.fallback, -EINVAL); + + /* A truncated world package leaves nothing to fall back to. */ + env->fs.files = 0; + fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size - 1); + KUNIT_EXPECT_EQ(test, select_run(env, "NZ"), -EINVAL); + KUNIT_EXPECT_PTR_EQ(test, env->file.handle, NULL); + KUNIT_EXPECT_EQ(test, env->fs.held, 0); +} + +/* The world fallback keeps the country and intersects the channels. */ +static void mt7932_policy_fallback_test(struct kunit *test) +{ + struct select_env *env = select_env(test); + struct mt7932_policy world; + size_t xz_size; + unsigned int i; + u8 *xz; + + xz = pkg_build(test, "XZ", &xz_size); + KUNIT_ASSERT_EQ(test, mt7932_policy_parse(&world, xz, xz_size, "XZ"), 0); + /* The world profile forbids 149; the country lists it. */ + memset(pkg_entry(&world, 149) + 1, MT7932_POLICY_NOT_PERMITTED, PKG_ENTRY_SIZE - 1); + fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); + + KUNIT_ASSERT_EQ(test, select_run(env, "CA"), 0); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); + KUNIT_EXPECT_EQ(test, env->choice.fallback, -ENOENT); + KUNIT_EXPECT_EQ(test, env->choice.version, 1); + KUNIT_EXPECT_MEMEQ(test, env->choice.alpha2, "CA", 2); + KUNIT_EXPECT_MEMEQ(test, env->reg.alpha2, "CA", 2); + KUNIT_EXPECT_MEMEQ(test, env->reg.domain, "XZ", 2); + KUNIT_EXPECT_MEMEQ(test, env->choice.domain, "XZ", 2); + /* 1, 6, 13 and 36 remain; 149 is gone. */ + KUNIT_EXPECT_EQ(test, env->reg.domain[8], 3); + KUNIT_EXPECT_EQ(test, env->reg.domain[9], 1); + KUNIT_ASSERT_EQ(test, env->reg.length, 12 + 4 * 8); + /* The world profile permits 14 and 165, which the country does not + * list: the fallback never adds a channel. + */ + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&env->policy, 14)); + KUNIT_EXPECT_TRUE(test, mt7932_policy_permits(&env->policy, 165)); + for (i = 0; i < 4; i++) { + u16 ch = get_unaligned_le16(env->reg.domain + 12 + i * 8); + + KUNIT_EXPECT_TRUE(test, ch != 14 && ch != 149 && ch != 165); + } + + /* A numeric cfg80211 code falls back with the reason recorded. */ + KUNIT_ASSERT_EQ(test, select_run(env, "98"), 0); + KUNIT_EXPECT_EQ(test, env->fs.requests, 1); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); + KUNIT_EXPECT_EQ(test, env->choice.fallback, -EOPNOTSUPP); + KUNIT_EXPECT_MEMEQ(test, env->reg.domain, "XZ", 2); + + /* 00 uses the same package as its own: no fallback is reported. */ + KUNIT_ASSERT_EQ(test, select_run(env, "00"), 0); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_WORLD); + KUNIT_EXPECT_EQ(test, env->choice.fallback, 0); + KUNIT_EXPECT_MEMEQ(test, env->reg.domain, "XZ", 2); + fake_put(&env->fs, &env->file); + KUNIT_EXPECT_EQ(test, env->fs.held, 0); +} + +/* Country changes in a row, as the notifier drives them without a reload. */ +static void mt7932_policy_switch_test(struct kunit *test) +{ + static const struct { + const char *alpha2; + enum mt7932_policy_source source; + u8 domain[2]; + int fallback; + } steps[] = { + { "AU", MT7932_POLICY_V1, "AU", 0 }, + { "US", MT7932_POLICY_V1, "US", 0 }, + { "CA", MT7932_POLICY_WORLD, "XZ", -ENOENT }, + { "00", MT7932_POLICY_WORLD, "XZ", 0 }, + { "AU", MT7932_POLICY_V1, "AU", 0 }, + { "US", MT7932_POLICY_V1, "US", 0 }, + }; + struct select_env *env = select_env(test); + u8 digest[3][SHA256_DIGEST_SIZE]; + size_t au_size, us_size, ca_size, xz_size; + u8 *au, *us, *ca, *xz; + unsigned int i; + + au = pkg_build(test, "AU", &au_size); + us = pkg_build(test, "US", &us_size); + ca = pkg_build(test, "CA", &ca_size); + xz = pkg_build(test, "XZ", &xz_size); + fake_add(&env->fs, "mediatek/mt7932/policy/AU.bin", au, au_size); + fake_add(&env->fs, "mediatek/mt7932/policy/US.bin", us, us_size); + fake_add(&env->fs, "mediatek/mt7932/policy/world-XZ.bin", xz, xz_size); + sha256(au, au_size, digest[0]); + sha256(us, us_size, digest[1]); + sha256(xz, xz_size, digest[2]); + + for (i = 0; i < ARRAY_SIZE(steps); i++) { + const u8 *expected = !memcmp(steps[i].domain, "AU", 2) ? digest[0] : + !memcmp(steps[i].domain, "US", 2) ? digest[1] : digest[2]; + + KUNIT_ASSERT_EQ_MSG(test, select_run(env, steps[i].alpha2), 0, "step %u", i); + KUNIT_EXPECT_EQ_MSG(test, env->choice.source, steps[i].source, "step %u", i); + KUNIT_EXPECT_MEMEQ_MSG(test, env->choice.domain, steps[i].domain, 2, + "step %u", i); + KUNIT_EXPECT_EQ_MSG(test, env->choice.fallback, steps[i].fallback, "step %u", i); + KUNIT_EXPECT_EQ_MSG(test, env->fs.held, 1, "step %u", i); + KUNIT_EXPECT_MEMEQ_MSG(test, env->choice.sha256, expected, SHA256_DIGEST_SIZE, + "step %u", i); + } + /* A package installed later is used on the next change. */ + fake_add(&env->fs, "mediatek/mt7932/policy/CA.bin", ca, ca_size); + KUNIT_ASSERT_EQ(test, select_run(env, "CA"), 0); + KUNIT_EXPECT_EQ(test, env->choice.source, MT7932_POLICY_V1); + KUNIT_EXPECT_EQ(test, env->choice.fallback, 0); + KUNIT_EXPECT_MEMEQ(test, env->choice.domain, "CA", 2); + fake_put(&env->fs, &env->file); + KUNIT_EXPECT_EQ(test, env->fs.held, 0); +} + static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_reg_relaxation_test), KUNIT_CASE(mt7932_policy_valid_test), @@ -418,6 +735,10 @@ static struct kunit_case mt7932_policy_test_cases[] = { KUNIT_CASE(mt7932_policy_permits_test), KUNIT_CASE(mt7932_policy_filter_test), KUNIT_CASE(mt7932_policy_corruption_test), + KUNIT_CASE(mt7932_policy_order_test), + KUNIT_CASE(mt7932_policy_version_test), + KUNIT_CASE(mt7932_policy_fallback_test), + KUNIT_CASE(mt7932_policy_switch_test), {} }; diff --git a/drivers/net/wireless/mediatek/mt7932/regulatory.h b/drivers/net/wireless/mediatek/mt7932/regulatory.h index 3a8c1ffc74015..7ffdc293b55f4 100644 --- a/drivers/net/wireless/mediatek/mt7932/regulatory.h +++ b/drivers/net/wireless/mediatek/mt7932/regulatory.h @@ -5,6 +5,7 @@ /* Private, versioned packaging of original-derived country policy. This is * not firmware code and does not replace cfg80211's regulatory database. */ +#include #include #include "channels.h" @@ -184,4 +185,69 @@ static inline void mt7932_policy_filter(struct mt7932_reg_snapshot *reg, reg->length = 12 + kept * 8; } +/* + * Country package lookup. policy/ holds J7RP version 1 packages (2.4 and + * 5 GHz, at most 16 KiB): the country's own one, then the world one. + */ +enum mt7932_policy_source { + MT7932_POLICY_NONE, + MT7932_POLICY_V1, /* policy/.bin */ + MT7932_POLICY_WORLD, /* policy/world-XZ.bin */ +}; + +#define MT7932_POLICY_CANDIDATES 2 +#define MT7932_POLICY_PATH_MAX 48 + +struct mt7932_policy_file { + const void *handle; + const u8 *data; + size_t size; +}; + +/* get() returns 0 with @file filled in, or an error; put() releases it. */ +struct mt7932_policy_loader { + int (*get)(void *ctx, const char *path, struct mt7932_policy_file *file); + void (*put)(void *ctx, struct mt7932_policy_file *file); + void *ctx; +}; + +/* What was applied, for the debugfs readback. */ +struct mt7932_policy_choice { + u8 alpha2[2], domain[2]; + enum mt7932_policy_source source; + u16 version; + u32 size; + u8 sha256[SHA256_DIGEST_SIZE]; + /* Why the country's own package was not used: -ENOENT, -EINVAL, or + * -EOPNOTSUPP for a numeric cfg80211 code. + */ + int fallback; + int error; + u32 generation; + unsigned int n_tried; + struct { + enum mt7932_policy_source source; + int error; + } tried[MT7932_POLICY_CANDIDATES]; +}; + +/* An ISO 3166 alpha2, which may have a package of its own. */ +static inline bool mt7932_policy_country(const u8 alpha2[2]) +{ + return alpha2[0] >= 'A' && alpha2[0] <= 'Z' && alpha2[1] >= 'A' && alpha2[1] <= 'Z'; +} + +unsigned int mt7932_policy_candidates(const u8 alpha2[2], + enum mt7932_policy_source sources[MT7932_POLICY_CANDIDATES]); +void mt7932_policy_path(char *buf, size_t len, enum mt7932_policy_source source, + const u8 alpha2[2]); +/* + * Load the first usable package for @reg's country, parse it into @policy + * and filter @reg's CID0f domain with it. On success @file holds the + * package @policy points into; the caller releases it with loader->put(). + */ +int mt7932_policy_select(const struct mt7932_policy_loader *loader, + struct mt7932_reg_snapshot *reg, struct mt7932_policy *policy, + struct mt7932_policy_file *file, struct mt7932_policy_choice *choice); + #endif diff --git a/drivers/net/wireless/mediatek/mt7932/scan.h b/drivers/net/wireless/mediatek/mt7932/scan.h index 217fc5c9d1e96..8b3356bb94fdc 100644 --- a/drivers/net/wireless/mediatek/mt7932/scan.h +++ b/drivers/net/wireless/mediatek/mt7932/scan.h @@ -86,6 +86,33 @@ static inline void mt7932_scan_disconnected_passive(u8 body[0x4d4]) put_unaligned_le16(233, body + 0x4ca); } +/* CID03 carries up to four directed SSIDs in 36-byte slots (le32 length, + * 32 bytes) at offset 8; selector 8 marks a directed request and byte 4 + * counts the populated slots. Wildcard entries occupy no slot. A directed + * request carries no probe IEs. + */ +#define MT7932_SCAN_SSIDS 4 + +static inline int mt7932_scan_ssid_slot(u8 body[0x4d4], const u8 *ssid, + size_t ssid_length) +{ + u8 slot = body[4]; + + if (!ssid_length) + return 0; + if (ssid_length > 32 || !body[2]) + return -EINVAL; + if (slot >= MT7932_SCAN_SSIDS) + return -ENOSPC; + body[3] = 8; + put_unaligned_le32(ssid_length, body + 8 + 36 * slot); + memcpy(body + 12 + 36 * slot, ssid, ssid_length); + body[4] = slot + 1; + memset(body + 0xe2, 0, 600); + put_unaligned_le16(0, body + 0xe0); + return 0; +} + /* This firmware requires its native wildcard template. Directed scans use * no probe IEs. Forwarding the normal supplicant extras stalled the scan * on J700; arbitrary caller probe-IE carriage remains unsupported here. @@ -102,17 +129,7 @@ static inline int mt7932_scan_options(u8 body[0x4d4], const u8 *ssid, return -EINVAL; offset += 2 + ies[offset + 1]; } - if (ssid_length) { - body[3] = 8; - body[4] = 1; - put_unaligned_le32(ssid_length, body + 8); - memcpy(body + 12, ssid, ssid_length); - } - if (ssid_length) { - memset(body + 0xe2, 0, 600); - put_unaligned_le16(0, body + 0xe0); - } - return 0; + return mt7932_scan_ssid_slot(body, ssid, ssid_length); } /* Original online-scan state retains the BSS/keys and supplies its home diff --git a/drivers/net/wireless/mediatek/mt7932/startup.c b/drivers/net/wireless/mediatek/mt7932/startup.c index 679c9193f85bc..8f1125f984f57 100644 --- a/drivers/net/wireless/mediatek/mt7932/startup.c +++ b/drivers/net/wireless/mediatek/mt7932/startup.c @@ -1,5 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include #include +#include #include "mt7932.h" @@ -10,6 +14,7 @@ static int mt_privacy_defaults(struct mt7932 *m); static int mt_startup_once(struct mt7932 *m); static void mt_startup_work(struct work_struct *work); static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request *request); +static const char *mt7932_policy_fallback_reason(int fallback); struct mt_config_record { u8 type, key_len, value_len, reserved; @@ -307,15 +312,62 @@ void mt_retry_missing_policy(struct mt7932 *m) spin_unlock_irqrestore(&m->response_lock, flags); } +static int mt_policy_get(void *ctx, const char *path, struct mt7932_policy_file *file) +{ + struct mt7932 *m = ctx; + const struct firmware *fw; + int ret; + + /* Every candidate but the last is optional: no warning on a miss. */ + ret = request_firmware_direct(&fw, path, &m->pdev->dev); + if (ret) { + dev_dbg(&m->pdev->dev, "REGULATORY_PACKAGE_MISSING: %s: %d\n", path, ret); + return ret; + } + file->handle = fw; + file->data = fw->data; + file->size = fw->size; + return 0; +} + +static void mt_policy_put(void *ctx, struct mt7932_policy_file *file) +{ + release_firmware(file->handle); + file->handle = NULL; +} + +static void mt_policy_report(struct mt7932 *m, const struct mt7932_policy_choice *choice, + int ret) +{ + char path[MT7932_POLICY_PATH_MAX]; + unsigned int i; + + for (i = 0; i < choice->n_tried; i++) { + if (choice->tried[i].error != -EINVAL) + continue; + mt7932_policy_path(path, sizeof(path), choice->tried[i].source, choice->alpha2); + dev_warn_ratelimited(&m->pdev->dev, "local input %s has invalid policy: %d\n", + path, -EINVAL); + } + if (ret || choice->source != MT7932_POLICY_WORLD || !choice->fallback) + return; + dev_info(&m->pdev->dev, "REGULATORY_FALLBACK: %c%c firmware=XZ reason=%s\n", + choice->alpha2[0], choice->alpha2[1], + mt7932_policy_fallback_reason(choice->fallback)); +} + static void mt_startup_work(struct work_struct *work) { struct mt7932 *m = container_of(work, struct mt7932, startup_work); + const struct mt7932_policy_loader loader = { + .get = mt_policy_get, .put = mt_policy_put, .ctx = m, + }; + struct mt7932_policy_choice choice; + struct mt7932_policy_file file; struct mt7932_reg_snapshot reg; struct mt7932_policy policy; - const struct firmware *file; unsigned long flags; u32 generation; - char path[64]; bool submitted; int ret; @@ -348,28 +400,18 @@ static void mt_startup_work(struct work_struct *work) ret = reg.error; if (READ_ONCE(m->retired_scan_seq)) ret = -EBUSY; - file = NULL; + memset(&file, 0, sizeof(file)); + memset(&choice, 0, sizeof(choice)); + memcpy(choice.alpha2, reg.alpha2, 2); submitted = false; if (READ_ONCE(m->policy_failed) || READ_ONCE(m->link_failed) || (m->startup_started && !READ_ONCE(m->rf_ready))) ret = -EIO; if (!ret) { - if (!memcmp(reg.alpha2, "00", 2)) - scnprintf(path, sizeof(path), - "mediatek/mt7932/policy/world-XZ.bin"); - else - scnprintf(path, sizeof(path), "mediatek/mt7932/policy/%c%c.bin", - reg.alpha2[0], reg.alpha2[1]); - ret = mt_request_input(m, &file, path); - } - if (!ret) { - ret = mt7932_policy_parse(&policy, file->data, file->size, reg.domain); - if (!ret) { - mt7932_policy_filter(®, &policy); + ret = mt7932_policy_select(&loader, ®, &policy, &file, &choice); + mt_policy_report(m, &choice, ret); + if (!ret) mt_policy_disable(m, &policy, generation); - } else { - dev_err_ratelimited(&m->pdev->dev, "local input %s has invalid policy: %d\n", path, ret); - } } mutex_lock(&m->command_mutex); if (READ_ONCE(m->stopping) || generation != READ_ONCE(m->reg_generation)) @@ -394,6 +436,11 @@ static void mt_startup_work(struct work_struct *work) !m->cal_state.error; if (!ret && generation == m->reg_generation) WRITE_ONCE(m->reg_pending, false); + if (generation == m->reg_generation) { + choice.generation = generation; + choice.error = ret; + m->policy_choice = choice; + } if (ret) dev_warn_ratelimited(&m->pdev->dev, "REGULATORY_BLOCKED: %c%c generation=%u error=%d recovery-required=%u\n", reg.alpha2[0], reg.alpha2[1], generation, ret, @@ -405,8 +452,8 @@ static void mt_startup_work(struct work_struct *work) spin_unlock_irqrestore(&m->response_lock, flags); next: mutex_unlock(&m->command_mutex); - if (file) - release_firmware(file); + if (file.handle) + mt_policy_put(m, &file); } } @@ -423,15 +470,20 @@ static void mt_regulatory_notify(struct wiphy *wiphy, struct regulatory_request if (!request) return; memcpy(reg.alpha2, request->alpha2, 2); - if (!((reg.alpha2[0] >= 'A' && reg.alpha2[0] <= 'Z' && - reg.alpha2[1] >= 'A' && reg.alpha2[1] <= 'Z') || - !memcmp(reg.alpha2, "00", 2))) + /* Countries, 00 and cfg80211's numeric codes (98 intersection, 99 + * custom); the numeric ones have no package and use the world one. + */ + if (!mt7932_policy_country(reg.alpha2) && + !(isdigit(reg.alpha2[0]) && isdigit(reg.alpha2[1]))) reg.error = -EOPNOTSUPP; memcpy(reg.domain, reg.alpha2, 2); /* Stock database-present startup resolves the absent 00 row to XZ. * Keep Linux's world channel restrictions and requested identity; only * the firmware country/domain and original power package use XZ. - * Never fall back for an arbitrary unsupported explicit country. + * A country without a usable package of its own falls back the same + * way in mt7932_policy_select(), keeping its own channel list: the + * firmware domain is the intersection of both, so the radio stays up + * and is never more permissive than the country or the world profile. */ if (!memcmp(reg.alpha2, "00", 2)) memcpy(reg.domain, "XZ", 2); @@ -548,7 +600,9 @@ int mt_register_regulatory_gate(struct mt7932 *m) { static const int rates[] = {10,20,55,110,60,90,120,180,240,360,480,540}; static const u32 ciphers[] = { WLAN_CIPHER_SUITE_CCMP }; - static const u32 akms[] = { WLAN_AKM_SUITE_PSK }; + static const u32 akms[] = { + WLAN_AKM_SUITE_PSK, WLAN_AKM_SUITE_SAE, WLAN_AKM_SUITE_OWE, + }; unsigned int i; int ret; @@ -567,7 +621,10 @@ int mt_register_regulatory_gate(struct mt7932 *m) m->wiphy->akm_suites = akms; m->wiphy->n_akm_suites = ARRAY_SIZE(akms); wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_4WAY_HANDSHAKE_STA_PSK); - m->wiphy->max_scan_ssids = 1; + wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_SAE_OFFLOAD); + wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_OWE_OFFLOAD); + wiphy_ext_feature_set(m->wiphy, NL80211_EXT_FEATURE_MFP_OPTIONAL); + m->wiphy->max_scan_ssids = MT7932_SCAN_SSIDS; m->wiphy->max_scan_ie_len = 600; m->wiphy->reg_notifier = mt_regulatory_notify; /* World roaming may relax NO_IR after observing an AP beacon without @@ -638,6 +695,7 @@ int mt_register_regulatory_gate(struct mt7932 *m) return ret; } m->wiphy_registered = true; + mt_policy_debugfs_init(m); ret = mt_register_interface(m); if (ret) return ret; @@ -646,3 +704,166 @@ int mt_register_regulatory_gate(struct mt7932 *m) dev_dbg(&m->pdev->dev, "REGULATORY_GATE_REGISTERED: cfg80211 country policy required\n"); return 0; } + +/* Country package selection. The lookup goes through a loader so the order + * and the world fallback can be tested without the firmware loader. + */ +static const char *mt7932_policy_fallback_reason(int fallback) +{ + switch (fallback) { + case 0: + return "none"; + case -ENOENT: + return "missing"; + case -EOPNOTSUPP: + return "no-country-code"; + default: + return "invalid"; + } +} + +static const char *const mt7932_policy_names[] = { + [MT7932_POLICY_NONE] = "none", + [MT7932_POLICY_V1] = "policy", + [MT7932_POLICY_WORLD] = "world", +}; + +unsigned int mt7932_policy_candidates(const u8 alpha2[2], + enum mt7932_policy_source sources[MT7932_POLICY_CANDIDATES]) +{ + unsigned int n = 0; + + /* 00 and the other numeric codes have no country package. */ + if (mt7932_policy_country(alpha2)) + sources[n++] = MT7932_POLICY_V1; + sources[n++] = MT7932_POLICY_WORLD; + return n; +} + +void mt7932_policy_path(char *buf, size_t len, enum mt7932_policy_source source, + const u8 alpha2[2]) +{ + switch (source) { + case MT7932_POLICY_V1: + scnprintf(buf, len, "mediatek/mt7932/policy/%c%c.bin", alpha2[0], alpha2[1]); + break; + case MT7932_POLICY_WORLD: + scnprintf(buf, len, "mediatek/mt7932/policy/world-XZ.bin"); + break; + default: + scnprintf(buf, len, "-"); + break; + } +} + +int mt7932_policy_select(const struct mt7932_policy_loader *loader, + struct mt7932_reg_snapshot *reg, struct mt7932_policy *policy, + struct mt7932_policy_file *file, struct mt7932_policy_choice *choice) +{ + enum mt7932_policy_source sources[MT7932_POLICY_CANDIDATES]; + unsigned int i, n = mt7932_policy_candidates(reg->alpha2, sources); + char path[MT7932_POLICY_PATH_MAX]; + int country = mt7932_policy_country(reg->alpha2) ? 0 : -EOPNOTSUPP; + int ret = -ENOENT; + + memset(choice, 0, sizeof(*choice)); + memcpy(choice->alpha2, reg->alpha2, 2); + memset(file, 0, sizeof(*file)); + for (i = 0; i < n; i++) { + const u8 *alpha2 = sources[i] == MT7932_POLICY_WORLD ? (const u8 *)"XZ" : + reg->alpha2; + + mt7932_policy_path(path, sizeof(path), sources[i], reg->alpha2); + choice->tried[i].source = sources[i]; + ret = loader->get(loader->ctx, path, file); + if (!ret) { + /* The parser accepts version 1 of at most 16 KiB only. */ + ret = mt7932_policy_parse(policy, file->data, file->size, alpha2); + if (ret) { + loader->put(loader->ctx, file); + memset(file, 0, sizeof(*file)); + } + } + choice->tried[i].error = ret; + choice->n_tried = i + 1; + if (!ret) { + choice->source = sources[i]; + break; + } + if (sources[i] != MT7932_POLICY_WORLD) + country = ret; + } + if (ret) + return ret; + + /* + * A country without a usable package of its own keeps its cfg80211 + * identity and channel list; the firmware gets the world profile and + * the CID0f domain loses every channel the world package forbids. The + * result is the intersection of both, never more permissive than + * either. + */ + if (choice->source == MT7932_POLICY_WORLD) { + memcpy(reg->domain, "XZ", 2); + if (memcmp(reg->alpha2, "00", 2)) + choice->fallback = country; + } + mt7932_policy_filter(reg, policy); + memcpy(choice->domain, reg->domain, 2); + choice->version = get_unaligned_le16(file->data + 4); + choice->size = file->size; + sha256(file->data, file->size, choice->sha256); + return 0; +} + +static int mt_policy_show(struct seq_file *s, void *unused) +{ + struct mt7932 *m = s->private; + struct mt7932_policy_choice *c; + char path[MT7932_POLICY_PATH_MAX]; + unsigned long flags; + unsigned int i; + bool pending; + + c = kmalloc_obj(*c, GFP_KERNEL); + if (!c) + return -ENOMEM; + spin_lock_irqsave(&m->response_lock, flags); + *c = m->policy_choice; + pending = m->reg_pending; + spin_unlock_irqrestore(&m->response_lock, flags); + + if (!c->alpha2[0]) { + seq_puts(s, "state: none\n"); + goto out; + } + mt7932_policy_path(path, sizeof(path), c->source, c->alpha2); + seq_printf(s, "country: %c%c\n", c->alpha2[0], c->alpha2[1]); + if (c->error) + seq_printf(s, "state: blocked (%d)\n", c->error); + else + seq_printf(s, "state: %s\n", pending ? "applied, update pending" : "applied"); + seq_printf(s, "generation: %u\n", c->generation); + if (c->source != MT7932_POLICY_NONE) { + seq_printf(s, "firmware_domain: %c%c\n", c->domain[0], c->domain[1]); + seq_printf(s, "source: %s\n", mt7932_policy_names[c->source]); + seq_printf(s, "path: %s\n", path); + seq_printf(s, "version: %u\n", c->version); + seq_printf(s, "size: %u\n", c->size); + seq_printf(s, "sha256: %8phN\n", c->sha256); + } + seq_printf(s, "fallback: %s\n", mt7932_policy_fallback_reason(c->fallback)); + for (i = 0; i < c->n_tried; i++) { + mt7932_policy_path(path, sizeof(path), c->tried[i].source, c->alpha2); + seq_printf(s, "tried: %s %d\n", path, c->tried[i].error); + } +out: + kfree(c); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(mt_policy); + +void mt_policy_debugfs_init(struct mt7932 *m) +{ + debugfs_create_file("mt7932_policy", 0400, m->wiphy->debugfsdir, m, &mt_policy_fops); +} diff --git a/drivers/net/wireless/mediatek/mt7932/station.c b/drivers/net/wireless/mediatek/mt7932/station.c index 83cee4b650926..d592b5029d637 100644 --- a/drivers/net/wireless/mediatek/mt7932/station.c +++ b/drivers/net/wireless/mediatek/mt7932/station.c @@ -1,6 +1,58 @@ // SPDX-License-Identifier: GPL-2.0-only #include "mt7932.h" +static const char mt_rx_stat_names[][ETH_GSTRING_LEN] = { + "host_rx_integrity_rejected", + "host_rx_malformed", + "host_rx_unsupported", + "host_rx_wrong_peer", + "host_rx_unprotected", + "host_rx_retry_duplicate", + "host_rx_no_memory", + "host_rx_retry_records", + "host_rx_amsdu_records", +}; + +static int mt_get_sset_count(struct net_device *netdev, int set) +{ + return set == ETH_SS_STATS ? ARRAY_SIZE(mt_rx_stat_names) : -EOPNOTSUPP; +} + +static void mt_get_strings(struct net_device *netdev, u32 set, u8 *data) +{ + if (set == ETH_SS_STATS) + memcpy(data, mt_rx_stat_names, sizeof(mt_rx_stat_names)); +} + +/* Host observations only: no firmware commands or invented ACK/BA counters. + * They span associations so a test can take deltas across a reconnect. Retry + * and A-MSDU counts are MSDU records, not over-air MPDU/aggregate counts. + */ +static void mt_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, u64 *data) +{ + struct mt7932 *m = *(struct mt7932 **)netdev_priv(netdev); + unsigned long flags; + + spin_lock_irqsave(&m->response_lock, flags); + data[0] = m->rx_integrity; + data[1] = m->rx_malformed; + data[2] = m->rx_unsupported; + data[3] = m->rx_wrong_peer; + data[4] = m->rx_unprotected; + data[5] = m->rx_duplicate; + data[6] = m->rx_no_memory; + data[7] = m->rx_retry; + data[8] = m->rx_amsdu; + spin_unlock_irqrestore(&m->response_lock, flags); +} + +const struct ethtool_ops mt_ethtool_ops = { + .get_sset_count = mt_get_sset_count, + .get_strings = mt_get_strings, + .get_ethtool_stats = mt_get_ethtool_stats, +}; + /* Report the owned, validated EE4d association, not an AP advertisement * or a transient off-channel scan. */ diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index d269457e839ea..fdcb91042f290 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -1346,8 +1346,10 @@ static int xennet_poll(struct napi_struct *napi, int budget) skb->data_len = rx->status; skb->len += rx->status; - if (unlikely(xennet_fill_frags(queue, skb, &tmpq))) + if (unlikely(xennet_fill_frags(queue, skb, &tmpq))) { + __skb_queue_head(&tmpq, skb); goto err; + } if (rx->flags & XEN_NETRXF_csum_blank) skb->ip_summed = CHECKSUM_PARTIAL; diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index d3ef47fa840e4..3a29c5db19fc8 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -198,6 +198,10 @@ struct apple_nvme_hw { bool has_queue_count; bool has_separate_nvmmu; bool needs_ioq_registers; + /* The coprocessor waits for AP readiness before acknowledging IOP power. */ + bool early_ap_power; + /* Enable with the firmware's CC fields (128-byte SQ entries) instead of the core's. */ + bool keep_cc_fields; u32 max_queue_depth; unsigned long quirks; }; @@ -343,6 +347,13 @@ static bool apple_nvme_can_adopt_rtkit(struct apple_nvme *anv) * and ready identifies a live session. Borrowed buffers still require * reserved RAM and protected SART ranges. */ + /* + * T6050: the loader hands over the controller disabled with the RTKit + * session live (it cannot be restarted after a stop). Adopt on RUN and + * BOOT_STATUS alone there; the enable below starts the controller. + */ + if (anv->hw->keep_cc_fields && (cc & NVME_CC_ENABLE)) + return !(csts & NVME_CSTS_CFS); return (cc & NVME_CC_ENABLE) && (csts & NVME_CSTS_RDY) && !(csts & NVME_CSTS_CFS); } @@ -360,7 +371,8 @@ static unsigned int apple_nvme_queue_depth(struct apple_nvme_queue *q) struct apple_nvme *anv = queue_to_apple_nvme(q); if (q->is_adminq) - return APPLE_NVME_AQ_DEPTH; + return anv->hw->keep_cc_fields ? anv->hw->max_queue_depth : + APPLE_NVME_AQ_DEPTH; return anv->hw->max_queue_depth; } @@ -1535,6 +1547,32 @@ static void apple_nvme_wait_freeze(struct apple_nvme *anv) } } + +/* + * T6050: the controller only reports ready when CC keeps the entry-size fields + * the firmware set (IOSQES 7, 128-byte entries), as the loader does. The core's + * nvme_enable_ctrl composes CC with the standard IOSQES 6. + */ +static int apple_nvme_enable_ctrl_keep_cc(struct apple_nvme *anv) +{ + u32 cc, csts, timeout_ms; + + anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); + cc = readl(anv->mmio_nvme + NVME_REG_CC); + cc &= ~(NVME_CC_SHN_MASK | NVME_CC_ENABLE); + cc |= NVME_CC_ENABLE; + anv->ctrl.ctrl_config = cc; + writel(cc, anv->mmio_nvme + NVME_REG_CC); + timeout_ms = min_t(u32, max_t(u32, (NVME_CAP_TIMEOUT(anv->ctrl.cap) + 1) * 500, 1000), 10000); + if (readl_poll_timeout(anv->mmio_nvme + NVME_REG_CSTS, csts, + csts & NVME_CSTS_RDY, 100, timeout_ms * 1000)) { + dev_err(anv->dev, "controller not ready after enable, CC=%#x CSTS=%#x\n", + readl(anv->mmio_nvme + NVME_REG_CC), readl(anv->mmio_nvme + NVME_REG_CSTS)); + return -ENODEV; + } + return 0; +} + static void apple_nvme_reset_work(struct work_struct *work) { unsigned int nr_io_queues = 1; @@ -1646,7 +1684,8 @@ static void apple_nvme_reset_work(struct work_struct *work) "post-M4 firmware handoff: restarting stopped ANS without reset\n"); writel(APPLE_ANS_COPROC_CPU_CONTROL_RUN, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); - ret = apple_rtkit_boot(anv->rtk); + ret = anv->hw->early_ap_power ? apple_rtkit_boot_early_ap(anv->rtk) : + apple_rtkit_boot(anv->rtk); goto booted; } @@ -1665,9 +1704,11 @@ static void apple_nvme_reset_work(struct work_struct *work) writel(APPLE_ANS_COPROC_CPU_CONTROL_RUN, anv->mmio_coproc + APPLE_ANS_COPROC_CPU_CONTROL); - ret = apple_rtkit_boot(anv->rtk); + ret = anv->hw->early_ap_power ? apple_rtkit_boot_early_ap(anv->rtk) : + apple_rtkit_boot(anv->rtk); } else { - ret = apple_rtkit_wake(anv->rtk); + ret = anv->hw->early_ap_power ? apple_rtkit_wake_early_ap(anv->rtk) : + apple_rtkit_wake(anv->rtk); } booted: @@ -1710,8 +1751,13 @@ static void apple_nvme_reset_work(struct work_struct *work) * Enable NVMMU and linear submission queues which is required * since T6000. */ - writel(APPLE_ANS_LINEAR_SQ_EN, - anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); + if (anv->hw->keep_cc_fields) + writel(readl(anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL) | + APPLE_ANS_LINEAR_SQ_EN, + anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); + else + writel(APPLE_ANS_LINEAR_SQ_EN, + anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); if (anv->hw->needs_ioq_registers) { u32 lsq_ctrl = readl(anv->mmio_nvme + APPLE_ANS_LINEAR_SQ_CTRL); @@ -1750,15 +1796,28 @@ static void apple_nvme_reset_work(struct work_struct *work) } /* Setup the admin queue */ - aqa = APPLE_NVME_AQ_DEPTH - 1; + aqa = apple_nvme_queue_depth(&anv->adminq) - 1; aqa |= aqa << 16; - writel(aqa, anv->mmio_nvme + NVME_REG_AQA); - apple_nvme_writeq(anv, anv->adminq.sq_dma_addr, - anv->mmio_nvme + NVME_REG_ASQ); - apple_nvme_writeq(anv, anv->adminq.cq_dma_addr, - anv->mmio_nvme + NVME_REG_ACQ); + if (anv->hw->keep_cc_fields) { + /* T6050: the loader's order, NVMMU TCB bases, ASQ, ACQ, then AQA. */ + apple_nvme_writeq(anv, anv->adminq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); + apple_nvme_writeq(anv, anv->ioq.tcb_dma_addr, + anv->mmio_nvmmu + APPLE_NVMMU_IOSQ_TCB_BASE); + apple_nvme_writeq(anv, anv->adminq.sq_dma_addr, + anv->mmio_nvme + NVME_REG_ASQ); + apple_nvme_writeq(anv, anv->adminq.cq_dma_addr, + anv->mmio_nvme + NVME_REG_ACQ); + writel(aqa, anv->mmio_nvme + NVME_REG_AQA); + } else { + writel(aqa, anv->mmio_nvme + NVME_REG_AQA); + apple_nvme_writeq(anv, anv->adminq.sq_dma_addr, + anv->mmio_nvme + NVME_REG_ASQ); + apple_nvme_writeq(anv, anv->adminq.cq_dma_addr, + anv->mmio_nvme + NVME_REG_ACQ); + } - if (anv->hw->has_lsq_nvmmu) { + if (anv->hw->has_lsq_nvmmu && !anv->hw->keep_cc_fields) { /* Setup NVMMU for both queues */ apple_nvme_writeq(anv, anv->adminq.tcb_dma_addr, anv->mmio_nvmmu + APPLE_NVMMU_ASQ_TCB_BASE); @@ -1771,7 +1830,8 @@ static void apple_nvme_reset_work(struct work_struct *work) anv->ctrl.cap = readq(anv->mmio_nvme + NVME_REG_CAP); dev_dbg(anv->dev, "Enabling controller now"); - ret = nvme_enable_ctrl(&anv->ctrl); + ret = anv->hw->keep_cc_fields ? apple_nvme_enable_ctrl_keep_cc(anv) : + nvme_enable_ctrl(&anv->ctrl); if (ret) goto out; @@ -2642,11 +2702,28 @@ static const struct apple_nvme_hw apple_nvme_t8140_hw = { .quirks = NVME_QUIRK_BROKEN_FUA, }; +/* T6050: T8140 queue setup; its ANS acknowledges IOP power only after AP readiness. */ +static const struct apple_nvme_hw apple_nvme_t6050_hw = { + .has_lsq_nvmmu = true, + .has_separate_nvmmu = true, + .needs_ioq_registers = true, + .max_queue_depth = 64, + /* + * An fsynced overwrite completed and later reverted after an abrupt + * reset; explicit flushes persisted the preceding data. Let the block + * layer follow each FUA write with a flush. + */ + .quirks = NVME_QUIRK_BROKEN_FUA, + .early_ap_power = false, + .keep_cc_fields = true, +}; + static const struct of_device_id apple_nvme_of_match[] = { { .compatible = "apple,t8015-nvme-ans2", .data = &apple_nvme_t8015_hw }, { .compatible = "apple,t8103-nvme-ans2", .data = &apple_nvme_t8103_hw }, { .compatible = "apple,t8132-nvme-ans2", .data = &apple_nvme_t8132_hw }, { .compatible = "apple,t8140-nvme-ans2", .data = &apple_nvme_t8140_hw }, + { .compatible = "apple,t6050-nvme-ans2", .data = &apple_nvme_t6050_hw }, { .compatible = "apple,nvme-ans2", .data = &apple_nvme_t8103_hw }, {}, }; diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 2aa9ded7d5514..0b1f800c6cc8f 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1310,7 +1310,7 @@ static int nvmet_req_alloc_p2pmem_sgls(struct pci_dev *p2p_dev, return 0; out_free_sg: - pci_p2pmem_free_sgl(req->p2p_dev, req->sg); + pci_p2pmem_free_sgl(p2p_dev, req->sg); out_err: return -ENOMEM; } diff --git a/drivers/nvme/target/pr.c b/drivers/nvme/target/pr.c index ad07f55e7f173..acb44df15fc7e 100644 --- a/drivers/nvme/target/pr.c +++ b/drivers/nvme/target/pr.c @@ -51,6 +51,8 @@ u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask) if (nsid != U32_MAX) { status = nvmet_req_find_ns(req); + if (status == (NVME_SC_INVALID_NS | NVME_STATUS_DNR)) + return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; if (status) return status; if (!req->ns->pr.enable) @@ -75,6 +77,8 @@ u16 nvmet_get_feat_resv_notif_mask(struct nvmet_req *req) u16 status; status = nvmet_req_find_ns(req); + if (status == (NVME_SC_INVALID_NS | NVME_STATUS_DNR)) + return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; if (status) return status; diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c index f9320c46c1232..36bee1acf5c3d 100644 --- a/drivers/pci/controller/pcie-apple.c +++ b/drivers/pci/controller/pcie-apple.c @@ -228,6 +228,20 @@ EXPORT_SYMBOL_GPL(apple_pcie_tunnel_needs_cold_init); #define PORT_MSIMAP_ENABLE BIT(31) #define PORT_MSIMAP_TARGET GENMASK(7, 0) +/* T6050: the root port's port logic, laid out as DesignWare's */ +#define RC_T6050_PORT_LINK_CTRL 0x710 +#define RC_T6050_LINK_MODE GENMASK(21, 16) +#define RC_T6050_PORT_DEBUG0 0x728 +#define RC_T6050_LTSSM_STATE GENMASK(5, 0) +#define RC_T6050_LTSSM_L1_IDLE 0x14 +#define RC_T6050_LINK_WIDTH_SPEED_CTRL 0x80c +#define RC_T6050_LINK_WIDTH GENMASK(12, 8) +#define RC_T6050_SPEED_CHANGE BIT(17) +#define RC_T6050_GEN3_RELATED 0x890 +#define RC_T6050_RATE_SHADOW_SEL GENMASK(25, 24) +#define RC_T6050_MISC_CONTROL_1 0x8bc +#define RC_T6050_DBI_RO_WR_EN BIT(0) + /* * The doorbell address must be in the bottom 4GB because the base register * is only 32 bits wide. Exclude it from the IOVA range and share the same @@ -248,6 +262,7 @@ struct hw_info { u32 pwren_on_ms; bool sid_indexed_rid2sid; bool retain_clocks; + bool adopt_link; bool root_bus_only; bool tunneled; }; @@ -293,6 +308,24 @@ static const struct hw_info t8140_hw = { .root_bus_only = true, }; +/* + * Bare-metal T6050 adoption only. Firmware owns power, PHY, reset and training. + * The 64-entry route table is never probed with test writes. + */ +static const struct hw_info t6050_hw = { + .port_msiaddr = PORT_T602X_MSIADDR, + .port_msiaddr_hi = PORT_T602X_MSIADDR_HI, + .port_perst = PORT_T602X_PERST, + .port_rid2sid = PORT_T602X_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + .max_rid2sid = 64, + .retain_clocks = true, + .adopt_link = true, +}; + +#define T6050_ROUTE(rid, sid) (PORT_RID2SID_VALID | \ + ((sid) << PORT_RID2SID_SID_SHIFT) | (rid)) + static const struct hw_info t8103_pciec_hw = { .port_msiaddr = PORT_MSIADDR, .port_perst = PORT_PERST, @@ -324,6 +357,7 @@ struct apple_pcie { void __iomem *intr2axi_base; void __iomem *oe_fabric_base; void __iomem *early_cfg; + void __iomem *t6050_phy; /* shared PHY, mapped by a cold train */ struct pci_config_window *cfg; struct apple_tunable *rc_tunable; struct apple_tunable *fabric_tunable; @@ -373,6 +407,10 @@ struct apple_pcie_port { bool needs_stop; bool link_failed; bool irq_disabled; + void __iomem *intr2axi; /* adopt_link: this port's */ + u8 adopted_bus; /* adopt_link: the secondary bus */ + unsigned long adopted_slots; /* adopt_link: RID2SID slots routed (slot == SID) */ + struct gpio_desc *perst; /* adopt_link: its PERST# line, for a root-port reset */ }; static void rmw_set(u32 set, void __iomem *addr) @@ -676,9 +714,20 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) struct apple_pcie *pcie = port->pcie; u32 val = 0; - /* FIXME: consider moving each interrupt under each port */ - port->irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), - port->idx); + if (pcie->hw->adopt_link) { + char name[16]; + int irq; + + snprintf(name, sizeof(name), "port%d", port->idx); + irq = of_irq_get_byname(pcie->dev->of_node, name); + + if (irq <= 0) + return irq ?: -ENXIO; + port->irq = irq; + } else { + /* FIXME: consider moving each interrupt under each port */ + port->irq = irq_of_parse_and_map(pcie->dev->of_node, port->idx); + } if (!port->irq) return -ENXIO; @@ -691,6 +740,9 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) return -ENOMEM; } + if (pcie->hw->adopt_link) + apple_pcie_port_writel(port, 0, PORT_MSICFG); + /* Disable all interrupts */ apple_pcie_port_writel(port, ~0, PORT_INTMSK); apple_pcie_port_writel(port, ~0, PORT_INTSTAT); @@ -723,7 +775,22 @@ static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) val = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; } + if (pcie->hw->adopt_link) { + if (apple_pcie_port_readl(port, pcie->hw->port_msiaddr) != DOORBELL_ADDR || + apple_pcie_port_readl(port, pcie->hw->port_msiaddr_hi)) + return -EIO; + for (int i = 0; i < pcie->nvecs; i++) + if (apple_pcie_port_readl(port, pcie->hw->port_msimap + 4 * i) != + (PORT_MSIMAP_ENABLE | (pcie->msi_vector_base + i))) + return -EIO; + /* Enable command is not readable; +0x84 is only a completion read. */ + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + readl(port->intr2axi + 0x84); + } apple_pcie_port_writel(port, val | PORT_MSICFG_EN, PORT_MSICFG); + if (pcie->hw->adopt_link && + !(apple_pcie_port_readl(port, PORT_MSICFG) & PORT_MSICFG_EN)) + return -EIO; return 0; } @@ -739,6 +806,8 @@ static irqreturn_t apple_pcie_port_irq(int irq, void *data) complete_all(&port->pcie->event); break; case PORT_INT_LINK_DOWN: + if (port->pcie->hw->adopt_link) + WRITE_ONCE(port->link_failed, true); /* A later link-up alone does not prove downstream state survived. */ if (port->pcie->hw->tunneled && READ_ONCE(port->started)) { WRITE_ONCE(port->link_failed, true); @@ -1385,9 +1454,19 @@ static void apple_pcie_port_teardown(struct apple_pcie_port *port) { apple_pcie_port_disable_irq(port); - if (port->base) + if (port->base && (port->domain || !port->pcie->hw->adopt_link)) apple_pcie_port_writel(port, ~0, PORT_INTMSK); + if (port->pcie->hw->adopt_link && port->domain) + apple_pcie_port_writel(port, 0, PORT_MSICFG); + if (port->adopted_slots) { + int slot; + + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + apple_pcie_rid2sid_write(port, slot, 0); + port->adopted_slots = 0; + } + if (port->pcie->hw->tunneled && (port->started || port->needs_stop)) apple_pcie_tunnel_stop(port); @@ -1466,6 +1545,806 @@ static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, return apple_pcie_port_readl(port, offset); } +/* + * The DART stream an adopted port's function `rid` maps to: one T6050 DART stream, nonzero + * and below the RID2SID table size (slot == SID), from the host's iommu-map. + */ +static int apple_pcie_adopted_sid(struct apple_pcie_port *port, u32 rid, u32 *sid) +{ + struct apple_pcie *pcie = port->pcie; + struct of_phandle_args iommu = {}; + int ret; + + if (rid >> 8 != port->adopted_bus) { + /* The PCI core numbers root ports' buses in devfn order: the bootloader must too */ + dev_err(pcie->dev, "port %d: RID %04x is off bus %02x, which the bootloader numbered\n", + port->idx, rid, port->adopted_bus); + return -ENODEV; + } + ret = of_map_iommu_id(pcie->dev->of_node, rid, &iommu); + if (ret) + return ret; + ret = iommu.args_count == 1 && iommu.args[0] && + iommu.args[0] < pcie->hw->max_rid2sid && + iommu.args[0] < BITS_PER_LONG && + of_device_is_compatible(iommu.np, "apple,t6050-dart") ? 0 : -EINVAL; + *sid = iommu.args[0]; + of_node_put(iommu.np); + return ret; +} + +/* Route an adopted port's function `rid` to its stream, or confirm the route is in place. */ +static int apple_pcie_adopted_route(struct apple_pcie_port *port, u32 rid) +{ + u32 sid, value, cur; + int ret = apple_pcie_adopted_sid(port, rid, &sid); + + if (ret) + return ret; + value = T6050_ROUTE(rid, sid); + cur = apple_pcie_port_readl(port, port_rid2sid_offset(port, sid)); + if (cur == value && test_bit(sid, &port->adopted_slots)) + return 0; + if (cur & PORT_RID2SID_VALID) + return -EBUSY; + set_bit(sid, &port->adopted_slots); + if (apple_pcie_rid2sid_write(port, sid, value) != value) + return -EIO; + dev_info(port->pcie->dev, "port %d: RID %04x -> SID %u, slot %u\n", + port->idx, rid, sid, sid); + return 0; +} + +/* + * Qualified J775D handoff of one bootloader-trained port: its root port + * 00:.0 and the endpoint on its secondary bus (function 0 routed here; a + * multi-function endpoint's other functions when they are enabled), with DMA and + * endpoint MSI disabled. Every resource comes from the DT by the port's index + * (port, phy, intr2axi, interrupt port, the iommu-map entry for the + * endpoint's RID). All checks precede the first PCIe register write. + */ +static int apple_pcie_adopt_link(struct apple_pcie_port *port) +{ + static const char *const names[] = { "port%d", "phy%d", "intr2axi%d" }; + struct apple_pcie *pcie = port->pcie; + struct platform_device *pdev = to_platform_device(pcie->dev); + void __iomem *rp, *ep; + char name[16]; + u32 value, bus; + u8 cap; + bool msi = false; + int ret, i; + + if (pcie->cfg->busr.start != 0 || + !platform_get_resource_byname(pdev, IORESOURCE_MEM, "config") || + !platform_get_resource_byname(pdev, IORESOURCE_MEM, "rc")) + return -EINVAL; + for (i = 0; i < ARRAY_SIZE(names); i++) { + snprintf(name, sizeof(name), names[i], port->idx); + if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, name)) + return -EINVAL; + } + + if (!(readl(pcie->base + CORE_RC_CTL) & CORE_RC_CTL_RUN) || + !(readl(pcie->base + CORE_RC_STAT) & CORE_RC_STAT_READY) || + !(apple_pcie_port_readl(port, PORT_STATUS) & PORT_STATUS_READY) || + (apple_pcie_port_readl(port, PORT_LINKSTS) & + (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP || + !(apple_pcie_port_readl(port, PORT_LTSSMCTL) & PORT_LTSSMCTL_START) || + (apple_pcie_port_readl(port, PORT_APPCLK) & 0x101) != 0x101 || + (apple_pcie_port_readl(port, pcie->hw->port_perst) & 0x10001) != 1 || + (readl(port->phy + PHY_LANE_CFG) & 0x1000061f) != 0x60f) + return -ENOLINK; + + /* The bootloader numbered the root port's buses: primary 0, one secondary. */ + rp = pcie->cfg->win + (PCI_DEVFN(port->idx, 0) << 12); + value = readl(rp + PCI_PRIMARY_BUS); + bus = (value >> 8) & 0xff; + if (readl(rp) != 0x100c106b || (value & 0xff) != 0 || !bus || + ((value >> 16) & 0xff) != bus || bus > pcie->cfg->busr.end || + (readw(rp + PCI_BRIDGE_CONTROL) & PCI_BRIDGE_CTL_BUS_RESET)) + return -ENODEV; + ep = pcie->cfg->win + (bus << 20); + value = readl(ep); + if (value == 0xffffffff || !(value & 0xffff) || + (readb(ep + PCI_HEADER_TYPE) & 0x7f) != PCI_HEADER_TYPE_NORMAL || + (readw(ep + PCI_COMMAND) & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY | + PCI_COMMAND_MASTER))) + return -ENODEV; + + port->adopted_bus = bus; + ret = apple_pcie_adopted_sid(port, bus << 8, &value); + if (ret) + return ret; + + snprintf(name, sizeof(name), "intr2axi%d", port->idx); + if (!port->intr2axi) + port->intr2axi = devm_platform_ioremap_resource_byname(pdev, name); + if (IS_ERR(port->intr2axi)) + return PTR_ERR(port->intr2axi); + /* + * Only a port whose endpoint needs a root-port reset names its PERST# line + * (reset-gpios, on the port node as on every other Apple host). The pin + * controller owns the pad; the line is taken as the bootloader left it, + * released, and driven only by the reset. + */ + if (!port->perst) + port->perst = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(port->np), + "reset", GPIOD_ASIS, "PERST#"); + if (IS_ERR(port->perst)) { + if (PTR_ERR(port->perst) != -ENOENT) + return PTR_ERR(port->perst); + port->perst = NULL; + } + + /* The currently qualified handoff is Gen3 x1, with no retrain in progress. */ + cap = readb(rp + PCI_CAPABILITY_LIST); + for (i = 0; cap && i < 48; i++) { + if (cap < 0x40 || (cap & 3)) + return -EINVAL; + if (readb(rp + cap + PCI_CAP_LIST_ID) == PCI_CAP_ID_EXP) + break; + cap = readb(rp + cap + PCI_CAP_LIST_NEXT); + } + if (!cap || i == 48) + return -EINVAL; + /* + * x1 at up to the Gen3 target the bootloader set. The endpoint picks the speed: the N1's + * ROM personality links at Gen1 and comes back at Gen3 after its root-port reset. + */ + value = readw(rp + cap + PCI_EXP_LNKSTA); + if ((value & (PCI_EXP_LNKSTA_NLW | PCI_EXP_LNKSTA_LT)) != (1 << PCI_EXP_LNKSTA_NLW_SHIFT) || + (value & PCI_EXP_LNKSTA_CLS) < PCI_EXP_LNKSTA_CLS_2_5GB || + (value & PCI_EXP_LNKSTA_CLS) > PCI_EXP_LNKSTA_CLS_8_0GB) + return -ENOLINK; + + /* Refuse an active endpoint or malformed capability chain. */ + cap = readb(ep + PCI_CAPABILITY_LIST); + for (i = 0; cap && i < 48; i++) { + if (cap < 0x40 || (cap & 3)) + return -EINVAL; + value = readw(ep + cap + PCI_MSI_FLAGS); + switch (readb(ep + cap + PCI_CAP_LIST_ID)) { + case PCI_CAP_ID_MSI: + if (value & PCI_MSI_FLAGS_ENABLE) + return -EBUSY; + msi = true; + break; + case PCI_CAP_ID_MSIX: + if (value & PCI_MSIX_FLAGS_ENABLE) + return -EBUSY; + msi = true; + break; + } + cap = readb(ep + cap + PCI_CAP_LIST_NEXT); + } + if (cap || !msi) + return -EINVAL; + + for (i = 0; i < pcie->hw->max_rid2sid; i++) + if (apple_pcie_port_readl(port, port_rid2sid_offset(port, i)) & + PORT_RID2SID_VALID) + return -EBUSY; + + /* The route goes in the slot whose index is the SID, as on J775D's adopted ports. */ + ret = apple_pcie_adopted_route(port, bus << 8); + if (!ret) + dev_info(pcie->dev, "adopted port %d, endpoint %02x:00\n", port->idx, bus); + return ret; +} + +/* + * T6050 port bring-up, for an adopted port whose endpoint must come back on a + * retrained link: port disable, the port reset table, port enable, root complex + * configuration for Gen3 x1, then the lane PHY, PERST# and the LTSSM. The port's + * registers are its port, phy and intr2axi blocks; the root complex's + * are the root port's config space. The boot loader copies the port's tunables + * onto its node. + */ +static const struct apple_pcie_reset_reg apple_pcie_t6050_reset_regs[] = { + { 0x088, 0x00000110 }, + { 0x100, 0xffffffff }, + { 0x148, 0xffffffff }, + { 0x210, 0xffffffff }, + { 0x080, 0x00000000 }, + { 0x084, 0x00000000 }, + { 0x104, 0xfffffff0 }, + { 0x124, 0x00000100 }, + { 0x16c, 0x00000000 }, + { 0x13c, 0x00000000 }, + { 0x800, 0x00100100 }, + { 0x808, 0x001001ff }, + { 0x82c, 0x00010000 }, +}; + +/* Written after the RID2SID and MSI map tables are cleared */ +static const struct apple_pcie_reset_reg apple_pcie_t6050_reset_regs_late[] = { + { 0x130, 0x03020000 }, + { 0x140, 0x00000010 }, + { 0x144, 0x00253770 }, + { 0x21c, 0x00000000 }, + { 0x834, 0x00000000 }, + { 0x83c, 0x00000000 }, +}; + +static void apple_pcie_t6050_rmw(void __iomem *addr, u32 clear, u32 set) +{ + writel((readl(addr) & ~clear) | set, addr); +} + +/* The root port's config space through the ECAM window: there is no pci_dev before enumeration */ +static void __iomem *apple_pcie_t6050_dbi(struct apple_pcie_port *port) +{ + return port->pcie->cfg->win + (PCI_DEVFN(port->idx, 0) << 12); +} + +static void apple_pcie_t6050_dbi_rmw(void __iomem *dbi, int where, u32 clear, u32 set) +{ + writel((readl(dbi + where) & ~clear) | set, dbi + where); +} + +static int apple_pcie_t6050_exp_cap(void __iomem *dbi) +{ + u8 cap = readb(dbi + PCI_CAPABILITY_LIST); + int i; + + for (i = 0; cap && i < 48; i++) { + if (cap < 0x40 || (cap & 3)) + return 0; + if (readb(dbi + cap + PCI_CAP_LIST_ID) == PCI_CAP_ID_EXP) + return cap; + cap = readb(dbi + cap + PCI_CAP_LIST_NEXT); + } + return 0; +} + +static int apple_pcie_t6050_wait(struct apple_pcie_port *port, void __iomem *base, + u32 offset, u32 mask, u32 value, unsigned int timeout_us) +{ + u32 stat; + int ret; + + ret = readl_poll_timeout(base + offset, stat, (stat & mask) == value, 1, timeout_us); + if (ret) + dev_err(port->pcie->dev, "port %d: %#x = %#x, waiting for %#x/%#x\n", + port->idx, offset, stat, value, mask); + return ret; +} + +/* + * A tunable list the boot loader copies onto the root port node, groups of + * (offset, 4, mask, value): for the port's registers, or with @rp for the root + * port's config space. A list it did not copy is skipped. + */ +static int apple_pcie_t6050_tune(struct apple_pcie_port *port, void __iomem *dbi, + const char *name) +{ + u32 cells[64]; + int count, ret, i; + + if (!of_property_present(port->np, name)) + return 0; + count = of_property_count_u32_elems(port->np, name); + if (count <= 0 || count > ARRAY_SIZE(cells) || count % 4) + return -EINVAL; + ret = of_property_read_u32_array(port->np, name, cells, count); + if (ret) + return ret; + for (i = 0; i < count; i += 4) + if (cells[i + 1] != 4 || (cells[i] & 3) || + cells[i] >= (dbi ? PCI_CFG_SPACE_EXP_SIZE : SZ_16K)) + return -EINVAL; + + for (i = 0; i < count; i += 4) + apple_pcie_t6050_rmw((dbi ?: port->base) + cells[i], cells[i + 2], cells[i + 3]); + return 0; +} + +/* + * PERST# through the pin controller: the pad as a GPIO output (its function bits + * cleared, mode output) at the line's logical level, which the DT's + * GPIO_ACTIVE_LOW turns into the pad level. + */ +static int apple_pcie_t6050_perst(struct apple_pcie_port *port, bool asserted) +{ + return gpiod_direction_output(port->perst, asserted); +} + +/* Let the link go idle, hold the endpoint in reset and stop the port and its lane clocks */ +static int apple_pcie_t6050_port_disable(struct apple_pcie_port *port, struct pci_dev *rp) +{ + void __iomem *perst = port->base + PORT_T602X_PERST; + void __iomem *lane = port->phy + PHY_LANE_CFG; + u32 stat; + int ret, err; + + if (!(readl(port->base + PORT_STATUS) & BIT(2))) { + ret = apple_pcie_t6050_wait(port, port->base, 0x820, BIT(2), BIT(2), 10000); + if (ret) + return ret; + } + ret = readl_poll_timeout_atomic(port->base + 0x820, stat, + !(stat & GENMASK(8, 0)) || !(stat & GENMASK(23, 16)), + 1, 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + 0x13c, 0, BIT(8)); + + /* The link out of L1 */ + ret = read_poll_timeout_atomic(pci_read_config_dword, err, + err || FIELD_GET(RC_T6050_LTSSM_STATE, stat) != + RC_T6050_LTSSM_L1_IDLE, + 5, 5000, false, rp, RC_T6050_PORT_DEBUG0, &stat); + if (ret || err) + return ret ?: pcibios_err_to_errno(err); + + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(0), BIT(0), 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, BIT(24)); + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(16), BIT(16), 2000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, BIT(16)); + udelay(1); + apple_pcie_t6050_rmw(perst, PORT_PERST_OFF, 0); + udelay(1); + apple_pcie_t6050_rmw(perst, BIT(24), 0); + apple_pcie_t6050_rmw(port->base + 0x13c, BIT(8), 0); + + apple_pcie_t6050_rmw(lane, BIT(31), 0); + apple_pcie_t6050_rmw(lane, BIT(10), 0); + apple_pcie_t6050_rmw(lane, BIT(9), 0); + apple_pcie_t6050_rmw(lane, 0, BIT(4)); + apple_pcie_t6050_rmw(lane, PHY_LANE_CFG_REFCLK0REQ, 0); + apple_pcie_t6050_rmw(lane, PHY_LANE_CFG_REFCLK1REQ, 0); + + writel(~0, port->base + PORT_INTSTAT); + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, PORT_APPCLK_EN, 0); + return apple_pcie_t6050_wait(port, port->base, PORT_STATUS, PORT_STATUS_READY, 0, + 250000); +} + +/* The port reset table, then the port clocked and out of reset, ready to train */ +static int apple_pcie_t6050_port_enable(struct apple_pcie_port *port) +{ + void __iomem *perst = port->base + PORT_T602X_PERST; + u32 stat; + int ret, i; + + for (i = 0; i < ARRAY_SIZE(apple_pcie_t6050_reset_regs); i++) + writel(apple_pcie_t6050_reset_regs[i].value, + port->base + apple_pcie_t6050_reset_regs[i].offset); + for (i = 0; i < port->pcie->hw->max_rid2sid; i++) + writel(0, port->base + port_rid2sid_offset(port, i)); + for (i = 0; i < 256; i++) + writel(0, port->base + PORT_T602X_MSIMAP + 4 * i); + for (i = 0; i < ARRAY_SIZE(apple_pcie_t6050_reset_regs_late); i++) + writel(apple_pcie_t6050_reset_regs_late[i].value, + port->base + apple_pcie_t6050_reset_regs_late[i].offset); + + if (!(readl(perst) & BIT(16))) + return -EIO; + ret = apple_pcie_t6050_tune(port, NULL, "apple,apcie-config-tunables"); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, 0, PORT_APPCLK_EN); + + ret = readl_poll_timeout_atomic(port->base + 0xa8, stat, + (stat & (BIT(16) | BIT(0))) != (BIT(16) | BIT(0)), + 1, 10000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, 0, PORT_PERST_OFF); + ret = apple_pcie_t6050_wait(port, port->base, 0xa8, BIT(0), BIT(0), 20000); + if (ret) + return ret; + apple_pcie_t6050_rmw(perst, BIT(16), 0); + ret = apple_pcie_t6050_wait(port, port->base, PORT_STATUS, PORT_STATUS_READY, + PORT_STATUS_READY, 250000); + if (ret) + return ret; + + apple_pcie_t6050_rmw(port->base + PORT_APPCLK, 0, PORT_APPCLK_CGDIS); + return apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, PORT_LINKSTS_BUSY, 0, + 250000); +} + +/* Root complex tunables and a Gen3 x1 link, with the read-only registers opened for it */ +static int apple_pcie_t6050_port_configure(struct apple_pcie_port *port) +{ + static const char * const shadow[] = { + "apple,pcie-rc-gen3-shadow-tunables", + "apple,pcie-rc-gen4-shadow-tunables", + "apple,pcie-rc-gen5-shadow-tunables", + }; + static const u32 shadow_sel[] = { 0, BIT(24), 0 }; + void __iomem *dbi = apple_pcie_t6050_dbi(port); + int cap = apple_pcie_t6050_exp_cap(dbi); + u16 ctl2; + int ret, i; + + if (!cap) + return -EIO; + apple_pcie_t6050_dbi_rmw(dbi, RC_T6050_MISC_CONTROL_1, 0, RC_T6050_DBI_RO_WR_EN); + ret = apple_pcie_t6050_tune(port, dbi, "apple,pcie-rc-tunables"); + for (i = 0; !ret && i < ARRAY_SIZE(shadow); i++) { + apple_pcie_t6050_dbi_rmw(dbi, RC_T6050_GEN3_RELATED, RC_T6050_RATE_SHADOW_SEL, + shadow_sel[i]); + ret = apple_pcie_t6050_tune(port, dbi, shadow[i]); + } + if (!ret) { + apple_pcie_t6050_dbi_rmw(dbi, 0x194, 0, BIT(0)); + if (!(readl(dbi + cap + PCI_EXP_LNKCAP2) & PCI_EXP_LNKCAP2_SLS_8_0GB)) + ret = -EIO; + } + if (!ret) { + ctl2 = readw(dbi + cap + PCI_EXP_LNKCTL2); + writew((ctl2 & ~PCI_EXP_LNKCTL2_TLS) | PCI_EXP_LNKCTL2_TLS_8_0GT, + dbi + cap + PCI_EXP_LNKCTL2); + apple_pcie_t6050_dbi_rmw(dbi, RC_T6050_PORT_LINK_CTRL, RC_T6050_LINK_MODE, + FIELD_PREP(RC_T6050_LINK_MODE, 1)); + apple_pcie_t6050_dbi_rmw(dbi, RC_T6050_LINK_WIDTH_SPEED_CTRL, RC_T6050_LINK_WIDTH, + FIELD_PREP(RC_T6050_LINK_WIDTH, 1)); + apple_pcie_t6050_dbi_rmw(dbi, cap + PCI_EXP_LNKCAP, PCI_EXP_LNKCAP_MLW, + FIELD_PREP(PCI_EXP_LNKCAP_MLW, 1)); + } + apple_pcie_t6050_dbi_rmw(dbi, RC_T6050_MISC_CONTROL_1, RC_T6050_DBI_RO_WR_EN, 0); + if (ret) + return ret; + + apple_pcie_t6050_dbi_rmw(dbi, cap + PCI_EXP_LNKCTL, 0, PCI_EXP_LNKCTL_LBMIE); + apple_pcie_t6050_dbi_rmw(dbi, RC_T6050_LINK_WIDTH_SPEED_CTRL, 0, RC_T6050_SPEED_CHANGE); + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + return 0; +} + +/* Disable, reset, enable and configure an adopted port whose endpoint is held in PERST# */ +static int apple_pcie_t6050_port_restart(struct apple_pcie_port *port, struct pci_dev *rp) +{ + return apple_pcie_t6050_port_disable(port, rp) ?: + apple_pcie_t6050_port_enable(port) ?: + apple_pcie_t6050_port_configure(port); +} + +/* Lane clocks, PERST# released, then the LTSSM to link up */ +static int apple_pcie_t6050_port_train(struct apple_pcie_port *port) +{ + void __iomem *lane = port->phy + PHY_LANE_CFG; + int ret; + + apple_pcie_t6050_rmw(lane, 0, PHY_LANE_CFG_REFCLK0REQ); + ret = apple_pcie_t6050_wait(port, port->phy, PHY_LANE_CFG, PHY_LANE_CFG_REFCLK0ACK, + PHY_LANE_CFG_REFCLK0ACK, 500000); + if (ret) + return ret; + apple_pcie_t6050_rmw(lane, 0, PHY_LANE_CFG_REFCLK1REQ); + ret = apple_pcie_t6050_wait(port, port->phy, PHY_LANE_CFG, PHY_LANE_CFG_REFCLK1ACK, + PHY_LANE_CFG_REFCLK1ACK, 500000); + if (ret) + return ret; + apple_pcie_t6050_rmw(lane, BIT(4), 0); + udelay(1); + apple_pcie_t6050_rmw(lane, 0, BIT(9)); + apple_pcie_t6050_rmw(lane, 0, BIT(10)); + + /* Tperst-clk is at least 100 us (PCIe CEM r5.0, 2.9.2) */ + usleep_range(100, 110); + ret = apple_pcie_t6050_perst(port, false); + if (ret) + return ret; + apple_pcie_t6050_rmw(port->base + PORT_LTSSMCTL, 0, PORT_LTSSMCTL_START); + ret = apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, PORT_LINKSTS_UP, + PORT_LINKSTS_UP, 500000); + if (ret) + return ret; + + /* Wait for 100ms after the link trains (PCIe r5.0, 6.6.1) */ + msleep(100); + return 0; +} + +/* + * A T6050 port the boot loader left untrained (the J714c's N1 on every boot): the shared + * PCIe PHY clocked and out of reset, the RC running, the endpoint powered (pwren-gpios) + * with PERST# held, then the port enable, configure and train sequences the root-port + * reset uses, and the bus numbers a boot loader leaves. The adopt checks then run as for + * a port trained before Linux. Order and values are m1n1's controller bring-up and the + * kernel's own T6050 port sequence. + */ +#define T6050_PHY_CTRL 0x0 +#define T6050_PHY_CTRL_CLK0REQ BIT(0) +#define T6050_PHY_CTRL_CLK1REQ BIT(1) +#define T6050_PHY_CTRL_CLK0ACK BIT(2) +#define T6050_PHY_CTRL_CLK1ACK BIT(3) +#define T6050_PHY_CTRL_RESET_T8132 BIT(4) +#define T6050_PHY_CTRL_RESET_T8103 BIT(7) +#define T6050_PHY_CTRL_LANES GENMASK(9, 8) +#define T6050_PHY_CFG 0x4 +#define T6050_PHY_STAT 0x8 +#define T6050_PHYCMN_CLK 0x0 +#define T6050_PHYCMN_CLK_MODE GENMASK(1, 0) +#define CORE_RC_T6050_PRE_RUN 0x54 /* m1n1 writes 0x140 here before RUN */ + +static int apple_pcie_t6050_phy_wait(struct apple_pcie *pcie, u32 mask, u32 value) +{ + u32 v; + int ret; + + ret = readl_poll_timeout(pcie->t6050_phy + T6050_PHY_CTRL, v, (v & mask) == value, + 10, 50000); + if (ret) + dev_err(pcie->dev, "shared PHY: %#x, waiting for %#x/%#x\n", v, value, mask); + return ret; +} + +/* + * A host tunable list (apple,tunable-, groups of (offset, 4, mask, value)) applied + * to the host window @win. The window may contain windows mapped elsewhere (the PHY + * window holds the port and shared PHYs), so it is mapped without claiming it. + */ +static int apple_pcie_t6050_tune_host(struct apple_pcie *pcie, const char *name, + const char *win) +{ + struct platform_device *pdev = to_platform_device(pcie->dev); + struct device_node *np = pcie->dev->of_node; + struct resource *res; + void __iomem *base; + char prop[48]; + u32 *cells; + int count, i, ret; + + snprintf(prop, sizeof(prop), "apple,tunable-%s", name); + if (!of_property_present(np, prop)) + return 0; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, win); + count = of_property_count_u32_elems(np, prop); + if (!res) + return dev_err_probe(pcie->dev, -EINVAL, "tunable %s names missing reg window %s\n", + prop, win); + if (count <= 0 || count % 4) + return dev_err_probe(pcie->dev, -EINVAL, "tunable %s has %d cells, want a 4-cell multiple\n", + prop, count); + cells = kmalloc_array(count, sizeof(*cells), GFP_KERNEL); + if (!cells) + return -ENOMEM; + ret = of_property_read_u32_array(np, prop, cells, count); + for (i = 0; !ret && i < count; i += 4) + if (cells[i + 1] != 4 || (cells[i] & 3) || cells[i] > resource_size(res) - 4) + ret = -EINVAL; + if (!ret) { + base = ioremap(res->start, resource_size(res)); + if (base) { + for (i = 0; i < count; i += 4) + apple_pcie_t6050_rmw(base + cells[i], cells[i + 2], cells[i + 3]); + iounmap(base); + } else { + ret = -ENOMEM; + } + } + kfree(cells); + if (ret) + dev_err(pcie->dev, "%s: %d\n", prop, ret); + return ret; +} + +static int apple_pcie_t6050_shared_phy_on(struct apple_pcie *pcie) +{ + struct platform_device *pdev = to_platform_device(pcie->dev); + const u32 up = T6050_PHY_CTRL_CLK0ACK | T6050_PHY_CTRL_CLK1ACK | T6050_PHY_CTRL_LANES; + void __iomem *phy, *cmn; + u32 v; + int ret; + + if (!pcie->t6050_phy) { + phy = devm_platform_ioremap_resource_byname(pdev, "phy-shared"); + if (IS_ERR(phy)) + return PTR_ERR(phy); + pcie->t6050_phy = phy; + } + phy = pcie->t6050_phy; + if ((readl(phy + T6050_PHY_CTRL) & (up | T6050_PHY_CTRL_RESET_T8132 | + T6050_PHY_CTRL_RESET_T8103)) == up) + return 0; + cmn = devm_platform_ioremap_resource_byname(pdev, "phy-common"); + if (IS_ERR(cmn)) + return PTR_ERR(cmn); + + /* Fabric, clock generator and PHY tunables, then the PHY clocks (m1n1's order) */ + ret = apple_pcie_t6050_tune_host(pcie, "axi2af", "axi2af") ?: + apple_pcie_t6050_tune_host(pcie, "cio3pllcore", "pcieclkgen") ?: + apple_pcie_t6050_tune_host(pcie, "pcieclkgen", "pcieclkgen") ?: + apple_pcie_t6050_tune_host(pcie, "phy", "phy-window"); + if (ret) + return ret; + apple_pcie_t6050_rmw(phy + T6050_PHY_CTRL, 0, T6050_PHY_CTRL_CLK0REQ); + ret = apple_pcie_t6050_phy_wait(pcie, T6050_PHY_CTRL_CLK0ACK, T6050_PHY_CTRL_CLK0ACK); + if (ret) + return ret; + apple_pcie_t6050_rmw(phy + T6050_PHY_CTRL, 0, T6050_PHY_CTRL_CLK1REQ); + ret = apple_pcie_t6050_phy_wait(pcie, T6050_PHY_CTRL_CLK1ACK, T6050_PHY_CTRL_CLK1ACK); + if (ret) + return ret; + apple_pcie_t6050_rmw(phy + T6050_PHY_CTRL, T6050_PHY_CTRL_RESET_T8132, 0); + ret = apple_pcie_t6050_phy_wait(pcie, T6050_PHY_CTRL_RESET_T8132, 0); + if (ret) + return ret; + apple_pcie_t6050_rmw(phy + T6050_PHY_CTRL, T6050_PHY_CTRL_RESET_T8103, 0); + ret = apple_pcie_t6050_phy_wait(pcie, T6050_PHY_CTRL_RESET_T8103, 0); + if (ret) + return ret; + apple_pcie_t6050_rmw(phy + T6050_PHY_CFG, 0, BIT(0)); + ret = apple_pcie_t6050_tune_host(pcie, "phy-ip-pll", "phy-ip") ?: + apple_pcie_t6050_tune_host(pcie, "phy-ip-auspma", "phy-ip"); + if (ret) + return ret; + apple_pcie_t6050_rmw(phy + T6050_PHY_CFG, 0, BIT(4)); + apple_pcie_t6050_rmw(cmn + T6050_PHYCMN_CLK, T6050_PHYCMN_CLK_MODE, + FIELD_PREP(T6050_PHYCMN_CLK_MODE, 1)); + if (readl_poll_timeout(phy + T6050_PHY_STAT, v, v & BIT(0), 10, 250000)) + dev_info(pcie->dev, "shared PHY: clock status %#x\n", v); + apple_pcie_t6050_rmw(phy + T6050_PHY_CTRL, 0, T6050_PHY_CTRL_LANES); + return 0; +} + +/* The port's DART answers once the root complex is up (apple,manual-availability) */ +static void apple_pcie_dart_available(struct apple_pcie_port *port) +{ + struct of_phandle_args iommu = {}; + + if (of_map_iommu_id(port->pcie->dev->of_node, port->adopted_bus << 8, &iommu) || + !iommu.np) + return; + apple_dart_make_available(iommu.np); + of_node_put(iommu.np); +} + +static int apple_pcie_t6050_cold_train(struct apple_pcie_port *port) +{ + struct apple_pcie *pcie = port->pcie; + struct platform_device *pdev = to_platform_device(pcie->dev); + struct gpio_desc *pwren; + void __iomem *dbi; + char name[16]; + u32 v, bus; + int ret; + + bus = pcie->cfg->busr.start + 1 + port->idx; + if (bus > pcie->cfg->busr.end) + return -EINVAL; + ret = apple_pcie_t6050_shared_phy_on(pcie); + if (ret) + return ret; + if (!(readl(pcie->base + CORE_RC_CTL) & CORE_RC_CTL_RUN)) { + writel(0x140, pcie->base + CORE_RC_T6050_PRE_RUN); + writel(CORE_RC_CTL_RUN, pcie->base + CORE_RC_CTL); + } + ret = readl_poll_timeout(pcie->base + CORE_RC_STAT, v, v & CORE_RC_STAT_READY, 10, 250000); + if (ret) + return ret; + + snprintf(name, sizeof(name), "intr2axi%d", port->idx); + if (!port->intr2axi) + port->intr2axi = devm_platform_ioremap_resource_byname(pdev, name); + if (IS_ERR(port->intr2axi)) + return PTR_ERR(port->intr2axi); + if (!port->perst) + port->perst = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(port->np), + "reset", GPIOD_ASIS, "PERST#"); + if (IS_ERR(port->perst)) + return PTR_ERR(port->perst); + ret = apple_pcie_t6050_perst(port, true); + if (ret) + return ret; + + /* The endpoint held, then powered: the boot loader leaves it off */ + pwren = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(port->np), "pwren", + GPIOD_OUT_LOW, "PWREN"); + if (IS_ERR(pwren)) + return PTR_ERR(pwren); + msleep(50); + ret = gpiod_set_value_cansleep(pwren, 1); + if (ret) + return ret; + msleep(150); + + ret = apple_pcie_t6050_port_enable(port) ?: + apple_pcie_t6050_port_configure(port) ?: + apple_pcie_t6050_port_train(port); + if (ret) + return ret; + + /* Primary bus 0, the port's secondary and subordinate bus, as a boot loader leaves them */ + dbi = apple_pcie_t6050_dbi(port); + v = readl(dbi + PCI_PRIMARY_BUS) & 0xff000000; + writel(v | (bus << 8) | (bus << 16), dbi + PCI_PRIMARY_BUS); + dev_info(pcie->dev, "port %d: trained by Linux, the boot loader left it down\n", port->idx); + return 0; +} + +/* + * Root-port reset of an adopted T6050 port, for an endpoint that must come back on a + * retrained link (the N1 after its ROM boot): PME_Turn_Off to L2, PERST# asserted, the + * port disabled, reset, enabled and configured, its routes and MSI map rewritten, then the + * lane PHY, PERST# release and LTSSM to a Gen3 x1 link. The caller has quiesced the + * endpoint and saved the root port's state; the PCI core restores it. + */ +static struct apple_pcie *apple_pcie_lookup(struct device *dev); + +static int apple_pcie_t6050_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *rp) +{ + struct pci_config_window *cfg = rp->sysdata; + struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); + struct apple_pcie_port *port = NULL, *p; + u32 routes[BITS_PER_LONG], value; + int ret, slot, i; + + if (!pcie || !pcie->hw->adopt_link || !pci_is_root_bus(rp->bus)) + return -EOPNOTSUPP; + list_for_each_entry(p, &pcie->ports, entry) + if (p->idx == PCI_SLOT(rp->devfn)) + port = p; + if (!port || !port->perst || !port->intr2axi) + return -EOPNOTSUPP; + + mutex_lock(&pcie->lock); + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + routes[slot] = apple_pcie_port_readl(port, port_rid2sid_offset(port, slot)); + + /* Bridge decoding off and the secondary bus unnumbered while the link is down */ + ret = pcibios_err_to_errno(pci_write_config_word(rp, PCI_COMMAND, 0)) ?: + pcibios_err_to_errno(pci_read_config_dword(rp, PCI_PRIMARY_BUS, &value)) ?: + pcibios_err_to_errno(pci_write_config_dword(rp, PCI_PRIMARY_BUS, + value & 0xff0000ff)); + if (ret) + goto out; + /* PME_Turn_Off, then L2 */ + apple_pcie_port_writel(port, apple_pcie_port_readl(port, PORT_APPCLK) | 0x100, PORT_APPCLK); + apple_pcie_port_writel(port, 0x11, 0x88); + ret = apple_pcie_t6050_wait(port, port->base, 0x88, 1, 0, 500000) ?: + apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, BIT(6), BIT(6), 500000); + if (ret) + goto out; + WRITE_ONCE(port->link_failed, true); + ret = apple_pcie_t6050_perst(port, true); + if (ret) + goto out; + writel(readl(port->phy) & ~BIT(31), port->phy); + ret = apple_pcie_t6050_port_restart(port, rp); + if (ret) + goto out; + + /* The reset table cleared the routes and the MSI setup: put back what Linux set */ + for_each_set_bit(slot, &port->adopted_slots, BITS_PER_LONG) + if (apple_pcie_rid2sid_write(port, slot, routes[slot]) != routes[slot]) + ret = -EIO; + apple_pcie_port_writel(port, lower_32_bits(DOORBELL_ADDR), pcie->hw->port_msiaddr); + if (pcie->hw->port_msiaddr_hi) + apple_pcie_port_writel(port, 0, pcie->hw->port_msiaddr_hi); + for (i = 0; i < pcie->nvecs; i++) + apple_pcie_port_writel(port, FIELD_PREP(PORT_MSIMAP_TARGET, pcie->msi_vector_base + i) | + PORT_MSIMAP_ENABLE, pcie->hw->port_msimap + 4 * i); + writel(PCIEC_INTR2AXI_ENABLE, port->intr2axi + PCIEC_INTR2AXI_CTRL); + readl(port->intr2axi + 0x84); + apple_pcie_port_writel(port, PORT_MSICFG_EN, PORT_MSICFG); + if (ret) + goto out; + + ret = apple_pcie_t6050_port_train(port); + if (ret) + goto out; + /* Link interrupts stay masked after the reset table: the link is checked on access */ + apple_pcie_port_writel(port, ~0, PORT_INTSTAT); + WRITE_ONCE(port->link_failed, false); + dev_info(pcie->dev, "port %d: link retrained after a root-port reset\n", port->idx); +out: + mutex_unlock(&pcie->lock); + if (ret) + dev_err(pcie->dev, "port %d: root-port reset stopped: %d\n", port->idx, ret); + return ret; +} + static int apple_pcie_setup_link(struct apple_pcie *pcie, struct apple_pcie_port *port, struct device_node *np) @@ -1695,6 +2574,17 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, goto err_teardown; } } + } else if (pcie->hw->adopt_link) { + ret = apple_pcie_adopt_link(port); + /* A port left down that names its endpoint's power line is trained here */ + if (ret == -ENOLINK && of_property_present(np, "pwren-gpios")) { + ret = apple_pcie_t6050_cold_train(port); + if (!ret) + ret = apple_pcie_adopt_link(port); + } + if (ret) + goto err_teardown; + apple_pcie_dart_available(port); } else { if (pcie->hw->retain_clocks) rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); @@ -1731,6 +2621,9 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, if (ret) goto err_teardown; + if (pcie->hw->adopt_link) + goto rid2sid_ready; + /* Reset all RID/SID mappings, and check for RAZ/WI registers */ for (i = 0; i < pcie->hw->max_rid2sid; i++) { if (apple_pcie_rid2sid_write(port, i, 0xbad1d) != 0xbad1d) @@ -1741,7 +2634,7 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, dev_dbg(pcie->dev, "%pOF: %d RID/SID mapping entries\n", np, i); port->sid_map_sz = i; - +rid2sid_ready: list_add_tail(&port->entry, &pcie->ports); init_completion(&pcie->event); @@ -1754,6 +2647,10 @@ static int apple_pcie_setup_port(struct apple_pcie *pcie, unsigned long timeout, left = 0; int attempt, attempts = 1; + if (pcie->hw->adopt_link) { + ret = -ENOLINK; + goto err_teardown; + } if (pcie->hw->tunneled) { if (pcie->kernel_init) { apple_pcie_tunnel_apply_tunable(port->base, @@ -1865,6 +2762,23 @@ static int apple_msi_init(struct apple_pcie *pcie) return ret; } + if (pcie->hw->adopt_link) { + ret = of_property_read_u32(to_of_node(fwnode), "apple,msi-vector-base", + &pcie->msi_vector_base); + /* + * The host's slice of apcie's 128 vectors (ADT msi-vector-offset 1824): the + * J716c's bridge owns 64-127 (AIC 1888, four interrupt cells), the J714c's + * pci-bridge0 owns 0-31 (AIC 1824, three cells). + */ + if (ret || (pcie->nvecs != 32 && pcie->nvecs != 64) || + (args.args_count != 3 && args.args_count != 4) || args.args[0] != 0 || + (args.args_count == 4 && args.args[1] != 0) || + args.args[args.args_count - 2] != 1824 + pcie->msi_vector_base || + args.args[args.args_count - 1] != IRQ_TYPE_EDGE_RISING || + DOORBELL_ADDR != 0xfffff000) + goto invalid_slice; + } + /* * The controller-wide ADT count (512) is not a per-host allocation. * Internal PCIe owns vectors 0..31. Each PCIe-C bridge owns a distinct @@ -1954,7 +2868,7 @@ static int apple_msi_init(struct apple_pcie *pcie) invalid_slice: of_node_put(args.np); return dev_err_probe(pcie->dev, -EINVAL, - "invalid or missing PCIe-C MSI slice\n"); + "invalid or missing MSI slice\n"); } static void apple_pcie_cleanup(void *data) @@ -2049,6 +2963,14 @@ static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_d of_node_put(iommu_spec.np); sid = iommu_spec.args[0]; + if (pcie->hw->adopt_link) { + if (READ_ONCE(port->link_failed)) + return -ENODEV; + mutex_lock(&pcie->lock); + err = apple_pcie_adopted_route(port, rid); + mutex_unlock(&pcie->lock); + return err; + } mutex_lock(&port->pcie->lock); if (port->pcie->hw->sid_indexed_rid2sid) { @@ -2090,7 +3012,7 @@ static void apple_pcie_disable_device(struct pci_host_bridge *bridge, struct pci int idx; port = apple_pcie_get_port(pdev); - if (!port) + if (!port || port->pcie->hw->adopt_link) return; mutex_lock(&port->pcie->lock); @@ -2126,9 +3048,13 @@ static int apple_pcie_init(struct pci_config_window *cfg) ret = apple_pcie_setup_port(pcie, of_port); if (ret) { dev_err(dev, "Port %pOF setup fail: %d\n", of_port, ret); - return ret; + /* Adopted ports are independent: one refused leaves the others. */ + if (!pcie->hw->adopt_link) + return ret; } } + if (pcie->hw->adopt_link && list_empty(&pcie->ports)) + return -ENODEV; /* The bus scan follows this callback. */ apple_pcie_tunnel_wait_ready(pcie); @@ -2148,6 +3074,27 @@ static void __iomem *apple_pcie_map_bus(struct pci_bus *bus, struct apple_pcie *pcie = apple_pcie_lookup(cfg->parent); struct apple_pcie_port *port; + if (pcie->hw->adopt_link) { + /* Only adopted ports' root ports, and the endpoint behind each. */ + list_for_each_entry(port, &pcie->ports, entry) { + if (bus->number == 0) { + if (devfn == PCI_DEVFN(port->idx, 0)) + return pci_ecam_map_bus(bus, devfn, where); + continue; + } + if (bus->number != port->adopted_bus) + continue; + if (PCI_SLOT(devfn) || READ_ONCE(port->link_failed)) + return NULL; + if ((apple_pcie_port_readl(port, PORT_LINKSTS) & + (PORT_LINKSTS_UP | PORT_LINKSTS_BUSY)) != PORT_LINKSTS_UP) { + WRITE_ONCE(port->link_failed, true); + return NULL; + } + return pci_ecam_map_bus(bus, devfn, where); + } + return NULL; + } if (!pcie->hw->root_bus_only) return pci_ecam_map_bus(bus, devfn, where); @@ -2383,11 +3330,24 @@ static int apple_pcie_tunnel_pci_notify(struct notifier_block *nb, switch (action) { case BUS_NOTIFY_ADD_DEVICE: + /* + * T6050 adopted ports: no ASPM at all. On the J714c the N1 in its ROM + * personality (no firmware yet) takes the SoC down 20-40 s after the link + * first idles into L1; pcie_aspm=off runs clean. + */ + if (pcie->hw->adopt_link) { + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) + pcie_aspm_remove_cap(pdev, PCI_EXP_LNKCAP_ASPM_L0S | + PCI_EXP_LNKCAP_ASPM_L1); + break; + } pm_runtime_get_noresume(&pdev->dev); if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) pcie_aspm_remove_cap(pdev, PCI_EXP_LNKCAP_ASPM_L1); break; case BUS_NOTIFY_DEL_DEVICE: + if (pcie->hw->adopt_link) + break; pm_runtime_put_noidle(&pdev->dev); break; } @@ -2670,7 +3630,38 @@ static int apple_pcie_probe(struct platform_device *pdev) * reset-gpios provider here rejects the valid synthesized port before * the tunneled setup path can consume that handoff. */ - if (!hw->tunneled) { + if (hw->adopt_link) { + /* + * An adopted host may name its ports' power domain. The driver core + * attaches it before probe; the boot loader left it on, so nothing + * changes, and the host becomes its consumer: genpd's sync_state no + * longer switches it off under a live link. + */ + if (!of_get_available_child_count(dev->of_node)) + return dev_err_probe(dev, -EINVAL, "adopted host has no available port node\n"); + for_each_available_child_of_node_scoped(dev->of_node, child) { + u32 reg; + + /* A root port 00:.0 on bus 0, n a GP port (0-3) */ + if (of_property_read_u32(child, "reg", ®) || reg & ~0xf800 || + (reg >> 11) > 3 || + !of_property_read_bool(child, "apple,link-trained-by-bootloader")) + return dev_err_probe(dev, -EINVAL, + "port %pOF: bad reg or missing apple,link-trained-by-bootloader\n", + child); + /* + * A port whose endpoint needs a root-port reset names its PERST# + * line: probe waits for the line's controller before any port is + * touched (a failed adoption would otherwise swallow the deferral). + */ + if (of_property_present(child, "reset-gpios")) { + ret = apple_pcie_probe_port(child); + if (ret) + return dev_err_probe(dev, ret, "Port %pOF probe fail\n", + child); + } + } + } else if (!hw->tunneled) { /* Check for probe dependencies for all ports first */ for_each_available_child_of_node(dev->of_node, of_port) { ret = apple_pcie_probe_port(of_port); @@ -2698,15 +3689,15 @@ static int apple_pcie_probe(struct platform_device *pdev) pcie->dev = dev; pcie->hw = hw; pcie->piodma_supplier = piodma_supplier; - /* Only this retained-DMA radio hierarchy forbids user reset/removal. */ - if (piodma_supplier) { + /* Firmware-owned links cannot be recovered by user reset/removal. */ + if (piodma_supplier || hw->adopt_link) { bridge->no_user_remove = true; bridge->reset_sensitive = true; bridge->native_pcie_hotplug = false; bridge->native_shpc_hotplug = false; bridge->native_aer = false; bridge->native_dpc = false; - dev_info_once(dev, "Neo radio host: native AER/DPC recovery disabled for retained-DMA bootstrap\n"); + dev_dbg(dev, "firmware-owned host: automatic AER/DPC recovery disabled\n"); } if (hw->root_bus_only && !piodma_supplier) dev_info(dev, "root-port probe only; downstream config is blocked\n"); @@ -2748,7 +3739,7 @@ static int apple_pcie_probe(struct platform_device *pdev) } /* Every function is added after this and removed before devres runs. */ - if (pcie->tunnel_cold_init) { + if (pcie->tunnel_cold_init || pcie->hw->adopt_link) { pcie->tunnel_pci_nb.notifier_call = apple_pcie_tunnel_pci_notify; ret = bus_register_notifier(&pci_bus_type, &pcie->tunnel_pci_nb); if (ret) @@ -2772,6 +3763,8 @@ static int apple_pcie_probe(struct platform_device *pdev) } } + if (hw->adopt_link) + bridge->reset_root_port = apple_pcie_t6050_reset_root_port; ret = pci_host_common_init(pdev, bridge, &apple_pcie_cfg_ecam_ops); if (ret) { apple_pcie_cleanup(pcie); @@ -3302,6 +4295,48 @@ static void apple_pcie_remove(struct platform_device *pdev) pci_unlock_rescan_remove(); } +/* + * A restart with an adopted port's link alive never reaches m1n1 on the J714c (hard + * reset needed). Put each adopted port back the way the boot stage leaves it on a + * cold start: bridge decoding off, link to L2, PERST# asserted, port and lane clocks + * stopped (the first half of apple_pcie_t6050_reset_root_port, without the restart). + */ +static void apple_pcie_shutdown(struct platform_device *pdev) +{ + struct pci_host_bridge *bridge = platform_get_drvdata(pdev); + struct apple_pcie *pcie = pci_host_bridge_priv(bridge); + struct apple_pcie_port *port; + struct pci_dev *rp; + u32 value; + + if (!pcie->hw->adopt_link) + return; + mutex_lock(&pcie->lock); + list_for_each_entry(port, &pcie->ports, entry) { + if (!port->perst || !port->base || !port->phy) + continue; + rp = pci_get_domain_bus_and_slot(pci_domain_nr(bridge->bus), 0, + PCI_DEVFN(port->idx, 0)); + if (!rp) + continue; + pci_write_config_word(rp, PCI_COMMAND, 0); + if (!pci_read_config_dword(rp, PCI_PRIMARY_BUS, &value)) + pci_write_config_dword(rp, PCI_PRIMARY_BUS, value & 0xff0000ff); + apple_pcie_port_writel(port, apple_pcie_port_readl(port, PORT_APPCLK) | 0x100, + PORT_APPCLK); + apple_pcie_port_writel(port, 0x11, 0x88); + apple_pcie_t6050_wait(port, port->base, 0x88, 1, 0, 500000); + apple_pcie_t6050_wait(port, port->base, PORT_LINKSTS, BIT(6), BIT(6), 500000); + WRITE_ONCE(port->link_failed, true); + apple_pcie_t6050_perst(port, true); + writel(readl(port->phy) & ~BIT(31), port->phy); + apple_pcie_t6050_port_disable(port, rp); + dev_info(pcie->dev, "port %d held for the restart\n", port->idx); + pci_dev_put(rp); + } + mutex_unlock(&pcie->lock); +} + static int apple_pcie_mark_power_lost(struct pci_dev *pdev, void *unused) { /* @@ -3414,6 +4449,14 @@ static int apple_pcie_suspend_noirq(struct device *dev) { struct apple_pcie *pcie = apple_pcie_lookup(dev); + /* + * Linux cannot retrain an adopted link: keep its power domain on + * through suspend-to-idle and leave the link exactly as it is. + */ + if (pcie->hw->adopt_link) { + device_set_awake_path(dev); + return 0; + } if (!pcie->hw->tunneled) return 0; pcie->reset_on_resume = false; @@ -3548,6 +4591,7 @@ static const struct of_device_id apple_pcie_of_match[] = { { .compatible = "apple,t6000-pciec", .data = &t8103_pciec_hw }, { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, { .compatible = "apple,t8140-pcie", .data = &t8140_hw }, + { .compatible = "apple,t6050-pcie", .data = &t6050_hw }, { .compatible = "apple,pcie", .data = &t8103_hw }, { } }; @@ -3556,6 +4600,7 @@ MODULE_DEVICE_TABLE(of, apple_pcie_of_match); static struct platform_driver apple_pcie_driver = { .probe = apple_pcie_probe, .remove = apple_pcie_remove, + .shutdown = apple_pcie_shutdown, .driver = { .name = "pcie-apple", .of_match_table = apple_pcie_of_match, diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 80d2338b2a0a8..c686c860e5027 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -4854,6 +4854,19 @@ void pci_reset_secondary_bus(struct pci_dev *dev) void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) { + struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); + int ret; + + if (pci_is_root_bus(dev->bus) && host->reset_root_port) { + ret = host->reset_root_port(host, dev); + if (ret) + pci_err(dev, "Failed to reset Root Port: %d\n", ret); + else + pci_restore_state(dev); + + return; + } + pci_reset_secondary_bus(dev); } diff --git a/drivers/pci/pcie/err.c b/drivers/pci/pcie/err.c index bebe4bc111d75..13b9d9eb714fc 100644 --- a/drivers/pci/pcie/err.c +++ b/drivers/pci/pcie/err.c @@ -256,11 +256,6 @@ pci_ers_result_t pcie_do_recovery(struct pci_dev *dev, } if (status == PCI_ERS_RESULT_NEED_RESET) { - /* - * TODO: Should call platform-specific - * functions to reset slot before calling - * drivers' slot_reset callbacks? - */ status = PCI_ERS_RESULT_RECOVERED; pci_dbg(bridge, "broadcast slot_reset message\n"); pci_walk_bridge(bridge, report_slot_reset, &status); diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index dd0abbc63e18d..47060843be7f5 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -3515,6 +3515,13 @@ void pci_lock_rescan_remove(void) } EXPORT_SYMBOL_GPL(pci_lock_rescan_remove); +/* Work cancelled by removal must not wait on the remover's own lock. */ +bool pci_trylock_rescan_remove(void) +{ + return mutex_trylock(&pci_rescan_remove_lock); +} +EXPORT_SYMBOL_GPL(pci_trylock_rescan_remove); + void pci_unlock_rescan_remove(void) { mutex_unlock(&pci_rescan_remove_lock); diff --git a/drivers/phy/apple/Kconfig b/drivers/phy/apple/Kconfig index 66c71acb00515..837bf5ee8f9cc 100644 --- a/drivers/phy/apple/Kconfig +++ b/drivers/phy/apple/Kconfig @@ -40,3 +40,17 @@ config PHY_APPLE_DPTX such as the M2. This driver provides support for DisplayPort and is used on the Mac mini (M2 and M2 Pro, 2023). + +config PHY_APPLE_T6050_USB2 + tristate "Apple T6050 Type-C port USB 2.0 host PHY" + depends on (ARM64 && ARCH_APPLE) || COMPILE_TEST + depends on USB_ROLE_SWITCH || !USB_ROLE_SWITCH + select GENERIC_PHY + select APPLE_TUNABLE + help + Puts a USB-C port of the Apple T6050 (M5 Pro/Max) into USB 2.0 + host mode for the generic xHCI platform driver. A port described + with its Type-C PHY windows also gets SuperSpeed host mode (normal + plug orientation). USB4 is not supported by this driver. + + If M is selected the module will be called 'phy-apple-t6050-usb2'. diff --git a/drivers/phy/apple/Makefile b/drivers/phy/apple/Makefile index b9e7bf3e4ac17..4562e0fa9aaeb 100644 --- a/drivers/phy/apple/Makefile +++ b/drivers/phy/apple/Makefile @@ -5,3 +5,6 @@ phy-apple-atc-y := atc.o obj-$(CONFIG_PHY_APPLE_DPTX) += phy-apple-dptx.o phy-apple-dptx-y += dptx.o + +obj-$(CONFIG_PHY_APPLE_T6050_USB2) += phy-apple-t6050-usb2.o +phy-apple-t6050-usb2-y := t6050-usb2.o diff --git a/drivers/phy/apple/atc-test.c b/drivers/phy/apple/atc-test.c index eb0dbb9037bab..9067059f0675d 100644 --- a/drivers/phy/apple/atc-test.c +++ b/drivers/phy/apple/atc-test.c @@ -991,7 +991,43 @@ static void atcphy_test_usb2_init_after_safe(struct kunit *test) KUNIT_EXPECT_EQ(test, atcphy->mode, APPLE_ATCPHY_MODE_OFF); } +static void atcphy_test_t8103_usb3_machine_scope(struct kunit *test) +{ + static const struct { + const char *compatible; + bool enabled; + } cases[] = { + { "apple,t8103", true }, + { "apple,t6000", false }, + { "apple,t6001", false }, + { "apple,t6002", false }, + { "apple,t8112", false }, + { "apple,t6020", false }, + { "apple,t6021", false }, + { "apple,t6022", false }, + { "apple,t8122", false }, + { "apple,t6030", false }, + { "apple,t6031", false }, + { "apple,t6034", false }, + { "apple,t8140", false }, + { "apple,t8142", false }, + }; + struct property compatible = { .name = "compatible" }; + struct device_node root = { .properties = &compatible }; + + for (size_t i = 0; i < ARRAY_SIZE(cases); i++) { + compatible.value = (void *)cases[i].compatible; + compatible.length = strlen(cases[i].compatible) + 1; + /* M2 and Pro/Max PHYs select this same fallback hardware row. */ + KUNIT_EXPECT_EQ(test, atcphy_has_t8103_usb3_workaround(&atcphy_hw_t8103, &root), + cases[i].enabled); + KUNIT_EXPECT_FALSE(test, + atcphy_has_t8103_usb3_workaround(&atcphy_hw_t8122, &root)); + } +} + static struct kunit_case atcphy_mux_test_cases[] = { + KUNIT_CASE(atcphy_test_t8103_usb3_machine_scope), KUNIT_CASE(atcphy_test_target_mode), KUNIT_CASE(atcphy_test_target_mode_fixed_hub), KUNIT_CASE(atcphy_test_noop_same_mode), diff --git a/drivers/phy/apple/atc-tunnel-test.c b/drivers/phy/apple/atc-tunnel-test.c index e5decf7480a68..15f1055707332 100644 --- a/drivers/phy/apple/atc-tunnel-test.c +++ b/drivers/phy/apple/atc-tunnel-test.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause #include +#include #include "atc-tunnel.h" @@ -90,7 +91,21 @@ static void atc_tunnel_wiring_gate(struct kunit *test) cases[i].expected); } +static void atc_tunnel_t6030_clock_sources(struct kunit *test) +{ + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_t6030_pclk(0, 0), 0); + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_t6030_pclk(2, 0), 0); + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_t6030_pclk(0, 1), 1); + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_t6030_pclk(2, 1), 1); + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_t6030_pclk(1, 0), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_t6030_pclk(4, 0), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_t6030_pclk(0, 2), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_t6030_pclk(~0U, 0), -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, apple_dp_tunnel_t6030_pclk(0, ~0U), -EOPNOTSUPP); +} + static struct kunit_case atc_tunnel_cases[] = { + KUNIT_CASE(atc_tunnel_t6030_clock_sources), KUNIT_CASE(atc_tunnel_soc_qualification), KUNIT_CASE(atc_tunnel_core_qualification), KUNIT_CASE(atc_tunnel_wiring_gate), diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c index 51e2e9d18e5ed..615d788fb09ab 100644 --- a/drivers/phy/apple/atc.c +++ b/drivers/phy/apple/atc.c @@ -722,13 +722,17 @@ struct atcphy_hw { * @tunnel_rate: DP link rate code the t8103 tunnel pixel clock is set up for * @tunnel_users: DP IN adapters (BIT(dpin)) whose tunnel pixel clock runs; on * t8103 and t600x both share the one clock - * @tunnel_dpin_rate: DP link rate code each T602X DP IN adapter's clock runs at + * @tunnel_dpin_rate: DP link rate code each T602X/T6030 DP IN clock runs at + * @tunnel_source: T6030 display source owning each DP IN clock * @dp_t8122: DisplayPort state of a T8122 generation PHY * @dp_t8122.aux: The AUX channel block is powered * @dp_t8122.pll: The AUSPLL runs for the DisplayPort main link * @dp_t8122.pairs: Lane pairs (bit mask) whose DisplayPort transmitters run * @dp_t8122.rate: Main link rate in Mb/s per lane, 0 while stopped * @pipe_state: Backend the PIPE mux ("pipehandler") is routed to + * @t8103_dummy_pending: A live T8103-generation handoff still needs its unlock ACK + * @t8103_usb3_retry: USB3 routing owed after a deferred role or parked lane change + * @t8103_usb3_workaround: Base M1 admission for the USB3 handoff workaround * @pd_nb: Power domain notifier, registered with restore_after_pd_off * @pd_was_off: The power domain was switched off since the state was last * brought back; set from the notifier, cleared under @lock @@ -794,6 +798,9 @@ struct apple_atcphy { int dp_link_rate; bool swap_lanes; enum atcphy_pipehandler_state pipe_state; + bool t8103_dummy_pending; + bool t8103_usb3_retry; + bool t8103_usb3_workaround; struct notifier_block pd_nb; bool pd_was_off; @@ -806,6 +813,7 @@ struct apple_atcphy { u8 tunnel_rate; u8 tunnel_users; u8 tunnel_dpin_rate[2]; + unsigned int tunnel_source[2]; /* T6030 source owning each DP IN clock */ struct { bool aux; @@ -1192,6 +1200,22 @@ static int atcphy_pipehandler_lock(struct apple_atcphy *atcphy) return ret; } +/* Retry only a live legacy PIPE: USB4 and stopped parking cannot clock the ACK. */ +static int atcphy_pipehandler_lock_live(struct apple_atcphy *atcphy) +{ + unsigned int attempts = atcphy->t8103_usb3_workaround ? 10 : 1; + int ret; + + for (unsigned int attempt = 0; attempt < attempts; attempt++) { + ret = atcphy_pipehandler_lock(atcphy); + if (!ret) + return 0; + if (attempt + 1 < attempts) + usleep_range(5000, 10000); + } + return ret; +} + static int atcphy_pipehandler_unlock(struct apple_atcphy *atcphy) { int ret; @@ -1621,14 +1645,18 @@ static int atcphy_configure_pipehandler_dummy(struct apple_atcphy *atcphy, bool static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) { enum atcphy_pipehandler_state state = atcphy_modes[atcphy->mode].pipehandler_state; + bool host_bist = host && !atcphy->t8103_usb3_workaround; int ret = -EINVAL; lockdep_assert_held(&atcphy->lock); + if (atcphy->t8103_dummy_pending) + return -EBUSY; switch (state) { case ATCPHY_PIPEHANDLER_STATE_USB3: atcphy->pipe_pending = true; - ret = atcphy_configure_pipehandler_usb3(atcphy, host); + /* Legacy host BIST can hide the first xHCI link-detection scan. */ + ret = atcphy_configure_pipehandler_usb3(atcphy, host_bist); break; case ATCPHY_PIPEHANDLER_STATE_USB4: ret = atcphy_configure_pipehandler_usb4(atcphy); @@ -1640,11 +1668,73 @@ static int atcphy_configure_pipehandler(struct apple_atcphy *atcphy, bool host) if (!ret) { atcphy->pipe_state = state; atcphy->pipe_pending = false; + if (state == ATCPHY_PIPEHANDLER_STATE_USB3) + atcphy->t8103_usb3_retry = false; } return ret; } +/* + * Park a live T8103 USB3 backend before a Type-C lane change. Unlike legacy + * shutdown, an unacknowledged handoff must not let the lane change continue. + */ +static int atcphy_park_live_t8103_pipehandler(struct apple_atcphy *atcphy) +{ + const u32 override_mask = PIPEHANDLER_OVERRIDE_RXVALID | + PIPEHANDLER_OVERRIDE_RXDETECT; + const u32 value_mask = PIPEHANDLER_OVERRIDE_VAL_RXDETECT0 | + PIPEHANDLER_OVERRIDE_VAL_RXDETECT1; + void __iomem *pipe = atcphy->regs.pipehandler; + u32 overrides, values; + int ret; + + lockdep_assert_held(&atcphy->lock); + if (!atcphy->t8103_usb3_workaround) + return -EOPNOTSUPP; + + if (!atcphy->t8103_dummy_pending) { + if (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY) + return 0; + if (atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_USB3) + return -EBUSY; + ret = atcphy_pipehandler_check(atcphy); + if (ret) + return ret; + overrides = readl(pipe + PIPEHANDLER_OVERRIDE); + values = readl(pipe + PIPEHANDLER_OVERRIDE_VALUES); + clear32(pipe + PIPEHANDLER_OVERRIDE_VALUES, value_mask); + set32(pipe + PIPEHANDLER_OVERRIDE, override_mask); + + ret = atcphy_pipehandler_lock_live(atcphy); + if (ret) { + /* The mux did not move: restore only the fields we touched. */ + mask32(pipe + PIPEHANDLER_OVERRIDE_VALUES, value_mask, + values & value_mask); + mask32(pipe + PIPEHANDLER_OVERRIDE, override_mask, + overrides & override_mask); + return ret; + } + atcphy_pipehandler_set_mux(atcphy, PIPEHANDLER_MUX_CTRL_DATA_DUMMY, + PIPEHANDLER_MUX_CTRL_CLK_DUMMY); + /* The physical mux moved even if completion now fails. */ + atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; + atcphy->t8103_dummy_pending = true; + atcphy->t8103_usb3_retry = true; + } + + /* Retry completion without requesting another lock on the dummy clock. */ + ret = atcphy_pipehandler_unlock(atcphy); + if (ret) + return ret; + mask32(pipe + PIPEHANDLER_NONSELECTED_OVERRIDE, + PIPEHANDLER_NATIVE_POWER_DOWN, FIELD_PREP(PIPEHANDLER_NATIVE_POWER_DOWN, 2)); + set32(pipe + PIPEHANDLER_NONSELECTED_OVERRIDE, PIPEHANDLER_NATIVE_RESET); + atcphy->t8103_dummy_pending = false; + atcphy->pipe_pending = false; + return 0; +} + /* Route the PIPE back to the dummy backend unless it is there already */ static int atcphy_park_pipehandler(struct apple_atcphy *atcphy, bool stopped) { @@ -1656,6 +1746,10 @@ static int atcphy_park_pipehandler(struct apple_atcphy *atcphy, bool stopped) dev_dbg(atcphy->dev, "PIPE park: mode=%u pipe=%u host=%u dirty=%u owned=%u idle=%u\n", atcphy->mode, atcphy->pipe_state, atcphy->host_active, atcphy->mode_dirty, atcphy->dp_owned_pmas, atcphy->dp_idle_pmas); + if (atcphy->t8103_dummy_pending || + (!stopped && atcphy->t8103_usb3_workaround && + atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3)) + return atcphy_park_live_t8103_pipehandler(atcphy); if (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && !atcphy->pipe_pending) return 0; @@ -1677,6 +1771,8 @@ static void atcphy_setup_pipehandler(struct apple_atcphy *atcphy) PIPEHANDLER_MUX_CTRL_CLK_DUMMY); atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY; atcphy->pipe_pending = false; + atcphy->t8103_dummy_pending = false; + atcphy->t8103_usb3_retry = false; if (atcphy_needs_dummy_phy(atcphy)) atcphy_enable_dummy_phy(atcphy); } @@ -2534,10 +2630,40 @@ static int atc_t8122_tunnel_stop(struct apple_atcphy *atcphy) } atcphy->tunnel_clock_on = false; atcphy->tunnel_rate = 0; + atcphy->tunnel_users = 0; + atcphy->tunnel_dpin_rate[0] = 0; + atcphy->tunnel_dpin_rate[1] = 0; return 0; } -static int atc_t8122_tunnel_start(struct apple_atcphy *atcphy, u8 rate) +static void atc_t8122_tunnel_pclk(struct apple_atcphy *atcphy, + unsigned int pclk, u32 selector) +{ + u32 enable = pclk ? DPTX_PCLK2_ENABLE : DPTX_PCLK1_ENABLE; + u32 select = pclk ? DPTX_PCLK2_SELECT : DPTX_PCLK1_SELECT; + u32 value = pclk ? FIELD_PREP(DPTX_PCLK2_SELECT, selector) : + FIELD_PREP(DPTX_PCLK1_SELECT, selector); + + lockdep_assert_held(&atcphy->lock); + /* PCLK1 is also DPIN1's prerequisite: retuning DPIN0 must not gate it. */ + if (!pclk && (atcphy->tunnel_users & BIT(1))) + value |= DPTX_PCLK1_ENABLE; + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + enable | select, value); + /* Enable the prerequisite before the second stream's pixel clock. */ + if (pclk) { + if (!(atcphy->tunnel_users & BIT(0))) + core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_SELECT, + FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK1_ENABLE); + } + core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, enable); +} + +static int atc_t8122_tunnel_start(struct apple_atcphy *atcphy, + unsigned int pclk, u8 rate) { u32 selector, value, saved_tx; int ret, cleanup; @@ -2563,13 +2689,7 @@ static int atc_t8122_tunnel_start(struct apple_atcphy *atcphy, u8 rate) /* A failed shutdown retains ownership, but is not a running clock. */ if (!atcphy->tunnel_rate) return -EBUSY; - if (atcphy->tunnel_rate == rate) - return 0; - core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, - DPTX_PCLK1_ENABLE | DPTX_PCLK1_SELECT, - FIELD_PREP(DPTX_PCLK1_SELECT, selector)); - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, - DPTX_PCLK1_ENABLE); + atc_t8122_tunnel_pclk(atcphy, pclk, selector); atcphy->tunnel_rate = rate; return 0; } @@ -2582,7 +2702,8 @@ static int atc_t8122_tunnel_start(struct apple_atcphy *atcphy, u8 rate) /* Inactive slots use selector zero; all gates stay off during PLL setup. */ core_mask32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, GENMASK(15, 4), - FIELD_PREP(DPTX_PCLK1_SELECT, selector)); + FIELD_PREP(DPTX_PCLK1_SELECT, selector) | + (pclk ? FIELD_PREP(DPTX_PCLK2_SELECT, selector) : 0)); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL); core_set32(atcphy, ACIOPHY_CFG0, ACIOPHY_CFG0_COMMON_SMALL_OV); udelay(1); @@ -2624,7 +2745,7 @@ static int atc_t8122_tunnel_start(struct apple_atcphy *atcphy, u8 rate) goto fail; core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTXPHY_PMA_LANE_RESET_N | DPTXPHY_PMA_LANE_RESET_N_OV); - core_set32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, DPTX_PCLK1_ENABLE); + atc_t8122_tunnel_pclk(atcphy, pclk, selector); atcphy->tunnel_rate = rate; return 0; @@ -3098,7 +3219,7 @@ static int atc_tunnel_set_t602x(struct apple_atcphy *atcphy, unsigned int dpin, return 0; } -/* T6030 (M3 Pro): T8122-generation PHYs, only the DP IN0 route so far. */ +/* T6030 (M3 Pro) uses T8122-generation PHYs for both DP IN clock outputs. */ static bool apple_atc_tunnel_is_t6030(struct apple_atcphy *atcphy) { return of_machine_is_compatible("apple,t6030") && @@ -3331,6 +3452,8 @@ static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) KUNIT_STATIC_STUB_REDIRECT(atcphy_configure, atcphy, mode); lockdep_assert_held(&atcphy->lock); + if (atcphy->t8103_dummy_pending) + return -EBUSY; if (atcphy->pipe_pending) return -EBUSY; if (atcphy->usb_off_pending && mode != APPLE_ATCPHY_MODE_OFF) @@ -3636,6 +3759,7 @@ static int atcphy_usb3_power_off(struct phy *phy) ret = atcphy_park_pipehandler(atcphy, true); if (ret) return ret; + atcphy->t8103_usb3_retry = false; /* * dwc3 can tear its host down while a DisplayPort alt mode link is up, * e.g. when a monitor's built-in hub re-enumerates after a hot-plug. @@ -3661,6 +3785,12 @@ static int atcphy_usb3_route(struct apple_atcphy *atcphy, bool host) int ret; lockdep_assert_held(&atcphy->lock); + if (atcphy->t8103_dummy_pending) { + ret = atcphy_park_live_t8103_pipehandler(atcphy); + if (ret) + return ret; + } + /* * We may get multiple calls to set_mode (for host mode e.g. at least one from the dwc3 glue @@ -3671,6 +3801,19 @@ static int atcphy_usb3_route(struct apple_atcphy *atcphy, bool host) /* A failed lane/domain transition must be restored before routing PIPE. */ if (atcphy->mode_dirty) return -EBUSY; + /* dwc3 can select its role before the debounced Type-C mux update. */ + if (atcphy->t8103_usb3_workaround && + atcphy->mode == APPLE_ATCPHY_MODE_OFF) { + /* Keep the valid role's route owed across a late mux update or failure. */ + atcphy->t8103_usb3_retry = true; + if (atcphy->typec_mode != APPLE_ATCPHY_MODE_USB3) + return 0; + if (atcphy->dp_only || !atcphy->ss_tunables) + return -EOPNOTSUPP; + ret = atcphy_configure(atcphy, APPLE_ATCPHY_MODE_USB3); + if (ret) + return ret; + } if (atcphy->pipe_pending) { ret = atcphy_park_pipehandler(atcphy, false); if (ret) @@ -3849,6 +3992,62 @@ static bool apple_atc_tunnel_is_t8103_style(void) of_machine_is_compatible("apple,t8112") || apple_atc_tunnel_is_t600x(); } +int apple_atc_dp_tunnel_rate_source(struct phy *phy, unsigned int source, + unsigned int dpin, u8 rate) +{ + struct apple_atcphy *atcphy; + int pclk, ret; + + if (!phy || phy->ops != &apple_atc_dp_phy_ops || dpin > 1) + return -EINVAL; + atcphy = phy_get_drvdata(phy); + pclk = apple_dp_tunnel_t6030_pclk(source, dpin); + if (!apple_atc_tunnel_is_t6030(atcphy) || pclk < 0) + return -EOPNOTSUPP; + guard(mutex)(&atcphy->lock); + atcphy_restore_after_pd_off(atcphy); + /* A stale source must not retune or stop another source's clock. */ + if ((atcphy->tunnel_users & BIT(dpin)) && + atcphy->tunnel_source[dpin] != source) + return -ESTALE; + if (!rate) { + if (!(atcphy->tunnel_users & BIT(dpin))) + return 0; + if (!(atcphy->tunnel_users & ~BIT(dpin))) + return atc_t8122_tunnel_stop(atcphy); + /* Release only this stream; the other stream keeps its PLL and gates. */ + if (pclk) + core_clear32(atcphy, ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0, + DPTX_PCLK2_ENABLE); + atcphy->tunnel_users &= ~BIT(dpin); + atcphy->tunnel_dpin_rate[dpin] = 0; + return 0; + } + if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && + atcphy->mode != APPLE_ATCPHY_MODE_TBT) + return -EBUSY; + if (atcphy->tunnel_clock_on && !atcphy->tunnel_rate) + return -EBUSY; + /* Keep this owner if a failed start cannot shut the PLL down. */ + if ((atcphy->tunnel_users & BIT(1 - dpin)) && + atcphy->tunnel_source[1 - dpin] == source) + return -EBUSY; + if ((atcphy->tunnel_users & BIT(dpin)) && + atcphy->tunnel_dpin_rate[dpin] == rate && atcphy->tunnel_rate) + return 0; + atcphy->tunnel_source[dpin] = source; + ret = atc_t8122_tunnel_start(atcphy, pclk, rate); + if (!ret || (atcphy->tunnel_clock_on && !atcphy->tunnel_rate)) { + atcphy->tunnel_users |= BIT(dpin); + atcphy->tunnel_dpin_rate[dpin] = ret ? 0 : rate; + } + dev_dbg(atcphy->dev, "DP tunnel source %u rate 0x%x: %d (TX_DP_CTRL0=%08x PCLK_STAT=%08x)\n", + source, rate, ret, + readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0), + readl(atcphy->regs.core + T8122_DP_PCLK_STATUS)); + return ret; +} +EXPORT_SYMBOL_GPL(apple_atc_dp_tunnel_rate_source); int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) { @@ -3859,23 +4058,8 @@ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate) return -EINVAL; atcphy = phy_get_drvdata(phy); /* Keep each supported SoC on its qualified clock sequence. */ - if (apple_atc_tunnel_is_t6030(atcphy)) { - if (dpin) - return -EOPNOTSUPP; - guard(mutex)(&atcphy->lock); - atcphy_restore_after_pd_off(atcphy); - if (!rate) - return atc_t8122_tunnel_stop(atcphy); - if (atcphy->mode != APPLE_ATCPHY_MODE_USB4 && - atcphy->mode != APPLE_ATCPHY_MODE_TBT) - return -EBUSY; - ret = atc_t8122_tunnel_start(atcphy, rate); - dev_dbg(atcphy->dev, "DP tunnel clock rate 0x%x: %d (TX_DP_CTRL0=%08x PCLK_STAT=%08x)\n", - rate, ret, - readl(atcphy->regs.core + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0), - readl(atcphy->regs.core + T8122_DP_PCLK_STATUS)); - return ret; - } + if (apple_atc_tunnel_is_t6030(atcphy)) + return apple_atc_dp_tunnel_rate_source(phy, 0, dpin, rate); if (!apple_atc_tunnel_is_t8103_style()) { bool valid_core = apple_atc_t602x_core_valid(atcphy->np, atcphy->res.core->start, resource_size(atcphy->res.core)); @@ -4013,6 +4197,7 @@ static int atcphy_dwc3_reset_assert(struct reset_controller_dev *rcdev, unsigned if (ret) return ret; + atcphy->t8103_usb3_retry = false; _atcphy_dwc3_reset_assert(atcphy); atcphy_usb2_power_off(atcphy); @@ -4025,6 +4210,12 @@ static int atcphy_dwc3_reset_deassert(struct reset_controller_dev *rcdev, unsign guard(mutex)(&atcphy->lock); atcphy_restore_after_pd_off(atcphy); + if (atcphy->t8103_dummy_pending) { + int ret = atcphy_park_live_t8103_pipehandler(atcphy); + + if (ret) + return ret; + } clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN); @@ -4467,7 +4658,8 @@ static enum atcphy_mode atcphy_typec_target_mode(struct apple_atcphy *atcphy, */ static bool atcphy_mux_in_mode(const struct apple_atcphy *atcphy, enum atcphy_mode target_mode) { - if (atcphy->mode != target_mode || atcphy->mode_dirty) + if (atcphy->mode != target_mode || atcphy->mode_dirty || + atcphy->t8103_dummy_pending || atcphy->t8103_usb3_retry) return false; return !(atcphy->fixed_usb2 && atcphy->host_active && @@ -4528,8 +4720,20 @@ static int atcphy_mux_switch(struct apple_atcphy *atcphy, enum atcphy_mode targe int ret; lockdep_assert_held(&atcphy->lock); + if (atcphy->t8103_dummy_pending) { + ret = atcphy_park_live_t8103_pipehandler(atcphy); + if (ret) + return ret; + } + if (atcphy->mode == target_mode && !atcphy->mode_dirty) { atcphy->typec_mode = target_mode; + /* A canceled lane change owes the old USB3 backend its reconnect. */ + if (atcphy->t8103_usb3_retry) { + if (atcphy_modes[target_mode].pipehandler_state == ATCPHY_PIPEHANDLER_STATE_USB3) + return atcphy_configure_pipehandler(atcphy, false); + atcphy->t8103_usb3_retry = false; + } /* Retry a failed PIPE setup without disturbing the configured lanes. */ if (atcphy->fixed_usb2 && atcphy->host_active && (atcphy->pipe_state != atcphy_modes[target_mode].pipehandler_state || @@ -4578,6 +4782,8 @@ static int atcphy_mux_switch(struct apple_atcphy *atcphy, enum atcphy_mode targe if (atcphy->host_active && atcphy->pipe_state != pipe_state) ret = atcphy_configure_pipehandler(atcphy, true); + if (!ret && pipe_state != ATCPHY_PIPEHANDLER_STATE_USB3) + atcphy->t8103_usb3_retry = false; return ret; } @@ -4594,8 +4800,28 @@ static int atcphy_mux_switch(struct apple_atcphy *atcphy, enum atcphy_mode targe * complain loudly. We can still try to switch modes and hope for the best though, * in the worst case the hardware will fall back to USB2-only. */ - WARN_ON_ONCE(atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY); - return atcphy_configure(atcphy, target_mode); + if (atcphy->t8103_usb3_workaround) { + if (atcphy->pipe_state == ATCPHY_PIPEHANDLER_STATE_USB3 && + atcphy_modes[target_mode].pipehandler_state == ATCPHY_PIPEHANDLER_STATE_DUMMY) { + ret = atcphy_park_live_t8103_pipehandler(atcphy); + if (ret) + return ret; + } + WARN_ON_ONCE(atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY && + atcphy_modes[target_mode].pipehandler_state == + ATCPHY_PIPEHANDLER_STATE_DUMMY); + } else { + WARN_ON_ONCE(atcphy->pipe_state != ATCPHY_PIPEHANDLER_STATE_DUMMY); + } + ret = atcphy_configure(atcphy, target_mode); + if (ret) + return ret; + if (atcphy->t8103_usb3_retry) { + if (atcphy_modes[target_mode].pipehandler_state == ATCPHY_PIPEHANDLER_STATE_USB3) + return atcphy_configure_pipehandler(atcphy, false); + atcphy->t8103_usb3_retry = false; + } + return 0; } /* @@ -5137,6 +5363,14 @@ static void atcphy_probe_pd_notifier(struct apple_atcphy *atcphy) ret); } +/* The T8103 PHY compatible is also a fallback on unqualified M1/M2 variants. */ +static bool atcphy_has_t8103_usb3_workaround(const struct atcphy_hw *hw, + const struct device_node *root) +{ + return hw->gen == ATCPHY_GENERATION_T8103 && + of_device_is_compatible(root, "apple,t8103"); +} + static int atcphy_probe(struct platform_device *pdev) { struct apple_atcphy *atcphy; @@ -5150,6 +5384,7 @@ static int atcphy_probe(struct platform_device *pdev) atcphy->hw = of_device_get_match_data(&pdev->dev); if (!atcphy->hw) return -EINVAL; + atcphy->t8103_usb3_workaround = atcphy_has_t8103_usb3_workaround(atcphy->hw, of_root); atcphy->dev = dev; atcphy->np = dev->of_node; @@ -5239,11 +5474,30 @@ static const struct atcphy_hw atcphy_hw_t8142 = { .has_usb4 = true, }; +static const struct atcphy_hw atcphy_hw_t6050 = { + /* + * M5 Pro/Max Type-C PHY. Same T8122-family core as the M5 Air (t8142), + * but the pipehandler mux uses the T8103 field layout - live silicon + * parks the dummy PHY at MUX_CTRL=0x22, which only decodes as + * DUMMY/DUMMY in the T8103 fields. The SuperSpeed/CIO tunables this + * generation does not ship are optional: USB2 comes up without them. + */ + .configure_post_tunable = atcphy_configure_post_tunable_all_modes, + .gen = ATCPHY_GENERATION_T8122, + .aciophy_lane_mode = ACIOPHY_LANE_MODE_T8122, + .aciophy_crossbar = ACIOPHY_CROSSBAR_T8122, + .pipe_mux_clk = PIPEHANDLER_MUX_CTRL_CLK_T8103, + .pipe_mux_data = PIPEHANDLER_MUX_CTRL_DATA_T8103, + .has_usb4 = true, + .optional_tunables = true, +}; + static const struct of_device_id atcphy_match[] = { { .compatible = "apple,t8103-atcphy", .data = &atcphy_hw_t8103 }, { .compatible = "apple,t8122-atcphy", .data = &atcphy_hw_t8122 }, { .compatible = "apple,t8142-atcphy", .data = &atcphy_hw_t8142 }, { .compatible = "apple,t8140-atcphy", .data = &atcphy_hw_t8140 }, + { .compatible = "apple,t6050-atcphy", .data = &atcphy_hw_t6050 }, {}, }; MODULE_DEVICE_TABLE(of, atcphy_match); diff --git a/drivers/phy/apple/t6050-usb2.c b/drivers/phy/apple/t6050-usb2.c new file mode 100644 index 0000000000000..517f280df0a25 --- /dev/null +++ b/drivers/phy/apple/t6050-usb2.c @@ -0,0 +1,1096 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Apple T6050 (M5 Pro/Max) Type-C port, USB 2.0 host only. + * + * The T6050 Type-C PHY is a new generation; its SuperSpeed and CIO parts are + * not driven here. USB 2.0 needs only the port's usb2phy block, the pipe + * handler and the dwc3 global registers. phy_init() puts the port into + * USB 2.0 host mode with the sequence the loader (m1n1 usb_t6050_host_prep) + * uses on J714c, so the port works whatever state the loader left it in. + * The xHCI registers are then driven by the generic xHCI platform driver, + * which calls phy_init() before its host controller reset. + * + * A port whose node has the "atc-power" window also gets SuperSpeed: after the + * USB 2.0 flip, phy_init() runs the port 1 SuperSpeed host sequence measured + * over the m1n1 proxy (kernel/USB-BRINGUP.md), steps 1-16, including the boot + * of the PHY CPU's RTKit firmware. The xHCI reset of the platform driver is + * step 17 (phy_calibrate() restores the glue after it), and the hub's USB 2.0 + * port reset is step 18. Normal plug orientation only. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define USB2PHY_USBCTL 0x00 +#define USB2PHY_USBCTL_RUN 2 +#define USB2PHY_CTL 0x04 +#define USB2PHY_CTL_RESET BIT(0) +#define USB2PHY_CTL_PORT_RESET BIT(1) +#define USB2PHY_SIG 0x08 +#define USB2PHY_SIG_DEFAULT 0x01c1000f +#define USB2PHY_SIG_HOST (7 << 12) +#define USB2PHY_MISCTUNE 0x1c +#define USB2PHY_MISCTUNE_DEFAULT 0x008c0813 + +#define PIPEHANDLER_OVERRIDE 0x00 +#define PIPEHANDLER_OVERRIDE_VALUES 0x04 +#define PIPEHANDLER_MUX_CTRL 0x0c +#define PIPEHANDLER_MUX_CTRL_DUMMY 0x22 +#define PIPEHANDLER_AON_GEN 0x1c +#define PIPEHANDLER_AON_GEN_DWC3_RESET_N BIT(0) +#define PIPEHANDLER_NONSELECTED_OVERRIDE 0x20 +#define PIPEHANDLER_NONSELECTED_DEFAULT 0x9332 + +#define DWC3_GCTL 0xc110 +#define DWC3_GCTL_CORESOFTRESET BIT(11) +#define DWC3_GCTL_PRTCAPDIR_MASK (3 << 12) +#define DWC3_GCTL_PRTCAP_HOST (1 << 12) +#define DWC3_GUSB2PHYCFG0 0xc200 +#define DWC3_GUSB2PHYCFG_SUSPHY BIT(6) +#define DWC3_GUSB3PIPECTL0 0xc2c0 +#define DWC3_GUSB3PIPECTL_SUSPHY BIT(17) +#define DWC3_DCTL 0xc704 +#define DWC3_DCTL_RUN_STOP BIT(31) +#define DWC3_DSTS 0xc70c +#define DWC3_DSTS_DEVCTRLHLT BIT(22) +#define DWC3_GDBGLTSSM 0xc164 + +/* xHCI CAPLENGTH is 0x20 on these controllers; PORTSC of port 1 at op + 0x400. */ +#define XHCI_USBCMD 0x20 +#define XHCI_USBCMD_HCRST BIT(1) +#define XHCI_USBSTS 0x24 +#define XHCI_USBSTS_CNR BIT(11) +#define XHCI_PORTSC1 0x420 +#define XHCI_PORTSC2 0x430 +#define XHCI_PORTSC_CCS BIT(0) +#define XHCI_PORTSC_PR BIT(4) + +/* SuperSpeed: pipe handler, AUSBC power control, Type-C PHY ("atc") windows. */ +#define PIPEHANDLER_LOCK_REQ 0x10 +#define PIPEHANDLER_LOCK_ACK 0x14 +#define PIPEHANDLER_LOCK BIT(0) +#define PIPEHANDLER_MUX_CLK GENMASK(6, 4) +#define PIPEHANDLER_MUX_DATA GENMASK(1, 0) + +#define PCTL_TXDETRX_REQ 0x18 +#define PCTL_TXDETRX_ACK 0x1c + +#define ATC_CFG0 0x0c +#define ATC_RX_PWR 0x10 +#define ATC_LANE_MODE 0x6c +#define ATC_XBAR 0x78 +#define ATC_TX_PWR 0x120 +#define ATC_UC_CTRL 0x260 +#define ATC_UC_TNBL_DONE BIT(10) +#define ATC_BIST_CFG1 0x29004 +#define ATC_BIST_CFG1_CLOCK_EN BIT(0) +#define ATC_BIST_CFG1_USB_EN BIT(3) +#define ATC_BIST_STATUS2 0x29104 +#define ATC_BIST_STATUS2_PHY_STATUS BIT(8) +#define ATC_BIST_OV1 0x2910c +#define ATC_BIST_OV1_PWR_DOWN GENMASK(28, 25) +#define ATC_BIST_OV1_PIPE_RESET_N BIT(20) +#define ATC_BIST_OV2 0x29110 + +#define ATC_PWR_CTRL 0x0 +#define ATC_PWR_STAT 0x4 +#define ATC_PWR_MISC 0x8 +#define ATC_PWR_SLEEP_SMALL BIT(0) +#define ATC_PWR_SLEEP_BIG BIT(1) +#define ATC_PWR_CLAMP BIT(2) +#define ATC_PWR_APB_RESET_N BIT(3) +#define ATC_PWR_PHY_RESET_N BIT(4) +#define ATC_PWR_MISC_USB_LN_SWAP BIT(1) + +/* MxWrap of the PHY CPU, with its RTKit mailbox at +0x8000. */ +#define MXW_CPU_CTRL 0x28 +#define MXW_CPU_CTRL_RUN 0x10 +#define MXW_IDLE_CTRL 0x40 +#define MXW_A2I_CTRL 0x8110 +#define MXW_I2A_CTRL 0x8114 +#define MXW_MBOX_FULL BIT(16) +#define MXW_MBOX_EMPTY BIT(17) +#define MXW_A2I_MSG0 0x8800 +#define MXW_A2I_MSG1 0x8808 +#define MXW_I2A_MSG0 0x8830 +#define MXW_I2A_MSG1 0x8838 + +#define RTK_TYPE GENMASK_ULL(59, 52) +#define RTK_HELLO 1 +#define RTK_HELLO_ACK 2 +#define RTK_START_EP 5 +#define RTK_IOP_PWR 6 +#define RTK_IOP_PWR_ACK 7 +#define RTK_EPMAP 8 +#define RTK_AP_PWR 0xb +#define RTK_HELLO_MIN GENMASK_ULL(15, 0) +#define RTK_HELLO_MAX GENMASK_ULL(31, 16) +#define RTK_EPMAP_LAST BIT_ULL(51) +#define RTK_EPMAP_BASE GENMASK_ULL(34, 32) +#define RTK_EPMAP_BITMAP GENMASK_ULL(31, 0) +#define RTK_EPMAP_MORE BIT_ULL(0) +#define RTK_EP GENMASK_ULL(39, 32) +#define RTK_START_EP_FLAG 2 +#define RTK_PWR_STATE GENMASK_ULL(15, 0) +#define RTK_PWR_ON 0x20 + +/* PMGR power states: target [3:0], actual [7:4]. */ +#define PS_TARGET GENMASK(3, 0) +#define PS_ACTUAL GENMASK(7, 4) +#define PS_ON 0xf + +/* atc1_common, auss, atc1_cio, atc1_cio_usb, atc1_cio_auss, parent first ("pmgr"). */ +static const u32 t6050_cio_domains[] = { 0x210, 0x248, 0x290, 0x358, 0x360 }; +/* atc1_phymxwrap ("pmgr-mxwrap"). */ +#define PS_PHYMXWRAP 0x170 + +/* + * Apple glue registers of the dwc3 core (pipe and reference clock enables) and the + * values iBoot leaves in them. The core soft reset of the USB 2.0 flip clears them, + * they take writes only with CIO off, and the xHCI reset may change them. + */ +static const u32 t6050_glue[] = { 0xcd28, 0xcd2c, 0xcd40, 0xcd44 }; +static const u32 t6050_glue_boot[] = { 0x886, 0x886, 0x3c20060c, 0x2000b }; + +struct t6050_tunable_desc { + const char *reg; + const char *prop; +}; + +/* In the order the proxy sequence applies them. */ +static const struct t6050_tunable_desc t6050_host_tunables[] = { + { "core", "apple,tunable-gbl" }, + { "drd-cfg", "apple,tunable-cfg" }, + { "core", "apple,tunable-link" }, + { "ausbc-debug", "apple,tunable-ausbc-debug" }, + { "bulk-fabric", "apple,tunable-bulk-fabric" }, + { "pipehandler", "apple,tunable-pipehandler" }, + { "core", "apple,tunable-cio" }, +}; + +static const struct t6050_tunable_desc t6050_phy_tunables[] = { + { "atc-common-cfg", "apple,tunable-atc-common-cfg" }, + { "atc-fabric", "apple,tunable-atc-fabric" }, + { "atc-axi2af", "apple,tunable-atc-axi2af" }, + { "atc", "apple,tunable-atc-pll" }, + { "atc-axi2af-lioa", "apple,tunable-atc-axi2af-lioa" }, + { "atc", "apple,tunable-atc-lanes" }, +}; + +struct t6050_tunable { + void __iomem *regs; + struct apple_tunable *tunable; +}; + +struct t6050_usb2 { + struct device *dev; + void __iomem *usb2phy; + void __iomem *pipehandler; + void __iomem *core; + + /* SuperSpeed only */ + bool ss; + struct work_struct ltssm_work; + struct device_node *hpm; + struct delayed_work replug_work; + unsigned int replug_tries; + struct delayed_work u2reset_work; + unsigned int u2resets; + bool u2reset_queued; + struct delayed_work standalone_work; + void __iomem *pctl; + void __iomem *atc; + void __iomem *mxw; + void __iomem *atc_pwr; + void __iomem *pmgr; + void __iomem *pmgr_mxw; + struct t6050_tunable host_tun[ARRAY_SIZE(t6050_host_tunables)]; + struct t6050_tunable phy_tun[ARRAY_SIZE(t6050_phy_tunables)]; + + /* USB 2.0 only: re-arm the usb2phy after its device left */ + struct delayed_work rearm_work; + bool had_device; + + /* USB 2.0 only: the Type-C controller's attach and detach */ + struct usb_role_switch *role_sw; + enum usb_role role; + bool detached; + bool initialized; +}; + +static void t6050_usb2_mask(void __iomem *reg, u32 clear, u32 set) +{ + writel((readl(reg) & ~clear) | set, reg); +} + +static void t6050_usb2_host_flip(struct t6050_usb2 *p) +{ + u32 val; + + /* usb2phy out of reset and running, as for device mode. */ + writel(USB2PHY_SIG_DEFAULT, p->usb2phy + USB2PHY_SIG); + writel(USB2PHY_CTL_RESET | USB2PHY_CTL_PORT_RESET, p->usb2phy + USB2PHY_CTL); + t6050_usb2_mask(p->usb2phy + USB2PHY_CTL, USB2PHY_CTL_RESET, 0); + t6050_usb2_mask(p->usb2phy + USB2PHY_CTL, USB2PHY_CTL_PORT_RESET, 0); + writel(USB2PHY_MISCTUNE_DEFAULT, p->usb2phy + USB2PHY_MISCTUNE); + writel(USB2PHY_USBCTL_RUN, p->usb2phy + USB2PHY_USBCTL); + + /* PIPE on the dummy backend, dwc3 out of reset. */ + writel(PIPEHANDLER_MUX_CTRL_DUMMY, p->pipehandler + PIPEHANDLER_MUX_CTRL); + writel(PIPEHANDLER_AON_GEN_DWC3_RESET_N, p->pipehandler + PIPEHANDLER_AON_GEN); + writel(PIPEHANDLER_NONSELECTED_DEFAULT, p->pipehandler + PIPEHANDLER_NONSELECTED_OVERRIDE); + + /* usb2phy host bits, gadget stopped, core soft reset, port capability host. */ + writel(USB2PHY_SIG_DEFAULT | USB2PHY_SIG_HOST, p->usb2phy + USB2PHY_SIG); + t6050_usb2_mask(p->core + DWC3_DCTL, DWC3_DCTL_RUN_STOP, 0); + if (readl_poll_timeout(p->core + DWC3_DSTS, val, val & DWC3_DSTS_DEVCTRLHLT, 100, 10000)) + dev_dbg(p->dev, "device controller did not halt (DSTS %#x)\n", val); + t6050_usb2_mask(p->core + DWC3_GCTL, 0, DWC3_GCTL_CORESOFTRESET); + msleep(10); + t6050_usb2_mask(p->core + DWC3_GCTL, DWC3_GCTL_PRTCAPDIR_MASK | DWC3_GCTL_CORESOFTRESET, + DWC3_GCTL_PRTCAP_HOST); + msleep(10); + + /* PIPE parked: link detection forced off through the overrides. */ + t6050_usb2_mask(p->pipehandler + PIPEHANDLER_OVERRIDE_VALUES, BIT(1) | BIT(2), 0); + t6050_usb2_mask(p->pipehandler + PIPEHANDLER_OVERRIDE, 0, BIT(0) | BIT(2)); + + /* No PHY auto-suspend: the xHCI reset needs the PHY clocks. */ + t6050_usb2_mask(p->core + DWC3_GUSB2PHYCFG0, DWC3_GUSB2PHYCFG_SUSPHY, 0); + t6050_usb2_mask(p->core + DWC3_GUSB3PIPECTL0, DWC3_GUSB3PIPECTL_SUSPHY, 0); +} + +/* Polls (reg & mask) == want; a timeout is only logged, as in the proxy sequence. */ +static bool t6050_wait(void __iomem *reg, u32 mask, u32 want, unsigned int ms) +{ + u32 val; + + return !readl_poll_timeout(reg, val, (val & mask) == want, 100, ms * 1000); +} + +static bool t6050_ps_on(void __iomem *reg) +{ + u32 val = readl(reg); + + if (FIELD_GET(PS_ACTUAL, val) == PS_ON) + return true; + writel((val & ~(PS_TARGET | GENMASK(9, 8))) | PS_ON, reg); + return t6050_wait(reg, PS_ACTUAL, PS_ACTUAL, 100); +} + +/* + * Timing of the SuperSpeed sequence, settable on the command line while the link + * training is not reliable: the gap after each register write (the proxy that + * measured the sequence took about 1 ms per write) and a pause after step 16. + */ +static unsigned int ss_pace_us; +module_param(ss_pace_us, uint, 0444); +MODULE_PARM_DESC(ss_pace_us, "SuperSpeed sequence: gap after each register write, us"); + +static unsigned int ss_settle_ms; +module_param(ss_settle_ms, uint, 0444); +MODULE_PARM_DESC(ss_settle_ms, "SuperSpeed sequence: pause after step 16, ms"); + +/* + * GUSB3PIPECTL SUSPHY: set by iBoot and set throughout the proxy runs that trained + * the link; the USB 2.0 flip clears it. + */ +static bool ss_susphy = true; +module_param(ss_susphy, bool, 0444); +MODULE_PARM_DESC(ss_susphy, "SuperSpeed sequence: set GUSB3PIPECTL SUSPHY again before step 1"); + +static bool ss_susphy_clear; +module_param(ss_susphy_clear, bool, 0444); +MODULE_PARM_DESC(ss_susphy_clear, "SuperSpeed sequence: clear GUSB3PIPECTL SUSPHY after the xHCI reset (step 17)"); + +/* Plug orientation: flipped uses lane 1 and the swapped crossbar (steps 7, 9, 10). */ +static bool ss_flip; +module_param(ss_flip, bool, 0444); +MODULE_PARM_DESC(ss_flip, "SuperSpeed sequence: flipped plug orientation"); + +/* + * A device plugged in at power-on no longer goes for SuperSpeed on a USB 2.0 port + * reset, but a fresh connection trains at once: after step 16 the port's Type-C + * controller simulates an unplug of this many seconds. Off by default: the device + * does not come back as a fresh connection (kernel/USB-BRINGUP.md). + */ +static unsigned int ss_replug_s; +module_param(ss_replug_s, uint, 0444); +MODULE_PARM_DESC(ss_replug_s, "SuperSpeed: simulated disconnect after step 16, s (0: off)"); + +/* + * Step 18 again: a device left at high speed by the hub's USB 2.0 resets often goes + * for SuperSpeed on a later one (the proxy: U0 on the 5th). From 3 s after the xHCI + * reset, while the SS port has no connection, reset the USB 2.0 port every 2 s. Off by + * default: under Linux each reset costs the high-speed device and none has trained yet. + */ +static unsigned int ss_u2resets; +module_param(ss_u2resets, uint, 0444); +MODULE_PARM_DESC(ss_u2resets, "SuperSpeed: extra USB 2.0 port resets until the SS port connects (0: off)"); + +/* + * Test without the xHCI driver (port 1's xHCI node disabled): the proxy run from + * here, steps 1-17, run, port power, then USB 2.0 resets until the SS port is in U0. + */ +static bool ss_standalone; +module_param(ss_standalone, bool, 0444); +MODULE_PARM_DESC(ss_standalone, "SuperSpeed test: sequence and USB 2.0 resets without the xHCI driver"); + +static void t6050_ss_mask(void __iomem *reg, u32 clear, u32 set) +{ + t6050_usb2_mask(reg, clear, set); + if (ss_pace_us) + usleep_range(ss_pace_us, ss_pace_us + ss_pace_us / 5 + 1); +} + +static void t6050_tunables_apply(struct t6050_tunable *t, unsigned int n) +{ + unsigned int i, j; + + for (i = 0; i < n; i++) + for (j = 0; j < t[i].tunable->sz; j++) + t6050_ss_mask(t[i].regs + t[i].tunable->values[j].offset, + t[i].tunable->values[j].mask, t[i].tunable->values[j].value); +} + +static bool t6050_rtk_send(struct t6050_usb2 *p, u64 msg) +{ + writeq(msg, p->mxw + MXW_A2I_MSG0); + writeq(0, p->mxw + MXW_A2I_MSG1); /* endpoint 0, management */ + return t6050_wait(p->mxw + MXW_A2I_CTRL, MXW_MBOX_FULL, 0, 100); +} + +/* + * Boots the PHY CPU's RTKit firmware, which iBoot left loaded, the way m1n1's + * StandardASC does: HELLO -> ack, EPMAP -> ack, start the system endpoints, + * AP power on, then IOP power on. Endpoint 1 (crashlog) asks for a buffer at + * a preallocated address, which needs no reply; other endpoints are drained. + */ +static void t6050_phy_cpu_boot(struct t6050_usb2 *p) +{ + unsigned long ap_end = jiffies + msecs_to_jiffies(3000); + unsigned long end = ap_end + msecs_to_jiffies(3000), settle = 0; + u32 eps = 0, ap = 0, iop = 0, ver = 0, other = 0, other_ep = 0; + u64 msg, other_msg = 0; + bool iop_sent = false; + unsigned int ep; + + t6050_usb2_mask(p->mxw + MXW_IDLE_CTRL, 0, BIT(0)); + msleep(10); + writel(MXW_CPU_CTRL_RUN, p->mxw + MXW_CPU_CTRL); + + while (settle ? time_before(jiffies, settle) : time_before(jiffies, end)) { + /* The proxy sent IOP power after 3 s even without the AP power ack. */ + if (!iop_sent && time_after(jiffies, ap_end)) { + t6050_rtk_send(p, FIELD_PREP(RTK_TYPE, RTK_IOP_PWR) | + FIELD_PREP(RTK_PWR_STATE, 0x220)); + iop_sent = true; + } + if (readl(p->mxw + MXW_I2A_CTRL) & MXW_MBOX_EMPTY) { + usleep_range(1000, 2000); + continue; + } + msg = readq(p->mxw + MXW_I2A_MSG0); + ep = readq(p->mxw + MXW_I2A_MSG1) & 0xff; + if (ep) { + other++; + other_ep = ep; + other_msg = msg; + continue; + } + + switch (FIELD_GET(RTK_TYPE, msg)) { + case RTK_HELLO: + ver = FIELD_GET(RTK_HELLO_MAX, msg); + t6050_rtk_send(p, FIELD_PREP(RTK_TYPE, RTK_HELLO_ACK) | + FIELD_PREP(RTK_HELLO_MIN, ver) | FIELD_PREP(RTK_HELLO_MAX, ver)); + break; + case RTK_EPMAP: + if (FIELD_GET(RTK_EPMAP_BASE, msg) == 0) + eps |= FIELD_GET(RTK_EPMAP_BITMAP, msg); + t6050_rtk_send(p, FIELD_PREP(RTK_TYPE, RTK_EPMAP) | (msg & RTK_EPMAP_LAST) | + (msg & RTK_EPMAP_BASE) | + ((msg & RTK_EPMAP_LAST) ? 0 : RTK_EPMAP_MORE)); + if (msg & RTK_EPMAP_LAST) { + for (ep = 1; ep < 0x10; ep++) + if (eps & BIT(ep)) + t6050_rtk_send(p, FIELD_PREP(RTK_TYPE, RTK_START_EP) | + FIELD_PREP(RTK_EP, ep) | RTK_START_EP_FLAG); + t6050_rtk_send(p, FIELD_PREP(RTK_TYPE, RTK_AP_PWR) | + FIELD_PREP(RTK_PWR_STATE, RTK_PWR_ON)); + } + break; + case RTK_AP_PWR: + ap = FIELD_GET(RTK_PWR_STATE, msg); + if (ap == RTK_PWR_ON && !iop_sent) { + t6050_rtk_send(p, FIELD_PREP(RTK_TYPE, RTK_IOP_PWR) | + FIELD_PREP(RTK_PWR_STATE, 0x220)); + iop_sent = true; + } + break; + case RTK_IOP_PWR_ACK: + iop = FIELD_GET(RTK_PWR_STATE, msg); + break; + } + /* Booted: keep draining for one more second, as the proxy did. */ + if (ap == RTK_PWR_ON && iop == RTK_PWR_ON && !settle) + settle = jiffies + msecs_to_jiffies(1000); + } + + dev_info(p->dev, "SS: PHY CPU RTKit v%u, endpoints %#x, AP %#x, IOP %#x, %u other message(s), last %#x:%#llx\n", + ver, eps, ap, iop, other, other_ep, other_msg); +} + +/* Steps 1-16 of the port 1 SuperSpeed host sequence (kernel/USB-BRINGUP.md). */ +static void t6050_usb3_up(struct t6050_usb2 *p) +{ + void __iomem *atc = p->atc, *pipe = p->pipehandler; + bool hcrst, cnr, cio = true, mxw, tnbl, s1, rise, fall, lock, ack, unlock, rel, wpr; + unsigned long wpr_us; + unsigned int l = ss_flip; /* active lane */ + unsigned int i, n = 0; + char trace[17 * 9 + 1]; + u32 val, tr[17]; + int len = 0; + + /* The glue back to the boot values while CIO is still off; tr[] = SS PORTSC after each step. */ + for (i = 0; i < ARRAY_SIZE(t6050_glue); i++) + writel(t6050_glue_boot[i], p->core + t6050_glue[i]); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + if (ss_susphy) + t6050_usb2_mask(p->core + DWC3_GUSB3PIPECTL0, 0, DWC3_GUSB3PIPECTL_SUSPHY); + + /* 1. xHCI reset while CIO is still off. */ + t6050_ss_mask(p->core + XHCI_USBCMD, 0, XHCI_USBCMD_HCRST); + hcrst = t6050_wait(p->core + XHCI_USBCMD, XHCI_USBCMD_HCRST, 0, 1000); + cnr = t6050_wait(p->core + XHCI_USBSTS, XHCI_USBSTS_CNR, 0, 1000); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 2. Controller tunables. 3. TXDETRX lock request. */ + t6050_tunables_apply(p->host_tun, ARRAY_SIZE(p->host_tun)); + t6050_ss_mask(p->pctl + PCTL_TXDETRX_REQ, 0, BIT(0)); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 4. CIO power domains, then the PHY power control; the PHY reset stays asserted. */ + for (i = 0; i < ARRAY_SIZE(t6050_cio_domains); i++) + cio &= t6050_ps_on(p->pmgr + t6050_cio_domains[i]); + mxw = t6050_ps_on(p->pmgr_mxw + PS_PHYMXWRAP); + tr[n++] = readl(p->core + XHCI_PORTSC2); + t6050_ss_mask(p->atc_pwr + ATC_PWR_CTRL, 0, ATC_PWR_SLEEP_SMALL); + t6050_wait(p->atc_pwr + ATC_PWR_STAT, ATC_PWR_SLEEP_SMALL, ATC_PWR_SLEEP_SMALL, 100); + t6050_ss_mask(p->atc_pwr + ATC_PWR_CTRL, 0, ATC_PWR_SLEEP_BIG); + t6050_wait(p->atc_pwr + ATC_PWR_STAT, ATC_PWR_SLEEP_BIG, ATC_PWR_SLEEP_BIG, 100); + t6050_ss_mask(p->atc_pwr + ATC_PWR_CTRL, ATC_PWR_CLAMP, 0); + t6050_ss_mask(p->atc_pwr + ATC_PWR_CTRL, 0, ATC_PWR_APB_RESET_N); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 5. PHY tunables. */ + t6050_tunables_apply(p->phy_tun, ARRAY_SIZE(p->phy_tun)); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 6. Port capability host, no core soft reset. */ + t6050_ss_mask(p->core + DWC3_GCTL, DWC3_GCTL_PRTCAPDIR_MASK, DWC3_GCTL_PRTCAP_HOST); + t6050_ss_mask(atc + ATC_LANE_MODE, 0xfc, (2 << 2) | (2 << 5)); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 7. Sleep control, USB3 mode: common block and lane 0 overrides to the FSM. */ + t6050_ss_mask(atc + ATC_CFG0, BIT(1) | BIT(3) | BIT(5), 0); + t6050_ss_mask(atc + ATC_RX_PWR, BIT(2 + l), 0); + t6050_ss_mask(atc + ATC_RX_PWR, BIT(6 + l), 0); + t6050_ss_mask(atc + ATC_CFG0, BIT(16 + l), 0); + t6050_ss_mask(atc + ATC_CFG0, BIT(8 + l), 0); + t6050_ss_mask(atc + ATC_CFG0, BIT(12 + l), 0); + t6050_ss_mask(atc + ATC_TX_PWR, BIT(6 + l), 0); + t6050_ss_mask(atc + ATC_TX_PWR, BIT(2 + l), 0); + t6050_ss_mask(atc + ATC_TX_PWR, BIT(10 + l), 0); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 8. MxWrap idle control. */ + t6050_ss_mask(p->mxw + MXW_IDLE_CTRL, 0, BIT(0)); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* + * 9. Crossbar, USB3 + DP: normal lane 0 mode 1, lane 1 mode 2; flipped lane 0 + * mode 2, lane 1 mode 1. + */ + if (l) + t6050_ss_mask(atc + ATC_LANE_MODE, 0xfc | BIT(22) | BIT(24), 0x28); + else + t6050_ss_mask(atc + ATC_LANE_MODE, 0xfc | BIT(23) | BIT(25), 0x44); + t6050_ss_mask(atc + ATC_XBAR, GENMASK(16, 0), 0x110 | l); + t6050_ss_mask(atc + ATC_LANE_MODE, GENMASK(1, 0), 2); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 10. BIST first half. */ + t6050_ss_mask(atc + ATC_BIST_CFG1, 0, ATC_BIST_CFG1_CLOCK_EN); + t6050_ss_mask(atc + ATC_BIST_OV1, ATC_BIST_OV1_PWR_DOWN | ATC_BIST_OV1_PIPE_RESET_N, + FIELD_PREP(ATC_BIST_OV1_PWR_DOWN, 2)); + t6050_ss_mask(atc + ATC_BIST_CFG1, 0, ATC_BIST_CFG1_USB_EN); + t6050_ss_mask(atc + ATC_BIST_CFG1, ATC_BIST_CFG1_USB_EN, 0); + t6050_ss_mask(atc + ATC_BIST_OV2, 0x660, 0x220); + t6050_ss_mask(atc + ATC_BIST_CFG1, 0, ATC_BIST_CFG1_USB_EN); + t6050_ss_mask(atc + ATC_BIST_CFG1, ATC_BIST_CFG1_USB_EN, 0); + t6050_ss_mask(p->atc_pwr + ATC_PWR_MISC, ATC_PWR_MISC_USB_LN_SWAP, + l ? ATC_PWR_MISC_USB_LN_SWAP : 0); + msleep(10); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 11. PHY CPU firmware, then its tunables-done flag. */ + t6050_phy_cpu_boot(p); + usleep_range(5000, 6000); + tnbl = t6050_wait(atc + ATC_UC_CTRL, ATC_UC_TNBL_DONE, ATC_UC_TNBL_DONE, 1000); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 12. PHY reset release. */ + t6050_ss_mask(p->atc_pwr + ATC_PWR_CTRL, 0, ATC_PWR_PHY_RESET_N); + msleep(100); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 13. BIST second half. */ + t6050_ss_mask(atc + ATC_BIST_OV1, 0, ATC_BIST_OV1_PIPE_RESET_N); + t6050_ss_mask(atc + ATC_BIST_CFG1, 0, ATC_BIST_CFG1_USB_EN); + t6050_ss_mask(atc + ATC_BIST_CFG1, ATC_BIST_CFG1_USB_EN, 0); + s1 = t6050_wait(atc + ATC_BIST_STATUS2, ATC_BIST_STATUS2_PHY_STATUS, 0, 1000); + t6050_ss_mask(atc + ATC_BIST_OV1, ATC_BIST_OV1_PWR_DOWN, FIELD_PREP(ATC_BIST_OV1_PWR_DOWN, 3)); + t6050_ss_mask(atc + ATC_BIST_CFG1, 0, ATC_BIST_CFG1_USB_EN); + t6050_ss_mask(atc + ATC_BIST_CFG1, ATC_BIST_CFG1_USB_EN, 0); + rise = t6050_wait(atc + ATC_BIST_STATUS2, ATC_BIST_STATUS2_PHY_STATUS, + ATC_BIST_STATUS2_PHY_STATUS, 1000); + fall = t6050_wait(atc + ATC_BIST_STATUS2, ATC_BIST_STATUS2_PHY_STATUS, 0, 1000); + /* + * The SS port has been in a warm reset since host mode (step 6); it completes + * with the PHY held here (power-down override 3). The proxy runs, which missed + * the PhyStatus pulse and waited out its 1 s timeout, always had it completed + * at this point and reached step 17 with a clean port; wait for it here. + */ + wpr_us = jiffies; + wpr = t6050_wait(p->core + XHCI_PORTSC2, XHCI_PORTSC_PR, 0, 1000); + wpr_us = jiffies_to_usecs(jiffies - wpr_us); + t6050_ss_mask(atc + ATC_BIST_OV2, 0x660, 0); + t6050_ss_mask(atc + ATC_BIST_CFG1, 0, ATC_BIST_CFG1_USB_EN); + t6050_ss_mask(atc + ATC_BIST_CFG1, ATC_BIST_CFG1_USB_EN, 0); + t6050_ss_mask(pipe + PIPEHANDLER_NONSELECTED_OVERRIDE, GENMASK(3, 0), 3); + t6050_ss_mask(pipe + PIPEHANDLER_NONSELECTED_OVERRIDE, BIT(12), 0); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 14. PIPE overrides and lock; the PHY answers the TXDETRX request. */ + t6050_ss_mask(pipe + PIPEHANDLER_OVERRIDE_VALUES, BIT(1) | BIT(2), 0); + t6050_ss_mask(pipe + PIPEHANDLER_OVERRIDE, 0, BIT(0)); + t6050_ss_mask(pipe + PIPEHANDLER_OVERRIDE, 0, BIT(2)); + t6050_ss_mask(pipe + PIPEHANDLER_LOCK_REQ, 0, PIPEHANDLER_LOCK); + lock = t6050_wait(pipe + PIPEHANDLER_LOCK_ACK, PIPEHANDLER_LOCK, PIPEHANDLER_LOCK, 100); + ack = t6050_wait(p->pctl + PCTL_TXDETRX_ACK, BIT(0), BIT(0), 50); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 15. PIPE to the USB3 PHY, overrides off, unlock. */ + t6050_ss_mask(pipe + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CLK, 0); + t6050_ss_mask(pipe + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_DATA, 0); + t6050_ss_mask(pipe + PIPEHANDLER_MUX_CTRL, PIPEHANDLER_MUX_CLK, + FIELD_PREP(PIPEHANDLER_MUX_CLK, 1)); + t6050_ss_mask(pipe + PIPEHANDLER_OVERRIDE, BIT(0), 0); + t6050_ss_mask(pipe + PIPEHANDLER_OVERRIDE, BIT(2), 0); + t6050_ss_mask(pipe + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK, 0); + unlock = t6050_wait(pipe + PIPEHANDLER_LOCK_ACK, PIPEHANDLER_LOCK, 0, 10); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + /* 16. TXDETRX lock release. */ + t6050_ss_mask(p->pctl + PCTL_TXDETRX_REQ, BIT(0), 0); + rel = t6050_wait(p->pctl + PCTL_TXDETRX_ACK, BIT(0), 0, 50); + tr[n++] = readl(p->core + XHCI_PORTSC2); + + val = readl(atc + ATC_BIST_STATUS2); + for (i = 0; i < n; i++) + len += scnprintf(trace + len, sizeof(trace) - len, " %x", tr[i]); + + dev_info(p->dev, "SS: HCRST %d CNR %d, CIO %d, mxwrap %d, UC_TNBL_DONE %d\n", + hcrst, cnr, cio, mxw, tnbl); + dev_info(p->dev, "SS: SS PORTSC from the write through step 16:%s\n", trace); + dev_info(p->dev, "SS: PhyStatus s1 %d rise %d fall %d (STATUS2 %#x), lock %d ack %d unlock %d release %d\n", + s1, rise, fall, val, lock, ack, unlock, rel); + dev_info(p->dev, "SS: warm reset in step 13 done %d after %lu us\n", wpr, wpr_us); + dev_info(p->dev, "SS: pace %u us, settle %u ms, susphy %d, flip %d\n", ss_pace_us, ss_settle_ms, + ss_susphy, ss_flip); + msleep(ss_settle_ms); + dev_info(p->dev, "SS: glue %#x %#x %#x %#x, PORTSC %#x / %#x\n", readl(p->core + t6050_glue[0]), + readl(p->core + t6050_glue[1]), readl(p->core + t6050_glue[2]), + readl(p->core + t6050_glue[3]), readl(p->core + XHCI_PORTSC1), + readl(p->core + XHCI_PORTSC2)); + if (p->hpm && ss_replug_s) + schedule_delayed_work(&p->replug_work, 0); +} + +/* Diagnostic around the simulated unplug: the port 1 registers of both sides. */ +static void t6050_port_state(struct t6050_usb2 *p, const char *when) +{ + dev_info(p->dev, "SS: %s: PORTSC %#x / %#x, usb2phy %#x %#x %#x, pipe mux %#x, GUSB2PHYCFG %#x, GUSB3PIPECTL %#x, power %#x / %#x, UC_CTRL %#x, STATUS2 %#x\n", + when, readl(p->core + XHCI_PORTSC1), readl(p->core + XHCI_PORTSC2), + readl(p->usb2phy + USB2PHY_USBCTL), readl(p->usb2phy + USB2PHY_CTL), + readl(p->usb2phy + USB2PHY_SIG), readl(p->pipehandler + PIPEHANDLER_MUX_CTRL), + readl(p->core + DWC3_GUSB2PHYCFG0), readl(p->core + DWC3_GUSB3PIPECTL0), + readl(p->atc_pwr + ATC_PWR_CTRL), readl(p->atc_pwr + ATC_PWR_STAT), + readl(p->atc + ATC_UC_CTRL), readl(p->atc + ATC_BIST_STATUS2)); +} + +/* Waits up to 10 s for the Type-C controller's driver, then simulates the unplug. */ +static void t6050_replug(struct work_struct *work) +{ + struct t6050_usb2 *p = container_of(to_delayed_work(work), struct t6050_usb2, replug_work); + int ret; + + if (!p->replug_tries) + t6050_port_state(p, "before the disconnect"); + ret = sn201202x_simulate_disconnect(p->hpm, min(ss_replug_s, 255U)); + if (ret == -EPROBE_DEFER && ++p->replug_tries < 100) { + schedule_delayed_work(&p->replug_work, msecs_to_jiffies(100)); + return; + } + dev_info(p->dev, "SS: simulated disconnect %u s: %d after %u tries\n", ss_replug_s, ret, + p->replug_tries + 1); + t6050_port_state(p, "after the disconnect"); +} + +/* + * Diagnostic while the link training is not reliable: for 3 s after the xHCI reset, + * log every change of the two PORTSC registers and of the controller's LTSSM state. + */ +static void t6050_ltssm_watch(struct work_struct *work) +{ + struct t6050_usb2 *p = container_of(work, struct t6050_usb2, ltssm_work); + u32 last[3] = { ~0U, ~0U, ~0U }, now[3]; + unsigned int i, changes = 0; + + for (i = 0; i < 6000 && changes < 60; i++) { + now[0] = readl(p->core + XHCI_PORTSC1); + now[1] = readl(p->core + XHCI_PORTSC2); + now[2] = readl(p->core + DWC3_GDBGLTSSM); + if (memcmp(now, last, sizeof(now))) { + dev_info(p->dev, "SS watch: PORTSC %#x / %#x, GDBGLTSSM %#x\n", + now[0], now[1], now[2]); + memcpy(last, now, sizeof(now)); + changes++; + } + usleep_range(500, 600); + } +} + +/* After the xHCI reset (step 17): the glue back to the iBoot values, host mode. */ +static int t6050_usb2_calibrate(struct phy *phy) +{ + struct t6050_usb2 *p = phy_get_drvdata(phy); + u32 rd[ARRAY_SIZE(t6050_glue)]; + unsigned int i, changed = 0; + + if (!p->ss) + return 0; + for (i = 0; i < ARRAY_SIZE(t6050_glue); i++) { + rd[i] = readl(p->core + t6050_glue[i]); + if (rd[i] == t6050_glue_boot[i]) + continue; + writel(t6050_glue_boot[i], p->core + t6050_glue[i]); + changed |= BIT(i); + } + t6050_usb2_mask(p->core + DWC3_GCTL, DWC3_GCTL_PRTCAPDIR_MASK, DWC3_GCTL_PRTCAP_HOST); + if (ss_susphy_clear) + t6050_usb2_mask(p->core + DWC3_GUSB3PIPECTL0, DWC3_GUSB3PIPECTL_SUSPHY, 0); + dev_info(p->dev, "SS: after xHCI reset: glue read %#x %#x %#x %#x, rewritten %#x, now %#x %#x %#x %#x\n", + rd[0], rd[1], rd[2], rd[3], changed, readl(p->core + t6050_glue[0]), + readl(p->core + t6050_glue[1]), readl(p->core + t6050_glue[2]), + readl(p->core + t6050_glue[3])); + dev_info(p->dev, "SS: after xHCI reset: USBSTS %#x, PORTSC %#x / %#x, GUSB2PHYCFG %#x, GUSB3PIPECTL %#x\n", + readl(p->core + XHCI_USBSTS), readl(p->core + XHCI_PORTSC1), + readl(p->core + XHCI_PORTSC2), readl(p->core + DWC3_GUSB2PHYCFG0), + readl(p->core + DWC3_GUSB3PIPECTL0)); + queue_work(system_long_wq, &p->ltssm_work); + /* phy_calibrate() runs again on a root hub resume */ + if (ss_u2resets && !p->u2reset_queued) { + p->u2reset_queued = true; + schedule_delayed_work(&p->u2reset_work, msecs_to_jiffies(3000)); + } + return 0; +} + +static void t6050_usb3_inherited(struct t6050_usb2 *p); + +static void t6050_standalone(struct work_struct *work) +{ + struct t6050_usb2 *p = container_of(to_delayed_work(work), struct t6050_usb2, + standalone_work); + u32 last[2] = { ~0U, ~0U }, now[2]; + unsigned int i, k, changes = 0; + bool hcrst, cnr; + + t6050_usb3_inherited(p); + t6050_usb3_up(p); + + /* 17. xHCI reset, glue, host mode; then run and port power, as the proxy runs. */ + t6050_usb2_mask(p->core + XHCI_USBCMD, 0, XHCI_USBCMD_HCRST); + hcrst = t6050_wait(p->core + XHCI_USBCMD, XHCI_USBCMD_HCRST, 0, 2000); + cnr = t6050_wait(p->core + XHCI_USBSTS, XHCI_USBSTS_CNR, 0, 2000); + for (i = 0; i < ARRAY_SIZE(t6050_glue); i++) + writel(t6050_glue_boot[i], p->core + t6050_glue[i]); + t6050_usb2_mask(p->core + DWC3_GCTL, DWC3_GCTL_PRTCAPDIR_MASK, DWC3_GCTL_PRTCAP_HOST); + t6050_usb2_mask(p->core + XHCI_USBCMD, 0, BIT(0)); + for (i = 0; i < 2; i++) { + void __iomem *reg = p->core + (i ? XHCI_PORTSC2 : XHCI_PORTSC1); + + writel((readl(reg) & 0x4e00ffe9) | BIT(9), reg); + } + dev_info(p->dev, "SS standalone: HCRST %d CNR %d, PORTSC %#x / %#x\n", hcrst, cnr, + readl(p->core + XHCI_PORTSC1), readl(p->core + XHCI_PORTSC2)); + msleep(100); + + /* 18. USB 2.0 port resets, 2 s apart, until the SS port is in U0 */ + for (k = 1; k <= 10; k++) { + u32 hs = readl(p->core + XHCI_PORTSC1); + + writel((hs & 0x0e00c200) | XHCI_PORTSC_PR, p->core + XHCI_PORTSC1); + dev_info(p->dev, "SS standalone: USB 2.0 reset %u, PORTSC %#x / %#x\n", k, hs, + readl(p->core + XHCI_PORTSC2)); + for (i = 0; i < 2000; i++) { + now[0] = readl(p->core + XHCI_PORTSC1); + now[1] = readl(p->core + XHCI_PORTSC2); + if (memcmp(now, last, sizeof(now)) && changes++ < 100) + dev_info(p->dev, "SS standalone: %u ms PORTSC %#x / %#x\n", i, + now[0], now[1]); + memcpy(last, now, sizeof(now)); + if ((now[1] & 0x1e1) == XHCI_PORTSC_CCS) { + dev_info(p->dev, "SS standalone: U0 after USB 2.0 reset %u\n", k); + return; + } + usleep_range(1000, 1100); + } + } + dev_info(p->dev, "SS standalone: no U0 after 10 USB 2.0 resets\n"); +} + +static void t6050_u2reset(struct work_struct *work) +{ + struct t6050_usb2 *p = container_of(to_delayed_work(work), struct t6050_usb2, u2reset_work); + u32 hs = readl(p->core + XHCI_PORTSC1), ss = readl(p->core + XHCI_PORTSC2); + + if (ss & XHCI_PORTSC_CCS) { + dev_info(p->dev, "SS: connected after %u extra USB 2.0 reset(s), PORTSC %#x / %#x\n", + p->u2resets, hs, ss); + return; + } + if (p->u2resets == ss_u2resets) { + dev_info(p->dev, "SS: no connection after %u extra USB 2.0 resets, PORTSC %#x / %#x\n", + p->u2resets, hs, ss); + return; + } + if (hs & XHCI_PORTSC_CCS) { + /* PP, port indicators and the wake enables kept; the change bits not cleared */ + writel((hs & 0x0e00c200) | XHCI_PORTSC_PR, p->core + XHCI_PORTSC1); + p->u2resets++; + dev_info(p->dev, "SS: extra USB 2.0 reset %u, PORTSC %#x / %#x\n", p->u2resets, hs, ss); + } + schedule_delayed_work(&p->u2reset_work, msecs_to_jiffies(2000)); +} + +/* + * The port 1 state the kernel inherits, to compare with the state the proven proxy + * runs started from. The core is read only with dwc3 out of reset. + */ +static void t6050_usb3_inherited(struct t6050_usb2 *p) +{ + void __iomem *pipe = p->pipehandler; + u32 aon = readl(pipe + PIPEHANDLER_AON_GEN); + + dev_info(p->dev, "SS: inherited pipehandler +0x0 %#x +0x4 %#x +0xc %#x +0x1c %#x +0x20 %#x\n", + readl(pipe + PIPEHANDLER_OVERRIDE), readl(pipe + PIPEHANDLER_OVERRIDE_VALUES), + readl(pipe + PIPEHANDLER_MUX_CTRL), aon, + readl(pipe + PIPEHANDLER_NONSELECTED_OVERRIDE)); + if (!(aon & PIPEHANDLER_AON_GEN_DWC3_RESET_N)) { + dev_info(p->dev, "SS: inherited dwc3 in reset, core not read; reset released\n"); + writel(PIPEHANDLER_AON_GEN_DWC3_RESET_N, pipe + PIPEHANDLER_AON_GEN); + return; + } + dev_info(p->dev, "SS: inherited GCTL %#x, GUSB2PHYCFG %#x, GUSB3PIPECTL %#x, glue %#x / %#x, PORTSC %#x / %#x\n", + readl(p->core + DWC3_GCTL), readl(p->core + DWC3_GUSB2PHYCFG0), + readl(p->core + DWC3_GUSB3PIPECTL0), readl(p->core + t6050_glue[0]), + readl(p->core + t6050_glue[2]), readl(p->core + XHCI_PORTSC1), + readl(p->core + XHCI_PORTSC2)); +} + +/* + * On J714c a USB 2.0-only port detects no new device once its device has left + * (the adapter in port 0 never re-enumerated on a replug). Running the usb2phy + * again (reset, tunes, run, host bits) while the port is empty brings detection + * back; the dwc3 core and xHCI are not touched. + */ +#define T6050_REARM_PERIOD (3 * HZ) +#define T6050_DETACH_DELAY (HZ / 2) + +static void t6050_usb2_phy_rerun(struct t6050_usb2 *p) +{ + writel(USB2PHY_CTL_RESET | USB2PHY_CTL_PORT_RESET, p->usb2phy + USB2PHY_CTL); + udelay(100); + t6050_usb2_mask(p->usb2phy + USB2PHY_CTL, USB2PHY_CTL_RESET, 0); + t6050_usb2_mask(p->usb2phy + USB2PHY_CTL, USB2PHY_CTL_PORT_RESET, 0); + writel(USB2PHY_MISCTUNE_DEFAULT, p->usb2phy + USB2PHY_MISCTUNE); + writel(USB2PHY_USBCTL_RUN, p->usb2phy + USB2PHY_USBCTL); + writel(USB2PHY_SIG_DEFAULT | USB2PHY_SIG_HOST, p->usb2phy + USB2PHY_SIG); +} + +static void t6050_usb2_rearm(struct work_struct *work) +{ + struct t6050_usb2 *p = container_of(to_delayed_work(work), struct t6050_usb2, rearm_work); + u32 portsc = readl(p->core + XHCI_PORTSC1); + /* A NONE that tipd followed with HOST again (resume, DP mode) is no detach. */ + bool detached = xchg(&p->detached, false) && READ_ONCE(p->role) == USB_ROLE_NONE; + + if ((portsc & XHCI_PORTSC_CCS) && detached) { + t6050_usb2_phy_rerun(p); + msleep(50); + dev_info(p->dev, "Type-C detach while connected: usb2phy run again, PORTSC %#x -> %#x\n", + portsc, readl(p->core + XHCI_PORTSC1)); + p->had_device = true; + } else if (portsc & XHCI_PORTSC_CCS) { + p->had_device = true; + } else if (p->had_device) { + t6050_usb2_phy_rerun(p); + } + schedule_delayed_work(&p->rearm_work, T6050_REARM_PERIOD); +} + +/* + * The Type-C controller reports attach and detach through this role switch, as + * tipd does for dwc3 on the other Apple machines. A device plugged in at + * power-on can stay connected in the xHCI after it left (J714c port 0, after + * a minute or two); running the usb2phy again then lets the xHCI see the + * disconnect. + */ +static int t6050_usb2_role_set(struct usb_role_switch *sw, enum usb_role role) +{ + struct t6050_usb2 *p = usb_role_switch_get_drvdata(sw); + + WRITE_ONCE(p->role, role); + if (role != USB_ROLE_NONE) { + WRITE_ONCE(p->detached, false); + } else if (READ_ONCE(p->initialized)) { + /* tipd may set HOST again in the same pass; the work checks the role */ + WRITE_ONCE(p->detached, true); + mod_delayed_work(system_wq, &p->rearm_work, T6050_DETACH_DELAY); + } + return 0; +} + +static enum usb_role t6050_usb2_role_get(struct usb_role_switch *sw) +{ + struct t6050_usb2 *p = usb_role_switch_get_drvdata(sw); + + return p->role; +} + +static void t6050_usb2_role_unregister(void *data) +{ + usb_role_switch_unregister(data); +} + +static int t6050_usb2_role_register(struct t6050_usb2 *p) +{ + struct usb_role_switch_desc desc = { + .fwnode = dev_fwnode(p->dev), + .set = t6050_usb2_role_set, + .get = t6050_usb2_role_get, + .driver_data = p, + }; + + p->role_sw = usb_role_switch_register(p->dev, &desc); + if (IS_ERR(p->role_sw)) + return dev_err_probe(p->dev, PTR_ERR(p->role_sw), "role switch\n"); + return devm_add_action_or_reset(p->dev, t6050_usb2_role_unregister, p->role_sw); +} + +static int t6050_usb2_init(struct phy *phy) +{ + struct t6050_usb2 *p = phy_get_drvdata(phy); + static const unsigned int wait_ms[] = { 50, 100, 100 }; + unsigned int attempt; + u32 portsc = 0; + + /* + * Port 1 starts the SuperSpeed sequence from the inherited state, as the + * proven proxy runs did: no USB 2.0 flip, step 6 sets host mode. + */ + if (p->ss) { + t6050_usb3_inherited(p); + t6050_usb3_up(p); + return 0; + } + + /* + * The loader saw on J714c that a device plugged in at power-on shows + * up only after a second bring-up and flip (it also redoes the dwc3 + * device-mode bring-up in between; this driver repeats the flip + * only). A port without a device stays in host mode after the last + * attempt. + */ + for (attempt = 0; attempt < ARRAY_SIZE(wait_ms); attempt++) { + t6050_usb2_host_flip(p); + if (!readl_poll_timeout(p->core + XHCI_PORTSC1, portsc, portsc & XHCI_PORTSC_CCS, + 1000, wait_ms[attempt] * 1000)) + break; + } + + dev_info(p->dev, "USB 2.0 host mode, PORTSC %#x after %u flip(s)\n", portsc, + min_t(unsigned int, attempt + 1, ARRAY_SIZE(wait_ms))); + p->had_device = portsc & XHCI_PORTSC_CCS; + WRITE_ONCE(p->initialized, true); + mod_delayed_work(system_wq, &p->rearm_work, T6050_REARM_PERIOD); + return 0; +} + +static const struct phy_ops t6050_usb2_ops = { + .init = t6050_usb2_init, + .calibrate = t6050_usb2_calibrate, + .owner = THIS_MODULE, +}; + +static void __iomem *t6050_usb2_map(struct platform_device *pdev, const char *name) +{ + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); + + if (!res) + return IOMEM_ERR_PTR(-EINVAL); + /* Not requested: the core window overlaps the xHCI node's registers. */ + return devm_ioremap(&pdev->dev, res->start, resource_size(res)) ?: IOMEM_ERR_PTR(-ENOMEM); +} + +static int t6050_tunables_parse(struct platform_device *pdev, const struct t6050_tunable_desc *desc, + struct t6050_tunable *t, unsigned int n) +{ + unsigned int i; + + for (i = 0; i < n; i++) { + struct resource *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, desc[i].reg); + + if (!res) + return dev_err_probe(&pdev->dev, -EINVAL, "%s registers\n", desc[i].reg); + t[i].regs = t6050_usb2_map(pdev, desc[i].reg); + if (IS_ERR(t[i].regs)) + return PTR_ERR(t[i].regs); + t[i].tunable = devm_apple_tunable_parse(&pdev->dev, pdev->dev.of_node, desc[i].prop, res); + if (IS_ERR(t[i].tunable)) + return dev_err_probe(&pdev->dev, PTR_ERR(t[i].tunable), "%s\n", desc[i].prop); + } + return 0; +} + +static int t6050_usb3_probe(struct platform_device *pdev, struct t6050_usb2 *p) +{ + struct { + void __iomem **regs; + const char *name; + } maps[] = { + { &p->pctl, "pctl" }, { &p->atc, "atc" }, { &p->mxw, "atc-mxwrap" }, + { &p->atc_pwr, "atc-power" }, { &p->pmgr, "pmgr" }, { &p->pmgr_mxw, "pmgr-mxwrap" }, + }; + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(maps); i++) { + *maps[i].regs = t6050_usb2_map(pdev, maps[i].name); + if (IS_ERR(*maps[i].regs)) + return dev_err_probe(&pdev->dev, PTR_ERR(*maps[i].regs), "%s registers\n", + maps[i].name); + } + ret = t6050_tunables_parse(pdev, t6050_host_tunables, p->host_tun, ARRAY_SIZE(p->host_tun)); + if (ret) + return ret; + ret = t6050_tunables_parse(pdev, t6050_phy_tunables, p->phy_tun, ARRAY_SIZE(p->phy_tun)); + if (ret) + return ret; + INIT_WORK(&p->ltssm_work, t6050_ltssm_watch); + INIT_DELAYED_WORK(&p->replug_work, t6050_replug); + INIT_DELAYED_WORK(&p->u2reset_work, t6050_u2reset); + INIT_DELAYED_WORK(&p->standalone_work, t6050_standalone); + if (ss_standalone) + schedule_delayed_work(&p->standalone_work, msecs_to_jiffies(1000)); + p->hpm = of_parse_phandle(pdev->dev.of_node, "apple,typec-controller", 0); + p->ss = true; + return 0; +} + +static int t6050_usb2_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct phy_provider *provider; + struct t6050_usb2 *p; + struct phy *phy; + int ret; + + p = devm_kzalloc(dev, sizeof(*p), GFP_KERNEL); + if (!p) + return -ENOMEM; + p->dev = dev; + ret = devm_delayed_work_autocancel(dev, &p->rearm_work, t6050_usb2_rearm); + if (ret) + return ret; + + p->usb2phy = t6050_usb2_map(pdev, "usb2phy"); + if (IS_ERR(p->usb2phy)) + return dev_err_probe(dev, PTR_ERR(p->usb2phy), "usb2phy registers\n"); + p->pipehandler = t6050_usb2_map(pdev, "pipehandler"); + if (IS_ERR(p->pipehandler)) + return dev_err_probe(dev, PTR_ERR(p->pipehandler), "pipehandler registers\n"); + p->core = t6050_usb2_map(pdev, "core"); + if (IS_ERR(p->core)) + return dev_err_probe(dev, PTR_ERR(p->core), "core registers\n"); + + if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "atc-power")) { + int ret = t6050_usb3_probe(pdev, p); + + if (ret) + return ret; + } + + if (IS_ENABLED(CONFIG_USB_ROLE_SWITCH) && !p->ss && + device_property_present(dev, "usb-role-switch")) { + ret = t6050_usb2_role_register(p); + if (ret) + return ret; + } + + phy = devm_phy_create(dev, NULL, &t6050_usb2_ops); + if (IS_ERR(phy)) + return PTR_ERR(phy); + phy_set_drvdata(phy, p); + + provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); + return PTR_ERR_OR_ZERO(provider); +} + +static const struct of_device_id t6050_usb2_of_match[] = { + { .compatible = "apple,t6050-usb2-phy" }, + {} +}; +MODULE_DEVICE_TABLE(of, t6050_usb2_of_match); + +static struct platform_driver t6050_usb2_driver = { + .probe = t6050_usb2_probe, + .driver = { + .name = "phy-apple-t6050-usb2", + .of_match_table = t6050_usb2_of_match, + }, +}; +module_platform_driver(t6050_usb2_driver); + +MODULE_DESCRIPTION("Apple T6050 Type-C port USB 2.0 host PHY"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/pinctrl/pinctrl-apple-gpio.c b/drivers/pinctrl/pinctrl-apple-gpio.c index eee7f5014ecc1..e08d50e697e1e 100644 --- a/drivers/pinctrl/pinctrl-apple-gpio.c +++ b/drivers/pinctrl/pinctrl-apple-gpio.c @@ -45,7 +45,7 @@ struct apple_gpio_pinctrl { #define REG_GPIO(x) (4 * (x)) #define REG_GPIOx_DATA BIT(0) -#define REG_GPIOx_MODE GENMASK(3, 1) +#define REG_GPIOx_MODE GENMASK(4, 1) #define REG_GPIOx_OUT 1 #define REG_GPIOx_IN_IRQ_HI 2 #define REG_GPIOx_IN_IRQ_LO 3 diff --git a/drivers/pmdomain/apple/Kconfig b/drivers/pmdomain/apple/Kconfig index b2f9b5113782f..5f0a9d95c196c 100644 --- a/drivers/pmdomain/apple/Kconfig +++ b/drivers/pmdomain/apple/Kconfig @@ -14,6 +14,17 @@ config APPLE_PMGR_PWRSTATE controls for SoC devices. This driver manages them through the generic power domain framework, and also provides reset support. +config APPLE_PMGR_PWRSTATE_KUNIT_TEST + bool "KUnit tests for the Apple PMGR power state driver" if !KUNIT_ALL_TESTS + depends on APPLE_PMGR_PWRSTATE && KUNIT=y + default KUNIT_ALL_TESTS + help + Check that a domain marked apple,no-auto-pm never gets automatic + power management when it is powered on or when a failed power-off + is undone, and that other domains still do. The tests run the + driver's power callbacks against an emulated power state register + and need no Apple hardware. + config APPLE_PMP_REPORT bool "Apple PMP report control" depends on OF @@ -26,7 +37,8 @@ config APPLE_PMP_REPORT report region it shares with the system, and the PMP's report of its own state. - On T8122 (M3) and T6031 (M3 Max) Macs whose boot loader hands the + On T8122 (M3), T6031 (M3 Max), and opted-in T6030 (M3 Pro) Macs + whose boot loader hands the internal display over (see APPLE_T6030_DISPLAY_GATE), iBoot leaves the PMP halted, and this driver starts it when the display driver first asks for it. Once its nodes are enabled the PMP may run until diff --git a/drivers/pmdomain/apple/pmgr-pwrstate-test.c b/drivers/pmdomain/apple/pmgr-pwrstate-test.c new file mode 100644 index 0000000000000..64eceb75ede6b --- /dev/null +++ b/drivers/pmdomain/apple/pmgr-pwrstate-test.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Included by pmgr-pwrstate.c: apple,no-auto-pm on the power-on and + * power-off paths. The power state register is emulated: its actual state + * follows the target written, unless the test holds it where it is. + */ +#include +#include + +struct pmgr_ps_test_reg { + u32 val; + bool stuck; /* the actual state no longer follows the target */ + unsigned int auto_writes; /* writes that enabled automatic PM */ +}; + +static int pmgr_ps_test_read(void *ctx, unsigned int reg, unsigned int *val) +{ + struct pmgr_ps_test_reg *r = ctx; + + *val = r->val; + return 0; +} + +static int pmgr_ps_test_write(void *ctx, unsigned int reg, unsigned int val) +{ + struct pmgr_ps_test_reg *r = ctx; + u32 actual = r->stuck ? FIELD_GET(APPLE_PMGR_PS_ACTUAL, r->val) : + FIELD_GET(APPLE_PMGR_PS_TARGET, val); + + if (val & APPLE_PMGR_AUTO_ENABLE) + r->auto_writes++; + r->val = (val & ~APPLE_PMGR_PS_ACTUAL) | FIELD_PREP(APPLE_PMGR_PS_ACTUAL, actual); + return 0; +} + +static const struct regmap_config pmgr_ps_test_regmap = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .reg_read = pmgr_ps_test_read, + .reg_write = pmgr_ps_test_write, + .fast_io = true, +}; + +/* A domain in power state @pstate, actual and target, on a device of its own */ +static struct apple_pmgr_ps *pmgr_ps_test_domain(struct kunit *test, u32 pstate, + bool no_auto_pm, struct pmgr_ps_test_reg **reg) +{ + static const char *const names[] = { "apple-pmgr-ps-auto", "apple-pmgr-ps-no-auto" }; + struct pmgr_ps_test_reg *r; + struct apple_pmgr_ps *ps; + struct device *dev; + + r = kunit_kzalloc(test, sizeof(*r), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, r); + r->val = FIELD_PREP(APPLE_PMGR_PS_TARGET, pstate) | + FIELD_PREP(APPLE_PMGR_PS_ACTUAL, pstate); + dev = kunit_device_register(test, names[no_auto_pm]); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + ps = kunit_kzalloc(test, sizeof(*ps), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, ps); + ps->dev = dev; + ps->genpd.name = "test"; + ps->no_auto_pm = no_auto_pm; + ps->regmap = devm_regmap_init(dev, NULL, r, &pmgr_ps_test_regmap); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ps->regmap); + *reg = r; + return ps; +} + +/* Powering on enables automatic PM, unless the domain is apple,no-auto-pm. */ +static void pmgr_ps_power_on_test(struct kunit *test) +{ + struct pmgr_ps_test_reg *r; + struct apple_pmgr_ps *ps; + bool no_auto_pm; + + for (no_auto_pm = false; ; no_auto_pm = true) { + ps = pmgr_ps_test_domain(test, APPLE_PMGR_PS_PWRGATE, no_auto_pm, &r); + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_on(&ps->genpd), 0); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_TARGET, r->val), + APPLE_PMGR_PS_ACTIVE); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_ACTUAL, r->val), + APPLE_PMGR_PS_ACTIVE); + KUNIT_EXPECT_EQ_MSG(test, !!(r->val & APPLE_PMGR_AUTO_ENABLE), !no_auto_pm, + "no_auto_pm %d", no_auto_pm); + KUNIT_EXPECT_EQ_MSG(test, r->auto_writes, no_auto_pm ? 0U : 1U, + "no_auto_pm %d", no_auto_pm); + if (no_auto_pm) + break; + } +} + +/* + * A power-off that does not complete is undone: the domain is requested on + * again, with automatic PM only when the domain allows it. + */ +static void pmgr_ps_power_off_undo_test(struct kunit *test) +{ + struct pmgr_ps_test_reg *r; + struct apple_pmgr_ps *ps; + bool no_auto_pm; + + for (no_auto_pm = false; ; no_auto_pm = true) { + ps = pmgr_ps_test_domain(test, APPLE_PMGR_PS_ACTIVE, no_auto_pm, &r); + r->stuck = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_off(&ps->genpd), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_TARGET, r->val), + APPLE_PMGR_PS_ACTIVE); + KUNIT_EXPECT_EQ_MSG(test, !!(r->val & APPLE_PMGR_AUTO_ENABLE), !no_auto_pm, + "no_auto_pm %d", no_auto_pm); + KUNIT_EXPECT_EQ_MSG(test, r->auto_writes, no_auto_pm ? 0U : 1U, + "no_auto_pm %d", no_auto_pm); + if (no_auto_pm) + break; + } +} + +/* A power-on that does not complete never enabled automatic PM on such a domain. */ +static void pmgr_ps_power_on_fails_test(struct kunit *test) +{ + struct pmgr_ps_test_reg *r; + struct apple_pmgr_ps *ps; + + ps = pmgr_ps_test_domain(test, APPLE_PMGR_PS_PWRGATE, true, &r); + r->stuck = true; + KUNIT_EXPECT_EQ(test, apple_pmgr_ps_power_on(&ps->genpd), -ETIMEDOUT); + /* Undone: requested off again */ + KUNIT_EXPECT_EQ(test, FIELD_GET(APPLE_PMGR_PS_TARGET, r->val), APPLE_PMGR_PS_PWRGATE); + KUNIT_EXPECT_EQ(test, r->auto_writes, 0U); +} + +static struct kunit_case pmgr_ps_test_cases[] = { + KUNIT_CASE(pmgr_ps_power_on_test), + KUNIT_CASE(pmgr_ps_power_off_undo_test), + KUNIT_CASE(pmgr_ps_power_on_fails_test), + {} +}; + +static struct kunit_suite pmgr_ps_test_suite = { + .name = "apple-pmgr-pwrstate", + .test_cases = pmgr_ps_test_cases, +}; + +kunit_test_suite(pmgr_ps_test_suite); diff --git a/drivers/pmdomain/apple/pmgr-pwrstate.c b/drivers/pmdomain/apple/pmgr-pwrstate.c index d0b75e0b4b671..c54bbb4dec8a3 100644 --- a/drivers/pmdomain/apple/pmgr-pwrstate.c +++ b/drivers/pmdomain/apple/pmgr-pwrstate.c @@ -16,6 +16,7 @@ #include #include #include +#include #define APPLE_PMGR_RESET BIT(31) #define APPLE_PMGR_AUTO_ENABLE BIT(28) @@ -50,11 +51,140 @@ struct apple_pmgr_ps { bool externally_clocked; bool system_sleep; bool firmware_owned; + /* Never enable automatic power management (apple,no-auto-pm). */ + bool no_auto_pm; }; #define genpd_to_apple_pmgr_ps(_genpd) container_of(_genpd, struct apple_pmgr_ps, genpd) #define rcdev_to_apple_pmgr_ps(_rcdev) container_of(_rcdev, struct apple_pmgr_ps, rcdev) +/* + * T6050 bring-up switches: t6050_noauto=1 skips the probe-time auto-PM enable, + * t6050_nomin=1 skips the probe-time minimum-state write. + */ +static bool t6050_noauto, t6050_nomin, t6050_nopower, t6050_noalways; + +/* + * t6050_nopower alone makes every power transition a no-op; t6050_nopower=a,b + * only those domains. t6050_pmlog prints every transition on the console. + */ +static char t6050_nopower_list[256]; +static bool t6050_pmlog, t6050_nooff, t6050_noon; + +static int __init t6050_nooff_setup(char *s) +{ + t6050_nooff = true; + return 1; +} +__setup("t6050_nooff", t6050_nooff_setup); + +static int __init t6050_noon_setup(char *s) +{ + t6050_noon = true; + return 1; +} +__setup("t6050_noon", t6050_noon_setup); + +static int __init t6050_nopower_setup(char *s) +{ + if (s && *s == '=') + s++; + if (s && *s) + strscpy(t6050_nopower_list, s, sizeof(t6050_nopower_list)); + else + t6050_nopower = true; + return 1; +} +__setup("t6050_nopower", t6050_nopower_setup); + +static int __init t6050_pmlog_setup(char *s) +{ + t6050_pmlog = true; + return 1; +} +__setup("t6050_pmlog", t6050_pmlog_setup); + +/* The transition log is compact: many entries per console line. */ +static char t6050_pmlog_buf[8192]; +static unsigned int t6050_pmlog_len, t6050_pmlog_shown; +static struct delayed_work t6050_pmlog_work; + +static void t6050_pmlog_add(const char *what, const char *name, bool skip) +{ + int n = snprintf(t6050_pmlog_buf + t6050_pmlog_len, + sizeof(t6050_pmlog_buf) - t6050_pmlog_len, "%s%s:%s ", + skip ? "-" : "", what, name); + if (n > 0 && t6050_pmlog_len + n < sizeof(t6050_pmlog_buf)) + t6050_pmlog_len += n; +} + +static void t6050_pmlog_flush(struct work_struct *work) +{ + while (t6050_pmlog_shown < t6050_pmlog_len) { + unsigned int n = min(140U, t6050_pmlog_len - t6050_pmlog_shown); + + pr_emerg("pm: %.*s\n", n, t6050_pmlog_buf + t6050_pmlog_shown); + t6050_pmlog_shown += n; + } + schedule_delayed_work(&t6050_pmlog_work, 10 * HZ); +} + +static int __init t6050_pmlog_start(void) +{ + if (!t6050_pmlog) + return 0; + INIT_DELAYED_WORK(&t6050_pmlog_work, t6050_pmlog_flush); + schedule_delayed_work(&t6050_pmlog_work, HZ); + return 0; +} +late_initcall_sync(t6050_pmlog_start); + +static bool t6050_skip(struct generic_pm_domain *genpd, const char *what) +{ + bool skip = t6050_nopower || (t6050_nooff && what[1] == 'f') || + (t6050_noon && what[1] == 'n'); + char buf[256], *p, *tok; + size_t n; + + /* The list takes exact names or prefixes ending in '*'. */ + if (!skip && t6050_nopower_list[0]) { + strscpy(buf, t6050_nopower_list, sizeof(buf)); + p = buf; + while ((tok = strsep(&p, ",")) != NULL) { + n = strlen(tok); + if (n && tok[n - 1] == '*' ? !strncmp(tok, genpd->name, n - 1) + : !strcmp(tok, genpd->name)) { + skip = true; + break; + } + } + } + if (t6050_pmlog) + t6050_pmlog_add(what, genpd->name, skip); + return skip; +} + +static int __init t6050_noalways_setup(char *s) +{ + t6050_noalways = true; + return 1; +} +__setup("t6050_noalways", t6050_noalways_setup); + +static int __init t6050_noauto_setup(char *s) +{ + t6050_noauto = true; + return 1; +} +__setup("t6050_noauto", t6050_noauto_setup); + +static int __init t6050_nomin_setup(char *s) +{ + t6050_nomin = true; + return 1; +} +__setup("t6050_nomin", t6050_nomin_setup); + static int apple_pmgr_ps_set(struct generic_pm_domain *genpd, u32 pstate, bool auto_enable) { int ret; @@ -164,10 +294,10 @@ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) int ret; /* Its coprocessor's firmware owns the state; Linux only holds a vote. */ - if (ps->firmware_owned) + if (ps->firmware_owned || t6050_skip(genpd, "on")) return 0; - ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, !ps->no_auto_pm); if (ret && !ps->system_sleep) apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); @@ -176,14 +306,15 @@ static int apple_pmgr_ps_power_on(struct generic_pm_domain *genpd) static int apple_pmgr_ps_power_off(struct generic_pm_domain *genpd) { + struct apple_pmgr_ps *ps = genpd_to_apple_pmgr_ps(genpd); int ret; - if (genpd_to_apple_pmgr_ps(genpd)->firmware_owned) + if (ps->firmware_owned || t6050_skip(genpd, "off")) return 0; ret = apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_PWRGATE, false); if (ret) - apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, true); + apple_pmgr_ps_set(genpd, APPLE_PMGR_PS_ACTIVE, !ps->no_auto_pm); return ret; } @@ -309,6 +440,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) int ret; const char *name; bool active; + u32 reg; regmap = syscon_node_to_regmap(node->parent); if (IS_ERR(regmap)) @@ -347,7 +479,8 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) ps->firmware_owned = of_property_read_bool(node, "apple,firmware-owned"); ret = of_property_read_u32(node, "apple,min-state", &ps->min_state); - if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE && !ps->firmware_owned) + if (ret == 0 && ps->min_state <= APPLE_PMGR_PS_ACTIVE && !ps->firmware_owned && + !t6050_nomin) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_PS_MIN, FIELD_PREP(APPLE_PMGR_PS_MIN, ps->min_state)); @@ -360,6 +493,12 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) if (of_property_read_bool(node, "apple,externally-clocked")) ps->externally_clocked = true; + /* + * A coprocessor CPU domain whose firmware Linux starts cold must not be + * gated automatically behind its back: keep it at the requested state. + */ + ps->no_auto_pm = of_property_read_bool(node, "apple,no-auto-pm"); + active = apple_pmgr_ps_is_active(ps); if (ps->firmware_owned) { /* Consumers (the boot framebuffer) need it; never try to turn it on. */ @@ -369,7 +508,7 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) active = true; } else if (of_property_read_bool(node, "apple,always-on")) { ps->genpd.flags |= GENPD_FLAG_ALWAYS_ON; - if (!active) { + if (!active && !t6050_noalways) { /* * The boot loader need not leave an always-on domain on, * so this is not an error: apple_pmgr_ps_set() reports a @@ -383,16 +522,26 @@ static int apple_pmgr_ps_probe(struct platform_device *pdev) * even if it does not get there in time. */ active = apple_pmgr_ps_set(&ps->genpd, APPLE_PMGR_PS_ACTIVE, - true) == 0; + !ps->no_auto_pm) == 0; } } else if (active) { ps->genpd.flags |= GENPD_FLAG_DEFER_OFF | GENPD_FLAG_ACTIVE_WAKEUP; } - /* Turn on auto-PM if the domain is already on */ - if (active && !ps->firmware_owned) + /* Turn on auto-PM if the domain is already on, unless it must stay off */ + if (active && !ps->firmware_owned && !t6050_noauto) regmap_update_bits(regmap, ps->offset, APPLE_PMGR_FLAGS | APPLE_PMGR_AUTO_ENABLE, - APPLE_PMGR_AUTO_ENABLE); + ps->no_auto_pm ? 0 : APPLE_PMGR_AUTO_ENABLE); + + /* + * Under auto-PM a domain counts as on when only its target is. Taken out + * of auto-PM, it must reach that target before a consumer touches it. + */ + if (active && ps->no_auto_pm && !ps->firmware_owned && !t6050_noauto && + regmap_read_poll_timeout(regmap, ps->offset, reg, + FIELD_GET(APPLE_PMGR_PS_ACTUAL, reg) == APPLE_PMGR_PS_ACTIVE, + 1, APPLE_PMGR_PS_SET_TIMEOUT)) + dev_warn(dev, "PS %s: not on after leaving auto-PM (now: 0x%x)\n", name, reg); ret = pm_genpd_init(&ps->genpd, NULL, !active); if (ret < 0) { @@ -475,3 +624,199 @@ MODULE_AUTHOR("Hector Martin "); MODULE_DESCRIPTION("PMGR power state driver for Apple SoCs"); module_platform_driver(apple_pmgr_ps_driver); + +#if IS_ENABLED(CONFIG_APPLE_PMGR_PWRSTATE_KUNIT_TEST) +#include "pmgr-pwrstate-test.c" +#endif + +/* + * T6050 development loop: t6050_wdt= on the command line arms the AP + * watchdog at the end of init so a boot that stays up resets by itself. The + * default, 0, leaves the watchdog alone. + */ +static unsigned int t6050_wdt_secs; + +static int __init t6050_wdt_setup(char *s) +{ + return kstrtouint(s, 0, &t6050_wdt_secs) == 0; +} +__setup("t6050_wdt=", t6050_wdt_setup); + +static int __init t6050_wdt_arm(void) +{ + void __iomem *w; + + if (!t6050_wdt_secs) + return 0; + /* The 24 MHz count is a 32-bit register: at most 178 s. */ + t6050_wdt_secs = min(t6050_wdt_secs, U32_MAX / 24000000U); + w = ioremap(0x28836c000, 0x20); + if (!w) + return 0; + writel_relaxed(t6050_wdt_secs * 24000000U, w + 0x14); + writel_relaxed(0, w + 0x10); + writel_relaxed(4, w + 0x1c); + iounmap(w); + pr_info("pmgr: AP watchdog armed, reset in %u s\n", t6050_wdt_secs); + return 0; +} +late_initcall(t6050_wdt_arm); + +/* + * t6050_wdtmon= prints the AP watchdog registers (WD0/WD1/WD2 at + * 0x28836c000) and the secondary watchdog word at 0x288330008 on the console + * at that interval, to see which one is counting on a boot that resets. + */ +#include +#include + +static unsigned int t6050_wdtmon, t6050_wdtmon_ping; + +static int __init t6050_wdtmon_setup(char *s) +{ + return kstrtouint(s, 0, &t6050_wdtmon) == 0; +} +__setup("t6050_wdtmon=", t6050_wdtmon_setup); + +static int __init t6050_wdtmon_ping_setup(char *s) +{ + return kstrtouint(s, 0, &t6050_wdtmon_ping) == 0; +} +__setup("t6050_wdtmon_ping=", t6050_wdtmon_ping_setup); + +static void __iomem *t6050_wdtmon_w, *t6050_wdtmon_s, *t6050_wdtmon_p, *t6050_wdtmon_q; + +/* t6050_peek=[,...] (up to 8): 32-bit words printed with the monitor. */ +static u64 t6050_peek_addr[8]; +static void __iomem *t6050_peek_map[8]; +static unsigned int t6050_peek_n; + +static int __init t6050_peek_setup(char *s) +{ + char *tok; + + while ((tok = strsep(&s, ",")) != NULL && t6050_peek_n < 8) + if (!kstrtoull(tok, 0, &t6050_peek_addr[t6050_peek_n])) + t6050_peek_n++; + return 1; +} +__setup("t6050_peek=", t6050_peek_setup); +static struct delayed_work t6050_wdtmon_work; + +static void t6050_wdtmon_fn(struct work_struct *work) +{ + void __iomem *w = t6050_wdtmon_w; + + pr_emerg("wdtmon %4lld s: wd0 %08x/%08x/%08x c%x wd1 %08x/%08x c%x wd2 %08x/%08x c%x sec %08x p20 %08x p2c %08x p5260 %08x\n", + (long long)ktime_get_boottime_seconds(), + readl_relaxed(w), readl_relaxed(w + 4), readl_relaxed(w + 8), readl_relaxed(w + 0xc), + readl_relaxed(w + 0x10), readl_relaxed(w + 0x14), readl_relaxed(w + 0x1c), + readl_relaxed(w + 0x20), readl_relaxed(w + 0x24), readl_relaxed(w + 0x2c), + readl_relaxed(t6050_wdtmon_s), readl_relaxed(t6050_wdtmon_p + 0x20), + readl_relaxed(t6050_wdtmon_p + 0x2c), readl_relaxed(t6050_wdtmon_q)); + for (unsigned int i = 0; i < t6050_peek_n; i++) + if (t6050_peek_map[i]) + pr_emerg("peek %#llx = %#010x\n", t6050_peek_addr[i], + readl_relaxed(t6050_peek_map[i])); + if (t6050_wdtmon_ping) + writel_relaxed(0, w + 0x10); + schedule_delayed_work(&t6050_wdtmon_work, t6050_wdtmon * HZ); +} + +static int __init t6050_wdtmon_start(void) +{ + if (!t6050_wdtmon) + return 0; + t6050_wdtmon_w = ioremap(0x28836c000, 0x40); + t6050_wdtmon_s = ioremap(0x288330008, 0x4); + t6050_wdtmon_p = ioremap(0x288330000, 0x40); + t6050_wdtmon_q = ioremap(0x288335260, 0x4); + if (!t6050_wdtmon_w || !t6050_wdtmon_s || !t6050_wdtmon_p || !t6050_wdtmon_q) + return 0; + for (unsigned int i = 0; i < t6050_peek_n; i++) + t6050_peek_map[i] = ioremap(t6050_peek_addr[i] & ~3ULL, 4); + INIT_DELAYED_WORK(&t6050_wdtmon_work, t6050_wdtmon_fn); + schedule_delayed_work(&t6050_wdtmon_work, 0); + return 0; +} +late_initcall(t6050_wdtmon_start); + +/* + * t6050_hcrst=: before the host driver probes, do an + * xHCI reset by hand from the kernel and print how long the reset bit takes + * to clear (the same experiment as the proxy script, from kernel context). + */ +static u64 t6050_hcrst_addr; + +static int __init t6050_hcrst_setup(char *s) +{ + return kstrtoull(s, 0, &t6050_hcrst_addr) == 0; +} +__setup("t6050_hcrst=", t6050_hcrst_setup); + +static void t6050_hcrst_try(const char *when) +{ + void __iomem *op; + u32 cmd, first; + int i; + + op = ioremap(t6050_hcrst_addr, 0x1000); + if (!op) + return; + pr_emerg("hcrst %s: before USBCMD %#x USBSTS %#x PORTSC0 %#x PORTSC1 %#x\n", when, + readl(op), readl(op + 4), readl(op + 0x400), readl(op + 0x410)); + writel(readl(op) & ~1, op); + for (i = 0; i < 2000 && !(readl(op + 4) & 1); i++) + udelay(10); + pr_emerg("hcrst %s: halted after %d x 10us, USBSTS %#x\n", when, i, readl(op + 4)); + cmd = readl(op) | 2; + writel(cmd, op); + first = readl(op); + for (i = 0; i < 20000 && (readl(op) & 2); i++) + udelay(1000); + pr_emerg("hcrst %s: read-back %#x, HCRST clear after %d ms (limit 20000)\n", when, first, i); + for (i = 0; i < 20000 && (readl(op + 4) & BIT(11)); i++) + udelay(1000); + pr_emerg("hcrst %s: CNR clear after %d ms, USBCMD %#x USBSTS %#x PORTSC0 %#x\n", when, i, + readl(op), readl(op + 4), readl(op + 0x400)); + iounmap(op); +} + +static int __init t6050_hcrst_early(void) +{ + if (t6050_hcrst_addr) + t6050_hcrst_try("early"); + /* t6050_peek= words once, before the drivers probe. */ + for (unsigned int i = 0; i < t6050_peek_n; i++) { + void __iomem *m = ioremap(t6050_peek_addr[i] & ~3ULL, 4); + + if (m) { + pr_emerg("peek-entry %#llx = %#010x\n", t6050_peek_addr[i], readl(m)); + iounmap(m); + } + } + /* + * The loader's host-prep record: per port attempts, PORTSC after prep, + * PORTSC before jump, ms until the connection showed. + */ + if (t6050_peek_n && of_chosen) { + u32 rec[32]; + int n = of_property_read_variable_u32_array(of_chosen, "apple,t6050-usbprep", + rec, 1, ARRAY_SIZE(rec)); + + for (int i = 0; i + 3 < n; i += 4) + if (rec[i]) + pr_emerg("usbprep port %d: %u attempts, ccs after %u ms, portsc after prep %#x, before jump %#x\n", + i / 4, rec[i], rec[i + 3], rec[i + 1], rec[i + 2]); + } + return 0; +} +subsys_initcall(t6050_hcrst_early); + +static int __init t6050_hcrst_late(void) +{ + if (t6050_hcrst_addr) + t6050_hcrst_try("late"); + return 0; +} +late_initcall_sync(t6050_hcrst_late); diff --git a/drivers/pmdomain/apple/pmp-report-start-test.c b/drivers/pmdomain/apple/pmp-report-start-test.c index aeb9e28f53b9e..36cf2c87eb128 100644 --- a/drivers/pmdomain/apple/pmp-report-start-test.c +++ b/drivers/pmdomain/apple/pmp-report-start-test.c @@ -343,7 +343,50 @@ static void pmp_start_entry_teardown_test(struct kunit *test) KUNIT_EXPECT_EQ(test, apple_pmp_report_entry_power_on(&ent->genpd), -ENODEV); } +/* T6030's real overlay: DISP (7), deferred external requests (8/9), + * and storage (16), which has no acknowledgement. Only DISP gates startup. + */ +static struct pst *pst_t6030(struct kunit *test) +{ + struct pst *t = pst_new(test); + + t->st.seed = BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9) | BIT_ULL(16); + t->st.ack = BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9); + t->st.startup_ack = BIT_ULL(7); + t->pmp.ack = BIT_ULL(7); + pst_write(t, pst_offsets.tgt_read, t->st.seed); + return t; +} + +static void pmp_start_t6030_deferred_external_test(struct kunit *test) +{ + struct pst *t = pst_t6030(test); + + KUNIT_ASSERT_EQ(test, pst_wait(t, 7, 5000), 0); + KUNIT_EXPECT_EQ(test, t->pmp.enables, 1U); + KUNIT_EXPECT_EQ(test, pst_wait(t, 8, 1), -EIO); + KUNIT_EXPECT_EQ(test, pst_wait(t, 9, 1), -EIO); + KUNIT_EXPECT_EQ(test, pst_wait(t, 16, 1), -EINVAL); + pst_write(t, pst_offsets.actual, BIT_ULL(7) | BIT_ULL(8)); + KUNIT_EXPECT_EQ(test, pst_wait(t, 8, 1), 0); + KUNIT_EXPECT_EQ(test, t->pmp.enables, 1U); + pst_drain(t); +} + +/* An external consumer must not bypass the internal startup admission. */ +static void pmp_start_t6030_external_first_test(struct kunit *test) +{ + struct pst *t = pst_t6030(test); + + t->pmp.ack = BIT_ULL(8); + KUNIT_EXPECT_EQ(test, pst_wait(t, 8, 5000), -EIO); + KUNIT_EXPECT_EQ(test, pst_wait(t, 7, 1), -EIO); + pst_drain(t); +} + static struct kunit_case pmp_start_cases[] = { + KUNIT_CASE(pmp_start_t6030_deferred_external_test), + KUNIT_CASE(pmp_start_t6030_external_first_test), KUNIT_CASE(pmp_start_ok_test), KUNIT_CASE(pmp_start_check_fails_test), KUNIT_CASE(pmp_start_busy_ptd_test), diff --git a/drivers/pmdomain/apple/pmp-report.c b/drivers/pmdomain/apple/pmp-report.c index beedc4d96eb7f..b5ae3bbd866e9 100644 --- a/drivers/pmdomain/apple/pmp-report.c +++ b/drivers/pmdomain/apple/pmp-report.c @@ -51,6 +51,9 @@ * @pwrstate: with @starts_pmp, the compatible of the SoC's PMGR power states * @devices_ordered: with @starts_pmp, soc-device record i holds device ID * i + 1 + * @start_at_probe: with @starts_pmp, start the PMP when the report probes, as + * macOS starts it at boot, rather than on the first consumer's + * apple_pmp_report_wait_ready() */ struct apple_pmp_report_offsets { u32 tgt_read; @@ -64,6 +67,7 @@ struct apple_pmp_report_offsets { const char *fallback; const char *pwrstate; bool devices_ordered; + bool start_at_probe; }; struct apple_pmp_start; @@ -79,6 +83,8 @@ struct apple_pmp_report { struct apple_pmp_start *start; }; +static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030; + /* Bound report regions, protected by RCU until managed teardown drains readers. */ static LIST_HEAD(apple_pmp_reports); static DEFINE_MUTEX(apple_pmp_reports_lock); @@ -175,6 +181,61 @@ static void apple_pmp_report_release(void *data) debugfs_remove(rep->debugfs); } +/* The unique, enabled and acknowledged T6030 display request (DISP, reg 7). */ +static struct device_node *apple_pmp_report_t6030_entry(struct device_node *report) +{ + struct device_node *child, *entry = NULL; + u32 reg; + + for_each_child_of_node(report, child) { + if (of_property_read_u32(child, "reg", ®) || reg != 7) + continue; + if (entry || of_property_count_u32_elems(child, "reg") != 1 || + !of_device_is_available(child) || + !of_device_is_compatible(child, "apple,t6000-pmp-v2-report-entry") || + of_property_read_bool(child, "apple,no-ack")) { + of_node_put(child); + of_node_put(entry); + return NULL; + } + entry = of_node_get(child); + } + return entry; +} + +/* The existing display domain seeds DISP; require its READY and ACK before AGX. */ +static int apple_pmp_report_wait_t6030_14(struct device *supplier, unsigned int timeout_ms) +{ + struct device_node *np, *report = NULL, *pmp, *entry; + int ret; + + if (!IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) || + !of_machine_is_compatible("apple,t6030") || + !of_device_is_compatible(supplier->of_node, "apple,t6030-pmp-v2")) + return -ENODEV; + for_each_compatible_node(np, NULL, "apple,t6030-pmp-v2-report") { + if (report) { + of_node_put(np); + of_node_put(report); + return -ENODEV; + } + report = of_node_get(np); + } + if (!report || !of_device_is_available(report) || + of_property_count_u32_elems(report, "apple,pmp") != 1) { + of_node_put(report); + return -ENODEV; + } + pmp = of_parse_phandle(report, "apple,pmp", 0); + entry = apple_pmp_report_t6030_entry(report); + ret = pmp == supplier->of_node && entry ? + apple_pmp_report_wait_ready(entry, msecs_to_jiffies(timeout_ms)) : -ENODEV; + of_node_put(entry); + of_node_put(pmp); + of_node_put(report); + return ret; +} + /* The native 25G83 PMP starts through its supplier topology, after requests * are seeded. Its report is pinned against removal while readiness is read. */ @@ -186,6 +247,9 @@ int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int t u64 status; int ret; + /* A declared report always takes the existing native25 path below. */ + if (!np && !of_property_present(supplier->of_node, "apple,pmp-report")) + return apple_pmp_report_wait_t6030_14(supplier, timeout_ms); if (!np || !of_device_is_compatible(np, "apple,j613-25g83-pmp-report")) { of_node_put(np); return -ENODEV; @@ -212,7 +276,7 @@ int apple_pmp_report_wait_supplier_ready(struct device *supplier, unsigned int t EXPORT_SYMBOL_GPL(apple_pmp_report_wait_supplier_ready); /* - * Starting a halted PMP (T8122, T6031) + * Starting a halted PMP (T8122, T6031, opted-in T6030) * * On these SoCs iBoot leaves the PMP loaded but halted, and the DCP firmware * of the 14.x system needs it running. A display gate that admits the boot @@ -401,6 +465,7 @@ static int apple_pmp_start_check_power(struct apple_pmp_report *rep) const char *label = apple_pmp_start_domains[i].label; struct device_node *np = apple_pmp_start_domain(rep, label); struct regmap *regmap; + bool fw_owned; u32 offset, val; int ret; @@ -411,6 +476,7 @@ static int apple_pmp_start_check_power(struct apple_pmp_report *rep) } ret = of_property_read_u32(np, "reg", &offset); regmap = syscon_node_to_regmap(np->parent); + fw_owned = of_property_read_bool(np, "apple,firmware-owned"); of_node_put(np); if (ret) return ret; @@ -425,8 +491,12 @@ static int apple_pmp_start_check_power(struct apple_pmp_report *rep) rep->offsets->name, label, val); return -EIO; } - /* The PMGR power state driver applies the floor when it probes. */ - if (apple_pmp_start_domains[i].floor && + /* + * The PMGR power state driver applies the floor when it probes. A + * firmware-owned state takes no AP writes (T6050's DCP CPU SErrors), + * so its floor is the firmware's. + */ + if (apple_pmp_start_domains[i].floor && !fw_owned && FIELD_GET(APPLE_PMGR_PS_MIN, val) != APPLE_PMGR_PS_ACTIVE) { dev_err(rep->dev, "%s PMP not started: the %s minimum state is not active (%#x)\n", rep->offsets->name, label, val); @@ -997,6 +1067,50 @@ int apple_pmp_report_wait_ready(struct device_node *entry, unsigned long timeout } EXPORT_SYMBOL_GPL(apple_pmp_report_wait_ready); +/* + * With @start_at_probe, the start waits on the first always-on entry, off the + * probe path, since the wait blocks until that entry's request is acknowledged. + */ +struct apple_pmp_report_boot { + struct work_struct work; + struct device *dev; + struct device_node *entry; +}; + +static void apple_pmp_report_boot_work(struct work_struct *work) +{ + struct apple_pmp_report_boot *b = + container_of(work, struct apple_pmp_report_boot, work); + int ret = apple_pmp_report_wait_ready(b->entry, msecs_to_jiffies(30000)); + + if (ret) + dev_err(b->dev, "PMP start at probe failed: %d\n", ret); + of_node_put(b->entry); + kfree(b); +} + +static void apple_pmp_report_boot(struct apple_pmp_report *rep, struct device_node *np) +{ + struct apple_pmp_report_boot *b; + struct device_node *child; + + if (!rep->start || !rep->offsets->start_at_probe) + return; + for_each_available_child_of_node(np, child) { + if (!of_property_read_bool(child, "apple,always-on")) + continue; + b = kzalloc(sizeof(*b), GFP_KERNEL); + if (b) { + b->dev = rep->dev; + b->entry = of_node_get(child); + INIT_WORK(&b->work, apple_pmp_report_boot_work); + queue_work(system_dfl_long_wq, &b->work); + } + of_node_put(child); + return; + } +} + static int apple_pmp_report_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1042,6 +1156,7 @@ static int apple_pmp_report_probe(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "failed to create child devices\n"); + apple_pmp_report_boot(rep, np); return 0; } @@ -1064,6 +1179,12 @@ static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030 = { .tgt_write = 0x108c0, .actual = 0x11c0, .status = 0x10, + /* Native G15 explicitly opts into the gate's halted-PMP handoff. */ + .starts_pmp = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK), + .name = "T6030", + .fallback = "apple_t6030_display.enable=0", + .pwrstate = "apple,t6030-pmgr-pwrstate", + .devices_ordered = true, }; static const struct apple_pmp_report_offsets apple_pmp_offsets_t8112 = { @@ -1125,12 +1246,31 @@ static const struct apple_pmp_report_offsets apple_pmp_offsets_t8140 = { .fast_die_lanes = 2, }; +/* + * T6050, from the J714c ADT ptd-range: SOC-DEV-PS-REQ at PTD index 0x1e0 + * (read 0x1e00, update 0x10000 + 0xf00), SOC-DEV-PS-ACK at 0x1e8 (0x1e80), + * PMP-STATUS at 0x1 (0x10). soc-device records are in ID order, as on T6030. + */ +static const struct apple_pmp_report_offsets apple_pmp_offsets_t6050 = { + .tgt_read = 0x1e00, + .tgt_write = 0x10f00, + .actual = 0x1e80, + .status = 0x10, + .starts_pmp = true, + .name = "T6050", + .fallback = "initcall_blacklist=__asahi_init", + .pwrstate = "apple,t6050-pmgr-pwrstate", + .devices_ordered = true, + .start_at_probe = true, +}; + static const struct of_device_id apple_pmp_report_of_match[] = { { .compatible = "apple,j613-25g83-pmp-report", .data = &apple_pmp_offsets_j613_25g83 }, { .compatible = "apple,t6000-pmp-v2-report", .data = &apple_pmp_offsets_t600x }, { .compatible = "apple,t6020-pmp-v2-report", .data = &apple_pmp_offsets_t602x }, { .compatible = "apple,t6030-pmp-v2-report", .data = &apple_pmp_offsets_t6030 }, { .compatible = "apple,t6031-pmp-v2-report", .data = &apple_pmp_offsets_t6031 }, + { .compatible = "apple,t6050-pmp-v2-report", .data = &apple_pmp_offsets_t6050 }, { .compatible = "apple,t8112-pmp-v2-report", .data = &apple_pmp_offsets_t8112 }, { .compatible = "apple,t8122-pmp-v2-report", .data = &apple_pmp_offsets_t8122 }, { .compatible = "apple,t8140-pmp-v2-report", .data = &apple_pmp_offsets_t8140 }, @@ -1294,6 +1434,7 @@ static int apple_pmp_report_entry_probe(struct platform_device *pdev) apple_pmp_report_entry_set_state(&ent->genpd, true); } + ent->genpd.flags |= GENPD_FLAG_IRQ_SAFE; ent->genpd.power_on = apple_pmp_report_entry_power_on; ent->genpd.power_off = apple_pmp_report_entry_power_off; diff --git a/drivers/pmdomain/imx/imx8m-blk-ctrl.c b/drivers/pmdomain/imx/imx8m-blk-ctrl.c index 99d100e1d923f..62c4f829605c6 100644 --- a/drivers/pmdomain/imx/imx8m-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx8m-blk-ctrl.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,12 @@ struct imx8m_blk_ctrl { struct regmap *regmap; struct imx8m_blk_ctrl_domain *domains; struct genpd_onecell_data onecell_data; + /* + * Serializes the blk-ctrl reset/clock sequence across sibling domains; + * their transitions interact through the shared VPUMIX bus domain, + * VPU_NOC and the not-ack-verified ADB400 handshake (ERR050531). + */ + struct mutex power_lock; }; struct imx8m_blk_ctrl_domain_data { @@ -98,6 +105,8 @@ static int imx8m_blk_ctrl_power_on(struct generic_pm_domain *genpd) struct imx8m_blk_ctrl *bc = domain->bc; int ret; + guard(mutex)(&bc->power_lock); + /* make sure bus domain is awake */ ret = pm_runtime_get_sync(bc->bus_power_dev); if (ret < 0) { @@ -164,6 +173,8 @@ static int imx8m_blk_ctrl_power_off(struct generic_pm_domain *genpd) const struct imx8m_blk_ctrl_domain_data *data = domain->data; struct imx8m_blk_ctrl *bc = domain->bc; + guard(mutex)(&bc->power_lock); + /* put devices into reset and disable clocks */ if (data->mipi_phy_rst_mask) regmap_clear_bits(bc->regmap, BLK_MIPI_RESET_DIV, data->mipi_phy_rst_mask); @@ -202,6 +213,10 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev) bc->dev = dev; + ret = devm_mutex_init(dev, &bc->power_lock); + if (ret) + return ret; + bc_data = of_device_get_match_data(dev); base = devm_platform_ioremap_resource(pdev, 0); diff --git a/drivers/pmdomain/rockchip/pm-domains.c b/drivers/pmdomain/rockchip/pm-domains.c index 490bbb1d1d8e8..b57581b367a90 100644 --- a/drivers/pmdomain/rockchip/pm-domains.c +++ b/drivers/pmdomain/rockchip/pm-domains.c @@ -745,7 +745,19 @@ static int rockchip_pd_attach_dev(struct generic_pm_domain *genpd, } i = 0; - while ((clk = of_clk_get(dev->of_node, i++)) && !IS_ERR(clk)) { + while (1) { + clk = of_clk_get(dev->of_node, i++); + if (IS_ERR(clk)) { + error = PTR_ERR(clk); + if (error == -ENOENT) + break; + + dev_err(dev, "failed to get clock %d: %d\n", i - 1, + error); + pm_clk_destroy(dev); + return error; + } + dev_dbg(dev, "adding clock '%pC' to list of PM clocks\n", clk); error = pm_clk_add_clk(dev, clk); if (error) { @@ -828,7 +840,7 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu, dev_err(pmu->dev, "%pOFn: failed to get clk at index %d: %d\n", node, i, error); - return error; + goto err_put_clocks; } } @@ -1004,7 +1016,9 @@ static int rockchip_pm_add_subdomain(struct rockchip_pmu *pmu, parent_domain->name, child_domain->name); } - rockchip_pm_add_subdomain(pmu, np); + error = rockchip_pm_add_subdomain(pmu, np); + if (error) + return error; } return 0; diff --git a/drivers/power/sequencing/pwrseq-pcie-m2.c b/drivers/power/sequencing/pwrseq-pcie-m2.c index ddfe6ca824945..c59c487c43e30 100644 --- a/drivers/power/sequencing/pwrseq-pcie-m2.c +++ b/drivers/power/sequencing/pwrseq-pcie-m2.c @@ -562,6 +562,7 @@ static int pwrseq_pcie_m2_probe(struct platform_device *pdev) mutex_destroy(&ctx->list_lock); err_free_regulators: regulator_bulk_free(ctx->num_vregs, ctx->regs); + kfree(ctx->regs); return ret; } @@ -575,6 +576,7 @@ static void pwrseq_pcie_m2_remove(struct platform_device *pdev) mutex_destroy(&ctx->list_lock); regulator_bulk_free(ctx->num_vregs, ctx->regs); + kfree(ctx->regs); } static const struct of_device_id pwrseq_pcie_m2_of_match[] = { diff --git a/drivers/power/supply/Kconfig b/drivers/power/supply/Kconfig index c535296dd5e80..ae3121a55ec55 100644 --- a/drivers/power/supply/Kconfig +++ b/drivers/power/supply/Kconfig @@ -1189,11 +1189,14 @@ config MACSMC_POWER config MACSMC_POWER_KUNIT_TEST bool "KUnit tests for Apple SMC battery replies" if !KUNIT_ALL_TESTS depends on MACSMC_POWER && KUNIT=y + depends on MFD_MACSMC_KUNIT_TEST default KUNIT_ALL_TESTS help - Build tests for malformed fixed-width battery replies. The tests - exercise zero-length and short CHTE replies and check that the - charge-behaviour reader rejects them. Static stubs replace SMC - reads, so the tests need no Apple hardware. + Build tests for malformed fixed-width battery replies and for the + charge settings. The tests exercise zero-length and short CHTE + replies, the charge thresholds and behaviours against a model of + their SMC keys, failures of each key access, and two concurrent + writers. Static stubs replace SMC reads and writes, so the tests + need no Apple hardware. endif # POWER_SUPPLY diff --git a/drivers/power/supply/macsmc-power-test.c b/drivers/power/supply/macsmc-power-test.c index 78f7f1b877beb..3ddb58970771c 100644 --- a/drivers/power/supply/macsmc-power-test.c +++ b/drivers/power/supply/macsmc-power-test.c @@ -1,6 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-only OR MIT -/* Included by macsmc-power.c; the charge-behaviour reader is stubbed. */ +/* + * Included by macsmc-power.c. The first suite stubs the CHTE reader; the + * charge suite replaces apple_smc_read() and apple_smc_write() with a model + * of the charge keys that can fail any one access. + */ #include +#include +#include +#include struct macsmc_power_test_context { struct macsmc_power power; @@ -24,6 +31,7 @@ static int macsmc_power_test_init(struct kunit *test) ctx = kunit_kzalloc(test, sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; + mutex_init(&ctx->power.charge_lock); ctx->power.has_chte = true; test->priv = ctx; kunit_activate_static_stub(test, macsmc_power_read_chte, @@ -55,4 +63,625 @@ static struct kunit_suite macsmc_power_test_suite = { .test_cases = macsmc_power_test_cases, }; -kunit_test_suite(macsmc_power_test_suite); +/* Charge settings against a model of the SMC's charge keys. */ + +#define CHARGE_TEST_MAX_OPS 32 + +struct macsmc_charge_op { + smc_key key; + bool write; + u32 value; +}; + +struct charge_test_writer; + +struct macsmc_charge_test { + struct kunit *test; + struct apple_smc smc; + struct macsmc_power power; + struct power_supply psy; + + /* The keys' values */ + u16 chls; + u8 ch0i; + u8 ch0c; + u32 chte; + + /* Fail the access to fail_key (a write if fail_write) with fail_err */ + smc_key fail_key; + bool fail_write; + int fail_err; + + /* Every access, in order */ + struct macsmc_charge_op ops[CHARGE_TEST_MAX_OPS]; + unsigned int nops; + + /* Charge-key accesses must hold charge_lock (set by the tests) */ + bool expect_locked; + + /* Runs once from within the next CHLS read, after the value is taken */ + void (*on_chls_read)(struct macsmc_charge_test *t); + + /* The concurrent writer of the serialization test */ + struct charge_test_writer *writer; + bool writer_early; + /* The test's own task; the writer records its findings for it to report */ + struct task_struct *task; + atomic_t writer_unlocked; +}; + +static bool charge_test_key_is_charge_setting(smc_key key) +{ + return key == SMC_KEY(CHLS) || key == SMC_KEY(CH0I) || + key == SMC_KEY(CH0C) || key == SMC_KEY(CHTE); +} + +static void charge_test_log(struct macsmc_charge_test *t, smc_key key, bool write, + u32 value) +{ + bool unlocked = t->expect_locked && charge_test_key_is_charge_setting(key) && + !mutex_is_locked(&t->power.charge_lock); + + /* KUnit reports only from the test's task; the writer leaves it a count. */ + if (current != t->task) { + if (unlocked) + atomic_inc(&t->writer_unlocked); + return; + } + KUNIT_EXPECT_FALSE_MSG(t->test, unlocked, "charge key %08x accessed without charge_lock", + key); + if (t->nops >= CHARGE_TEST_MAX_OPS) { + KUNIT_FAIL(t->test, "more than %d charge key accesses", CHARGE_TEST_MAX_OPS); + return; + } + t->ops[t->nops++] = (struct macsmc_charge_op){ key, write, value }; +} + +static int charge_test_read(struct apple_smc *smc, smc_key key, void *buf, size_t size) +{ + struct macsmc_charge_test *t = container_of(smc, struct macsmc_charge_test, smc); + void (*hook)(struct macsmc_charge_test *t) = NULL; + u32 value; + + if (key == SMC_KEY(CHLS) && size == sizeof(u16)) { + value = t->chls; + hook = t->on_chls_read; + t->on_chls_read = NULL; + } else if (key == SMC_KEY(CH0I) && size == sizeof(u8)) { + value = t->ch0i; + } else if (key == SMC_KEY(CH0C) && size == sizeof(u8)) { + value = t->ch0c; + } else if (key == SMC_KEY(CHTE) && size == sizeof(u32)) { + value = t->chte; + } else { + KUNIT_FAIL(t->test, "unexpected read of %08x, %zu bytes", key, size); + return -EINVAL; + } + + charge_test_log(t, key, false, value); + if (t->fail_key == key && !t->fail_write) + return t->fail_err; + /* Little-endian, like the SMC's own replies */ + memcpy(buf, &value, size); + /* Another writer gets its turn after this read has taken the value */ + if (hook) + hook(t); + return size; +} + +static int charge_test_write(struct apple_smc *smc, smc_key key, const void *buf, + size_t size) +{ + struct macsmc_charge_test *t = container_of(smc, struct macsmc_charge_test, smc); + u32 value = 0; + + memcpy(&value, buf, min(size, sizeof(value))); + charge_test_log(t, key, true, value); + if (t->fail_key == key && t->fail_write) + return t->fail_err; + + if (key == SMC_KEY(CHLS) && size == sizeof(u16)) + t->chls = value; + else if (key == SMC_KEY(CH0I) && size == sizeof(u8)) + t->ch0i = value; + else if (key == SMC_KEY(CH0C) && size == sizeof(u8)) + t->ch0c = value; + else if (key == SMC_KEY(CHTE) && size == sizeof(u32)) + t->chte = value; + else + KUNIT_FAIL(t->test, "unexpected write of %08x, %zu bytes", key, size); + /* An SMC write reply may echo the request length; it is not an error. */ + return size; +} + +static int macsmc_charge_test_init(struct kunit *test) +{ + struct macsmc_charge_test *t; + + t = kunit_kzalloc(test, sizeof(*t), GFP_KERNEL); + if (!t) + return -ENOMEM; + t->test = test; + t->task = current; + atomic_set(&t->writer_unlocked, 0); + t->power.smc = &t->smc; + mutex_init(&t->power.charge_lock); + t->psy.drv_data = &t->power; + t->expect_locked = true; + test->priv = t; + kunit_activate_static_stub(test, apple_smc_read, charge_test_read); + kunit_activate_static_stub(test, apple_smc_write, charge_test_write); + return 0; +} + +static int charge_test_set(struct macsmc_charge_test *t, enum power_supply_property psp, + int value) +{ + union power_supply_propval val = { .intval = value }; + + return macsmc_battery_set_property(&t->psy, psp, &val); +} + +static int charge_test_get(struct macsmc_charge_test *t, enum power_supply_property psp, + int *value) +{ + union power_supply_propval val = {}; + int ret = macsmc_battery_get_property(&t->psy, psp, &val); + + *value = val.intval; + return ret; +} + +static unsigned int charge_test_writes(struct macsmc_charge_test *t) +{ + unsigned int i, n = 0; + + for (i = 0; i < t->nops; i++) + n += t->ops[i].write; + return n; +} + +static void charge_test_expect_op(struct kunit *test, struct macsmc_charge_test *t, + unsigned int i, smc_key key, bool write, u32 value) +{ + if (i >= t->nops) { + KUNIT_FAIL(test, "access %u missing, only %u", i, t->nops); + return; + } + KUNIT_EXPECT_EQ_MSG(test, t->ops[i].key, key, "access %u", i); + KUNIT_EXPECT_EQ_MSG(test, t->ops[i].write, write, "access %u", i); + if (write) + KUNIT_EXPECT_EQ_MSG(test, t->ops[i].value, value, "access %u", i); +} + +/* The behaviours a battery offers follow from its keys alone. */ +static void macsmc_charge_behaviours_test(struct kunit *test) +{ + struct macsmc_charge_test *t = test->priv; + struct macsmc_power *p = &t->power; + unsigned int combo; + + for (combo = 0; combo < 32; combo++) { + unsigned int got; + + p->has_ch0i = combo & BIT(0); + p->has_ch0c = combo & BIT(1); + p->has_chte = combo & BIT(2); + p->has_chls = combo & BIT(3); + p->has_chwa = combo & BIT(4); + got = macsmc_battery_charge_behaviours(p); + + KUNIT_EXPECT_EQ_MSG(test, !!(got & BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO)), + p->has_ch0i || p->has_ch0c || p->has_chte || p->has_chls, + "combo %u", combo); + KUNIT_EXPECT_EQ_MSG(test, + !!(got & BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE)), + p->has_ch0i, "combo %u", combo); + KUNIT_EXPECT_EQ_MSG(test, + !!(got & BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE)), + p->has_chte || p->has_ch0c, "combo %u", combo); + KUNIT_EXPECT_EQ_MSG(test, + !!(got & BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE)), + p->has_chls, "combo %u", combo); + KUNIT_EXPECT_EQ_MSG(test, + got & BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE_AWAKE), + 0U, "combo %u", combo); + } + + /* CHWA alone offers no behaviour, as before; CHLS alone offers both autos */ + p->has_ch0i = false; + p->has_ch0c = false; + p->has_chte = false; + p->has_chls = false; + p->has_chwa = true; + KUNIT_EXPECT_EQ(test, macsmc_battery_charge_behaviours(p), 0U); + p->has_chwa = false; + p->has_chls = true; + KUNIT_EXPECT_EQ(test, macsmc_battery_charge_behaviours(p), + (unsigned int)(BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO) | + BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE))); + KUNIT_EXPECT_EQ(test, t->nops, 0U); +} + +/* The end threshold keeps the force-discharge bit only while there is a limit. */ +static void macsmc_charge_end_threshold_test(struct kunit *test) +{ + static const struct { + u16 before; + int threshold; + u16 after; + } cases[] = { + { 0x0000, 80, 0x0050 }, + { 0x0150, 70, 0x0146 }, /* auto-discharge stays */ + { 0x0150, 99, 0x0163 }, + { 0x0000, 5, 0x000a }, /* below the minimum: the minimum */ + { 0x0150, 0, 0x010a }, + { 0x0000, -1, 0x000a }, + { 0x0350, 60, 0x013c }, /* other high bits are not kept */ + { 0x0150, 100, 0x0000 }, /* no limit ends auto-discharge */ + { 0x0050, 100, 0x0000 }, + { 0x0150, 150, 0x0000 }, + }; + struct macsmc_charge_test *t = test->priv; + unsigned int i; + + t->power.has_chls = true; + for (i = 0; i < ARRAY_SIZE(cases); i++) { + int ret; + + t->chls = cases[i].before; + t->nops = 0; + ret = charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD, + cases[i].threshold); + KUNIT_EXPECT_EQ_MSG(test, ret, 0, "case %u", i); + KUNIT_EXPECT_EQ_MSG(test, t->chls, cases[i].after, "case %u", i); + KUNIT_EXPECT_EQ_MSG(test, t->nops, 2U, "case %u", i); + charge_test_expect_op(test, t, 0, SMC_KEY(CHLS), false, 0); + charge_test_expect_op(test, t, 1, SMC_KEY(CHLS), true, cases[i].after); + } +} + +/* A failed CHLS read changes nothing; a failed write is reported. */ +static void macsmc_charge_end_threshold_fault_test(struct kunit *test) +{ + struct macsmc_charge_test *t = test->priv; + + t->power.has_chls = true; + t->chls = 0x0150; + t->fail_key = SMC_KEY(CHLS); + t->fail_err = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD, + 60), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, charge_test_writes(t), 0U); + KUNIT_EXPECT_EQ(test, t->chls, 0x0150); + + t->nops = 0; + t->fail_write = true; + t->fail_err = -EIO; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD, + 60), -EIO); + KUNIT_EXPECT_EQ(test, t->chls, 0x0150); +} + +/* auto-discharge clears the inhibitors, then sets the bit next to the limit. */ +static void macsmc_charge_auto_discharge_test(struct kunit *test) +{ + struct macsmc_charge_test *t = test->priv; + int value; + + t->power.has_ch0i = true; + t->power.has_chte = true; + t->power.has_chls = true; + t->chls = 0x0050; + t->ch0i = 1; + t->chte = 1; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE), 0); + KUNIT_EXPECT_EQ(test, t->nops, 4U); + charge_test_expect_op(test, t, 0, SMC_KEY(CHLS), false, 0); + charge_test_expect_op(test, t, 1, SMC_KEY(CH0I), true, 0); + charge_test_expect_op(test, t, 2, SMC_KEY(CHTE), true, 0); + charge_test_expect_op(test, t, 3, SMC_KEY(CHLS), true, 0x0150); + KUNIT_EXPECT_EQ(test, charge_test_get(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, &value), 0); + KUNIT_EXPECT_EQ(test, value, POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE); + + /* The threshold reads past the bit */ + t->expect_locked = false; + KUNIT_EXPECT_EQ(test, charge_test_get(t, POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD, + &value), 0); + KUNIT_EXPECT_EQ(test, value, 80); + KUNIT_EXPECT_EQ(test, charge_test_get(t, POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD, + &value), 0); + KUNIT_EXPECT_EQ(test, value, 75); + t->expect_locked = true; + + /* auto clears it again and keeps the limit */ + t->nops = 0; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO), 0); + KUNIT_EXPECT_EQ(test, t->chls, 0x0050); + KUNIT_EXPECT_EQ(test, charge_test_get(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, &value), 0); + KUNIT_EXPECT_EQ(test, value, POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO); +} + +/* Without a limit there is nothing to discharge to: refused, nothing written. */ +static void macsmc_charge_auto_discharge_no_limit_test(struct kunit *test) +{ + static const u16 no_limit[] = { 0x0000, 0x0009, 0x0064, 0x00ff, 0x0100 }; + struct macsmc_charge_test *t = test->priv; + unsigned int i; + + t->power.has_ch0i = true; + t->power.has_chte = true; + t->power.has_chls = true; + for (i = 0; i < ARRAY_SIZE(no_limit); i++) { + t->chls = no_limit[i]; + t->ch0i = 1; + t->nops = 0; + KUNIT_EXPECT_EQ_MSG(test, + charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE), + -EINVAL, "chls %04x", no_limit[i]); + KUNIT_EXPECT_EQ_MSG(test, charge_test_writes(t), 0U, "chls %04x", no_limit[i]); + KUNIT_EXPECT_EQ(test, t->ch0i, 1); + KUNIT_EXPECT_EQ(test, t->chls, no_limit[i]); + } + + /* The lowest and highest limits are limits */ + t->chls = 0x000a; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE), 0); + KUNIT_EXPECT_EQ(test, t->chls, 0x010a); + t->chls = 0x0063; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE), 0); + KUNIT_EXPECT_EQ(test, t->chls, 0x0163); + + /* auto is always allowed, and clears a bit left without a limit */ + t->chls = 0x0100; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO), 0); + KUNIT_EXPECT_EQ(test, t->chls, 0x0000); +} + +/* Without CHLS, auto-discharge is not offered and touches no key. */ +static void macsmc_charge_auto_discharge_unsupported_test(struct kunit *test) +{ + struct macsmc_charge_test *t = test->priv; + + t->power.has_ch0i = true; + t->power.has_chte = true; + t->power.has_chwa = true; + t->ch0i = 1; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE), + -EOPNOTSUPP); + KUNIT_EXPECT_EQ(test, t->nops, 0U); + KUNIT_EXPECT_EQ(test, t->ch0i, 1); +} + +/* An unreadable CHLS leaves every inhibitor as it was. */ +static void macsmc_charge_auto_chls_read_fault_test(struct kunit *test) +{ + static const int modes[] = { + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE, + }; + struct macsmc_charge_test *t = test->priv; + unsigned int i; + + t->power.has_ch0i = true; + t->power.has_chte = true; + t->power.has_chls = true; + t->fail_key = SMC_KEY(CHLS); + t->fail_err = -ETIMEDOUT; + for (i = 0; i < ARRAY_SIZE(modes); i++) { + t->chls = 0x0150; + t->ch0i = 1; + t->chte = 1; + t->nops = 0; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + modes[i]), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, charge_test_writes(t), 0U); + KUNIT_EXPECT_EQ(test, t->ch0i, 1); + KUNIT_EXPECT_EQ(test, t->chte, 1U); + KUNIT_EXPECT_EQ(test, t->chls, 0x0150); + } +} + +/* + * A failure partway through is reported and stops there. The keys are + * separate SMC writes, so the ones already written stay written. + */ +static void macsmc_charge_auto_partial_failure_test(struct kunit *test) +{ + struct macsmc_charge_test *t = test->priv; + + t->power.has_ch0i = true; + t->power.has_chte = true; + t->power.has_chls = true; + t->chls = 0x0050; + t->ch0i = 1; + t->chte = 1; + t->fail_key = SMC_KEY(CHTE); + t->fail_write = true; + t->fail_err = -EIO; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE), -EIO); + KUNIT_EXPECT_EQ(test, t->ch0i, 0); + KUNIT_EXPECT_EQ(test, t->chte, 1U); + KUNIT_EXPECT_EQ(test, t->chls, 0x0050); + KUNIT_EXPECT_EQ(test, t->nops, 3U); + + t->nops = 0; + t->fail_key = SMC_KEY(CHLS); + t->chte = 1; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE), -EIO); + KUNIT_EXPECT_EQ(test, t->chte, 0U); + KUNIT_EXPECT_EQ(test, t->chls, 0x0050); +} + +/* Reported behaviour: CH0I, then the inhibitors, then CHLS; failures propagate. */ +static void macsmc_charge_get_precedence_test(struct kunit *test) +{ + struct macsmc_charge_test *t = test->priv; + + t->power.has_ch0i = true; + t->power.has_ch0c = true; + t->power.has_chls = true; + t->chls = 0x0150; + t->ch0i = 1; + t->ch0c = 1; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&t->power), + POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE); + t->ch0i = 0; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&t->power), + POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE); + t->ch0c = 0; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&t->power), + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE); + t->chls = 0x0050; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&t->power), + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO); + /* The bit without a limit, as an earlier kernel could leave it, reads as auto */ + t->nops = 0; + t->chls = 0x0100; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&t->power), + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO); + t->chls = 0x0164; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&t->power), + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO); + t->fail_key = SMC_KEY(CHLS); + t->fail_err = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, macsmc_battery_get_charge_behaviour(&t->power), -ETIMEDOUT); + KUNIT_EXPECT_EQ(test, charge_test_writes(t), 0U); +} + +/* + * Two writers to CHLS at once: the end threshold's read-modify-write and the + * behaviour's. The behaviour's writer starts while the threshold's has read + * CHLS but not yet written it. Serialized, it waits, and both settings + * survive; without the lock its bit would be lost. + */ +struct charge_test_writer { + struct macsmc_charge_test *t; + struct completion done; + int ret; +}; + +static int charge_test_writer_fn(void *data) +{ + struct charge_test_writer *w = data; + + current->kunit_test = w->t->test; + w->ret = charge_test_set(w->t, POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE); + current->kunit_test = NULL; + complete(&w->done); + return 0; +} + +static void charge_test_start_writer(struct macsmc_charge_test *t) +{ + struct charge_test_writer *w = t->writer; + struct task_struct *task; + + task = kthread_run(charge_test_writer_fn, w, "macsmc-charge-test"); + if (IS_ERR(task)) { + KUNIT_FAIL(t->test, "no writer thread: %ld", PTR_ERR(task)); + w->ret = PTR_ERR(task); + complete(&w->done); + return; + } + /* Serialized, the writer cannot finish while this setter runs */ + t->writer_early = wait_for_completion_timeout(&w->done, msecs_to_jiffies(200)); +} + +static void macsmc_charge_serialized_test(struct kunit *test) +{ + struct macsmc_charge_test *t = test->priv; + struct charge_test_writer *w; + + w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, w); + w->t = t; + init_completion(&w->done); + t->writer = w; + + t->power.has_chls = true; + t->chls = 0x0050; + t->on_chls_read = charge_test_start_writer; + KUNIT_EXPECT_EQ(test, charge_test_set(t, POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD, + 60), 0); + /* + * Always wait for the writer, which uses this test's memory: a writer that never + * finishes is left to the KUnit timeout. + */ + wait_for_completion(&w->done); + KUNIT_EXPECT_FALSE(test, t->writer_early); + KUNIT_EXPECT_EQ(test, atomic_read(&t->writer_unlocked), 0); + KUNIT_EXPECT_EQ(test, w->ret, 0); + /* Both the new limit and auto-discharge */ + KUNIT_EXPECT_EQ(test, t->chls, 0x013c); +} + +/* Probe clears force-discharge left without a limit, and only that. */ +static void macsmc_charge_clear_stray_discharge_test(struct kunit *test) +{ + static const struct { + u16 before, after; + unsigned int writes; + } cases[] = { + { 0x0100, 0x0000, 1 }, + { 0x0164, 0x0064, 1 }, + { 0x0105, 0x0005, 1 }, + { 0x0150, 0x0150, 0 }, /* auto-discharge with a limit stays */ + { 0x0050, 0x0050, 0 }, + { 0x0000, 0x0000, 0 }, + }; + struct macsmc_charge_test *t = test->priv; + unsigned int i; + + t->power.has_chls = true; + for (i = 0; i < ARRAY_SIZE(cases); i++) { + t->chls = cases[i].before; + t->nops = 0; + KUNIT_EXPECT_EQ_MSG(test, macsmc_battery_clear_stray_discharge(&t->power), 0, + "case %u", i); + KUNIT_EXPECT_EQ_MSG(test, t->chls, cases[i].after, "case %u", i); + KUNIT_EXPECT_EQ_MSG(test, charge_test_writes(t), cases[i].writes, "case %u", i); + } + + t->chls = 0x0100; + t->fail_key = SMC_KEY(CHLS); + t->fail_err = -ETIMEDOUT; + KUNIT_EXPECT_EQ(test, macsmc_battery_clear_stray_discharge(&t->power), -ETIMEDOUT); + t->fail_write = true; + t->fail_err = -EIO; + KUNIT_EXPECT_EQ(test, macsmc_battery_clear_stray_discharge(&t->power), -EIO); + KUNIT_EXPECT_EQ(test, t->chls, 0x0100); +} + +static struct kunit_case macsmc_charge_test_cases[] = { + KUNIT_CASE(macsmc_charge_behaviours_test), + KUNIT_CASE(macsmc_charge_end_threshold_test), + KUNIT_CASE(macsmc_charge_end_threshold_fault_test), + KUNIT_CASE(macsmc_charge_auto_discharge_test), + KUNIT_CASE(macsmc_charge_auto_discharge_no_limit_test), + KUNIT_CASE(macsmc_charge_auto_discharge_unsupported_test), + KUNIT_CASE(macsmc_charge_auto_chls_read_fault_test), + KUNIT_CASE(macsmc_charge_auto_partial_failure_test), + KUNIT_CASE(macsmc_charge_get_precedence_test), + KUNIT_CASE(macsmc_charge_serialized_test), + KUNIT_CASE(macsmc_charge_clear_stray_discharge_test), + {} +}; + +static struct kunit_suite macsmc_charge_test_suite = { + .name = "macsmc-power-charge", + .init = macsmc_charge_test_init, + .test_cases = macsmc_charge_test_cases, +}; + +kunit_test_suites(&macsmc_power_test_suite, &macsmc_charge_test_suite); diff --git a/drivers/power/supply/macsmc-power.c b/drivers/power/supply/macsmc-power.c index be10128414758..9e8388f9a824b 100644 --- a/drivers/power/supply/macsmc-power.c +++ b/drivers/power/supply/macsmc-power.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -72,6 +73,12 @@ struct macsmc_power { struct device *dev; struct apple_smc *smc; + /* + * Serializes the charge settings: the CHLS threshold and behaviour + * setters each read, modify and write bits of the same key. + */ + struct mutex charge_lock; + struct power_supply_desc ac_desc; struct power_supply_desc batt_desc; @@ -326,11 +333,42 @@ static int macsmc_power_read_chte(struct macsmc_power *power, u8 *buf) return apple_smc_read(power->smc, SMC_KEY(CHTE), buf, 4); } +/* + * Whether a CHLS value holds a charge limit, as the threshold getter reads + * it. Force discharge is only ever written together with a limit: without + * one there is nothing to discharge to. + */ +static bool macsmc_chls_has_limit(u16 chls) +{ + u8 limit = chls & 0xff; + + return limit >= CHLS_MIN_END_THRESHOLD && limit < 100; +} + +static unsigned int macsmc_battery_charge_behaviours(struct macsmc_power *power) +{ + unsigned int behaviours = 0; + + if (power->has_ch0i || power->has_ch0c || power->has_chte || power->has_chls) + behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO); + if (power->has_ch0i) + behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE); + if (power->has_chte || power->has_ch0c) + behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE); + if (power->has_chls) + behaviours |= BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE); + + return behaviours; +} + static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) { int ret; u8 val8; u8 chte_buf[4]; + u16 vu16; + + guard(mutex)(&power->charge_lock); if (power->has_ch0i) { ret = apple_smc_read_u8(power->smc, SMC_KEY(CH0I), &val8); @@ -359,31 +397,73 @@ static int macsmc_battery_get_charge_behaviour(struct macsmc_power *power) return POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE; } + if (power->has_chls) { + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16); + if (ret) + return ret; + /* The bit without a limit discharges to nothing: report auto */ + if ((vu16 & CHLS_FORCE_DISCHARGE) && macsmc_chls_has_limit(vu16)) + return POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE; + } + return POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO; } static int macsmc_battery_set_charge_behaviour(struct macsmc_power *power, int val) { + u16 vu16 = 0; int ret; + lockdep_assert_held(&power->charge_lock); + switch (val) { + case POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE: + if (!power->has_chls) + return -EOPNOTSUPP; + fallthrough; case POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO: - /* Reset all inhibitors to a known-good 'auto' state */ + /* + * Read the shared limit before changing any inhibitor, so that a + * refusal leaves every setting as it was. + */ + if (power->has_chls) { + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16); + if (ret) + return ret; + if (val == POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE && + !macsmc_chls_has_limit(vu16)) + return -EINVAL; + } + + /* + * Reset all inhibitors to a known-good 'auto' state. A write + * reply carries the SMC's response size, which may echo the + * request length: only a negative value is a failure. + */ if (power->has_ch0i) { ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0I), 0); - if (ret) + if (ret < 0) return ret; } if (power->has_chte) { ret = apple_smc_write_u32(power->smc, SMC_KEY(CHTE), 0); - if (ret) + if (ret < 0) return ret; } else if (power->has_ch0c) { ret = apple_smc_write_u8(power->smc, SMC_KEY(CH0C), 0); - if (ret) + if (ret < 0) return ret; } + + /* Set or clear force-discharge to the CHLS charge limit */ + if (power->has_chls) { + if (val == POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE) + vu16 |= CHLS_FORCE_DISCHARGE; + else + vu16 &= ~CHLS_FORCE_DISCHARGE; + return apple_smc_write_u16(power->smc, SMC_KEY(CHLS), vu16); + } return 0; case POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE: @@ -640,11 +720,11 @@ static int macsmc_battery_get_property(struct power_supply *psy, return ret; } -static int macsmc_battery_set_property(struct power_supply *psy, - enum power_supply_property psp, - const union power_supply_propval *val) +static int macsmc_battery_set_charge_property(struct macsmc_power *power, + enum power_supply_property psp, + const union power_supply_propval *val) { - struct macsmc_power *power = power_supply_get_drvdata(psy); + lockdep_assert_held(&power->charge_lock); switch (psp) { case POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR: @@ -658,12 +738,20 @@ static int macsmc_battery_set_property(struct power_supply *psy, return 0; case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD: if (power->has_chls) { - u16 kval = 0; - /* TODO: Make CHLS_FORCE_DISCHARGE configurable */ + u16 kval; + int ret; + + /* Preserve the force-discharge (auto-discharge) charge_behaviour */ + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &kval); + if (ret) + return ret; + kval &= CHLS_FORCE_DISCHARGE; if (val->intval < CHLS_MIN_END_THRESHOLD) - kval = CHLS_FORCE_DISCHARGE | CHLS_MIN_END_THRESHOLD; + kval |= CHLS_MIN_END_THRESHOLD; else if (val->intval < 100) - kval = CHLS_FORCE_DISCHARGE | (val->intval & 0xff); + kval |= val->intval & 0xff; + else + kval = 0; /* No limit: auto-discharge ends with it */ return apple_smc_write_u16(power->smc, SMC_KEY(CHLS), kval); } else if (power->has_chwa) { return apple_smc_write_flag(power->smc, SMC_KEY(CHWA), @@ -676,6 +764,38 @@ static int macsmc_battery_set_property(struct power_supply *psy, } } +static int macsmc_battery_set_property(struct power_supply *psy, + enum power_supply_property psp, + const union power_supply_propval *val) +{ + struct macsmc_power *power = power_supply_get_drvdata(psy); + int ret; + + guard(mutex)(&power->charge_lock); + ret = macsmc_battery_set_charge_property(power, psp, val); + /* A write's reply size is not an error, and nothing for sysfs to see */ + return ret < 0 ? ret : 0; +} + +/* + * An earlier kernel could leave force-discharge set in CHLS without a charge limit, which this + * driver never asks of the SMC. Clear the bit once at probe; the limit byte is kept. + */ +static int macsmc_battery_clear_stray_discharge(struct macsmc_power *power) +{ + u16 chls; + int ret; + + guard(mutex)(&power->charge_lock); + ret = apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &chls); + if (ret) + return ret; + if (!(chls & CHLS_FORCE_DISCHARGE) || macsmc_chls_has_limit(chls)) + return 0; + ret = apple_smc_write_u16(power->smc, SMC_KEY(CHLS), chls & ~CHLS_FORCE_DISCHARGE); + return ret < 0 ? ret : 0; +} + static int macsmc_battery_property_is_writeable(struct power_supply *psy, enum power_supply_property psp) { @@ -875,13 +995,23 @@ static int macsmc_power_probe(struct platform_device *pdev) power->dev = dev; power->smc = smc; + ret = devm_mutex_init(dev, &power->charge_lock); + if (ret) + return ret; dev_set_drvdata(dev, power); - INIT_WORK(&power->critical_work, macsmc_power_critical_work); + /* devm_work_autocancel() initialises critical_work */ ret = devm_work_autocancel(dev, &power->critical_work, macsmc_power_critical_work); if (ret) return ret; + /* + * Must be ready before the notifier is registered below: an SMC event + * can arrive as soon as it is, and re-initialising a work item that + * is already queued corrupts the workqueue. + */ + INIT_DELAYED_WORK(&power->dbg_log_work, macsmc_dbg_work); + /* * Check for battery presence. * B0AV is a fundamental key. @@ -988,28 +1118,24 @@ static int macsmc_power_probe(struct platform_device *pdev) apple_smc_write_u8(smc, SMC_KEY(CH0K), 0); apple_smc_write_u8(smc, SMC_KEY(CH0B), 0); - /* Configure charge behaviour if supported */ - if (power->has_ch0i || power->has_ch0c || power->has_chte) { - props[nprops++] = POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR; - - power->batt_desc.charge_behaviours = - BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO); - - if (power->has_ch0i) - power->batt_desc.charge_behaviours |= - BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE); - - if (power->has_chte || power->has_ch0c) - power->batt_desc.charge_behaviours |= - BIT(POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE); - } - /* Detect charge limit method (CHWA vs CHLS) */ if (apple_smc_read_flag(power->smc, SMC_KEY(CHWA), &flag) == 0) power->has_chwa = true; else if (apple_smc_read_u16(power->smc, SMC_KEY(CHLS), &vu16) >= 0) power->has_chls = true; + if (power->has_chls && macsmc_battery_clear_stray_discharge(power)) + dev_warn(dev, "Failed to clear force-discharge without a charge limit\n"); + + /* + * CHLS carries auto-discharge, so a CHLS-only Mac can select auto + * to leave it as well. + */ + power->batt_desc.charge_behaviours = + macsmc_battery_charge_behaviours(power); + if (power->batt_desc.charge_behaviours) + props[nprops++] = POWER_SUPPLY_PROP_CHARGE_BEHAVIOUR; + if (power->has_chwa || power->has_chls) { props[nprops++] = POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD; props[nprops++] = POWER_SUPPLY_PROP_CHARGE_CONTROL_START_THRESHOLD; @@ -1090,9 +1216,6 @@ static int macsmc_power_probe(struct platform_device *pdev) power->nb.notifier_call = macsmc_power_event; blocking_notifier_chain_register(&smc->event_handlers, &power->nb); - INIT_WORK(&power->critical_work, macsmc_power_critical_work); - INIT_DELAYED_WORK(&power->dbg_log_work, macsmc_dbg_work); - g_power = power; if (log_power) diff --git a/drivers/power/supply/power_supply_sysfs.c b/drivers/power/supply/power_supply_sysfs.c index f30a7b9ccd5e9..d22541f4a4ecf 100644 --- a/drivers/power/supply/power_supply_sysfs.c +++ b/drivers/power/supply/power_supply_sysfs.c @@ -146,6 +146,7 @@ static const char * const POWER_SUPPLY_CHARGE_BEHAVIOUR_TEXT[] = { [POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE] = "inhibit-charge", [POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE_AWAKE] = "inhibit-charge-awake", [POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE] = "force-discharge", + [POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE] = "auto-discharge", }; static struct power_supply_attr power_supply_attrs[] __ro_after_init = { diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c index 3d26ec1f7b9ec..a8cba2f64b98a 100644 --- a/drivers/ptp/ptp_ocp.c +++ b/drivers/ptp/ptp_ocp.c @@ -1825,7 +1825,7 @@ ptp_ocp_watchdog(struct timer_list *t) static void ptp_ocp_estimate_pci_timing(struct ptp_ocp *bp) { - ktime_t start, end, delay = U64_MAX; + ktime_t start, end, delay = KTIME_MAX; u32 ctrl; int i; diff --git a/drivers/regulator/ltc3676.c b/drivers/regulator/ltc3676.c index 768cdce0d4ecf..1ec218951829d 100644 --- a/drivers/regulator/ltc3676.c +++ b/drivers/regulator/ltc3676.c @@ -232,7 +232,7 @@ static const struct regulator_desc ltc3676_regulators[LTC3676_NUM_REGULATORS] = LTC3676_FIXED_REG(LDO4, ldo4, LDOB, 2), }; -static bool ltc3676_readable_writeable_reg(struct device *dev, unsigned int reg) +static bool ltc3676_writeable_reg(struct device *dev, unsigned int reg) { switch (reg) { case LTC3676_BUCK1 ... LTC3676_IRQSTAT: @@ -243,6 +243,15 @@ static bool ltc3676_readable_writeable_reg(struct device *dev, unsigned int reg) return false; } +static bool ltc3676_readable_reg(struct device *dev, unsigned int reg) +{ + switch (reg) { + case LTC3676_BUCK1 ... LTC3676_IRQSTAT: + return true; + } + return false; +} + static bool ltc3676_volatile_reg(struct device *dev, unsigned int reg) { switch (reg) { @@ -255,8 +264,8 @@ static bool ltc3676_volatile_reg(struct device *dev, unsigned int reg) static const struct regmap_config ltc3676_regmap_config = { .reg_bits = 8, .val_bits = 8, - .writeable_reg = ltc3676_readable_writeable_reg, - .readable_reg = ltc3676_readable_writeable_reg, + .writeable_reg = ltc3676_writeable_reg, + .readable_reg = ltc3676_readable_reg, .volatile_reg = ltc3676_volatile_reg, .max_register = LTC3676_CLIRQ, .use_single_read = true, diff --git a/drivers/regulator/of_regulator.c b/drivers/regulator/of_regulator.c index c0cc6cc0afd8b..785b7a11dfc66 100644 --- a/drivers/regulator/of_regulator.c +++ b/drivers/regulator/of_regulator.c @@ -935,15 +935,15 @@ static int is_supply_name(const char *name) int of_regulator_bulk_get_all(struct device *dev, struct device_node *np, struct regulator_bulk_data **consumers) { - int num_consumers = 0; + int num_consumers = 0, names_len = 0; struct regulator *tmp; struct regulator_bulk_data *_consumers = NULL; struct property *prop; + char *names; int i, n = 0, ret; - char name[64]; /* - * first pass: get numbers of xxx-supply + * first pass: get numbers of xxx-supply and the room their names take * second pass: fill consumers */ restart: @@ -953,16 +953,19 @@ int of_regulator_bulk_get_all(struct device *dev, struct device_node *np, continue; if (!_consumers) { num_consumers++; + names_len += i + 1; continue; } else { - memcpy(name, prop->name, i); - name[i] = '\0'; - tmp = regulator_get(dev, name); + memcpy(names, prop->name, i); + names[i] = '\0'; + tmp = regulator_get(dev, names); if (IS_ERR(tmp)) { ret = PTR_ERR(tmp); goto error; } + _consumers[n].supply = names; _consumers[n].consumer = tmp; + names += i + 1; n++; continue; } @@ -973,9 +976,16 @@ int of_regulator_bulk_get_all(struct device *dev, struct device_node *np, } if (num_consumers == 0) return 0; - _consumers = kmalloc_objs(struct regulator_bulk_data, num_consumers); + /* + * The supply names are kept in the same allocation as the array, so + * that they share its lifetime and the caller has nothing extra to + * free. + */ + _consumers = kzalloc(size_add(size_mul(num_consumers, sizeof(*_consumers)), + names_len), GFP_KERNEL); if (!_consumers) return -ENOMEM; + names = (char *)(_consumers + num_consumers); goto restart; error: diff --git a/drivers/regulator/tps6594-regulator.c b/drivers/regulator/tps6594-regulator.c index 31a5218d55105..503b89a2f5f81 100644 --- a/drivers/regulator/tps6594-regulator.c +++ b/drivers/regulator/tps6594-regulator.c @@ -241,21 +241,21 @@ static const struct regulator_desc tps65224_buck_regs[] = { TPS65224_MASK_BUCKS_VSET, TPS6594_REG_BUCKX_CTRL(1), TPS6594_BIT_BUCK_EN, 0, 0, tps65224_bucks_2_3_4_ranges, - 4, 0, 0, NULL, 0, 0), + ARRAY_SIZE(tps65224_bucks_2_3_4_ranges), 0, 0, NULL, 0, 0), TPS6594_REGULATOR("BUCK3", "buck3", TPS6594_BUCK_3, REGULATOR_VOLTAGE, tps6594_bucks_ops, TPS65224_MASK_BUCKS_VSET, TPS6594_REG_BUCKX_VOUT_1(2), TPS65224_MASK_BUCKS_VSET, TPS6594_REG_BUCKX_CTRL(2), TPS6594_BIT_BUCK_EN, 0, 0, tps65224_bucks_2_3_4_ranges, - 4, 0, 0, NULL, 0, 0), + ARRAY_SIZE(tps65224_bucks_2_3_4_ranges), 0, 0, NULL, 0, 0), TPS6594_REGULATOR("BUCK4", "buck4", TPS6594_BUCK_4, REGULATOR_VOLTAGE, tps6594_bucks_ops, TPS65224_MASK_BUCKS_VSET, TPS6594_REG_BUCKX_VOUT_1(3), TPS65224_MASK_BUCKS_VSET, TPS6594_REG_BUCKX_CTRL(3), TPS6594_BIT_BUCK_EN, 0, 0, tps65224_bucks_2_3_4_ranges, - 4, 0, 0, NULL, 0, 0), + ARRAY_SIZE(tps65224_bucks_2_3_4_ranges), 0, 0, NULL, 0, 0), }; static const struct tps6594_regulator_irq_type tps6594_buck1_irq_types[] = { diff --git a/drivers/scsi/ch.c b/drivers/scsi/ch.c index 4010fdbf813cc..c36539c83ed1a 100644 --- a/drivers/scsi/ch.c +++ b/drivers/scsi/ch.c @@ -112,7 +112,6 @@ typedef struct { int minor; char name[8]; struct scsi_device *device; - struct scsi_device **dt; /* ptrs to data transfer elements */ u_int firsts[CH_TYPES]; u_int counts[CH_TYPES]; u_int voltags; @@ -363,15 +362,10 @@ ch_readconfig(scsi_changer *ch) vendor_labels[i]); } - /* look up the devices of the data transfer elements */ - ch->dt = kzalloc_objs(*ch->dt, ch->counts[CHET_DT]); - - if (!ch->dt) { - kfree(buffer); - return -ENOMEM; - } - + /* report the devices of the data transfer elements */ for (elem = 0; elem < ch->counts[CHET_DT]; elem++) { + struct scsi_device *sdev; + id = -1; lun = 0; if (elem < CH_DT_MAX && -1 != dt_id[elem]) { @@ -387,10 +381,8 @@ ch_readconfig(scsi_changer *ch) VPRINTK(KERN_INFO, "dt 0x%x: ",elem+ch->firsts[CHET_DT]); if (data[6] & 0x80) { VPRINTK(KERN_CONT, "not this SCSI bus\n"); - ch->dt[elem] = NULL; } else if (0 == (data[6] & 0x30)) { VPRINTK(KERN_CONT, "ID/LUN unknown\n"); - ch->dt[elem] = NULL; } else { id = ch->device->id; lun = 0; @@ -400,18 +392,16 @@ ch_readconfig(scsi_changer *ch) } if (-1 != id) { VPRINTK(KERN_CONT, "ID %i, LUN %i, ",id,lun); - ch->dt[elem] = - scsi_device_lookup(ch->device->host, - ch->device->channel, - id,lun); - if (!ch->dt[elem]) { + sdev = scsi_device_lookup(ch->device->host, + ch->device->channel, + id, lun); + if (!sdev) { /* should not happen */ VPRINTK(KERN_CONT, "Huh? device not found!\n"); } else { VPRINTK(KERN_CONT, "name: %8.8s %16.16s %4.4s\n", - ch->dt[elem]->vendor, - ch->dt[elem]->model, - ch->dt[elem]->rev); + sdev->vendor, sdev->model, sdev->rev); + scsi_device_put(sdev); } } } @@ -575,7 +565,6 @@ static void ch_destroy(struct kref *ref) scsi_changer *ch = container_of(ref, scsi_changer, ref); ch->device = NULL; - kfree(ch->dt); kfree(ch); } diff --git a/drivers/scsi/qedi/qedi_main.c b/drivers/scsi/qedi/qedi_main.c index 227ff7bd1bdc9..f394fcbb97e7e 100644 --- a/drivers/scsi/qedi/qedi_main.c +++ b/drivers/scsi/qedi/qedi_main.c @@ -2652,8 +2652,6 @@ static int __qedi_probe(struct pci_dev *pdev, int mode) snprintf(host_buf, sizeof(host_buf), "host_%d", qedi->shost->host_no); qedi_ops->common->set_name(qedi->cdev, host_buf); - qedi_ops->register_ops(qedi->cdev, &qedi_cb_ops, qedi); - memset(¶ms, 0, sizeof(params)); params.mtu = DEF_PATH_MTU + IPV6_HDR_LEN + TCP_HDR_LEN; qedi->ll2_mtu = DEF_PATH_MTU; @@ -2670,6 +2668,7 @@ static int __qedi_probe(struct pci_dev *pdev, int mode) spin_lock_init(&qedi->task_idx_lock); mutex_init(&qedi->stats_lock); } + qedi_ops->register_ops(qedi->cdev, &qedi_cb_ops, qedi); qedi_ops->ll2->register_cb_ops(qedi->cdev, &qedi_ll2_cb_ops, qedi); qedi_ops->ll2->start(qedi->cdev, ¶ms); diff --git a/drivers/soc/apple/Kconfig b/drivers/soc/apple/Kconfig index 0295ceac3c3fa..af85f95e1a201 100644 --- a/drivers/soc/apple/Kconfig +++ b/drivers/soc/apple/Kconfig @@ -234,6 +234,15 @@ config APPLE_T6030_DISPLAY_GATE Say 'y' if you want the native display on M3 Pro MacBook Pros. +config APPLE_T6030_GPU_PMP_LINK + bool + depends on APPLE_T6030_DISPLAY_GATE + default n + help + Link an opted-in T6030 GPU to the PMP admitted by the display gate. + GPU power commands wait for that supplier's acknowledged display + request through the existing PMP report startup and removal rules. + config APPLE_DISPLAY_GATE_KUNIT_TEST bool "KUnit tests for the Apple display gate" if !KUNIT_ALL_TESTS depends on KUNIT=y && APPLE_T6030_DISPLAY_GATE diff --git a/drivers/soc/apple/aop.rs b/drivers/soc/apple/aop.rs index 6641e1d501362..b556dba8d3371 100644 --- a/drivers/soc/apple/aop.rs +++ b/drivers/soc/apple/aop.rs @@ -37,6 +37,7 @@ use kernel::{ IoSysMap, Mmio, Region, // + SysMem, }, module_platform_driver, new_condvar, @@ -436,6 +437,12 @@ impl AFKEndpoint { msg: u64, ) -> Result<()> { let opc = msg >> 48; + dev_dbg!( + client.dev, + "AFK endpoint {:#04x} message {:#x}", + self.index, + msg + ); match opc { AFK_OPC_INIT => { rtkit.send_message(self.index, AFK_MSG_INIT_ACK)?; @@ -683,6 +690,17 @@ impl AFKEndpoint { self.drop_message(&client.dev, fmt!("short message ({} bytes)", msg_buf.len())) } Some((header, msg)) => { + let channel = qeh.channel; + dev_dbg!( + client.dev, + "AFK endpoint {:#04x} EPIC category {:#x} subtype {:#x} tag {} channel {} len {}", + self.index, + header.category, + header.subtype, + header.tag, + channel, + msg.len() + ); if let Err(e) = self.handle_ipc(client, &qeh, &header, msg) { let channel = qeh.channel; self.drop_message( @@ -740,6 +758,13 @@ impl AFKEndpoint { let name = &data[..EPIC_ANNOUNCE_NAME_LEN]; let name = &name[..name.iter().position(|x| *x == 0).unwrap_or(name.len())]; let chan = le_u32(data, EPIC_ANNOUNCE_CHANNEL_OFFSET); + dev_info!( + client.dev, + "AFK endpoint {:#04x} channel {} announces {}", + self.index, + chan, + core::str::from_utf8(name).unwrap_or("?") + ); return Into::>::into(client).register_service(self, chan, name); } else if subtype == EPIC_SUBTYPE_FAKEHID_REPORT { return client.process_fakehid_report(self, qhdr.channel, data); @@ -1000,6 +1025,8 @@ struct SetupState { /// The next unassigned page of the arena. next_page: usize, endpoints: [Option; SETUP_ENDPOINTS.len()], + /// The service endpoints this SoC's firmware serves on the port. + expected: &'static [u8], map_done: bool, ap_ready: bool, power_sent: bool, @@ -1008,12 +1035,13 @@ struct SetupState { } impl SetupState { - fn new() -> SetupState { + fn new(expected: &'static [u8]) -> SetupState { SetupState { mbox: None, arena: None, next_page: 0, endpoints: [None; SETUP_ENDPOINTS.len()], + expected, map_done: false, ap_ready: false, power_sent: false, @@ -1055,7 +1083,7 @@ impl SetupState { /// Boot is complete once the AP power state is on and every service /// endpoint has its buffers. fn ready(&self) -> bool { - self.ap_ready && self.endpoint_count() == SETUP_ENDPOINTS.len() + self.ap_ready && self.endpoint_count() == self.expected.len() } /// Sends one word to a setup-port endpoint. Callers run in process @@ -1189,6 +1217,8 @@ struct AopData { dev: ARef, /// The firmware speaks EPIC with version 4 sub-headers. epic_v4: bool, + /// See [`AopHwConfig::early_ap`]. + early_ap: bool, /// Runs the service registrations one at a time; drained on removal. registration_queue: OwnedQueue, /// Serializes queueing a registration with the start of removal, so that @@ -1344,6 +1374,7 @@ impl AopData { AopData { dev: dev.as_ref().into(), epic_v4: cfg.epic_v4, + early_ap: cfg.early_ap, registration_queue, registration_gate <- new_mutex!(()), removing: Atomic::new(false), @@ -1353,7 +1384,7 @@ impl AopData { required_endpoints: cfg.required_endpoints, setup_queue, setup_lost: Atomic::new(false), - setup <- new_mutex!(SetupState::new()), + setup <- new_mutex!(SetupState::new(cfg.setup_endpoints)), setup_cv <- new_condvar!(), rtkit <- new_mutex!(None), endpoints <- pin_init::pin_init_array_from_fn(|i| { @@ -1378,11 +1409,16 @@ impl AopData { fn wake(&self) -> Result<()> { let mut guard = self.rtkit.lock(); let mut rtk = guard.as_mut().as_pin_mut().ok_or(ENODEV)?; - rtk.as_mut().wake() + if self.early_ap { + rtk.as_mut().wake_early_ap() + } else { + rtk.as_mut().wake() + } } /// Starts the AFK handshake on every advertised endpoint. An endpoint /// the match data does not require may fail to start; it is skipped. fn start_afk(&self) -> Result<()> { + let mut started_count = 0u32; for ep in 0..AFK_ENDPOINT_COUNT as usize { let rtk_ep_num = AFK_ENDPOINT_START + ep as u8; let mut guard = self.rtkit.lock(); @@ -1399,7 +1435,10 @@ impl AopData { Ok(()) }); match started { - Ok(()) => {} + Ok(()) => { + started_count += 1; + dev_dbg!(self.dev, "AFK endpoint {:#04x} started, init sent", rtk_ep_num); + } Err(e) if !required => { dev_warn!( self.dev, @@ -1411,6 +1450,7 @@ impl AopData { Err(e) => return Err(e), } } + dev_info!(self.dev, "{} AFK endpoints started", started_count); Ok(()) } fn register_service( @@ -1939,10 +1979,7 @@ impl AopData { /// from the arena. fn setup_endpoint_request(&self, st: &mut SetupState, ep: u8, word: u64) -> Result<()> { let requested = (word & 0xffff_ffff) as usize; - let slot = SETUP_ENDPOINTS - .iter() - .position(|e| *e == ep) - .ok_or(EINVAL)?; + let slot = st.expected.iter().position(|e| *e == ep).ok_or(EINVAL)?; if st.endpoints[slot].is_some() || (requested != 0x400 && requested != 0x1000) { return Err(EINVAL); } @@ -2180,7 +2217,7 @@ impl AopData { "setup port: boot incomplete (AP power on: {}, endpoints: {}/{}): {:?}", st.ap_ready, st.endpoint_count(), - SETUP_ENDPOINTS.len(), + st.expected.len(), e ); return Err(e); @@ -2189,20 +2226,77 @@ impl AopData { } } -struct NoBuffer; -impl rtkit::Buffer for NoBuffer { +/// Address bits of a buffer the co-processor places itself; the firmware sets +/// higher bits in its requests. +const RTKIT_BUFFER_ADDR_MASK: u64 = (1 << 40) - 1; + +/// A shared buffer handed to RTKit: the host's coherent allocation for a +/// request at address 0 (ioreport, oslog), or firmware memory at the address +/// the firmware names (its crashlog region, which the ADT lists as +/// crashlog-region under iop-aop-nub), reached through memremap. +enum AopShmem { + Host { mem: Coherent<[u8]> }, + Firmware { ptr: *mut u8, size: usize, iova: usize }, +} + +// SAFETY: the raw pointer is a memremap mapping owned by this value alone and +// only dereferenced through `buf()`; the RTKit core serializes its use. +unsafe impl Send for AopShmem {} + +impl Drop for AopShmem { + fn drop(&mut self) { + if let AopShmem::Firmware { ptr, .. } = *self { + // SAFETY: `ptr` came from `memremap` and is not used after this. + unsafe { bindings::memunmap(ptr.cast()) }; + } + } +} + +impl rtkit::Buffer for AopShmem { fn iova(&self) -> Result { - unreachable!() + Ok(match self { + AopShmem::Host { mem } => mem.dma_handle() as usize, + AopShmem::Firmware { iova, .. } => *iova, + }) } fn buf(&mut self) -> Result> { - unreachable!() + let (ptr, size) = match self { + AopShmem::Host { mem } => (mem.as_mut_ptr().cast::(), mem.size()), + AopShmem::Firmware { ptr, size, .. } => (*ptr, *size), + }; + // SAFETY: `ptr` maps `size` bytes, 4-byte aligned, for as long as `self` lives. + Ok(IoSysMap::Sys(unsafe { SysMem::new(Region::ptr_from_raw_parts_mut(ptr, size)) })) } } #[vtable] impl rtkit::Operations for AopData { type Data = Arc; - type Buffer = NoBuffer; + type Buffer = AopShmem; + /// The T6050 AOP places its crashlog buffer itself (a request with the + /// address set, on endpoint 1, which its endpoint map does not list); a + /// placed buffer gets no reply. Other buffers it asks the host for. + const COPROC_PLACES_BUFFERS: bool = true; + + fn shmem_alloc(data: ::Borrowed<'_>, size: usize) -> Result { + // SAFETY: this runs from the RTKit receive worker, which `AopData::remove()` + // stops before the device is unbound, so the device is bound here. + let bound_dev = unsafe { data.dev.as_bound() }; + let mem = Coherent::::zeroed_slice(bound_dev, size, GFP_KERNEL)?; + dev_dbg!(data.dev, "RTKit buffer: {} bytes allocated at iova {:#x}", size, mem.dma_handle()); + Ok(AopShmem::Host { mem }) + } + + fn shmem_map(data: ::Borrowed<'_>, iova: usize, size: usize) -> Result { + let phys = (iova as u64) & RTKIT_BUFFER_ADDR_MASK; + dev_dbg!(data.dev, "RTKit buffer: firmware places {} bytes at {:#x}", size, phys); + // SAFETY: maps firmware memory the firmware points the host at; nothing else uses it. + let ptr = unsafe { bindings::memremap(phys as _, size, bindings::MEMREMAP_WB as _) }; + if ptr.is_null() { + return Err(ENOMEM); + } + Ok(AopShmem::Firmware { ptr: ptr.cast(), size, iova }) + } fn recv_message(data: ::Borrowed<'_>, ep: u8, msg: u64) { let Some(index) = afk_endpoint_index(ep) else { @@ -2246,6 +2340,12 @@ struct AopHwConfig { epic_v4: bool, /// The firmware boots through a second, "setup", mailbox as well. setup_port: bool, + /// The setup port's service endpoints; boot completes once each has its + /// buffers. T8140 has five, T6050 three (its TRUSTED compartment's). + setup_endpoints: &'static [u8], + /// Wake with the AP powered before the IOP's power acknowledgment + /// (T6050). + early_ap: bool, /// Shut the co-processor down on unbind and retain its buffers if the /// shutdown cannot be confirmed. quiesce_on_unbind: bool, @@ -2260,6 +2360,8 @@ const HW_CFG_T8103: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + setup_endpoints: &[], + early_ap: false, quiesce_on_unbind: false, required_endpoints: &[], }; @@ -2270,6 +2372,8 @@ const HW_CFG_T8112: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + setup_endpoints: &[], + early_ap: false, quiesce_on_unbind: false, required_endpoints: &[], }; @@ -2280,6 +2384,8 @@ const HW_CFG_T6000: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + setup_endpoints: &[], + early_ap: false, quiesce_on_unbind: false, required_endpoints: &[], }; @@ -2290,6 +2396,8 @@ const HW_CFG_T6020: AopHwConfig = AopHwConfig { patch_bootargs: true, epic_v4: false, setup_port: false, + setup_endpoints: &[], + early_ap: false, quiesce_on_unbind: false, required_endpoints: &[], }; @@ -2305,6 +2413,26 @@ const HW_CFG_T8140: AopHwConfig = AopHwConfig { patch_bootargs: false, epic_v4: true, setup_port: true, + setup_endpoints: &SETUP_ENDPOINTS, + early_ap: false, + quiesce_on_unbind: true, + required_endpoints: &[0x20, 0x21, 0x22, 0x23, 0x2b], +}; + +/// T6050 (J714c ADT: iop,ascwrap-v6, iop-aop-nub rtbuddy-v2, firmware +/// pre-loaded): T8140's protocol, without the setup port. The ADT's +/// aop-exclave-mailbox aperture (0x4d067c000) hangs the AP on its first +/// read, and the firmware boots and starts its AFK endpoints without it +/// (J714c: RTKit protocol 12, 13 endpoints). +const HW_CFG_T6050: AopHwConfig = AopHwConfig { + ec0p: 0, + aopt: 0, + alig: 0, + patch_bootargs: false, + epic_v4: true, + setup_port: true, + setup_endpoints: &[0x20, 0x23, 0x32], + early_ap: true, quiesce_on_unbind: true, required_endpoints: &[0x20, 0x21, 0x22, 0x23, 0x2b], }; @@ -2319,6 +2447,7 @@ kernel::of_device_table!( (of::DeviceId::new(c_str!("apple,t6000-aop")), &HW_CFG_T6000), (of::DeviceId::new(c_str!("apple,t6020-aop")), &HW_CFG_T6020), (of::DeviceId::new(c_str!("apple,t8140-aop")), &HW_CFG_T8140), + (of::DeviceId::new(c_str!("apple,t6050-aop")), &HW_CFG_T6050), ] ); @@ -2378,8 +2507,10 @@ impl platform::Driver for AopDriver { data.setup_finish_boot()?; // Before any client reaches the ALS service: a failure here costs // the sensor, not the AOP. - if let Err(e) = data.setup_als_calibration() { - dev_warn!(pdev.as_ref(), "ALS calibration failed ({:?})", e); + if cfg.setup_endpoints.contains(&SETUP_ALS_EP) { + if let Err(e) = data.setup_als_calibration() { + dev_warn!(pdev.as_ref(), "ALS calibration failed ({:?})", e); + } } data.start_afk()?; } else { diff --git a/drivers/soc/apple/mailbox.c b/drivers/soc/apple/mailbox.c index 5be648a40fe03..79b87b4a67af1 100644 --- a/drivers/soc/apple/mailbox.c +++ b/drivers/soc/apple/mailbox.c @@ -60,6 +60,11 @@ #define APPLE_T8140_GFX1_WRAPPER_BASE 0x482e00000ULL #define APPLE_T8140_GFX1_IOP_VBAR_BASE 0x482850000ULL +#define APPLE_T6050_GFX_WRAPPER_BASE 0x2302600000ULL +#define APPLE_T6050_GFX_IOP_VBAR_BASE 0x2302050000ULL +#define APPLE_T6050_GFX1_WRAPPER_BASE 0x2302e00000ULL +#define APPLE_T6050_GFX1_IOP_VBAR_BASE 0x2302850000ULL + static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_A2I_CONTROL == 0x8110); static_assert(APPLE_ASCWRAP_V6_MBOX_OFFSET + APPLE_ASC_MBOX_I2A_CONTROL == @@ -118,6 +123,12 @@ struct apple_mbox_hw { bool is_ascwrap_v6; bool ap_initializes_mailboxes; + /* ASCWrap v6 only: the fixed CPU-physical wrapper/iop-vbar bases. */ + u64 gfx_wrapper_base; + u64 gfx_iop_vbar_base; + u64 gfx1_wrapper_base; + u64 gfx1_iop_vbar_base; + unsigned int control_full; unsigned int control_empty; @@ -670,16 +681,16 @@ static int apple_mbox_probe(struct platform_device *pdev) if (!strcmp(firmware_role, "GFX")) { mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX; - if (res->start != APPLE_T8140_GFX_WRAPPER_BASE || - iop_vbar_res->start != APPLE_T8140_GFX_IOP_VBAR_BASE) + if (res->start != mbox->hw->gfx_wrapper_base || + iop_vbar_res->start != mbox->hw->gfx_iop_vbar_base) return dev_err_probe(dev, -EINVAL, - "GFX ASCWrap v6 resources do not match T8140\n"); + "GFX ASCWrap v6 resources do not match the SoC descriptor\n"); } else if (!strcmp(firmware_role, "GFX1")) { mbox->ascwrap_v6.role = APPLE_MBOX_ASCWRAP_V6_ROLE_GFX1; - if (res->start != APPLE_T8140_GFX1_WRAPPER_BASE || - iop_vbar_res->start != APPLE_T8140_GFX1_IOP_VBAR_BASE) + if (res->start != mbox->hw->gfx1_wrapper_base || + iop_vbar_res->start != mbox->hw->gfx1_iop_vbar_base) return dev_err_probe(dev, -EINVAL, - "GFX1 ASCWrap v6 resources do not match T8140\n"); + "GFX1 ASCWrap v6 resources do not match the SoC descriptor\n"); } else { return dev_err_probe(dev, -EINVAL, "invalid ASCWrap v6 firmware role\n"); @@ -842,6 +853,35 @@ static const struct apple_mbox_hw apple_mbox_ascwrap_v6_hw = { .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + .gfx_wrapper_base = APPLE_T8140_GFX_WRAPPER_BASE, + .gfx_iop_vbar_base = APPLE_T8140_GFX_IOP_VBAR_BASE, + .gfx1_wrapper_base = APPLE_T8140_GFX1_WRAPPER_BASE, + .gfx1_iop_vbar_base = APPLE_T8140_GFX1_IOP_VBAR_BASE, + + .has_irq_controls = false, +}; + +/* T6050 (M5 Max): same ASCWrap v6 block as T8140, different CPU-physical bases. */ +static const struct apple_mbox_hw apple_mbox_t6050_ascwrap_v6_hw = { + .reg_offset = APPLE_ASCWRAP_V6_MBOX_OFFSET, + .is_ascwrap_v6 = true, + + .control_full = APPLE_ASC_MBOX_CONTROL_FULL, + .control_empty = APPLE_ASC_MBOX_CONTROL_EMPTY, + + .a2i_control = APPLE_ASC_MBOX_A2I_CONTROL, + .a2i_send0 = APPLE_ASC_MBOX_A2I_SEND0, + .a2i_send1 = APPLE_ASC_MBOX_A2I_SEND1, + + .i2a_control = APPLE_ASC_MBOX_I2A_CONTROL, + .i2a_recv0 = APPLE_ASC_MBOX_I2A_RECV0, + .i2a_recv1 = APPLE_ASC_MBOX_I2A_RECV1, + + .gfx_wrapper_base = APPLE_T6050_GFX_WRAPPER_BASE, + .gfx_iop_vbar_base = APPLE_T6050_GFX_IOP_VBAR_BASE, + .gfx1_wrapper_base = APPLE_T6050_GFX1_WRAPPER_BASE, + .gfx1_iop_vbar_base = APPLE_T6050_GFX1_IOP_VBAR_BASE, + .has_irq_controls = false, }; @@ -882,6 +922,7 @@ static const struct apple_mbox_hw apple_mbox_m3_hw = { static const struct of_device_id apple_mbox_of_match[] = { { .compatible = "apple,t6031-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, { .compatible = "apple,t8122-agx-asc-mailbox", .data = &apple_mbox_asc_hw }, + { .compatible = "apple,t6050-ascwrap-v6", .data = &apple_mbox_t6050_ascwrap_v6_hw }, { .compatible = "apple,t8140-ascwrap-v6", .data = &apple_mbox_ascwrap_v6_hw }, { .compatible = "apple,t8140-asc-mailbox", .data = &apple_mbox_t8140_asc_hw }, { .compatible = "apple,t8140-aop-setup-mailbox", .data = &apple_mbox_t8140_aop_setup_hw }, diff --git a/drivers/soc/apple/pmp.rs b/drivers/soc/apple/pmp.rs index 10b884006978b..fd7354aeacff8 100644 --- a/drivers/soc/apple/pmp.rs +++ b/drivers/soc/apple/pmp.rs @@ -160,6 +160,9 @@ impl PmpData { fn start(&self) -> Result<()> { let mut guard = self.rtkit.lock(); let mut rtk = guard.as_mut().as_pin_mut().unwrap(); + // Plain wake(), as macsmc and m1n1's rtkit_boot do: INIT, then AP-ON only + // after IOP-ON. On T6050 pmp-report binds this driver once the PTD + // requests are seeded; wake_early_ap/boot_early_ap did not help without it. rtk.as_mut().wake()?; rtk.start_endpoint(PMP_ENDPOINT) } @@ -499,6 +502,13 @@ const HW_CFG_T8140: PmpHwConfig = PmpHwConfig { dma_mask: Some(DmaMask::new::<42>()), }; +/// T6050: dart-pmp0's window is [0x10000000000, 0x11000000000) at 1 TiB, so the +/// rtkit buffers need a mask wide enough to reach it. The generic +/// apple,t6000-pmp-v2 default sets none and leaves the mask at 32-bit. +const HW_CFG_T6050: PmpHwConfig = PmpHwConfig { + dma_mask: Some(DmaMask::new::<42>()), +}; + impl Drop for PmpDriver { fn drop(&mut self) { // SAFETY: The Arc remains live until unregister has joined any caller. @@ -512,6 +522,7 @@ kernel::of_device_table!( ::IdInfo, [ (of::DeviceId::new(c"apple,t8140-pmp-v2"), &HW_CFG_T8140), + (of::DeviceId::new(c"apple,t6050-pmp-v2"), &HW_CFG_T6050), (of::DeviceId::new(c"apple,t6000-pmp-v2"), &HW_CFG_DEFAULT), ] ); @@ -545,20 +556,30 @@ impl platform::Driver for PmpDriver { .property_read(c"apple,dram-vendor-id") .required_by(&dev)?; let bdid = node.property_read(c"apple,board-id").required_by(&dev)?; - match node.property_read(c"apple,dram-capacity").optional() { - Some(dcap) => data.patch_bootargs( - &pmp_mmio, - &[ - (from_fourcc(b"BDID"), bdid), - (from_fourcc(b"DCAP"), dcap), - (from_fourcc(b"DVID"), dvid), - ], - )?, - None => data.patch_bootargs( - &pmp_mmio, - &[(from_fourcc(b"BDID"), bdid), (from_fourcc(b"DVID"), dvid)], - )?, + let mut patches = [(0u32, 0u32); 7]; + let mut count = 0; + let mut add = |tag: &[u8; 4], value: u32| { + patches[count] = (from_fourcc(tag), value); + count += 1; }; + add(b"BDID", bdid); + add(b"DVID", dvid); + if let Some(dcap) = node.property_read(c"apple,dram-capacity").optional() { + add(b"DCAP", dcap); + } + // macOS's ApplePMPFirmware: PMC_ = pmgr "pmc", PMCV = "pmc-pmgr" bit 0, + // PMCB = "pmc-pmgr" bit 3, CVAR = the PMP nub's "soc-chip-variant". + if let Some(pmc) = node.property_read(c"apple,pmc").optional() { + add(b"PMC_", pmc); + } + if let Some(pmc_pmgr) = node.property_read::(c"apple,pmc-pmgr").optional() { + add(b"PMCV", pmc_pmgr & 1); + add(b"PMCB", (pmc_pmgr >> 3) & 1); + } + if let Some(cvar) = node.property_read(c"apple,soc-chip-variant").optional() { + add(b"CVAR", cvar); + } + data.patch_bootargs(&pmp_mmio, &patches[..count])?; let rtkit = rtkit::RtKit::::new(&dev, None, 0, data.clone())?; data.rtkit.lock().as_mut().set(Some(rtkit)); asc_mmio.update(ASC_CPU_CONTROL, |r| r.with_const_cpu_run::<1>()); diff --git a/drivers/soc/apple/rtkit-crashlog.c b/drivers/soc/apple/rtkit-crashlog.c index 5defa12862b15..0984a6da34f9d 100644 --- a/drivers/soc/apple/rtkit-crashlog.c +++ b/drivers/soc/apple/rtkit-crashlog.c @@ -13,6 +13,23 @@ #define APPLE_RTKIT_CRASHLOG_MBOX FOURCC('C', 'm', 'b', 'x') #define APPLE_RTKIT_CRASHLOG_TIME FOURCC('C', 't', 'i', 'm') #define APPLE_RTKIT_CRASHLOG_REGS FOURCC('C', 'r', 'g', '8') +#define APPLE_RTKIT_CRASHLOG_CORE_STATE FOURCC('C', 's', 't', 'a') +#define APPLE_RTKIT_CRASHLOG_STACK FOURCC('C', 'c', 's', 't') +#define APPLE_RTKIT_CRASHLOG_RTKIT FOURCC('C', 'r', 't', 'k') +#define APPLE_RTKIT_CRASHLOG_CCDP FOURCC('C', 'c', 'd', 'p') +#define APPLE_RTKIT_CRASHLOG_CPCT FOURCC('C', 'p', 'c', 't') + +/* + * Saved core context of the interrupted thread (PMP firmware, t6050). + * Offsets follow the firmware exception dispatcher context layout: + * pc at +0x100, far at +0x328, esr (u32) at +0x338. The esr is checked + * against the esr the Crg8 section dumped; on mismatch only the values + * print with a layout-unverified tag. + */ +#define CSTA_PC_OFF 0x100 +#define CSTA_FAR_OFF 0x328 +#define CSTA_ESR_OFF 0x338 + /* For COMPILE_TEST on non-ARM64 architectures */ #ifndef PSR_MODE_EL0t @@ -100,6 +117,11 @@ static void apple_rtkit_crashlog_dump_mailbox(struct apple_rtkit *rtk, u8 *bfr, size_t n_messages; struct apple_rtkit_crashlog_mbox_entry entry; + if (size < 28) { + dev_warn(rtk->dev, "RTKit: Mailbox section too small: 0x%zx", size); + return; + } + memcpy(&type, bfr + 16, 4); memcpy(&index, bfr + 24, 4); n_messages = (size - 28) / sizeof(entry); @@ -126,6 +148,7 @@ static void apple_rtkit_crashlog_dump_regs(struct apple_rtkit *rtk, u8 *bfr, } regs = (struct apple_rtkit_crashlog_regs *)bfr; + rtk->last_dumped_esr = regs->esr; switch (regs->psr & PSR_MODE_MASK) { case PSR_MODE_EL0t: @@ -173,11 +196,85 @@ static void apple_rtkit_crashlog_dump_regs(struct apple_rtkit *rtk, u8 *bfr, dev_warn(rtk->dev, "\n"); } +/* A crashlog can carry several KB per section; the console is slow, so bound the output. */ +#define CRASHLOG_HEX_MAX_WORDS 32 +#define CRASHLOG_MAX_SECTIONS 32 + +static void apple_rtkit_crashlog_dump_hexwords(struct apple_rtkit *rtk, + u8 *bfr, size_t size) +{ + size_t i, n = min_t(size_t, size / 8, CRASHLOG_HEX_MAX_WORDS); + u64 v[4]; + + for (i = 0; i < n; i += 4) { + size_t k, cnt = min_t(size_t, n - i, 4); + + for (k = 0; k < cnt; k++) + memcpy(&v[k], bfr + (i + k) * 8, 8); + for (; k < 4; k++) + v[k] = 0; + dev_warn(rtk->dev, "RTKit: +%03zx = %016llx %016llx %016llx %016llx", + i * 8, v[0], v[1], v[2], v[3]); + } + if (size / 8 > n) + dev_warn(rtk->dev, "RTKit: (%zu more words not shown)", size / 8 - n); +} + +static void apple_rtkit_crashlog_dump_core_state(struct apple_rtkit *rtk, + u8 *bfr, size_t size) +{ + u32 version, esr = 0; + u64 pc = 0, far = 0; + + if (size < 4) { + dev_warn(rtk->dev, "RTKit: Csta section too small: 0x%zx", + size); + return; + } + + memcpy(&version, bfr, 4); + if (size >= 4 + CSTA_ESR_OFF + 4) { + memcpy(&pc, bfr + 4 + CSTA_PC_OFF, 8); + memcpy(&far, bfr + 4 + CSTA_FAR_OFF, 8); + memcpy(&esr, bfr + 4 + CSTA_ESR_OFF, 4); + } + + if (esr == rtk->last_dumped_esr && esr) + dev_warn(rtk->dev, + "RTKit: Interrupted context: ELR = 0x%llx FAR = 0x%llx (ESR 0x%x matches dump)", + pc, far, esr); + else + dev_warn(rtk->dev, + "RTKit: Interrupted context (layout unverified, ESR 0x%x != dump 0x%llx): ELR = 0x%llx FAR = 0x%llx", + esr, rtk->last_dumped_esr, pc, far); + + apple_rtkit_crashlog_dump_hexwords(rtk, bfr + 4, size - 4); +} + +static void apple_rtkit_crashlog_dump_stack(struct apple_rtkit *rtk, + u8 *bfr, size_t size) +{ + u32 idx, flags; + + if (size < 8) { + dev_warn(rtk->dev, "RTKit: Ccst section too small: 0x%zx", + size); + return; + } + + memcpy(&idx, bfr, 4); + memcpy(&flags, bfr + 4, 4); + dev_warn(rtk->dev, "RTKit: Crashed stack (thread=%u flags=%x):", + idx, flags); + apple_rtkit_crashlog_dump_hexwords(rtk, bfr + 8, size - 8); +} + void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) { size_t offset; - u32 section_fourcc, section_size; + u32 section_fourcc, section_size, payload; struct apple_rtkit_crashlog_header header; + unsigned int n_sections = 0; memcpy(&header, bfr, sizeof(header)); if (header.fourcc != APPLE_RTKIT_CRASHLOG_HEADER_FOURCC) { @@ -196,8 +293,26 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) offset = sizeof(header); while (offset < size) { + if (size - offset < 16) { + dev_warn(rtk->dev, "RTKit: Truncated crashlog section header at 0x%zx", + offset); + return; + } memcpy(§ion_fourcc, bfr + offset, 4); memcpy(§ion_size, bfr + offset + 12, 4); + /* The size covers the 16 byte section header; handlers get the payload. */ + if (section_size < 16 || section_size > size - offset) { + dev_warn(rtk->dev, + "RTKit: Bad crashlog section %x size 0x%x at 0x%zx", + section_fourcc, section_size, offset); + return; + } + payload = section_size - 16; + if (++n_sections > CRASHLOG_MAX_SECTIONS) { + dev_warn(rtk->dev, "RTKit: Crashlog has more than %d sections, stopping", + CRASHLOG_MAX_SECTIONS); + return; + } switch (section_fourcc) { case APPLE_RTKIT_CRASHLOG_HEADER_FOURCC: @@ -205,23 +320,39 @@ void apple_rtkit_crashlog_dump(struct apple_rtkit *rtk, u8 *bfr, size_t size) return; case APPLE_RTKIT_CRASHLOG_STR: apple_rtkit_crashlog_dump_str(rtk, bfr + offset + 16, - section_size); + payload); break; case APPLE_RTKIT_CRASHLOG_VERSION: apple_rtkit_crashlog_dump_version( - rtk, bfr + offset + 16, section_size); + rtk, bfr + offset + 16, payload); break; case APPLE_RTKIT_CRASHLOG_MBOX: apple_rtkit_crashlog_dump_mailbox( - rtk, bfr + offset + 16, section_size); + rtk, bfr + offset + 16, payload); break; case APPLE_RTKIT_CRASHLOG_TIME: apple_rtkit_crashlog_dump_time(rtk, bfr + offset + 16, - section_size); + payload); break; case APPLE_RTKIT_CRASHLOG_REGS: apple_rtkit_crashlog_dump_regs(rtk, bfr + offset + 16, - section_size); + payload); + break; + case APPLE_RTKIT_CRASHLOG_CORE_STATE: + apple_rtkit_crashlog_dump_core_state( + rtk, bfr + offset + 16, payload); + break; + case APPLE_RTKIT_CRASHLOG_STACK: + apple_rtkit_crashlog_dump_stack(rtk, bfr + offset + 16, + payload); + break; + case APPLE_RTKIT_CRASHLOG_RTKIT: + case APPLE_RTKIT_CRASHLOG_CCDP: + case APPLE_RTKIT_CRASHLOG_CPCT: + dev_warn(rtk->dev, "RTKit: State section (size=0x%x):", + payload); + apple_rtkit_crashlog_dump_hexwords(rtk, bfr + offset + 16, + payload); break; default: dev_warn(rtk->dev, diff --git a/drivers/soc/apple/rtkit-helper.c b/drivers/soc/apple/rtkit-helper.c index 5f03bed772f26..d5d2b473219f2 100644 --- a/drivers/soc/apple/rtkit-helper.c +++ b/drivers/soc/apple/rtkit-helper.c @@ -169,7 +169,8 @@ static int apple_rtkit_helper_probe(struct platform_device *pdev) helper->asc_base + APPLE_ASC_CPU_CONTROL); /* Works for both wake and boot */ - if (of_device_is_compatible(dev->of_node, "apple,t8142-mtp")) + if (of_device_is_compatible(dev->of_node, "apple,t8142-mtp") || + of_device_is_compatible(dev->of_node, "apple,t6050-mtp")) ret = apple_rtkit_wake_early_ap(helper->rtk); else ret = apple_rtkit_wake(helper->rtk); diff --git a/drivers/soc/apple/rtkit-internal.h b/drivers/soc/apple/rtkit-internal.h index 417d4550ef3ae..210924cc7be72 100644 --- a/drivers/soc/apple/rtkit-internal.h +++ b/drivers/soc/apple/rtkit-internal.h @@ -69,6 +69,9 @@ struct apple_rtkit { struct apple_rtkit_shmem kdebug_buffer0; struct apple_rtkit_shmem kdebug_buffer1; + /* The T6050 PMP's CoreAnalytics buffer (non-standard system endpoint 7). */ + struct apple_rtkit_shmem ep7_buffer; + struct apple_rtkit_shmem syslog_buffer; char *syslog_msg_buffer; size_t syslog_n_entries; @@ -78,6 +81,9 @@ struct apple_rtkit { bool crashlog_inherited; bool oslog_inherited; + /* ESR of the last crashlog register dump, to tell a matching Csta apart. */ + u64 last_dumped_esr; + struct workqueue_struct *wq; }; diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c index 11334b3f68faa..9eae593de7803 100644 --- a/drivers/soc/apple/rtkit.c +++ b/drivers/soc/apple/rtkit.c @@ -230,7 +230,20 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) if (!(msg & APPLE_RTKIT_MGMT_EPMAP_LAST)) return; - for_each_set_bit(ep, rtk->endpoints, APPLE_RTKIT_APP_ENDPOINT_START) { + /* + * T6050: start the system endpoints in the loader's order, 3 1 2 4 8. The + * loader never starts tracekit; the ANS firmware runs without it. + */ + static const u8 t6050_order[] = { APPLE_RTKIT_EP_DEBUG, APPLE_RTKIT_EP_CRASHLOG, + APPLE_RTKIT_EP_SYSLOG, APPLE_RTKIT_EP_IOREPORT, + APPLE_RTKIT_EP_OSLOG }; + bool loader_order = of_machine_is_compatible("apple,t6050"); + int oi; + + for (oi = 0; oi < (loader_order ? ARRAY_SIZE(t6050_order) : APPLE_RTKIT_APP_ENDPOINT_START); oi++) { + ep = loader_order ? t6050_order[oi] : oi; + if (!test_bit(ep, rtk->endpoints)) + continue; switch (ep) { /* the management endpoint is started by default */ case APPLE_RTKIT_EP_MGMT: @@ -270,6 +283,21 @@ static void apple_rtkit_management_rx_epmap(struct apple_rtkit *rtk, u64 msg) } } + /* + * The T6050 PMP firmware has a non-standard system endpoint 7 + * (CoreAnalytics) that t6050_order omits. It does not acknowledge IOP + * power until that endpoint is started and its buffer granted. + */ + if (loader_order && test_bit(7, rtk->endpoints)) { + dev_dbg(rtk->dev, "RTKit: Starting non-standard system endpoint 0x07\n"); + ret = apple_rtkit_start_ep(rtk, 7); + if (ret) { + rtk->boot_result = ret; + complete_all(&rtk->epmap_completion); + return; + } + } + rtk->boot_result = 0; complete_all(&rtk->epmap_completion); } @@ -403,11 +431,11 @@ static int apple_rtkit_common_rx_get_buffer(struct apple_rtkit *rtk, buffer->size >> 12); reply |= FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, buffer->iova); + /* An endpoint 7 grant is class 1 (bits 54-55). */ + if (ep == 7) + reply |= BIT_ULL(54); } apple_rtkit_send_message(rtk, ep, reply, NULL, false); - } else if (apple_rtkit_coproc_places_buffers(rtk)) { - /* acknowledge the co-processor's own buffer with its request */ - apple_rtkit_send_message(rtk, ep, msg, NULL, false); } return 0; @@ -635,6 +663,29 @@ static void apple_rtkit_ioreport_rx(struct apple_rtkit *rtk, u64 msg) } } +/* + * The T6050 PMP firmware's system endpoint 7 is CoreAnalytics, and its power-on + * blocks until the buffer it requests there is granted. Bits 54-55 hold the + * message class (as macOS's RTBuddyCoreAnalyticsEndpoint reads them): 0 = buffer + * request (bits 52-53 size unit, 4 KiB for this request; 44-51 count; 0-43 IOVA), + * granted as class 1 with the same fields; 3 = event for the entry in bits 32-47, + * acknowledged as class 3 with that entry. + */ +static void apple_rtkit_ep7_rx(struct apple_rtkit *rtk, u64 msg) +{ + switch ((msg >> 54) & 3) { + case 0: + apple_rtkit_common_rx_get_buffer(rtk, &rtk->ep7_buffer, 7, msg); + break; + case 3: + apple_rtkit_send_message(rtk, 7, (msg & GENMASK_ULL(47, 32)) | (3ULL << 54), + NULL, false); + break; + default: + dev_warn(rtk->dev, "RTKit: Unknown ep7 message: %llx\n", msg); + } +} + static void apple_rtkit_syslog_rx_init(struct apple_rtkit *rtk, u64 msg) { size_t entries = FIELD_GET(APPLE_RTKIT_SYSLOG_N_ENTRIES, msg); @@ -756,7 +807,8 @@ static void apple_rtkit_oslog_rx(struct apple_rtkit *rtk, u64 msg) /* Preserve the tested Neo/M5 and synchronous-log no-ACK policy. */ if (type == APPLE_RTKIT_OSLOG_LOG && (rtk->sync_syslog_ack || of_machine_is_compatible("apple,t8140") || - of_machine_is_compatible("apple,t8142"))) { + of_machine_is_compatible("apple,t8142") || + of_machine_is_compatible("apple,t6050"))) { dev_dbg(rtk->dev, "RTKit: oslog notification: %016llx\n", msg); return; } @@ -841,6 +893,13 @@ static void apple_rtkit_rx_work(struct work_struct *work) "Received unexpected message to EP%02d: %llx\n", rtk_work->ep, rtk_work->msg); break; + /* The T6050 PMP's non-standard CoreAnalytics endpoint. */ + case 7: + if (of_machine_is_compatible("apple,t6050")) { + apple_rtkit_ep7_rx(rtk, rtk_work->msg); + break; + } + fallthrough; default: dev_warn(rtk->dev, "RTKit: message to unknown endpoint %02x: %llx\n", @@ -1261,6 +1320,7 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); apple_rtkit_free_buffer(rtk, &rtk->kdebug_buffer0); apple_rtkit_free_buffer(rtk, &rtk->kdebug_buffer1); + apple_rtkit_free_buffer(rtk, &rtk->ep7_buffer); kfree(rtk->syslog_msg_buffer); @@ -1313,6 +1373,18 @@ static int apple_rtkit_set_ap_power_state(struct apple_rtkit *rtk, if (ret) return ret; + /* + * T6050: the loader never waits for this acknowledgement, and not every + * coprocessor sends it. Give it a short window so a caller that starts an + * endpoint right away (macsmc) sees the AP state; carry on without it. + */ + if (of_machine_is_compatible("apple,t6050")) { + if (!wait_for_completion_timeout(&rtk->ap_pwr_ack_completion, + msecs_to_jiffies(200))) + dev_dbg(rtk->dev, "RTKit: no AP power state ACK\n"); + return 0; + } + ret = apple_rtkit_wait(rtk, &rtk->ap_pwr_ack_completion); if (ret) return ret; @@ -1379,6 +1451,28 @@ static int apple_rtkit_boot_common(struct apple_rtkit *rtk, bool early_ap_power) return ret; } + /* + * T6050: the loader only proceeds once the IOP reports ON, and + * intermediate states can be acknowledged first. Wait for ON here too. + */ + if (of_machine_is_compatible("apple,t6050")) { + int tries = 10; + + while ((rtk->iop_power_state & 0xff) != APPLE_RTKIT_PWR_STATE_ON && tries--) { + dev_dbg(rtk->dev, "RTKit: IOP power state 0x%x, waiting for ON\n", + rtk->iop_power_state); + reinit_completion(&rtk->iop_pwr_ack_completion); + if ((rtk->iop_power_state & 0xff) == APPLE_RTKIT_PWR_STATE_ON) + break; + ret = apple_rtkit_wait(rtk, &rtk->iop_pwr_ack_completion); + if (ret) { + dev_err(rtk->dev, "RTKit: IOP never reached ON (state 0x%x): %d\n", + rtk->iop_power_state, ret); + return ret; + } + } + } + return early_ap_power ? 0 : apple_rtkit_set_ap_power_state(rtk, APPLE_RTKIT_PWR_STATE_ON); } @@ -1389,6 +1483,12 @@ int apple_rtkit_boot(struct apple_rtkit *rtk) } EXPORT_SYMBOL_GPL(apple_rtkit_boot); +int apple_rtkit_boot_early_ap(struct apple_rtkit *rtk) +{ + return apple_rtkit_boot_common(rtk, true); +} +EXPORT_SYMBOL_GPL(apple_rtkit_boot_early_ap); + int apple_rtkit_adopt_running(struct apple_rtkit *rtk) { if (rtk->crashed) @@ -1522,6 +1622,7 @@ void apple_rtkit_free(struct apple_rtkit *rtk) apple_rtkit_free_buffer(rtk, &rtk->syslog_buffer); apple_rtkit_free_buffer(rtk, &rtk->kdebug_buffer0); apple_rtkit_free_buffer(rtk, &rtk->kdebug_buffer1); + apple_rtkit_free_buffer(rtk, &rtk->ep7_buffer); kfree(rtk->syslog_msg_buffer); apple_rtkit_release_rx(rtk); diff --git a/drivers/soc/apple/sart.c b/drivers/soc/apple/sart.c index b0b208c92741d..e2450cbfa09bc 100644 --- a/drivers/soc/apple/sart.c +++ b/drivers/soc/apple/sart.c @@ -62,6 +62,18 @@ #define APPLE_SART3_FLAGS_ALLOW 0xff +/* SARTv4: the T6050 layout, config at 0x00, paddr at 0x60, size at 0xc0 */ +#define APPLE_SART4_CONFIG(idx) (0x00 + 4 * (idx)) + +#define APPLE_SART4_PADDR(idx) (0x60 + 4 * (idx)) +#define APPLE_SART4_PADDR_SHIFT 12 + +#define APPLE_SART4_SIZE(idx) (0xc0 + 4 * (idx)) +#define APPLE_SART4_SIZE_SHIFT 12 +#define APPLE_SART4_SIZE_MAX GENMASK(29, 0) + +#define APPLE_SART4_FLAGS_ALLOW 0xff + struct apple_sart_ops { void (*get_entry)(struct apple_sart *sart, int index, u8 *flags, phys_addr_t *paddr, size_t *size); @@ -178,6 +190,34 @@ static struct apple_sart_ops sart_ops_v3 = { .size_max = APPLE_SART3_SIZE_MAX, }; +static void sart4_get_entry(struct apple_sart *sart, int index, u8 *flags, + phys_addr_t *paddr, size_t *size) +{ + phys_addr_t paddr_ = readl(sart->regs + APPLE_SART4_PADDR(index)); + size_t size_ = readl(sart->regs + APPLE_SART4_SIZE(index)); + + *flags = readl(sart->regs + APPLE_SART4_CONFIG(index)); + *size = size_ << APPLE_SART4_SIZE_SHIFT; + *paddr = paddr_ << APPLE_SART4_PADDR_SHIFT; +} + +static void sart4_set_entry(struct apple_sart *sart, int index, u8 flags, + phys_addr_t paddr_shifted, size_t size_shifted) +{ + writel(paddr_shifted, sart->regs + APPLE_SART4_PADDR(index)); + writel(size_shifted, sart->regs + APPLE_SART4_SIZE(index)); + writel(flags, sart->regs + APPLE_SART4_CONFIG(index)); +} + +static struct apple_sart_ops sart_ops_v4 = { + .get_entry = sart4_get_entry, + .set_entry = sart4_set_entry, + .flags_allow = APPLE_SART4_FLAGS_ALLOW, + .size_shift = APPLE_SART4_SIZE_SHIFT, + .paddr_shift = APPLE_SART4_PADDR_SHIFT, + .size_max = APPLE_SART4_SIZE_MAX, +}; + static int apple_sart_probe(struct platform_device *pdev) { int i; @@ -372,6 +412,10 @@ static void apple_sart_shutdown(struct platform_device *pdev) } static const struct of_device_id apple_sart_of_match[] = { + { + .compatible = "apple,t6050-sart", + .data = &sart_ops_v4, + }, { .compatible = "apple,t6000-sart", .data = &sart_ops_v3, diff --git a/drivers/soc/apple/shim.h b/drivers/soc/apple/shim.h index 0c3d680185685..0425bf2cdaa6e 100644 --- a/drivers/soc/apple/shim.h +++ b/drivers/soc/apple/shim.h @@ -52,6 +52,7 @@ long sep_store_read(void *handle, long long off, void *buf, size_t len); long sep_store_write(void *handle, long long off, const void *buf, size_t len); int sep_store_sync(void *handle); +int sep_store_sync_parent(void *handle); /* -- sha_shim.c --------------------------------------------------------- */ diff --git a/drivers/soc/apple/shim.rs b/drivers/soc/apple/shim.rs index 7e7f489e59481..8e30c3f54b817 100644 --- a/drivers/soc/apple/shim.rs +++ b/drivers/soc/apple/shim.rs @@ -16,6 +16,7 @@ extern "C" { fn sep_store_read(handle: *mut c_void, off: i64, buf: *mut c_void, len: usize) -> c_long; fn sep_store_write(handle: *mut c_void, off: i64, buf: *const c_void, len: usize) -> c_long; fn sep_store_sync(handle: *mut c_void) -> c_int; + fn sep_store_sync_parent(handle: *mut c_void) -> c_int; fn sep_random_bytes(buf: *mut c_void, len: usize) -> c_int; } @@ -184,6 +185,13 @@ impl StoreFile { // SAFETY: `handle` is live per the type invariant. kernel::error::to_result(unsafe { sep_store_sync(self.handle) }) } + + /// Flushes the directory entry of this file, which [`StoreFile::sync`] + /// alone does not make durable for a newly created file. + pub(crate) fn sync_parent(&self) -> Result<()> { + // SAFETY: `handle` is live per the type invariant. + kernel::error::to_result(unsafe { sep_store_sync_parent(self.handle) }) + } } impl Drop for StoreFile { diff --git a/drivers/soc/apple/store_shim.c b/drivers/soc/apple/store_shim.c index 4c89f92eb5f13..0eabe7305b049 100644 --- a/drivers/soc/apple/store_shim.c +++ b/drivers/soc/apple/store_shim.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -142,3 +143,35 @@ int sep_store_sync(void *handle) { return vfs_fsync((struct file *)handle, 0); } + +/* + * Flushes the directory that holds an open store file. fsync() of the file + * itself is not guaranteed to make its name in that directory durable, so a + * file created this boot could be missing after a crash. Opened under kernel + * credentials, as the file was. + */ +int sep_store_sync_parent(void *handle) +{ + struct file *f = handle; + struct cred *kern; + struct file *dir; + struct path parent; + int ret; + + kern = prepare_kernel_cred(&init_task); + if (!kern) + return -ENOMEM; + + /* The file holds its mount; the parent dentry is pinned until dput(). */ + parent.mnt = f->f_path.mnt; + parent.dentry = dget_parent(f->f_path.dentry); + dir = dentry_open(&parent, O_RDONLY | O_DIRECTORY, kern); + dput(parent.dentry); + put_cred(kern); + if (IS_ERR(dir)) + return PTR_ERR(dir); + + ret = vfs_fsync(dir, 0); + fput(dir); + return ret; +} diff --git a/drivers/soc/apple/t6030-display-gate.c b/drivers/soc/apple/t6030-display-gate.c index 82c69f53cfcae..2f73104b05c8c 100644 --- a/drivers/soc/apple/t6030-display-gate.c +++ b/drivers/soc/apple/t6030-display-gate.c @@ -811,18 +811,27 @@ static int __init gate_pmp_copy_values(struct of_changeset *cs, struct device_no return n; } -static int __init gate_t8122_gpu_pmp_link(struct of_changeset *cs, struct device_node *pmp) +static int __init gate_gpu_pmp_link(struct of_changeset *cs, struct device_node *pmp) { struct device_node *gpu, *linked; + bool t6030 = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && gate_soc == &gate_t6030; int ret = 0; - if (gate_soc != &gate_t8122) + if (gate_soc != &gate_t8122 && !t6030) return 0; - gpu = of_find_compatible_node(NULL, NULL, "apple,agx-t8122"); + gpu = t6030 ? gate_find_one("apple,agx-t6030") : + of_find_compatible_node(NULL, NULL, "apple,agx-t8122"); if (!gpu || !of_device_is_available(gpu)) goto out; + /* Do not replace a malformed or unresolved pre-existing GPU supplier. */ + if (t6030 && of_property_present(gpu, "apple,pmp") && + of_property_count_u32_elems(gpu, "apple,pmp") != 1) { + ret = -EINVAL; + goto out; + } linked = of_parse_phandle(gpu, "apple,pmp", 0); - if (!pmp || !pmp->phandle || (linked && linked != pmp)) + if (!pmp || !pmp->phandle || (linked && linked != pmp) || + (t6030 && of_property_present(gpu, "apple,pmp") && !linked)) ret = -EINVAL; else if (!linked) ret = of_changeset_add_prop_u32(cs, gpu, "apple,pmp", pmp->phandle); @@ -868,7 +877,7 @@ static int __init gate_reuse_apply(struct device_node **ps) ret = gate_add_bool(&gate_pmp_cs, r->ans, "apple,no-ack"); /* The current14 T8122 GPU needs an inner AGX vote from this Mac's PMP. */ if (!ret) - ret = gate_t8122_gpu_pmp_link(&gate_pmp_cs, r->pmp); + ret = gate_gpu_pmp_link(&gate_pmp_cs, r->pmp); /* The display domains' floor, applied by the PMGR driver when it probes. */ for (i = PMP_PS_DISP_SYS; i <= PMP_PS_DISP_CPU && !ret; i++) ret = gate_set_u32(&gate_pmp_cs, ps[i], "apple,min-state", PMGR_PS_ACTIVE); @@ -941,9 +950,9 @@ static int __init gate_pmp_apply(struct device_node *dcp, struct device_node **p if (!ret) ret = of_changeset_add_prop_u32(&gate_pmp_cs, dcp, "apple,pmp-report", disp->phandle); - /* The current14 T8122 GPU needs an inner AGX vote from this Mac's PMP. */ + /* T8122 and an opted-in T6030 GPU need this Mac's PMP AGX vote. */ if (!ret) - ret = gate_t8122_gpu_pmp_link(&gate_pmp_cs, pmp); + ret = gate_gpu_pmp_link(&gate_pmp_cs, pmp); /* The DART and mailbox interrupt parent lives in the base tree. */ if (!ret) ret = of_changeset_add_prop_u32(&gate_pmp_cs, dart, "interrupt-parent", diff --git a/drivers/soc/apple/xart_guard.rs b/drivers/soc/apple/xart_guard.rs index db6a6bbc5654e..661253c5b0146 100644 --- a/drivers/soc/apple/xart_guard.rs +++ b/drivers/soc/apple/xart_guard.rs @@ -27,9 +27,12 @@ //! a guard that blocks matching from one that has refused nothing yet. //! //! The list entry is made durable before the enclave's write reaches the -//! gigalocker. A crash between the two leaves an entry for a record that was -//! never written, which is harmless. It can never leave a Linux record that -//! the next boot treats as another OS's. +//! gigalocker, and the list's own directory entry is flushed before anything +//! is adopted; the key is listed in memory only once its entry is written and +//! flushed. A crash between the two durable points leaves an entry for a +//! record that was never written, which is harmless, and a failed adoption is +//! retried in the same slot rather than skipped. It can never leave a Linux +//! record that the next boot treats as another OS's. use crate::shim::StoreFile; use crate::xart_store::{crc32_ieee, Key}; @@ -167,6 +170,8 @@ pub(crate) trait ListFile { fn read_exact(&self, off: u64, buf: &mut [u8]) -> Result<()>; fn write_all(&self, off: u64, buf: &[u8]) -> Result<()>; fn sync(&self) -> Result<()>; + /// Makes the file's directory entry durable. + fn sync_parent(&self) -> Result<()>; } impl ListFile for StoreFile { @@ -185,6 +190,10 @@ impl ListFile for StoreFile { fn sync(&self) -> Result<()> { StoreFile::sync(self) } + + fn sync_parent(&self) -> Result<()> { + StoreFile::sync_parent(self) + } } pub(crate) struct Guard { @@ -223,6 +232,11 @@ impl Guard { existed: existing, refused: 0, }; + // A list created this boot (or by a crashed one) is only safe to adopt + // into once its name is durable: fsync() of the file alone does not + // guarantee that. If the directory cannot be flushed, the file is + // dropped and nothing is adopted, as without a list. + let file = file.and_then(|file| file.sync_parent().map(|()| file)); match file { Ok(file) => { if let Err(e) = guard.load(&file) { @@ -302,10 +316,18 @@ impl Guard { if usize::try_from(self.end).map_or(true, |end| end / ENTRY_SIZE >= MAX_ENTRIES) { return Err(ENOSPC); } + // Reserve room for the in-memory key before writing, so an entry made + // durable is always listed in memory as well: no allocation can fail + // after a record is written. + self.owned.reserve(1, GFP_KERNEL)?; file.write_all(self.end, &encode(key))?; file.sync()?; + // Only now that the entry is written and flushed: list it (into the + // reserved slot, which cannot fail) and then advance the end. A failed + // write or flush above leaves the end unchanged, so the next adoption + // overwrites this slot, which holds no record the enclave kept. + self.owned.push_within_capacity(*key).map_err(|_| ENOMEM)?; self.end += ENTRY_SIZE as u64; - self.owned.push(*key, GFP_KERNEL)?; Ok(()) } @@ -346,7 +368,7 @@ impl Guard { #[kernel::macros::kunit_tests(apple_sep_xart_guard)] mod tests { use super::*; - use core::cell::RefCell; + use core::cell::{Cell, RefCell}; fn session(byte: u8) -> Key { Key::new(4, [byte; 16]) @@ -442,8 +464,34 @@ mod tests { assert_eq!(decode(&encode(&key)[..31]), None); } - /// A list file in memory, kept across the guards of successive boots. - struct MemFile<'a>(&'a RefCell>); + /// Failures a [`MemFile`] reports, switchable while a guard holds it. + #[derive(Default)] + struct Faults { + /// The write fails and changes nothing. + write: Cell, + /// The write lands, but the flush after it fails. + sync: Cell, + /// The directory entry cannot be flushed. + parent: Cell, + } + + /// A list file in memory, kept across the guards of successive boots, with + /// optional injected faults. + struct MemFile<'a>(&'a RefCell>, Option<&'a Faults>); + + fn mem(bytes: &RefCell>) -> MemFile<'_> { + MemFile(bytes, None) + } + + fn faulty<'a>(bytes: &'a RefCell>, faults: &'a Faults) -> MemFile<'a> { + MemFile(bytes, Some(faults)) + } + + impl MemFile<'_> { + fn fails(&self, fault: impl Fn(&Faults) -> &Cell) -> bool { + self.1.is_some_and(|faults| fault(faults).get()) + } + } impl ListFile for MemFile<'_> { fn size(&self) -> Result { @@ -459,6 +507,9 @@ mod tests { } fn write_all(&self, off: u64, buf: &[u8]) -> Result<()> { + if self.fails(|f| &f.write) { + return Err(EIO); + } let mut bytes = self.0.borrow_mut(); let off = usize::try_from(off).map_err(|_| EINVAL)?; let end = off.checked_add(buf.len()).ok_or(EINVAL)?; @@ -470,6 +521,16 @@ mod tests { } fn sync(&self) -> Result<()> { + if self.fails(|f| &f.sync) { + return Err(EIO); + } + Ok(()) + } + + fn sync_parent(&self) -> Result<()> { + if self.fails(|f| &f.parent) { + return Err(EIO); + } Ok(()) } } @@ -496,7 +557,7 @@ mod tests { fn a_guard_refuses_another_os_record() -> Result<()> { let bytes = RefCell::new(KVec::new()); let foreign = session(1); - let mut guard = Guard::with_file(1, keys(&[foreign])?, Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, keys(&[foreign])?, Ok(mem(&bytes))); assert_eq!(guard.effective_level(), 1); assert!(guard.list_loaded()); assert_eq!(guard.check(&foreign, true), Err(EPERM)); @@ -512,7 +573,7 @@ mod tests { let mut list = KVec::new(); list.extend_from_slice(&encode(&own), GFP_KERNEL)?; let bytes = RefCell::new(list); - let mut guard = Guard::with_file(1, keys(&[own])?, Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, keys(&[own])?, Ok(mem(&bytes))); assert_eq!(guard.check(&own, true), Ok(())); assert_eq!(guard.check(&own, false), Ok(())); // An entry that is already listed is not written again. @@ -527,13 +588,13 @@ mod tests { let created = session(3); // First boot: the enclave creates a record, which is listed first. - let mut guard = Guard::with_file(1, keys(&[foreign])?, Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, keys(&[foreign])?, Ok(mem(&bytes))); assert_eq!(guard.check(&created, true), Ok(())); assert_eq!(listed(&bytes).as_slice(), &[Some(created)]); drop(guard); // Next boot: both records predate the guard; only Linux's is writable. - let mut guard = Guard::with_file(1, keys(&[foreign, created])?, Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, keys(&[foreign, created])?, Ok(mem(&bytes))); assert_eq!(guard.check(&created, true), Ok(())); assert_eq!(guard.check(&created, false), Ok(())); assert_eq!(guard.check(&foreign, true), Err(EPERM)); @@ -551,7 +612,7 @@ mod tests { list.extend_from_slice(&encode(&second)[..10], GFP_KERNEL)?; let bytes = RefCell::new(list); - let mut guard = Guard::with_file(1, keys(&[first])?, Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, keys(&[first])?, Ok(mem(&bytes))); assert_eq!(guard.check(&first, true), Ok(())); assert_eq!(guard.check(&second, true), Ok(())); assert_eq!(listed(&bytes).as_slice(), &[Some(first), Some(second)]); @@ -580,14 +641,14 @@ mod tests { all.push(numbered(i), GFP_KERNEL)?; } let bytes = RefCell::new(list_of(&all)?); - let mut guard = Guard::with_file(1, KVec::new(), Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, KVec::new(), Ok(mem(&bytes))); assert!(guard.file.is_some()); assert_eq!(guard.check(&numbered(MAX_ENTRIES), true), Err(ENOSPC)); assert_eq!(bytes.borrow().len(), MAX_ENTRIES * ENTRY_SIZE); drop(guard); // Reopened: still within what load() accepts, and Linux's records // are still Linux's. - let mut guard = Guard::with_file(1, keys(&[numbered(7)])?, Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, keys(&[numbered(7)])?, Ok(mem(&bytes))); assert!(guard.file.is_some()); assert_eq!(guard.check(&numbered(7), true), Ok(())); Ok(()) @@ -603,13 +664,13 @@ mod tests { all.push(numbered(i % distinct), GFP_KERNEL)?; } let bytes = RefCell::new(list_of(&all)?); - let mut guard = Guard::with_file(1, KVec::new(), Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, KVec::new(), Ok(mem(&bytes))); assert_eq!(guard.owned.len(), distinct); // Appending would make a list the next boot refuses. assert_eq!(guard.check(&numbered(MAX_ENTRIES), true), Err(ENOSPC)); assert_eq!(bytes.borrow().len(), MAX_ENTRIES * ENTRY_SIZE); drop(guard); - let guard = Guard::with_file(1, KVec::new(), Ok(MemFile(&bytes))); + let guard = Guard::with_file(1, KVec::new(), Ok(mem(&bytes))); assert!(guard.file.is_some()); Ok(()) } @@ -624,13 +685,13 @@ mod tests { list.extend_from_slice(&encode(&numbered(MAX_ENTRIES))[..10], GFP_KERNEL)?; let bytes = RefCell::new(list); - let mut guard = Guard::with_file(1, KVec::new(), Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, KVec::new(), Ok(mem(&bytes))); // The torn tail is overwritten by the one entry that still fits. assert_eq!(guard.check(&numbered(MAX_ENTRIES), true), Ok(())); assert_eq!(bytes.borrow().len(), MAX_ENTRIES * ENTRY_SIZE); assert_eq!(guard.check(&numbered(MAX_ENTRIES + 1), true), Err(ENOSPC)); drop(guard); - let mut guard = Guard::with_file(1, KVec::new(), Ok(MemFile(&bytes))); + let mut guard = Guard::with_file(1, KVec::new(), Ok(mem(&bytes))); assert!(guard.file.is_some()); assert_eq!(guard.check(&numbered(MAX_ENTRIES), true), Ok(())); Ok(()) @@ -649,6 +710,66 @@ mod tests { Ok(()) } + #[test] + fn a_failed_write_lists_nothing() -> Result<()> { + let bytes = RefCell::new(KVec::new()); + let faults = Faults::default(); + let created = session(3); + let mut guard = Guard::with_file(1, KVec::new(), Ok(faulty(&bytes, &faults))); + faults.write.set(true); + // The write fails: nothing is listed and the end did not advance. + assert_eq!(guard.check(&created, true), Err(EIO)); + assert_eq!(bytes.borrow().len(), 0); + faults.write.set(false); + // The same slot takes the record once the write lands. + assert_eq!(guard.check(&created, true), Ok(())); + assert_eq!(listed(&bytes).as_slice(), &[Some(created)]); + Ok(()) + } + + #[test] + fn a_failed_flush_is_retried_in_the_same_slot() -> Result<()> { + let bytes = RefCell::new(KVec::new()); + let faults = Faults::default(); + let refused = session(3); + let created = session(4); + let mut guard = Guard::with_file(1, KVec::new(), Ok(faulty(&bytes, &faults))); + faults.sync.set(true); + // The write lands but the flush fails: the record is not listed, so it + // is not writable on a retry either. + assert_eq!(guard.check(&refused, true), Err(EIO)); + assert_eq!(guard.check(&refused, true), Err(EIO)); + faults.sync.set(false); + // The next adoption reuses the same slot the unflushed write took, so + // the list holds no duplicate. + assert_eq!(guard.check(&created, true), Ok(())); + assert_eq!(guard.check(&refused, true), Ok(())); + assert_eq!(listed(&bytes).as_slice(), &[Some(created), Some(refused)]); + drop(guard); + + // The next boot reads exactly those two, in order. + let mut guard = Guard::with_file(1, keys(&[created, refused])?, Ok(mem(&bytes))); + assert_eq!(guard.check(&created, true), Ok(())); + assert_eq!(guard.check(&refused, true), Ok(())); + Ok(()) + } + + #[test] + fn without_a_durable_directory_entry_nothing_is_adopted() -> Result<()> { + let own = session(2); + let bytes = RefCell::new(list_of(&[own])?); + let faults = Faults::default(); + faults.parent.set(true); + let mut guard = Guard::with_file(1, keys(&[own])?, Ok(faulty(&bytes, &faults))); + // The directory flush fails, so the file is dropped: as without a + // list, every existing record is protected and nothing is listed. + assert!(!guard.list_loaded()); + assert_eq!(guard.check(&own, true), Err(EPERM)); + assert!(guard.check(&session(3), true).is_err()); + assert_eq!(bytes.borrow().len(), ENTRY_SIZE); + Ok(()) + } + #[test] fn the_default_follows_the_platform() { use crate::profile; diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c index 4425132432ec8..e0c88e9f81833 100644 --- a/drivers/spi/spi-nxp-fspi.c +++ b/drivers/spi/spi-nxp-fspi.c @@ -898,10 +898,10 @@ static int nxp_fspi_select_mem(struct nxp_fspi *f, struct spi_device *spi, dev_dbg(f->dev, "Target device [CS:%x] selected\n", spi_get_chipselect(spi, 0)); /* - * Per the FlexSPI reference manual (initialization sequence), MCR0 and - * the DLL control registers should be configured while the module is in - * stop mode (MCR0[MDIS] = 1). Enter stop mode before reconfiguring the - * RX sample clock source and the DLL, then exit stop mode afterwards. + * Per the FlexSPI reference manual (initialization sequence), MCR0 + * should be configured while the module is in stop mode (MCR0[MDIS] = 1). + * Enter stop mode before reconfiguring the RX sample clock source, then + * exit stop mode afterwards. */ reg = fspi_readl(f, f->iobase + FSPI_MCR0); fspi_writel(f, reg | FSPI_MCR0_MDIS, f->iobase + FSPI_MCR0); @@ -936,6 +936,16 @@ static int nxp_fspi_select_mem(struct nxp_fspi *f, struct spi_device *spi, if (ret) return ret; + /* + * Exit stop mode before running the DLL calibration. The DLL + * reference clock is derived from the module clock domain, which is + * gated while MCR0[MDIS] = 1. Calibrating the DLL in stop mode leaves + * the STS2 lock bits stuck at 0 (the DLL never runs), so clear MDIS + * first and let the module and reference clock run. + */ + reg = fspi_readl(f, f->iobase + FSPI_MCR0); + fspi_writel(f, reg & ~FSPI_MCR0_MDIS, f->iobase + FSPI_MCR0); + /* * If clock rate > 100MHz, then switch from DLL override mode to * DLL calibration mode. @@ -945,10 +955,6 @@ static int nxp_fspi_select_mem(struct nxp_fspi *f, struct spi_device *spi, else nxp_fspi_dll_override(f); - /* Exit stop mode now that MCR0 and the DLL have been reconfigured. */ - reg = fspi_readl(f, f->iobase + FSPI_MCR0); - fspi_writel(f, reg & ~FSPI_MCR0_MDIS, f->iobase + FSPI_MCR0); - f->pre_op_rate = op->max_freq; f->selected = spi_get_chipselect(spi, 0); diff --git a/drivers/spi/spi-sg2044-nor.c b/drivers/spi/spi-sg2044-nor.c index f3bcb8a1b92bb..bc9a8ee9bce44 100644 --- a/drivers/spi/spi-sg2044-nor.c +++ b/drivers/spi/spi-sg2044-nor.c @@ -307,15 +307,15 @@ static ssize_t sg2044_spifmc_tran_cmd(struct sg2044_spifmc *spifmc, return 0; } -static void sg2044_spifmc_trans(struct sg2044_spifmc *spifmc, - const struct spi_mem_op *op) +static int sg2044_spifmc_trans(struct sg2044_spifmc *spifmc, + const struct spi_mem_op *op) { if (op->data.dir == SPI_MEM_DATA_IN) - sg2044_spifmc_read(spifmc, op); + return sg2044_spifmc_read(spifmc, op); else if (op->data.dir == SPI_MEM_DATA_OUT) - sg2044_spifmc_write(spifmc, op); + return sg2044_spifmc_write(spifmc, op); else - sg2044_spifmc_tran_cmd(spifmc, op); + return sg2044_spifmc_tran_cmd(spifmc, op); } static ssize_t sg2044_spifmc_trans_reg(struct sg2044_spifmc *spifmc, @@ -385,29 +385,64 @@ static ssize_t sg2044_spifmc_trans_reg(struct sg2044_spifmc *spifmc, return 0; } +static int sg2044_spifmc_set_speed(struct sg2044_spifmc *spifmc, + unsigned int speed_hz) +{ + unsigned long rate = clk_get_rate(spifmc->clk); + u64 div; + u32 reg; + + if (!rate) + return -EINVAL; + + /* SCK = HCLK / (2 * (SckDiv + 1)); round down the resulting SCK. */ + div = speed_hz ? DIV_ROUND_UP_ULL(rate, 2ULL * speed_hz) : 1; + if (div > SPIFMC_CTRL_SCK_DIV_MASK + 1) + return -EINVAL; + + reg = readl(spifmc->io_base + SPIFMC_CTRL); + reg &= ~SPIFMC_CTRL_SCK_DIV_MASK; + reg |= div - 1; + writel(reg, spifmc->io_base + SPIFMC_CTRL); + + return 0; +} + static int sg2044_spifmc_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) { struct sg2044_spifmc *spifmc; + int ret; spifmc = spi_controller_get_devdata(mem->spi->controller); mutex_lock(&spifmc->lock); + ret = sg2044_spifmc_set_speed(spifmc, op->max_freq ? + min(op->max_freq, mem->spi->max_speed_hz) : + mem->spi->max_speed_hz); + if (ret) + goto out_unlock; + if (op->addr.nbytes == 0) - sg2044_spifmc_trans_reg(spifmc, op); + ret = sg2044_spifmc_trans_reg(spifmc, op); else - sg2044_spifmc_trans(spifmc, op); + ret = sg2044_spifmc_trans(spifmc, op); +out_unlock: mutex_unlock(&spifmc->lock); - return 0; + return ret; } static const struct spi_controller_mem_ops sg2044_spifmc_mem_ops = { .exec_op = sg2044_spifmc_exec_op, }; +static const struct spi_controller_mem_caps sg2044_spifmc_mem_caps = { + .per_op_freq = true, +}; + static void sg2044_spifmc_init(struct sg2044_spifmc *spifmc) { u32 tran_csr; @@ -447,6 +482,10 @@ static int sg2044_spifmc_probe(struct platform_device *pdev) if (IS_ERR(spifmc->clk)) return dev_err_probe(dev, PTR_ERR(spifmc->clk), "Cannot get and enable AHB clock\n"); + ctrl->max_speed_hz = clk_get_rate(spifmc->clk) / 2; + if (!ctrl->max_speed_hz) + return dev_err_probe(dev, -EINVAL, "Invalid AHB clock rate\n"); + spifmc->dev = &pdev->dev; spifmc->ctrl = ctrl; @@ -458,6 +497,7 @@ static int sg2044_spifmc_probe(struct platform_device *pdev) ctrl->bits_per_word_mask = SPI_BPW_MASK(8); ctrl->auto_runtime_pm = false; ctrl->mem_ops = &sg2044_spifmc_mem_ops; + ctrl->mem_caps = &sg2044_spifmc_mem_caps; ctrl->mode_bits = SPI_RX_DUAL | SPI_TX_DUAL | SPI_RX_QUAD | SPI_TX_QUAD; ret = devm_mutex_init(dev, &spifmc->lock); diff --git a/drivers/tee/optee/ffa_abi.c b/drivers/tee/optee/ffa_abi.c index 633715b98625c..d027fe7ce86fd 100644 --- a/drivers/tee/optee/ffa_abi.c +++ b/drivers/tee/optee/ffa_abi.c @@ -198,7 +198,8 @@ static int to_msg_param_ffa_mem(struct optee_msg_param *mp, if (shm) { u64 shm_offs = p->u.memref.shm_offs; - mp->u.fmem.internal_offs = shm->offset; + mp->u.fmem.internal_offs = tee_shm_get_page_offset(shm) & + (FFA_PAGE_SIZE - 1); mp->u.fmem.offs_low = shm_offs; mp->u.fmem.offs_high = shm_offs >> 32; @@ -284,14 +285,26 @@ static int optee_ffa_shm_register(struct tee_context *ctx, struct tee_shm *shm, .nattrs = 1, }; struct sg_table sgt; + size_t ffa_offs; int rc; + if (!num_pages) + return -EINVAL; + rc = optee_check_mem_type(start, num_pages); if (rc) return rc; - rc = sg_alloc_table_from_pages(&sgt, pages, num_pages, 0, - num_pages * PAGE_SIZE, GFP_KERNEL); + /* + * Start the FF-A descriptor at the 4 KiB page containing the shared + * buffer, skipping unused leading 4 KiB pages when PAGE_SIZE is + * larger. Same approach as optee_fill_pages_list() in the SMC ABI. + * This leaves only the offset into that 4 KiB page for internal_offs. + */ + ffa_offs = round_down(tee_shm_get_page_offset(shm), FFA_PAGE_SIZE); + rc = sg_alloc_table_from_pages(&sgt, pages, num_pages, ffa_offs, + num_pages * PAGE_SIZE - ffa_offs, + GFP_KERNEL); if (rc) return rc; args.sg = sgt.sgl; @@ -458,7 +471,8 @@ static void handle_ffa_rpc_func_cmd_shm_alloc(struct tee_context *ctx, .attr = OPTEE_MSG_ATTR_TYPE_FMEM_OUTPUT, .u.fmem.size = tee_shm_get_size(shm), .u.fmem.global_id = shm->sec_world_id, - .u.fmem.internal_offs = shm->offset, + .u.fmem.internal_offs = tee_shm_get_page_offset(shm) & + (FFA_PAGE_SIZE - 1), }; arg->ret = TEEC_SUCCESS; @@ -1123,14 +1137,6 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev) optee_set_dev_group(optee); - rc = tee_device_register(optee->teedev); - if (rc) - goto err_unreg_supp_teedev; - - rc = tee_device_register(optee->supp_teedev); - if (rc) - goto err_unreg_supp_teedev; - rc = rhashtable_init(&optee->ffa.global_ids, &shm_rhash_params); if (rc) goto err_unreg_supp_teedev; @@ -1159,6 +1165,14 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev) if (optee_ffa_protmem_pool_init(optee, sec_caps)) pr_info("Protected memory service not available\n"); + rc = tee_device_register(optee->teedev); + if (rc) + goto err_unregister_devices; + + rc = tee_device_register(optee->supp_teedev); + if (rc) + goto err_unregister_devices; + rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES); if (rc) goto err_unregister_devices; diff --git a/drivers/tee/optee/optee_msg.h b/drivers/tee/optee/optee_msg.h index 7d9b12e71c035..6c3043f8da33c 100644 --- a/drivers/tee/optee/optee_msg.h +++ b/drivers/tee/optee/optee_msg.h @@ -136,8 +136,8 @@ struct optee_msg_param_rmem { * struct optee_msg_param_fmem - FF-A memory reference parameter * @offs_low: lower bits of offset into shared memory reference * @offs_high: higher bits of offset into shared memory reference - * @internal_offs: internal offset into the first page of shared memory - * reference + * @internal_offs: offset into the first 4 KiB page of the FF-A shared + * memory region * @size: size of the buffer * @global_id: global identifier of the shared memory */ diff --git a/drivers/tee/optee/smc_abi.c b/drivers/tee/optee/smc_abi.c index b8a2bdac3208f..51624443359bf 100644 --- a/drivers/tee/optee/smc_abi.c +++ b/drivers/tee/optee/smc_abi.c @@ -1849,14 +1849,6 @@ static int optee_probe(struct platform_device *pdev) optee_set_dev_group(optee); - rc = tee_device_register(optee->teedev); - if (rc) - goto err_unreg_supp_teedev; - - rc = tee_device_register(optee->supp_teedev); - if (rc) - goto err_unreg_supp_teedev; - optee_cq_init(&optee->call_queue, thread_count); optee_supp_init(&optee->supp); optee->smc.memremaped_shm = memremaped_shm; @@ -1916,6 +1908,14 @@ static int optee_probe(struct platform_device *pdev) if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) pr_info("dynamic shared memory is enabled\n"); + rc = tee_device_register(optee->teedev); + if (rc) + goto err_disable_shm_cache; + + rc = tee_device_register(optee->supp_teedev); + if (rc) + goto err_disable_shm_cache; + rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES); if (rc) goto err_disable_shm_cache; @@ -1942,7 +1942,6 @@ static int optee_probe(struct platform_device *pdev) optee_shm_arg_cache_uninit(optee); optee_supp_uninit(&optee->supp); mutex_destroy(&optee->call_queue.mutex); -err_unreg_supp_teedev: tee_device_unregister(optee->supp_teedev); err_unreg_teedev: tee_device_unregister(optee->teedev); diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c index 6742b3579c86d..daa4af1e089e5 100644 --- a/drivers/tee/tee_shm.c +++ b/drivers/tee/tee_shm.c @@ -343,6 +343,10 @@ int tee_dyn_shm_alloc_helper(struct tee_shm *shm, size_t size, size_t align, unsigned int i; int rc = 0; + /* get_order(0) is undefined and exceeds MAX_PAGE_ORDER. */ + if (!nr_pages) + return -EINVAL; + /* * Ignore alignment since this is already going to be page aligned * and there's no need for any larger alignment. diff --git a/drivers/thunderbolt/apple-dpin-state.c b/drivers/thunderbolt/apple-dpin-state.c index b07d65ad531a7..18817e784822c 100644 --- a/drivers/thunderbolt/apple-dpin-state.c +++ b/drivers/thunderbolt/apple-dpin-state.c @@ -28,6 +28,8 @@ const struct apple_dpin_policy apple_dpin_m2 = { const struct apple_dpin_policy apple_dpin_m3 = { .flow = APPLE_DPIN_CHANGED, + .capacity_retry = true, + .defer_new_bringup = true, .host_policy = TB_HOST_DP_NOTIFY, }; diff --git a/drivers/thunderbolt/apple-dpin-test.c b/drivers/thunderbolt/apple-dpin-test.c index 46f23b72fb0e0..c22da168bd8bc 100644 --- a/drivers/thunderbolt/apple-dpin-test.c +++ b/drivers/thunderbolt/apple-dpin-test.c @@ -192,7 +192,7 @@ static void run_trace(struct kunit *test, const struct apple_dpin_policy *profil } } -static void apple_dpin_m1_capacity_trace(struct kunit *test) +static void apple_dpin_capacity_trace(struct kunit *test) { const struct dpin_trace trace[] = { { APPLE_DPIN_UP, false, 0, APPLE_DPIN_QUEUE, APPLE_DPIN_ACTIVATING }, @@ -217,6 +217,7 @@ static void apple_dpin_m1_capacity_trace(struct kunit *test) }; run_trace(test, &apple_dpin_m1, trace, ARRAY_SIZE(trace)); + run_trace(test, &apple_dpin_m3, trace, ARRAY_SIZE(trace)); } static void apple_dpin_t602x_rearm_trace(struct kunit *test) @@ -388,10 +389,11 @@ static void apple_dpin_all_policy_fields(struct kunit *test) TB_HOST_DP_HPD_ON_ACTIVATE | TB_HOST_DP_ACTIVE_BEFORE_DPRX | TB_HOST_DP_KEEP_DPRX_TIMEOUT | TB_HOST_DP_ADAPTER_QUIRKS | TB_HOST_DP_INITIAL_BW_GRANT }, - { APPLE_DPIN_CHANGED, false, false, false, false, TB_HOST_DP_NOTIFY }, + { APPLE_DPIN_CHANGED, true, false, false, true, TB_HOST_DP_NOTIFY }, }; unsigned int i, queue, display; + KUNIT_ASSERT_EQ(test, ARRAY_SIZE(actual), ARRAY_SIZE(expected)); for (i = 0; i < ARRAY_SIZE(actual); i++) { KUNIT_EXPECT_EQ(test, actual[i]->flow, expected[i].flow); KUNIT_EXPECT_EQ(test, actual[i]->capacity_retry, expected[i].capacity_retry); @@ -529,7 +531,7 @@ static void apple_dpin_sleep_handed_and_policy_scope(struct kunit *test) struct apple_dpin_state s = { .alive = true, .handed = true, .phase = APPLE_DPIN_HANDED }; const struct apple_dpin_policy * const unaffected[] = { - &apple_dpin_m2, &apple_dpin_m3, + &apple_dpin_m2, }; unsigned int i, actions; @@ -602,7 +604,8 @@ static void apple_dpin_sleep_dprx_retention(struct kunit *test) KUNIT_EXPECT_FALSE(test, apple_dpin_awaits_display(&s, &apple_dpin_m1)); s.handed = false; KUNIT_EXPECT_FALSE(test, apple_dpin_awaits_display(&s, &apple_dpin_m2)); - KUNIT_EXPECT_FALSE(test, apple_dpin_awaits_display(&s, &apple_dpin_m3)); + apple_dpin_step(&s, &apple_dpin_m3, APPLE_DPIN_UP, false, 0); + KUNIT_EXPECT_TRUE(test, apple_dpin_awaits_display(&s, &apple_dpin_m3)); } static void apple_dpin_sleep_complete_before_work(struct kunit *test) @@ -1086,7 +1089,75 @@ static void apple_dpin_provider_unavailable(struct kunit *test) KUNIT_EXPECT_EQ(test, fake.references, 0U); } +static void apple_dpin_m3_capacity_suspend(struct kunit *test) +{ + struct apple_dpin_state s = {}; + struct apple_dpin_tokens tokens = {}; + const struct apple_dpin_policy *p = &apple_dpin_m3; + + apple_dpin_step(&s, p, APPLE_DPIN_UP, false, 0); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_WORK, false, 0), + APPLE_DPIN_ATTACH); + apple_dpin_step(&s, p, APPLE_DPIN_ADMITTED, true, 0); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_RESULT, true, -EBUSY), + APPLE_DPIN_FIRST_WAIT | APPLE_DPIN_ARM_RETRY); + KUNIT_EXPECT_TRUE(test, apple_dpin_awaits_display(&s, p)); + apple_dpin_step(&s, p, APPLE_DPIN_PAUSE, true, 0); + KUNIT_EXPECT_TRUE(test, apple_dpin_admission_blocked(&s, p, false)); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_RETRY, true, 0), 0U); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_WORK, true, 0), 0U); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_RESUME, true, 0), + APPLE_DPIN_ARM_RETRY); + KUNIT_EXPECT_FALSE(test, apple_dpin_admission_blocked(&s, p, false)); + apple_dpin_step(&s, p, APPLE_DPIN_RETRY, true, 0); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_WORK, true, 0), + APPLE_DPIN_ATTACH); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_RESULT, true, 0), + APPLE_DPIN_RECOVERED | APPLE_DPIN_LOG_CONNECTED | APPLE_DPIN_AGAIN); + KUNIT_EXPECT_FALSE(test, apple_dpin_awaits_display(&s, p)); + /* A handed callback remains valid while new admission is paused. */ + apple_dpin_token_request(&tokens, 1, true); + apple_dpin_token_admit(&tokens, 1); + apple_dpin_token_complete(&tokens, 1); + apple_dpin_step(&s, p, APPLE_DPIN_PAUSE, true, 0); + KUNIT_EXPECT_TRUE(test, apple_dpin_token_access(&tokens, 1)); + KUNIT_EXPECT_FALSE(test, apple_dpin_awaits_display(&s, p)); + /* A replacement request revokes the old token during the same gate. */ + apple_dpin_token_request(&tokens, 2, true); + KUNIT_EXPECT_FALSE(test, apple_dpin_token_access(&tokens, 1)); + KUNIT_EXPECT_FALSE(test, apple_dpin_token_complete(&tokens, 1)); + apple_dpin_step(&s, p, APPLE_DPIN_DOWN, true, 0); + apple_dpin_step(&s, p, APPLE_DPIN_WORK, true, 0); + apple_dpin_step(&s, p, APPLE_DPIN_DROPPED, false, 0); + apple_dpin_step(&s, p, APPLE_DPIN_UP, false, 0); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_WORK, false, 0), 0U); + KUNIT_EXPECT_TRUE(test, apple_dpin_awaits_display(&s, p)); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_RESUME, false, 0), + APPLE_DPIN_QUEUE); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_WORK, false, 0), + APPLE_DPIN_ATTACH); + apple_dpin_token_admit(&tokens, 2); + KUNIT_EXPECT_TRUE(test, apple_dpin_token_access(&tokens, 2)); + /* Removal drains the request, then resets a gate with no complete(). */ + apple_dpin_step(&s, p, APPLE_DPIN_PAUSE, true, 0); + apple_dpin_step(&s, p, APPLE_DPIN_DOWN, true, 0); + apple_dpin_step(&s, p, APPLE_DPIN_WORK, true, 0); + apple_dpin_step(&s, p, APPLE_DPIN_DROPPED, false, 0); + apple_dpin_token_revoke(&tokens, 2); + apple_dpin_step(&s, p, APPLE_DPIN_END_PM_GATE, false, 0); + KUNIT_EXPECT_FALSE(test, s.paused); + KUNIT_EXPECT_FALSE(test, apple_dpin_token_access(&tokens, 2)); + apple_dpin_step(&s, p, APPLE_DPIN_UP, false, 0); + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_WORK, false, 0), + APPLE_DPIN_ATTACH); + /* An unsupported DP IN ends the wait instead of retaining DPRX forever. */ + KUNIT_EXPECT_EQ(test, apple_dpin_step(&s, p, APPLE_DPIN_RESULT, true, -EOPNOTSUPP), + APPLE_DPIN_CANCEL_RETRY | APPLE_DPIN_WARN); + KUNIT_EXPECT_FALSE(test, apple_dpin_awaits_display(&s, p)); +} + static struct kunit_case apple_dpin_cases[] = { + KUNIT_CASE(apple_dpin_m3_capacity_suspend), KUNIT_CASE(apple_dpin_provider_handed_lifetime), KUNIT_CASE(apple_dpin_provider_failed_retry), KUNIT_CASE(apple_dpin_provider_unavailable), @@ -1111,7 +1182,7 @@ static struct kunit_case apple_dpin_cases[] = { KUNIT_CASE(apple_dpin_sleep_handed_and_policy_scope), KUNIT_CASE(apple_dpin_sleep_complete_races_deferral), KUNIT_CASE(apple_dpin_exhaustive_equivalence), - KUNIT_CASE(apple_dpin_m1_capacity_trace), + KUNIT_CASE(apple_dpin_capacity_trace), KUNIT_CASE(apple_dpin_t602x_rearm_trace), KUNIT_CASE(apple_dpin_preserve_paused_event), KUNIT_CASE(apple_dpin_preserve_coalesced_replace), diff --git a/drivers/thunderbolt/apple.c b/drivers/thunderbolt/apple.c index 6c7b38e3bf422..c6625c5cbb5d3 100644 --- a/drivers/thunderbolt/apple.c +++ b/drivers/thunderbolt/apple.c @@ -733,7 +733,7 @@ static void apple_dpin_retry_fn(struct work_struct *work) /* * Only where both the display allocator and the connection manager can hold - * a tunnel for a pipeline (M1-family hosts, see + * a tunnel for a pipeline (capacity-retry hosts, see * apple_nhi_dp_tunnel_awaits_display()) does a refused tunnel wait for one. */ static void apple_dpin_work_fn(struct work_struct *work) @@ -2218,8 +2218,8 @@ static void apple_nhi_remove(struct platform_device *pdev) * seconds. Not while devices suspend and resume: stop asking before any of * them suspends, let an attempt in flight finish (a wait for appledrm to * load gives way, see apple_dpin_connect()), and ask again once all of them - * have resumed. The tunnel keeps waiting meanwhile. New M1-family handoffs are - * deferred until complete, including after an aborted suspend. Established + * have resumed. The tunnel keeps waiting meanwhile. Suspend-gated handoffs + * are deferred until complete, including after an aborted suspend. Established * handed callbacks and synchronous tunnel teardown remain permitted. */ static void apple_dpin_pause_retries(struct apple_cio *acio) diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index b4e6e231b603a..5a8233150bb62 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -62,6 +62,8 @@ MODULE_PARM_DESC(asym_threshold, * after cfg has been paused. * @remove_work: Work used to remove any unplugged routers after * runtime resume + * @dp_tunnel_work: Re-pair DP resources after retired DPRX work releases its paths + * @dp_tunnel_pending: A pairing request not yet serviced, retained across sleep * @groups: Bandwidth groups used in this domain. */ struct tb_cm { @@ -70,6 +72,8 @@ struct tb_cm { bool hotplug_active; bool stopping; struct delayed_work remove_work; + struct work_struct dp_tunnel_work; + bool dp_tunnel_pending; struct tb_bandwidth_group groups[MAX_GROUPS]; }; @@ -2260,6 +2264,57 @@ static void tb_switch_exit_redrive(struct tb_switch *sw) } } +/* + * A failed DPRX callback can retire the list's tunnel reference while its + * worker still owns the paths. Pairing here, on the same ordered domain + * queue, waits until that worker returns and drops its reference, releasing + * the old fixed HopIDs. Keep only the domain, never an unplugged port. + */ +static void tb_dp_tunnel_work(struct work_struct *work) +{ + struct tb_cm *tcm = container_of(work, struct tb_cm, dp_tunnel_work); + struct tb *tb = tcm_to_tb(tcm); + int ret; + + /* Domain removal drains this queue before unregistering the domain. */ + if (!READ_ONCE(tcm->hotplug_active)) + goto put_domain; + + /* Runtime resume can acquire the domain lock, so do it first. */ + ret = pm_runtime_resume_and_get(&tb->dev); + if (ret < 0) + goto put_domain; + + mutex_lock(&tb->lock); + if (tcm->hotplug_active && tb->root_switch) { + tcm->dp_tunnel_pending = false; + tb_tunnel_dp(tb); + } + mutex_unlock(&tb->lock); + + pm_runtime_mark_last_busy(&tb->dev); + pm_runtime_put_autosuspend(&tb->dev); +put_domain: + tb_domain_put(tb); +} + +static void tb_queue_dp_tunnels(struct tb *tb) +{ + struct tb_cm *tcm = tb_priv(tb); + + lockdep_assert_held(&tb->lock); + if (!tcm->hotplug_active) + return; + + /* An inactive worker leaves this set for thaw or resume. */ + tcm->dp_tunnel_pending = true; + + /* One reference for each queued execution; coalescing adds none. */ + tb_domain_get(tb); + if (!queue_work(tb->wq, &tcm->dp_tunnel_work)) + tb_domain_put(tb); +} + static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port, const char *reason) { @@ -2288,7 +2343,7 @@ static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port, * to create another tunnel. */ tb_recalc_estimated_bandwidth(tb); - tb_tunnel_dp(tb); + tb_queue_dp_tunnels(tb); } static void tb_dp_resource_available(struct tb *tb, struct tb_port *port) @@ -3064,6 +3119,7 @@ static void tb_stop(struct tb *tb) tb_switch_remove(tb->root_switch); tb->root_switch = NULL; tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ + tcm->dp_tunnel_pending = false; } static void tb_deinit(struct tb *tb) @@ -3330,6 +3386,8 @@ static int tb_resume_noirq(struct tb *tb) tb_switch_enter_redrive(tb->root_switch); /* Allow tb_handle_hotplug to progress events */ tcm->hotplug_active = true; + if (tcm->dp_tunnel_pending) + tb_queue_dp_tunnels(tb); tb_dbg(tb, "resume finished\n"); return 0; @@ -3348,6 +3406,8 @@ static int tb_thaw_noirq(struct tb *tb) struct tb_cm *tcm = tb_priv(tb); tcm->hotplug_active = true; + if (tcm->dp_tunnel_pending) + tb_queue_dp_tunnels(tb); return 0; } @@ -3390,6 +3450,8 @@ static void tb_restore_dp_resources(struct tb_switch *sw) static void tb_complete(struct tb *tb) { + struct tb_cm *tcm = tb_priv(tb); + /* * Unregister unplugged XDomains and if there is a case where * another domain is swapped in place of unplugged XDomain we @@ -3404,11 +3466,16 @@ static void tb_complete(struct tb *tb) * Routers kept awake through system sleep send no plug events for * the DP resources released at suspend, so pair them up again here. */ + guard(mutex)(&tb->lock); if (tb->nhi->quirks & QUIRK_NO_SYSTEM_SLEEP) { - guard(mutex)(&tb->lock); tb_restore_dp_resources(tb->root_switch); + tcm->dp_tunnel_pending = false; tb_tunnel_dp(tb); } + + /* A noirq replay may have failed while runtime PM was disabled. */ + if (tcm->dp_tunnel_pending) + tb_queue_dp_tunnels(tb); } static int tb_runtime_suspend(struct tb *tb) @@ -3463,6 +3530,8 @@ static int tb_runtime_resume(struct tb *tb) tb_tunnel_activate(tunnel); tb_switch_enter_redrive(tb->root_switch); tcm->hotplug_active = true; + if (tcm->dp_tunnel_pending) + tb_queue_dp_tunnels(tb); mutex_unlock(&tb->lock); /* @@ -3581,6 +3650,7 @@ struct tb *tb_probe(struct tb_nhi *nhi) INIT_LIST_HEAD(&tcm->tunnel_list); INIT_LIST_HEAD(&tcm->dp_resources); INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work); + INIT_WORK(&tcm->dp_tunnel_work, tb_dp_tunnel_work); tb_init_bandwidth_groups(tcm); tb_dbg(tb, "using software connection manager\n"); diff --git a/drivers/ufs/core/ufs-mcq.c b/drivers/ufs/core/ufs-mcq.c index 13b60a2d06db6..967b51ca57a0e 100644 --- a/drivers/ufs/core/ufs-mcq.c +++ b/drivers/ufs/core/ufs-mcq.c @@ -306,20 +306,24 @@ static int ufshcd_mcq_get_tag(struct ufs_hba *hba, struct cq_entry *cqe) return div_u64(addr, ufshcd_get_ucd_size(hba)); } +static void ufshcd_mcq_compl_cqe(struct ufs_hba *hba, struct cq_entry *cqe) +{ + int tag = ufshcd_mcq_get_tag(hba, cqe); + + ufshcd_compl_one_cqe(hba, tag, cqe); + /* After processing the CQE, mark it as an empty (invalid) entry. */ + cqe->command_desc_base_addr = 0; +} + static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, struct ufs_hw_queue *hwq) { struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); - if (cqe->command_desc_base_addr) { - int tag = ufshcd_mcq_get_tag(hba, cqe); - - ufshcd_compl_one_cqe(hba, tag, cqe); - /* After processed the cqe, mark it empty (invalid) entry */ - cqe->command_desc_base_addr = 0; - } else { + if (cqe->command_desc_base_addr) + ufshcd_mcq_compl_cqe(hba, cqe); + else dev_err(hba->dev, "Abnormal CQ entry!\n"); - } } /* @@ -327,7 +331,7 @@ static void ufshcd_mcq_process_cqe(struct ufs_hba *hba, * controller disabled (HCE = 0). Reading host controller registers, e.g. the * CQ tail pointer (CQTPy), may not be safe with the host controller disabled. * Hence, iterate over all completion queue entries. This won't result in - * double completions because ufshcd_mcq_process_cqe() clears a CQE after it + * double completions because ufshcd_mcq_compl_cqe() clears a CQE after it * has been processed. */ void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba, @@ -338,13 +342,13 @@ void ufshcd_mcq_compl_all_cqes_lock(struct ufs_hba *hba, spin_lock_irqsave(&hwq->cq_lock, flags); while (entries > 0) { - ufshcd_mcq_process_cqe(hba, hwq); + struct cq_entry *cqe = ufshcd_mcq_cur_cqe(hwq); + + if (cqe->command_desc_base_addr) + ufshcd_mcq_compl_cqe(hba, cqe); ufshcd_mcq_inc_cq_head_slot(hwq); entries--; } - - ufshcd_mcq_update_cq_tail_slot(hwq); - hwq->cq_head_slot = hwq->cq_tail_slot; spin_unlock_irqrestore(&hwq->cq_lock, flags); } diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index bc33da89af267..0a9366f21483f 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -5975,8 +5975,6 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) if (blk_mq_is_reserved_rq(rq) || !hwq) return true; - ufshcd_mcq_compl_all_cqes_lock(hba, hwq); - /* * For those cmds of which the cqes are not present in the cq, complete * them explicitly. @@ -5992,19 +5990,6 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv) return true; } -static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) -{ - struct scsi_device *sdev = rq->q->queuedata; - struct Scsi_Host *shost = sdev->host; - struct ufs_hba *hba = shost_priv(shost); - struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq); - - if (!blk_mq_is_reserved_rq(rq) && hwq) - ufshcd_mcq_poll_cqe_lock(hba, hwq); - - return true; -} - /** * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is * invoked from the error handler context or ufshcd_host_reset_and_restore() @@ -6019,10 +6004,18 @@ static bool ufshcd_mcq_compl_one(struct request *rq, void *priv) static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba, bool force_compl) { - blk_mq_tagset_busy_iter(&hba->host->tag_set, - force_compl ? ufshcd_mcq_force_compl_one : - ufshcd_mcq_compl_one, - NULL); + int i; + + for (i = 0; i < hba->nr_hw_queues; i++) { + if (force_compl) + ufshcd_mcq_compl_all_cqes_lock(hba, &hba->uhq[i]); + else + ufshcd_mcq_poll_cqe_lock(hba, &hba->uhq[i]); + } + + if (force_compl) + blk_mq_tagset_busy_iter(&hba->host->tag_set, + ufshcd_mcq_force_compl_one, NULL); } /** @@ -9491,6 +9484,7 @@ static void ufshcd_async_scan(void *data, async_cookie_t cookie) ret = ufshcd_add_lus(hba); out: + ufshcd_release(hba); pm_runtime_put_sync(hba->dev); if (ret) @@ -11213,6 +11207,9 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) ufshcd_init_clk_gating(hba); + /* Released by ufshcd_async_scan(), or by out_release on failure. */ + ufshcd_hold(hba); + ufshcd_init_clk_scaling(hba); /* @@ -11233,7 +11230,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); if (err) { dev_err(hba->dev, "request irq failed\n"); - goto out_disable; + goto out_release; } else { hba->is_irq_enabled = true; } @@ -11249,7 +11246,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) dev_err(hba->dev, "Host controller enable failed\n"); ufshcd_print_evt_hist(hba); ufshcd_print_host_state(hba); - goto out_disable; + goto out_release; } INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work); @@ -11263,7 +11260,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) err = ufshcd_add_scsi_host(hba); if (err) - goto out_disable; + goto out_release; /* Hold auto suspend until async scan completes */ pm_runtime_get_sync(dev); @@ -11283,7 +11280,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) err = ufshcd_link_startup(hba); if (err) - goto out_disable; + goto out_release; if (hba->mcq_enabled) ufshcd_config_mcq(hba); @@ -11300,23 +11297,23 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) /* Verify device initialization by sending NOP OUT UPIU */ err = ufshcd_verify_dev_init(hba); if (err) - goto out_disable; + goto out_release; /* Initiate UFS initialization, and waiting until completion */ err = ufshcd_complete_dev_init(hba); if (err) - goto out_disable; + goto out_release; err = ufshcd_device_params_init(hba); if (err) - goto out_disable; + goto out_release; err = ufshcd_post_device_init(hba); initialized: ufshcd_process_probe_result(hba, probe_start, err); if (err) - goto out_disable; + goto out_release; ufs_sysfs_add_nodes(hba->dev); hba->dme_qos_sysfs_handle = sysfs_get_dirent(hba->dev->kobj.sd, @@ -11327,6 +11324,8 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) ufshcd_pm_qos_init(hba); return 0; +out_release: + ufshcd_release(hba); out_disable: hba->is_irq_enabled = false; ufshcd_hba_exit(hba); diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c index 3991a51263a6d..a2defbe3b7483 100644 --- a/drivers/ufs/host/ufs-mediatek.c +++ b/drivers/ufs/host/ufs-mediatek.c @@ -509,6 +509,7 @@ static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on) struct ufs_mtk_host *host = ufshcd_get_variant(hba); struct phy *mphy = host->mphy; struct arm_smccc_res res; + int err; int ret = 0; if (!mphy || !(on ^ host->mphy_powered_on)) @@ -523,7 +524,18 @@ static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on) usleep_range(200, 210); ufs_mtk_va09_pwr_ctrl(res, 1); } - phy_power_on(mphy); + ret = phy_power_on(mphy); + if (ret) { + if (ufs_mtk_is_va09_supported(hba)) { + ufs_mtk_va09_pwr_ctrl(res, 0); + err = regulator_disable(host->reg_va09); + if (err) + dev_err(hba->dev, + "failed to unwind va09: %d\n", err); + } + dev_err(hba->dev, "failed to enable mphy: %d\n", ret); + return ret; + } } else { phy_power_off(mphy); if (ufs_mtk_is_va09_supported(hba)) { @@ -1295,7 +1307,9 @@ static int ufs_mtk_init(struct ufs_hba *hba) * * Enable phy clocks specifically here. */ - ufs_mtk_mphy_power_on(hba, true); + err = ufs_mtk_mphy_power_on(hba, true); + if (err) + goto out_variant_clear; if (ufs_mtk_is_rtff_mtcmos(hba)) { /* First Restore here, to avoid backup unexpected value */ diff --git a/drivers/usb/host/xhci-plat.c b/drivers/usb/host/xhci-plat.c index b82ed7dfdc864..1d85f895ca10d 100644 --- a/drivers/usb/host/xhci-plat.c +++ b/drivers/usb/host/xhci-plat.c @@ -294,6 +294,8 @@ int xhci_plat_probe(struct platform_device *pdev, struct device *sysdev, const s } device_set_wakeup_capable(&pdev->dev, true); + if (device_property_read_bool(&pdev->dev, "wakeup-source")) + device_wakeup_enable(&pdev->dev); xhci->main_hcd = hcd; diff --git a/drivers/usb/typec/mux.c b/drivers/usb/typec/mux.c index 3b72317e338ef..1df58b4f9012e 100644 --- a/drivers/usb/typec/mux.c +++ b/drivers/usb/typec/mux.c @@ -83,7 +83,7 @@ static void *typec_switch_match(const struct fwnode_handle *fwnode, */ struct typec_switch *fwnode_typec_switch_get(struct fwnode_handle *fwnode) { - struct typec_switch_dev *sw_devs[TYPEC_MUX_MAX_DEVS]; + struct typec_switch_dev *sw_devs[TYPEC_MUX_MAX_DEVS] = {}; struct typec_switch *sw; int count; int err; @@ -433,7 +433,7 @@ static void *typec_mux_match(const struct fwnode_handle *fwnode, */ struct typec_mux *fwnode_typec_mux_get(struct fwnode_handle *fwnode) { - struct typec_mux_dev *mux_devs[TYPEC_MUX_MAX_DEVS]; + struct typec_mux_dev *mux_devs[TYPEC_MUX_MAX_DEVS] = {}; struct typec_mux *mux; int count; int err; diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index cd7b544cc6dfb..0c796597ff48c 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -146,6 +146,7 @@ int tps6598x_block_read(struct tps6598x *tps, u8 reg, void *val, size_t len) memcpy(val, &data[1], len); return 0; } +EXPORT_SYMBOL_GPL(tps6598x_block_read); static int tps6598x_block_write(struct tps6598x *tps, u8 reg, const void *val, size_t len) @@ -398,6 +399,7 @@ int tps6598x_exec_cmd_tmo(struct tps6598x *tps, const char *cmd, return 0; } +EXPORT_SYMBOL_GPL(tps6598x_exec_cmd_tmo); static int tps6598x_exec_cmd(struct tps6598x *tps, const char *cmd, size_t in_len, const u8 *in_data, diff --git a/drivers/usb/typec/tipd/spmi.c b/drivers/usb/typec/tipd/spmi.c index d70a617d4084a..1b5177307ae4a 100644 --- a/drivers/usb/typec/tipd/spmi.c +++ b/drivers/usb/typec/tipd/spmi.c @@ -1,12 +1,14 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include #include #include #include #include #include +#include #include "tps6598x.h" @@ -278,6 +280,89 @@ static int sn201202x_probe(struct spmi_device *device) return ret; } +/* MODE, STATUS, POWER_STATUS and DATA_STATUS (0x03, 0x1a, 0x3f, 0x5f), for the log. */ +static u32 sn201202x_state(struct tps6598x *tps, const char *when) +{ + u32 mode = 0, status = 0, data = 0; + u16 power = 0; + + tps6598x_block_read(tps, 0x03, &mode, sizeof(mode)); + tps6598x_block_read(tps, 0x1a, &status, sizeof(status)); + tps6598x_block_read(tps, 0x3f, &power, sizeof(power)); + tps6598x_block_read(tps, 0x5f, &data, sizeof(data)); + dev_info(tps->dev, "%s: mode %.4s, status %#x, power %#x, data %#x\n", when, + (char *)&mode, status, power, data); + return status; +} + +/* + * "DISC" in the debug mode: unlocked with @key ("apple,unlock-key", the ADT + * target-type's first four characters reversed, as m1n1 does), "DBMa" entered + * for the command and left at once, while the port is still disconnected. + * Called with tps->lock held. + */ +static int sn201202x_disc(struct tps6598x *tps, const char *key, u8 seconds) +{ + static const u8 lock_key[4]; + int lock, dbma = -ENODATA, disc = -ENODATA, leave = -ENODATA, relock; + u32 status, last; + unsigned int i; + u8 en = 1; + + sn201202x_state(tps, "before DISC"); + lock = tps6598x_exec_cmd_tmo(tps, "LOCK", 4, (const u8 *)key, 0, NULL, 1000, 0); + if (!lock) + dbma = tps6598x_exec_cmd_tmo(tps, "DBMa", sizeof(en), &en, 0, NULL, 1000, 0); + if (!lock && !dbma) { + disc = tps6598x_exec_cmd_tmo(tps, "DISC", sizeof(seconds), &seconds, 0, NULL, + 1000 * (seconds + 1), 0); + sn201202x_state(tps, "DISC done"); + en = 0; + leave = tps6598x_exec_cmd_tmo(tps, "DBMa", sizeof(en), &en, 0, NULL, 1000, 0); + } + relock = tps6598x_exec_cmd_tmo(tps, "LOCK", sizeof(lock_key), lock_key, 0, NULL, 1000, 0); + dev_info(tps->dev, "simulated disconnect: LOCK %d, DBMa %d, DISC %d, DBMa 0 %d, LOCK 0 %d\n", + lock, dbma, disc, leave, relock); + + /* Diagnostic: every STATUS change for 25 s. */ + last = sn201202x_state(tps, "locked again"); + for (i = 0; i < 50; i++) { + msleep(500); + if (!tps6598x_block_read(tps, 0x1a, &status, sizeof(status)) && status != last) + last = sn201202x_state(tps, "after"); + } + return lock ?: dbma ?: disc; +} + +/* + * Simulated disconnect ("DISC"): the port drops its partner, VBUS included, for + * @seconds and then sees it again as freshly plugged in. -EPROBE_DEFER until + * the controller at @np is bound. + */ +int sn201202x_simulate_disconnect(struct device_node *np, u8 seconds) +{ + struct spmi_device *sdev = spmi_find_device_by_of_node(np); + struct tps6598x *tps; + const char *key; + int ret = -EPROBE_DEFER; + + if (!sdev) + return ret; + if (device_is_bound(&sdev->dev)) { + tps = spmi_device_get_drvdata(sdev); + guard(mutex)(&tps->lock); + if (!tps->dbma_at_probe && !of_property_read_string(np, "apple,unlock-key", &key) && + strlen(key) == 4) + ret = sn201202x_disc(tps, key, seconds); + else + ret = tps6598x_exec_cmd_tmo(tps, "DISC", sizeof(seconds), &seconds, 0, + NULL, 1000, 0); + } + put_device(&sdev->dev); + return ret; +} +EXPORT_SYMBOL_GPL(sn201202x_simulate_disconnect); + static void sn201202x_remove(struct spmi_device *device) { struct tps6598x *tps = spmi_device_get_drvdata(device); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 7fe0a63d690a3..579b82e27b5ac 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -9622,7 +9622,6 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, struct completion sync_reads; unsigned long i = 0; struct btrfs_bio *bbio; - int ret; /* * Fast path for synchronous reads which completes in this call, io_uring @@ -9669,10 +9668,10 @@ int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, if (uring_ctx) { if (refcount_dec_and_test(&priv->pending_refs)) { - ret = blk_status_to_errno(READ_ONCE(priv->status)); - btrfs_uring_read_extent_endio(uring_ctx, ret); + int error = blk_status_to_errno(READ_ONCE(priv->status)); + + btrfs_uring_read_extent_endio(uring_ctx, error); kfree(priv); - return ret; } return -EIOCBQUEUED; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index a537036303a47..833052306964b 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -4559,7 +4559,7 @@ static void btrfs_uring_read_finished(struct io_tw_req tw_req, io_tw_token_t tw) size_t page_offset; ssize_t ret; - /* The inode lock has already been acquired in btrfs_uring_read_extent. */ + /* The inode lock has already been acquired in btrfs_encoded_read(). */ btrfs_lockdep_inode_acquire(inode, i_rwsem); if (priv->err) { @@ -4625,7 +4625,6 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, struct iovec *iov, struct io_uring_cmd *cmd) { struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); - struct extent_io_tree *io_tree = &inode->io_tree; struct page **pages = NULL; struct btrfs_uring_priv *priv = NULL; unsigned long nr_pages; @@ -4681,8 +4680,6 @@ static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, return -EIOCBQUEUED; out_fail: - btrfs_unlock_extent(io_tree, start, lockend, &cached_state); - btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); kfree(priv); for (int i = 0; i < nr_pages; i++) { if (pages[i]) @@ -4795,7 +4792,7 @@ static int btrfs_uring_encoded_read(struct io_uring_cmd *cmd, unsigned int issue ret = btrfs_encoded_read(&kiocb, &data->iter, &data->args, &cached_state, &disk_bytenr, &disk_io_size); if (ret == -EAGAIN) - goto out_acct; + goto out_free; if (ret < 0 && ret != -EIOCBQUEUED) goto out_free; @@ -4823,8 +4820,10 @@ static int btrfs_uring_encoded_read(struct io_uring_cmd *cmd, unsigned int issue cached_state, disk_bytenr, disk_io_size, count, data->args.compression, data->iov, cmd); - - goto out_acct; + if (ret == -EIOCBQUEUED) + goto out_acct; + btrfs_unlock_extent(io_tree, start, lockend, &cached_state); + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); } out_free: @@ -4835,8 +4834,10 @@ static int btrfs_uring_encoded_read(struct io_uring_cmd *cmd, unsigned int issue add_rchar(current, ret); inc_syscr(current); - if (ret != -EIOCBQUEUED && ret != -EAGAIN) + if (ret != -EIOCBQUEUED) { kfree(data); + bc->data = NULL; + } return ret; } @@ -4865,6 +4866,11 @@ static int btrfs_uring_encoded_write(struct io_uring_cmd *cmd, unsigned int issu goto out_acct; } + if (issue_flags & IO_URING_F_NONBLOCK) { + ret = -EAGAIN; + goto out_acct; + } + if (!data) { data = kzalloc_obj(*data, GFP_NOFS); if (!data) { @@ -4933,11 +4939,6 @@ static int btrfs_uring_encoded_write(struct io_uring_cmd *cmd, unsigned int issu } } - if (issue_flags & IO_URING_F_NONBLOCK) { - ret = -EAGAIN; - goto out_acct; - } - pos = data->args.offset; ret = rw_verify_area(WRITE, file, &pos, data->args.len); if (ret < 0) @@ -4963,8 +4964,8 @@ static int btrfs_uring_encoded_write(struct io_uring_cmd *cmd, unsigned int issu add_wchar(current, ret); inc_syscw(current); - if (ret != -EAGAIN) - kfree(data); + kfree(data); + bc->data = NULL; return ret; } diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 3b3f04521369d..9db8f8b6d3d54 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -550,8 +550,10 @@ static int scrub_print_warning_inode(u64 inum, u64 offset, u64 num_bytes, } ret = paths_from_inode(inum, ipath); - if (ret < 0) + if (ret < 0) { + btrfs_put_root(local_root); goto err; + } /* * we deliberately ignore the bit ipath might have been too small to diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c index ab55d10bd71fd..cb001a211f557 100644 --- a/fs/btrfs/xattr.c +++ b/fs/btrfs/xattr.c @@ -161,6 +161,7 @@ int btrfs_setxattr(struct btrfs_trans_handle *trans, struct inode *inode, const u16 old_data_len = btrfs_dir_data_len(leaf, di); const u32 item_size = btrfs_item_size(leaf, slot); const u32 data_size = sizeof(*di) + name_len + size; + unsigned long name_ptr; unsigned long data_ptr; char *ptr; @@ -189,8 +190,16 @@ int btrfs_setxattr(struct btrfs_trans_handle *trans, struct inode *inode, ptr = btrfs_item_ptr(leaf, slot, char); ptr += btrfs_item_size(leaf, slot) - data_size; di = (struct btrfs_dir_item *)ptr; + memzero_extent_buffer(leaf, (unsigned long)ptr + + offsetof(struct btrfs_dir_item, location), + sizeof(struct btrfs_disk_key)); + btrfs_set_dir_flags(leaf, di, BTRFS_FT_XATTR); + btrfs_set_dir_transid(leaf, di, trans->transid); + btrfs_set_dir_name_len(leaf, di, name_len); btrfs_set_dir_data_len(leaf, di, size); + name_ptr = (unsigned long)(di + 1); data_ptr = ((unsigned long)(di + 1)) + name_len; + write_extent_buffer(leaf, name, name_ptr, name_len); write_extent_buffer(leaf, value, data_ptr, size); } else { /* @@ -320,10 +329,8 @@ ssize_t btrfs_listxattr(struct dentry *dentry, char *buffer, size_t size) if (!size) goto next; - if (!buffer || (name_len + 1) > size_left) { - iter_ret = -ERANGE; - break; - } + if (!buffer || (name_len + 1) > size_left) + return -ERANGE; read_extent_buffer(leaf, buffer, name_ptr, name_len); buffer[name_len] = '\0'; diff --git a/fs/dcache.c b/fs/dcache.c index 073c2ce2d4448..d6fbb4fa7dbc5 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -1583,6 +1583,8 @@ EXPORT_SYMBOL(path_has_submounts); * * Only one of d_invalidate() and d_set_mounted() must succeed. For * this reason take rename_lock and d_lock on dentry and ancestors. + * Likewise for dont_mount() which marks a dentry that is being removed + * under d_lock. */ int d_set_mounted(struct dentry *dentry) { @@ -1599,7 +1601,7 @@ int d_set_mounted(struct dentry *dentry) spin_unlock(&p->d_lock); } spin_lock(&dentry->d_lock); - if (!d_unlinked(dentry)) { + if (!d_unlinked(dentry) && !cant_mount(dentry)) { ret = -EBUSY; if (!d_mountpoint(dentry)) { dentry->d_flags |= DCACHE_MOUNTED; diff --git a/fs/fcntl.c b/fs/fcntl.c index c158f082f1da4..9b71bc9d71d15 100644 --- a/fs/fcntl.c +++ b/fs/fcntl.c @@ -372,6 +372,9 @@ static long fcntl_set_rw_hint(struct file *file, unsigned long arg) u64 __user *argp = (u64 __user *)arg; u64 hint; + /* nothing is ever written to it */ + if (IS_IMMUTABLE(inode)) + return -EPERM; if (!inode_owner_or_capable(file_mnt_idmap(file), inode)) return -EPERM; diff --git a/fs/fs_context.c b/fs/fs_context.c index 23ad66cd94e1f..a50584df97bea 100644 --- a/fs/fs_context.c +++ b/fs/fs_context.c @@ -315,6 +315,10 @@ struct fs_context *fs_context_for_reconfigure(struct dentry *dentry, unsigned int sb_flags, unsigned int sb_flags_mask) { + /* kernel-internal superblocks are nobody's to reconfigure */ + if (dentry->d_sb->s_flags & SB_NOUSER) + return ERR_PTR(-EINVAL); + return alloc_fs_context(dentry->d_sb->s_type, dentry, sb_flags, sb_flags_mask, FS_CONTEXT_FOR_RECONFIGURE); } diff --git a/fs/inode.c b/fs/inode.c index c6b08e8c5f627..efa801f3eb110 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -763,21 +763,18 @@ void clear_inode(struct inode *inode) fsverity_cleanup_inode(inode); /* - * We have to cycle the i_pages lock here because reclaim can be in the - * process of removing the last page (in __filemap_remove_folio()) - * and we must not free the mapping under it. + * We have to cycle the i_pages lock here because reclaim + * can be in the process of removing the last page (in + * __filemap_remove_folio()) and we must not free the mapping + * under it. We also remove nodes which are empty; these + * can occur in two different ways. The first is that radix + * tree expansion can fail partway and the second is that THP + * collapse_file() can allocate some temporary nodes and not + * clean them up. */ - xa_lock_irq(&inode->i_data.i_pages); + xa_destroy(&inode->i_data.i_pages); + BUG_ON(inode->i_data.nrpages); - /* - * Almost always, mapping_empty(&inode->i_data) here; but there are - * two known and long-standing ways in which nodes may get left behind - * (when deep radix-tree node allocation failed partway; or when THP - * collapse_file() failed). Until those two known cases are cleaned up, - * or a cleanup function is called here, do not BUG_ON(!mapping_empty), - * nor even WARN_ON(!mapping_empty). - */ - xa_unlock_irq(&inode->i_data.i_pages); BUG_ON(!(inode_state_read_once(inode) & I_FREEING)); BUG_ON(inode_state_read_once(inode) & I_CLEAR); BUG_ON(!list_empty(&inode->i_wb_list)); diff --git a/fs/mount.h b/fs/mount.h index 94fcc306d21e7..2223fb141499f 100644 --- a/fs/mount.h +++ b/fs/mount.h @@ -33,7 +33,8 @@ struct mnt_namespace { } __randomize_layout; struct mnt_pcp { - int mnt_count; + unsigned int mnt_gets; + unsigned int mnt_puts; int mnt_writers; }; @@ -233,6 +234,9 @@ static inline struct mnt_namespace *to_mnt_ns(struct ns_common *ns) #ifdef CONFIG_FSNOTIFY static inline void mnt_notify_add(struct mount *m) { + /* queued already under this namespace_sem hold */ + if (!list_empty(&m->to_notify)) + return; /* Optimize the case where there are no watches */ if ((m->mnt_ns && m->mnt_ns->n_fsnotify_marks) || (m->prev_ns && m->prev_ns->n_fsnotify_marks)) diff --git a/fs/namei.c b/fs/namei.c index 19ce43c9a6e66..27e0637bfdda2 100644 --- a/fs/namei.c +++ b/fs/namei.c @@ -2086,6 +2086,33 @@ static noinline const char *pick_link(struct nameidata *nd, struct path *link, return NULL; } +/* + * Be careful in case the dentry is unlinked or renamed. Any mounts + * stacked on top of it are going away. We need to make sure that we + * don't reveal the underyling dentry during refwalk. In rcuwalk we + * catch this via d_seq and another lookup for the name. Give the same + * guarantee in refwalk. + */ +static bool unlink_may_reveal(struct nameidata *nd, int flags, + struct path *path, struct dentry *dentry) +{ + /* ".." and LOOKUP_DOWN may land on an unhashed directory */ + if (flags & WALK_NOFOLLOW) + return false; + if (nd->flags & LOOKUP_REVAL) + return false; + /* We crossed onto a mount and the name led us here while it still existed */ + if (path->mnt != nd->path.mnt) + return false; + /* only a name on its way out is flagged */ + if (likely(!cant_mount(dentry))) + return false; + if (!d_unlinked(dentry)) + return false; + dput(no_free_ptr(path->dentry)); + return true; +} + /* * Do we need to follow links? We _really_ want to be able * to do this check without having to look at inode->i_op, @@ -2115,6 +2142,8 @@ static noinline const char *step_into_slowpath(struct nameidata *nd, int flags, if (unlikely(!inode)) return ERR_PTR(-ENOENT); } else { + if (unlikely(unlink_may_reveal(nd, flags, &path, dentry))) + return ERR_PTR(-ESTALE); dput(nd->path.dentry); if (nd->path.mnt != path.mnt) mntput(nd->path.mnt); diff --git a/fs/namespace.c b/fs/namespace.c index 3d5cd5bf3b053..ec93bdac9e789 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -130,7 +130,6 @@ static void mnt_ns_release(struct mnt_namespace *ns) { /* keep alive for {list,stat}mount() */ if (ns && refcount_dec_and_test(&ns->passive)) { - fsnotify_mntns_delete(ns); put_user_ns(ns->user_ns); kfree(ns); } @@ -249,16 +248,24 @@ void mnt_release_group_id(struct mount *mnt) mnt->mnt_group_id = 0; } -/* - * vfsmount lock must be held for read - */ -static inline void mnt_add_count(struct mount *mnt, int n) +static inline void mnt_inc_count(struct mount *mnt) { #ifdef CONFIG_SMP - this_cpu_add(mnt->mnt_pcp->mnt_count, n); + this_cpu_inc(mnt->mnt_pcp->mnt_gets); #else preempt_disable(); - mnt->mnt_count += n; + mnt->mnt_count++; + preempt_enable(); +#endif +} + +static inline void mnt_dec_count(struct mount *mnt) +{ +#ifdef CONFIG_SMP + this_cpu_inc(mnt->mnt_pcp->mnt_puts); +#else + preempt_disable(); + mnt->mnt_count--; preempt_enable(); #endif } @@ -269,14 +276,17 @@ static inline void mnt_add_count(struct mount *mnt, int n) int mnt_get_count(struct mount *mnt) { #ifdef CONFIG_SMP - int count = 0; + unsigned int gets = 0, puts = 0; int cpu; - for_each_possible_cpu(cpu) { - count += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_count; - } + /* puts first, so a put counted here has its get counted below */ + for_each_possible_cpu(cpu) + puts += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_puts; + smp_mb(); /* pairs with the smp_wmb() in mntput_no_expire() */ + for_each_possible_cpu(cpu) + gets += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_gets; - return count; + return gets - puts; #else return mnt->mnt_count; #endif @@ -305,7 +315,7 @@ static struct mount *alloc_vfsmnt(const char *name) if (!mnt->mnt_pcp) goto out_free_devname; - this_cpu_add(mnt->mnt_pcp->mnt_count, 1); + this_cpu_inc(mnt->mnt_pcp->mnt_gets); #else mnt->mnt_count = 1; mnt->mnt_writers = 0; @@ -322,6 +332,9 @@ static struct mount *alloc_vfsmnt(const char *name) INIT_HLIST_NODE(&mnt->mnt_mp_list); INIT_HLIST_HEAD(&mnt->mnt_stuck_children); INIT_HLIST_NODE(&mnt->mnt_ns_visible); +#ifdef CONFIG_FSNOTIFY + INIT_LIST_HEAD(&mnt->to_notify); +#endif RB_CLEAR_NODE(&mnt->mnt_node); mnt->mnt.mnt_idmap = &nop_mnt_idmap; } @@ -746,13 +759,13 @@ int __legitimize_mnt(struct vfsmount *bastard, unsigned seq) if (bastard == NULL) return 0; mnt = real_mount(bastard); - mnt_add_count(mnt, 1); - smp_mb(); // see mntput_no_expire() and do_umount() + mnt_inc_count(mnt); + smp_mb(); /* see mntput_no_expire_slowpath() and do_umount() */ if (likely(!read_seqretry(&mount_lock, seq))) return 0; lock_mount_hash(); if (unlikely(bastard->mnt_flags & (MNT_SYNC_UMOUNT | MNT_DOOMED))) { - mnt_add_count(mnt, -1); + mnt_dec_count(mnt); unlock_mount_hash(); return 1; } @@ -1058,7 +1071,10 @@ static void attach_mnt(struct mount *mnt, struct mount *parent, void mnt_change_mountpoint(struct mount *parent, struct mountpoint *mp, struct mount *mnt) { struct mountpoint *old_mp = mnt->mnt_mp; + struct mount *old_parent = mnt->mnt_parent; + if (old_parent->overmount == mnt) + old_parent->overmount = NULL; list_del_init(&mnt->mnt_child); hlist_del_init(&mnt->mnt_mp_list); hlist_del_init_rcu(&mnt->mnt_hash); @@ -1254,6 +1270,7 @@ static struct mount *clone_mnt(struct mount *old, struct dentry *root, mnt->mnt.mnt_flags = READ_ONCE(old->mnt.mnt_flags) & ~MNT_INTERNAL_FLAGS; + mnt->mnt_t_flags = old->mnt_t_flags & T_UNBINDABLE; if (flag & (CL_SLAVE | CL_PRIVATE)) mnt->mnt_group_id = 0; /* not a peer of original */ @@ -1347,7 +1364,7 @@ static void noinline mntput_no_expire_slowpath(struct mount *mnt) * mount_lock, we'll see their refcount increment here. */ smp_mb(); - mnt_add_count(mnt, -1); + mnt_dec_count(mnt); count = mnt_get_count(mnt); if (count != 0) { WARN_ON(count < 0); @@ -1404,7 +1421,8 @@ static void mntput_no_expire(struct mount *mnt) * non-NULL under rcu_read_lock(), the reference * we are dropping is not the final one. */ - mnt_add_count(mnt, -1); + smp_wmb(); /* pairs with the smp_mb() in mnt_get_count() */ + mnt_dec_count(mnt); rcu_read_unlock(); return; } @@ -1426,7 +1444,7 @@ EXPORT_SYMBOL(mntput); struct vfsmount *mntget(struct vfsmount *mnt) { if (mnt) - mnt_add_count(real_mount(mnt), 1); + mnt_inc_count(real_mount(mnt)); return mnt; } EXPORT_SYMBOL(mntget); @@ -1783,6 +1801,8 @@ static void umount_tree(struct mount *mnt, enum umount_tree_flags how) /* Gather the mounts to umount */ for (p = mnt; p; p = next_mnt(p, mnt)) { + /* A mount is unmounted once. */ + VFS_WARN_ON_ONCE(p->mnt.mnt_flags & MNT_UMOUNT); p->mnt.mnt_flags |= MNT_UMOUNT; if (mnt_ns_attached(p)) move_from_ns(p); @@ -1976,6 +1996,9 @@ static int do_umount(struct mount *mnt, int flags) * detach_mounts allows lazily unmounting those mounts instead of * leaking them. * + * The dentry is unhashed before the mounts go so that no lookup finds + * what they covered. The caller removes it for good afterwards. + * * The caller may hold dentry->d_inode->i_rwsem. */ void __detach_mounts(struct dentry *dentry) @@ -1989,6 +2012,8 @@ void __detach_mounts(struct dentry *dentry) if (!lookup_mountpoint(dentry, &mp)) return; + /* the name goes first, what covered it goes second */ + d_drop(dentry); event++; while (mp.node.next) { mnt = hlist_entry(mp.node.next, struct mount, mnt_mp_list); @@ -2345,6 +2370,18 @@ bool has_locked_children(struct mount *mnt, struct dentry *dentry) return __has_locked_children(mnt, dentry); } +/* locks: namespace_shared && pinned(mnt) || mount_locked_reader */ +static bool __has_children(struct mount *mnt, struct dentry *dentry) +{ + struct mount *child; + + list_for_each_entry(child, &mnt->mnt_mounts, mnt_child) { + if (is_subdir(child->mnt_mountpoint, dentry)) + return true; + } + return false; +} + /* * Check that there aren't references to earlier/same mount namespaces in the * specified subtree. Such references can act as pins for mount namespaces @@ -2564,11 +2601,11 @@ enum mnt_tree_flags_t { static int attach_recursive_mnt(struct mount *source_mnt, const struct pinned_mountpoint *dest) { - struct user_namespace *user_ns = current->nsproxy->mnt_ns->user_ns; struct mount *dest_mnt = dest->parent; struct mountpoint *dest_mp = dest->mp; HLIST_HEAD(tree_list); struct mnt_namespace *ns = dest_mnt->mnt_ns; + struct user_namespace *user_ns = ns->user_ns; struct pinned_mountpoint root = {}; struct mountpoint *shorter = NULL; struct mount *child, *p; @@ -2577,13 +2614,31 @@ static int attach_recursive_mnt(struct mount *source_mnt, int err = 0; bool moving = mnt_has_parent(source_mnt); + /* + * A caller in an unprivileged mount namespaces may trigger an + * automount and propagate locked mounts into privileged mount + * namespaces. Take ownership from the target mount namespace. + * It's equivalent for everything but the automount case. + * + * Detached trees in anonymous mount namespaces by be handed + * over via SCM_RIGHTS or inherited in other ways on purpose + * the attaching task's mount namespace is authoritative, not + * the creator of the detached tree. + */ + if (is_anon_ns(ns)) + user_ns = current->nsproxy->mnt_ns->user_ns; + else + user_ns = ns->user_ns; + /* * Preallocate a mountpoint in case the new mounts need to be * mounted beneath mounts on the same mountpoint. */ - for (top = source_mnt; unlikely(top->overmount); top = top->overmount) { + for (top = source_mnt; ; top = top->overmount) { if (!shorter && is_mnt_ns_file(top->mnt.mnt_root)) shorter = top->mnt_mp; + if (likely(!top->overmount)) + break; } err = get_mountpoint(top->mnt.mnt_root, &root); if (err) @@ -2654,15 +2709,17 @@ static int attach_recursive_mnt(struct mount *source_mnt, /* * If @q was locked it was meant to hide * whatever was under it. Let @child take over - * that job and lock it, then we can unlock @q. - * That'll allow another namespace to shed @q - * and reveal @child. Clearly, that mounter - * consented to this by not severing the mount - * relationship. Otherwise, what's the point. + * that job and lock it. If @child is the mount + * the caller placed we can then unlock @q: + * nothing another namespace does removes it + * again. A propagated copy goes away when the + * mounter of the original unmounts it, so @q + * keeps its lock. */ if (IS_MNT_LOCKED(q)) { child->mnt.mnt_flags |= MNT_LOCKED; - q->mnt.mnt_flags &= ~MNT_LOCKED; + if (child == source_mnt) + q->mnt.mnt_flags &= ~MNT_LOCKED; } mnt_change_mountpoint(r, mp, q); } @@ -2986,6 +3043,13 @@ static struct mount *__do_loopback(const struct path *old_path, if (!may_copy_tree(old_path)) return ERR_PTR(-EINVAL); + /* a pseudo dentry is freed without an RCU delay, no walk may find it */ + if (old_path->dentry->d_flags & DCACHE_NORCU) + return ERR_PTR(-EINVAL); + + if (recurse && !old->mnt_ns) + return ERR_PTR(-EINVAL); + if (!recurse && __has_locked_children(old, old_path->dentry)) return ERR_PTR(-EINVAL); @@ -3025,7 +3089,11 @@ static int do_loopback(const struct path *path, const char *old_name, if (IS_ERR(mnt)) return PTR_ERR(mnt); - err = graft_tree(mnt, &mp); + /* the copy may carry mount namespace files from below the source */ + if (recurse && !check_for_nsfs_mounts(mnt)) + err = -EINVAL; + else + err = graft_tree(mnt, &mp); if (err) { lock_mount_hash(); umount_tree(mnt, UMOUNT_SYNC); @@ -3107,12 +3175,18 @@ static struct mnt_namespace *create_new_namespace(struct path *path, struct mount *mnt; unsigned int copy_flags = 0; bool locked = false, recurse = flags & MOUNT_COPY_RECURSIVE; + bool foreign = user_ns != ns->user_ns; if (unlikely(!d_can_lookup(path->dentry))) return ERR_PTR(-ENOTDIR); - if (user_ns != ns->user_ns) - copy_flags |= CL_SLAVE; + /* + * Without privileges over the mount namespace the copy is made from + * nothing mounted below @path may be left out. It would reveal what + * it covers. That's what unshare() gives such a caller as well. + */ + if (foreign) + copy_flags |= CL_SLAVE | CL_COPY_UNBINDABLE; new_ns = alloc_mnt_ns(user_ns, false); if (IS_ERR(new_ns)) @@ -3146,10 +3220,14 @@ static struct mnt_namespace *create_new_namespace(struct path *path, /* * We don't emulate unshare()ing a mount namespace. We stick to * the restrictions of creating detached bind-mounts. It has a - * lot saner and simpler semantics. + * lot saner and simpler semantics. A caller without privileges + * over the mount namespace can't leave out any child though. */ if (flags & MOUNT_COPY_NEW) mnt = clone_mnt(real_mount(path->mnt), path->dentry, copy_flags); + else if (foreign && !recurse && + __has_children(real_mount(path->mnt), path->dentry)) + mnt = ERR_PTR(-EINVAL); else mnt = __do_loopback(path, recurse, copy_flags); scoped_guard(mount_writer) { @@ -3166,7 +3244,7 @@ static struct mnt_namespace *create_new_namespace(struct path *path, * of the real rootfs we created. */ attach_mnt(mnt, new_ns_root, mp.mp); - if (user_ns != ns->user_ns) + if (foreign) lock_mnt_tree(new_ns_root); } @@ -3464,7 +3542,7 @@ static int do_set_group(const struct path *from_path, const struct path *to_path return -EINVAL; /* Setting sharing groups is only allowed on private mounts */ - if (IS_MNT_SHARED(to) || IS_MNT_SLAVE(to)) + if (IS_MNT_SHARED(to) || IS_MNT_SLAVE(to) || IS_MNT_UNBINDABLE(to)) return -EINVAL; /* From should not be private */ @@ -3736,7 +3814,7 @@ static int do_add_mount(struct mount *newmnt, const struct pinned_mountpoint *mp if (!(mnt_flags & MNT_SHRINKABLE)) return -EINVAL; /* ... and for those we'd better have mountpoint still alive */ - if (!parent->mnt_ns) + if (!is_mounted(&parent->mnt)) return -EINVAL; } @@ -3946,6 +4024,8 @@ void mark_mounts_for_expiry(struct list_head *mounts) list_for_each_entry_safe(mnt, next, mounts, mnt_expire) { if (!is_mounted(&mnt->mnt)) continue; + /* lock_mnt_tree() leaves expirable mounts alone */ + VFS_WARN_ON_ONCE(IS_MNT_LOCKED(mnt)); if (!xchg(&mnt->mnt_expiry_mark, 1) || propagate_mount_busy(mnt, 1)) continue; @@ -3953,6 +4033,11 @@ void mark_mounts_for_expiry(struct list_head *mounts) } while (!list_empty(&graveyard)) { mnt = list_first_entry(&graveyard, struct mount, mnt_expire); + /* an earlier umount_tree() may have moved a busy mount here */ + if (propagate_mount_busy(mnt, 1)) { + list_move(&mnt->mnt_expire, mounts); + continue; + } touch_mnt_namespace(mnt->mnt_ns); umount_tree(mnt, UMOUNT_PROPAGATE|UMOUNT_SYNC); } @@ -3960,17 +4045,38 @@ void mark_mounts_for_expiry(struct list_head *mounts) EXPORT_SYMBOL_GPL(mark_mounts_for_expiry); +/* + * Unmount @mnt if it's a shrinkable mount without children that nobody uses. + * + * mount_lock must be held for write + */ +static bool shrink_submount(struct mount *mnt) +{ + /* not the kernel's to remove either */ + if (IS_MNT_LOCKED(mnt) || propagate_mount_busy(mnt, 1)) + return false; + touch_mnt_namespace(mnt->mnt_ns); + umount_tree(mnt, UMOUNT_PROPAGATE|UMOUNT_SYNC); + return true; +} + /* * Ripoff of 'select_parent()' * - * search the list of submounts for a given mountpoint, and move any - * shrinkable submounts to the 'graveyard' list. + * unmount the shrinkable submounts of @parent that aren't busy, children + * before their parent, and say whether anything went + * + * The cursor into the children of @this_parent survives the umount of a + * child mount without child mounts. The mounts that get umounted together with + * it are located under receiving mounts of @this_parent and never under + * @this_parent itself. The one exception is @this_parent getting unmounted + * then the walk starts over. */ -static int select_submounts(struct mount *parent, struct list_head *graveyard) +static bool __shrink_submounts(struct mount *parent) { struct mount *this_parent = parent; struct list_head *next; - int found = 0; + bool shrunk = false; repeat: next = this_parent->mnt_mounts.next; @@ -3989,42 +4095,45 @@ static int select_submounts(struct mount *parent, struct list_head *graveyard) this_parent = mnt; goto repeat; } - - if (!propagate_mount_busy(mnt, 1)) { - list_move_tail(&mnt->mnt_expire, graveyard); - found++; - } + if (!shrink_submount(mnt)) + continue; + shrunk = true; + if (unlikely(this_parent->mnt.mnt_flags & MNT_UMOUNT)) + return true; } /* * All done at this level ... ascend and resume the search */ if (this_parent != parent) { - next = this_parent->mnt_child.next; - this_parent = this_parent->mnt_parent; + struct mount *mnt = this_parent; + + next = mnt->mnt_child.next; + this_parent = mnt->mnt_parent; + /* its children are gone, maybe it can go as well */ + if (shrink_submount(mnt)) { + shrunk = true; + if (unlikely(this_parent->mnt.mnt_flags & MNT_UMOUNT)) + return true; + } goto resume; } - return found; + return shrunk; } /* - * process a list of expirable mountpoints with the intent of discarding any - * submounts of a specific parent mountpoint + * unmount the shrinkable submounts of @mnt that aren't busy + * + * The busy check and the umount of a mount are adjacent. An umount can + * still empty or move a mount in a part of the tree that was walked + * already, so walk again until nothing goes. * * mount_lock must be held for write */ static void shrink_submounts(struct mount *mnt) { - LIST_HEAD(graveyard); - struct mount *m; - - /* extract submounts of 'mountpoint' from the expiration list */ - while (select_submounts(mnt, &graveyard)) { - while (!list_empty(&graveyard)) { - m = list_first_entry(&graveyard, struct mount, - mnt_expire); - touch_mnt_namespace(m->mnt_ns); - umount_tree(m, UMOUNT_PROPAGATE|UMOUNT_SYNC); - } + for (;;) { + if (!__shrink_submounts(mnt)) + break; } } @@ -4189,6 +4298,8 @@ static void free_mnt_ns(struct mnt_namespace *ns) if (!is_anon_ns(ns)) ns_common_free(ns); dec_mnt_namespaces(ns->ucounts); + /* the last active reference is gone, no mark can show up anymore */ + fsnotify_mntns_delete(ns); mnt_ns_tree_remove(ns); } @@ -4690,14 +4801,15 @@ int path_pivot_root(struct path *new, struct path *old) new_mnt = real_mount(new->mnt); root_mnt = real_mount(root.mnt); + /* only a mounted mount has a parent that namespace_sem pins */ + if (!check_mnt(root_mnt) || !check_mnt(new_mnt)) + return -EINVAL; ex_parent = new_mnt->mnt_parent; root_parent = root_mnt->mnt_parent; if (IS_MNT_SHARED(old_mnt) || IS_MNT_SHARED(ex_parent) || IS_MNT_SHARED(root_parent)) return -EINVAL; - if (!check_mnt(root_mnt) || !check_mnt(new_mnt)) - return -EINVAL; if (new_mnt->mnt.mnt_flags & MNT_LOCKED) return -EINVAL; if (d_unlinked(new->dentry)) @@ -4948,6 +5060,11 @@ static int do_mount_setattr(const struct path *path, struct mount_kattr *kattr) * propagation. */ namespace_lock(); + /* invent_group_ids() walks the tree, only walk a mounted one */ + if (!anon_ns_root(mnt) && !check_mnt(mnt)) { + namespace_unlock(); + return -EINVAL; + } if (kattr->propagation == MS_SHARED) { err = invent_group_ids(mnt, kattr->kflags & MOUNT_KATTR_RECURSE); if (err) { @@ -4960,6 +5077,7 @@ static int do_mount_setattr(const struct path *path, struct mount_kattr *kattr) err = -EINVAL; lock_mount_hash(); + /* Checked under namespace_sem already if the propagation changes. */ if (!anon_ns_root(mnt) && !check_mnt(mnt)) goto out; @@ -5312,15 +5430,25 @@ static void statmount_sb_basic(struct kstatmount *s) s->sm.sb_flags = sb->s_flags & (SB_RDONLY|SB_SYNCHRONOUS|SB_DIRSYNC|SB_LAZYTIME); } +static void statmount_mnt_parent(struct kstatmount *s, const struct mount *m) +{ + s->sm.mnt_parent_id = m->mnt_parent->mnt_id_unique; + s->sm.mnt_parent_id_old = m->mnt_parent->mnt_id; +} + static void statmount_mnt_basic(struct kstatmount *s) { struct mount *m = real_mount(s->mnt); s->sm.mask |= STATMOUNT_MNT_BASIC; s->sm.mnt_id = m->mnt_id_unique; - s->sm.mnt_parent_id = m->mnt_parent->mnt_id_unique; s->sm.mnt_id_old = m->mnt_id; - s->sm.mnt_parent_id_old = m->mnt_parent->mnt_id; + /* An unmounted mount is cut loose from its parent under mount_lock alone. */ + if (likely(is_mounted(s->mnt))) + statmount_mnt_parent(s, m); + else + scoped_guard(mount_locked_reader) + statmount_mnt_parent(s, m); s->sm.mnt_attr = mnt_to_attr_flags(&m->mnt); s->sm.mnt_propagation = mnt_to_propagation_flags(m); s->sm.mnt_peer_group = m->mnt_group_id; diff --git a/fs/nullfs.c b/fs/nullfs.c index fdbd3e5d3d71f..20d3aa627a47d 100644 --- a/fs/nullfs.c +++ b/fs/nullfs.c @@ -30,8 +30,8 @@ static int nullfs_fs_fill_super(struct super_block *s, struct fs_context *fc) make_empty_dir_inode(inode); simple_inode_init_ts(inode); inode->i_ino = 1; - /* ... and immutable. */ - inode->i_flags |= S_IMMUTABLE; + /* ... and immutable, reading it leaves no trace either. */ + inode->i_flags |= S_IMMUTABLE | S_NOATIME; s->s_root = d_make_root(inode); if (!s->s_root) diff --git a/fs/pnode.c b/fs/pnode.c index 5d91c3e58d2a3..2cd667958efe1 100644 --- a/fs/pnode.c +++ b/fs/pnode.c @@ -410,19 +410,99 @@ bool propagation_would_overmount(const struct mount *from, return false; } +/* Does @m receive propagation from @parent? */ +static bool receives_from(struct mount *m, struct mount *parent) +{ + if (m == parent) + return false; + for (; m; m = m->mnt_master) + if (m == parent || peers(m, parent)) + return true; + return false; +} + +/* + * Does @m receive propagation from the victim's parent as well? If so, then + * the mount at the victim's mountpoint inside of @m is a umount candidate as + * well. So it's the next candidate in the chain. Otherwise the chain ends at + * @m. + */ +static struct mount *next_candidate(struct mount *m, struct mount *victim) +{ + if (!receives_from(m, victim->mnt_parent)) + return NULL; + return __lookup_mnt(&m->mnt, victim->mnt_mountpoint); +} + +/* + * Would propagate_umount() pull out a mount of the chain of candidates that + * starts at @c, and does that mount have references beyond its own? + * + * This mirrors how trim_one(), trim_ancestors() and handle_locked() handle a + * synchronous umount: + * + * - single victim + * - without children + * - with MNT_LOCKED already cleared on every candidate by propagate_mount_unlock() + * + * A copy of the victim gets unmounted when each of its children is + * the next candidate in the chain or its overmount, unless the next + * unmount candidate is not its overmount and some unmount candidate further + * down has a child outside the chain. Keep this in sync with + * Documentation/filesystems/propagate_umount.txt. + */ +static bool chain_busy(struct mount *c, struct mount *victim) +{ + struct mount *m, *n, *next, *deepest = NULL; + bool above; + + /* the deepest candidate with a child outside the chain */ + for (m = c; m; m = next) { + next = next_candidate(m, victim); + list_for_each_entry(n, &m->mnt_mounts, mnt_child) { + if (n != next && n != victim) { + deepest = m; + break; + } + } + } + + above = deepest != NULL; /* @deepest is at or below @m */ + for (m = c; m; m = next) { + bool goes = true; + + next = next_candidate(m, victim); + list_for_each_entry(n, &m->mnt_mounts, mnt_child) { + if (n != next && n != m->overmount && n != victim) { + goes = false; + break; + } + } + if (goes && next && next != m->overmount && above && m != deepest) + goes = false; + if (m == deepest) + above = false; + if (goes && do_refcount_check(m, 1)) + return true; + } + return false; +} + /* * check if the mount 'mnt' can be unmounted successfully. * @mnt: the mount to be checked for unmount * NOTE: unmounting 'mnt' would naturally propagate to all * other mounts its parent propagates to. - * Check if any of these mounts that **do not have submounts** - * have more references than 'refcnt'. If so return busy. + * Check if any of the mounts that propagate_umount() would pull out + * along with it have more references than their own. If so return busy. * * vfsmount lock must be held for write */ int propagate_mount_busy(struct mount *mnt, int refcnt) { struct mount *parent = mnt->mnt_parent; + struct dentry *mp = mnt->mnt_mountpoint; + struct mount *m; /* * quickly check if the current mount can be unmounted. @@ -435,24 +515,16 @@ int propagate_mount_busy(struct mount *mnt, int refcnt) if (mnt == parent) return 0; - for (struct mount *m = propagation_next(parent, parent); m; - m = propagation_next(m, parent)) { - struct list_head *head; - struct mount *child = __lookup_mnt(&m->mnt, mnt->mnt_mountpoint); + /* the candidates are the mounts at @mp below the receivers */ + for (m = propagation_next(parent, parent); m; + m = propagation_next(m, parent)) { + struct mount *c = __lookup_mnt(&m->mnt, mp); - if (!child) + /* each chain once, from its top: skip receivers that are candidates */ + if (!c || (mnt_has_parent(m) && m->mnt_mountpoint == mp && + receives_from(m->mnt_parent, parent))) continue; - - head = &child->mnt_mounts; - if (!list_empty(head)) { - /* - * a mount that covers child completely wouldn't prevent - * it being pulled out; any other would. - */ - if (!list_is_singular(head) || !child->overmount) - continue; - } - if (do_refcount_check(child, 1)) + if (chain_busy(c, mnt)) return 1; } return 0; diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h index 881d46f25e087..6f85f63861aaa 100644 --- a/fs/verity/fsverity_private.h +++ b/fs/verity/fsverity_private.h @@ -11,6 +11,7 @@ #define pr_fmt(fmt) "fs-verity: " fmt #include +#include #include /* @@ -76,7 +77,19 @@ struct merkle_tree_params { struct fsverity_info { struct rhash_head rhash_head; struct merkle_tree_params tree_params; - u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE]; + union { + u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE]; + + /* + * Used for final freeing of the struct, in which case root_hash + * is no longer needed. Specifically, the only fields needed + * besides the rcu_head are those needed by fsverity_free_info() + * itself, and those that can be accessed by fsverity_get_info() + * (rhash_head and inode) during the RCU-protected rhashtable + * chain search trying to find another inode's fsverity_info. + */ + struct rcu_head rcu_head; + }; u8 file_digest[FS_VERITY_MAX_DIGEST_SIZE]; struct inode *inode; unsigned long *hash_block_verified; diff --git a/fs/verity/open.c b/fs/verity/open.c index d0c56a7faa3bb..52c8cd0149216 100644 --- a/fs/verity/open.c +++ b/fs/verity/open.c @@ -399,11 +399,22 @@ void fsverity_free_info(struct fsverity_info *vi) kmem_cache_free(fsverity_info_cachep, vi); } +static void fsverity_free_info_rcu(struct rcu_head *head) +{ + fsverity_free_info(container_of(head, struct fsverity_info, rcu_head)); +} + void fsverity_remove_info(struct fsverity_info *vi) { rhashtable_remove_fast(&fsverity_info_hash, &vi->rhash_head, fsverity_info_hash_params); - fsverity_free_info(vi); + /* + * The freeing must be RCU-delayed because __fsverity_get_info() for a + * *different* inode's fsverity_info in the same rhashtable hash chain + * can still be accessing *this* fsverity_info's rhash_head and inode + * fields as part of the RCU-protected hash chain search. + */ + call_rcu(&vi->rcu_head, fsverity_free_info_rcu); } void fsverity_cleanup_inode(struct inode *inode) diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index 49eda8d0994de..37595f579b635 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -49,7 +49,7 @@ struct xfs_exchmaps_adjacent { .right2 = { .br_startblock = HOLESTARTBLOCK }, \ } -/* Information to reset reflink flag / CoW fork state after an exchange. */ +/* Information to reset CoW fork state after an exchange. */ /* * If the reflink flag is set on either inode, make sure it has an incore CoW @@ -127,9 +127,7 @@ xmi_has_more_exchange_work(const struct xfs_exchmaps_intent *xmi) static inline bool xmi_has_postop_work(const struct xfs_exchmaps_intent *xmi) { - return xmi->xmi_flags & (XFS_EXCHMAPS_CLEAR_INO1_REFLINK | - XFS_EXCHMAPS_CLEAR_INO2_REFLINK | - __XFS_EXCHMAPS_INO2_SHORTFORM); + return xmi->xmi_flags & __XFS_EXCHMAPS_INO2_SHORTFORM; } /* Check all mappings to make sure we can actually exchange them. */ @@ -525,18 +523,6 @@ xfs_exchmaps_link_to_sf( return error; } -/* Clear the reflink flag after an exchange. */ -static inline void -xfs_exchmaps_clear_reflink( - struct xfs_trans *tp, - struct xfs_inode *ip) -{ - trace_xfs_reflink_unset_inode_flag(ip); - - ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; - xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); -} - /* Finish whatever work might come after an exchange operation. */ static int xfs_exchmaps_do_postop_work( @@ -557,16 +543,6 @@ xfs_exchmaps_do_postop_work( return error; } - if (xmi->xmi_flags & XFS_EXCHMAPS_CLEAR_INO1_REFLINK) { - xfs_exchmaps_clear_reflink(tp, xmi->xmi_ip1); - xmi->xmi_flags &= ~XFS_EXCHMAPS_CLEAR_INO1_REFLINK; - } - - if (xmi->xmi_flags & XFS_EXCHMAPS_CLEAR_INO2_REFLINK) { - xfs_exchmaps_clear_reflink(tp, xmi->xmi_ip2); - xmi->xmi_flags &= ~XFS_EXCHMAPS_CLEAR_INO2_REFLINK; - } - return 0; } @@ -948,56 +924,14 @@ xfs_exchmaps_intent_destroy_cache(void) } /* - * Decide if we will exchange the reflink flags between the two files after the - * exchange. The only time we want to do this is if we're exchanging all - * mappings under EOF and the inode reflink flags have different states. + * Allocate and initialize a new incore intent item from a request. */ -static inline bool -xmi_can_exchange_reflink_flags( - const struct xfs_exchmaps_req *req, - unsigned int reflink_state) -{ - struct xfs_mount *mp = req->ip1->i_mount; - - /* - * The INO1_WRITTEN optimization can skip exchanging hole and - * unwritten mappings, which means we cannot guarantee that all - * shared extents actually moved to the other file. Clearing the - * reflink flag of an inode that still holds shared extents breaks - * the CoW write path, so refuse to exchange the flags in that case. - */ - if (req->flags & XFS_EXCHMAPS_INO1_WRITTEN) - return false; - - /* - * The INO1_WRITTEN optimization can skip exchanging hole and - * unwritten mappings, which means we cannot guarantee that all - * shared extents actually moved to the other file. Clearing the - * reflink flag of an inode that still holds shared extents breaks - * the CoW write path, so refuse to exchange the flags in that case. - */ - if (req->flags & XFS_EXCHMAPS_INO1_WRITTEN) - return false; - - if (hweight32(reflink_state) != 1) - return false; - if (req->startoff1 != 0 || req->startoff2 != 0) - return false; - if (req->blockcount != XFS_B_TO_FSB(mp, req->ip1->i_disk_size)) - return false; - if (req->blockcount != XFS_B_TO_FSB(mp, req->ip2->i_disk_size)) - return false; - return true; -} - - -/* Allocate and initialize a new incore intent item from a request. */ struct xfs_exchmaps_intent * xfs_exchmaps_init_intent( const struct xfs_exchmaps_req *req) { struct xfs_exchmaps_intent *xmi; - unsigned int rs = 0; + struct xfs_mount *mp = req->ip1->i_mount; xmi = kmem_cache_zalloc(xfs_exchmaps_intent_cache, GFP_NOFS | __GFP_NOFAIL); @@ -1015,27 +949,18 @@ xfs_exchmaps_init_intent( return xmi; } - if (req->flags & XFS_EXCHMAPS_SET_SIZES) { - xmi->xmi_flags |= XFS_EXCHMAPS_SET_SIZES; - xmi->xmi_isize1 = req->ip2->i_disk_size; - xmi->xmi_isize2 = req->ip1->i_disk_size; - } - - /* Record the state of each inode's reflink flag before the op. */ - if (xfs_is_reflink_inode(req->ip1)) - rs |= 1; - if (xfs_is_reflink_inode(req->ip2)) - rs |= 2; - /* - * Figure out if we're clearing the reflink flags (which effectively - * exchanges them) after the operation. + * If the caller wanted, set each file's size to that file's exchange + * offset + length exchanged from the other file because the ranges in + * each file might be different lengths. */ - if (xmi_can_exchange_reflink_flags(req, rs)) { - if (rs & 1) - xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO1_REFLINK; - if (rs & 2) - xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO2_REFLINK; + if (req->flags & XFS_EXCHMAPS_SET_SIZES) { + loff_t off1 = XFS_FSB_TO_B(mp, xmi->xmi_startoff1); + loff_t off2 = XFS_FSB_TO_B(mp, xmi->xmi_startoff2); + + xmi->xmi_flags |= XFS_EXCHMAPS_SET_SIZES; + xmi->xmi_isize1 = off1 + (req->ip2->i_disk_size - off2); + xmi->xmi_isize2 = off2 + (req->ip1->i_disk_size - off1); } if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode) || diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index a4e1b3eb425cd..6e620380d5dc8 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -927,10 +927,8 @@ struct xfs_xmi_log_format { */ #define XFS_EXCHMAPS_INO1_WRITTEN (1ULL << 2) -/* Clear the reflink flag from inode1 after the operation. */ +/* No longer set, but older intents may carry them; keep them valid. */ #define XFS_EXCHMAPS_CLEAR_INO1_REFLINK (1ULL << 3) - -/* Clear the reflink flag from inode2 after the operation. */ #define XFS_EXCHMAPS_CLEAR_INO2_REFLINK (1ULL << 4) #define XFS_EXCHMAPS_LOGGED_FLAGS (XFS_EXCHMAPS_ATTR_FORK | \ diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index c88b9ade7389d..2ea6507678050 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -1514,7 +1514,6 @@ xfs_swap_extents( struct xfs_bstat *sbp = &sxp->sx_stat; int src_log_flags, target_log_flags; int error = 0; - uint64_t f; int resblks = 0; unsigned int flags = 0; struct timespec64 ctime, mtime; @@ -1670,14 +1669,12 @@ xfs_swap_extents( if (error) goto out_trans_cancel; - /* Do we have to swap reflink flags? */ - if ((ip->i_diflags2 & XFS_DIFLAG2_REFLINK) ^ - (tip->i_diflags2 & XFS_DIFLAG2_REFLINK)) { - f = ip->i_diflags2 & XFS_DIFLAG2_REFLINK; - ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; - ip->i_diflags2 |= tip->i_diflags2 & XFS_DIFLAG2_REFLINK; - tip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; - tip->i_diflags2 |= f & XFS_DIFLAG2_REFLINK; + /* Either inode may now own shared blocks; flag both, with CoW forks. */ + if ((ip->i_diflags2 | tip->i_diflags2) & XFS_DIFLAG2_REFLINK) { + ip->i_diflags2 |= XFS_DIFLAG2_REFLINK; + tip->i_diflags2 |= XFS_DIFLAG2_REFLINK; + xfs_ifork_init_cow(ip); + xfs_ifork_init_cow(tip); } /* Swap the cow forks. */ diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index fafb4e3f065c7..935b561dfd3c5 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -290,17 +290,20 @@ xfs_exchrange_mappings( goto out_unlock; /* - * If the caller wanted us to exchange the contents of two complete - * files of unequal length, exchange the incore sizes now. This should - * be safe because we flushed both files' page caches, exchanged all - * the mappings, and updated the ondisk sizes. + * If the caller wanted, set each file's size to that file's exchange + * offset + length exchanged from the other file because the ranges in + * each file might be different lengths. This should be safe because + * we flushed both files' page caches, exchanged all the mappings, and + * updated the ondisk sizes. */ if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) { - loff_t temp; + loff_t old_ip1_size = i_size_read(VFS_I(ip1)); + loff_t old_ip2_size = i_size_read(VFS_I(ip2)); - temp = i_size_read(VFS_I(ip2)); - i_size_write(VFS_I(ip2), i_size_read(VFS_I(ip1))); - i_size_write(VFS_I(ip1), temp); + i_size_write(VFS_I(ip2), fxr->file2_offset + + (old_ip1_size - fxr->file1_offset)); + i_size_write(VFS_I(ip1), fxr->file1_offset + + (old_ip2_size - fxr->file2_offset)); } out_unlock: @@ -445,6 +448,30 @@ xfs_exchange_range_checks( return blen == fxr->length ? 0 : -EINVAL; } +/* Flush all relevant dirty pagecache near the range to be exchanged. */ +static inline int +xfs_exchange_range_flush( + struct xfs_inode *ip, + const struct xfs_exchrange *fxr, + loff_t offset) +{ + /* + * If TO_EOF is set, the ondisk file size update logic depends on the + * ondisk file size matching the incore file size. Flush everything. + */ + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) + return filemap_write_and_wait(VFS_I(ip)->i_mapping); + + /* + * Because we're updating the ondisk mappings, flush all dirty data + * between the ondisk size and the exchange offset to reduce the chance + * of zeroed file contents in that gap after a crash. + */ + return filemap_write_and_wait_range(VFS_I(ip)->i_mapping, + min(offset, ip->i_disk_size), + offset + fxr->length - 1); +} + /* * Check that the two inodes are eligible for range exchanges, the ranges make * sense, and then flush all dirty data. Caller must ensure that the inodes @@ -470,15 +497,11 @@ xfs_exchange_range_prep( if (!same_inode) inode_dio_wait(inode2); - error = filemap_write_and_wait_range(inode1->i_mapping, - fxr->file1_offset, - fxr->file1_offset + fxr->length - 1); + error = xfs_exchange_range_flush(XFS_I(inode1), fxr, fxr->file1_offset); if (error) return error; - error = filemap_write_and_wait_range(inode2->i_mapping, - fxr->file2_offset, - fxr->file2_offset + fxr->length - 1); + error = xfs_exchange_range_flush(XFS_I(inode2), fxr, fxr->file2_offset); if (error) return error; diff --git a/include/linux/apple-centauri.h b/include/linux/apple-centauri.h new file mode 100644 index 0000000000000..0766371a60d17 --- /dev/null +++ b/include/linux/apple-centauri.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _LINUX_APPLE_CENTAURI_H +#define _LINUX_APPLE_CENTAURI_H + +#include + +struct pci_dev; + +/* The caller must hold a managed device link to the Control supplier. */ +bool apple_centauri_control_ready(struct pci_dev *pdev); + +#endif diff --git a/include/linux/can/core.h b/include/linux/can/core.h index 3287232e3cad9..2de74c2b78b6c 100644 --- a/include/linux/can/core.h +++ b/include/linux/can/core.h @@ -17,6 +17,7 @@ #include #include #include +#include #define DNAME(dev) ((dev) ? (dev)->name : "any") @@ -58,7 +59,7 @@ extern void can_rx_unregister(struct net *net, struct net_device *dev, void *data); extern int can_send(struct sk_buff *skb, int loop); -void can_set_skb_uid(struct sk_buff *skb); +void can_set_skb_uid(struct can_skb_ext *csx); void can_sock_destruct(struct sock *sk); #endif /* !_CAN_CORE_H */ diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index a70a029670711..5d27843862fca 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -43,8 +43,10 @@ static inline struct can_skb_ext *can_skb_ext_add(struct sk_buff *skb) struct can_skb_ext *csx = skb_ext_add(skb, SKB_EXT_CAN); /* skb_ext_add() returns uninitialized space */ - if (csx) + if (csx) { csx->can_gw_hops = 0; + csx->can_skb_uid = 0; + } return csx; } diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index c7a7a7a6889f8..f83da0af60d1d 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -316,6 +316,8 @@ enum { IO_RING_F_DRAIN_DISABLED = BIT(10), IO_RING_F_COMPAT = BIT(11), IO_RING_F_IOWQ_LIMITS_SET = BIT(12), + /* restrictions were inherited from the task that created the ring */ + IO_RING_F_RESTRICT_INHERITED = BIT(13), }; struct iou_ctx {}; diff --git a/include/linux/mount.h b/include/linux/mount.h index acfe7ef86a1b3..b16ba5d456057 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -52,7 +52,7 @@ enum mount_flags { MNT_ATIME_MASK = MNT_NOATIME | MNT_NODIRATIME | MNT_RELATIME, MNT_INTERNAL_FLAGS = MNT_INTERNAL | MNT_DOOMED | - MNT_SYNC_UMOUNT | MNT_LOCKED + MNT_SYNC_UMOUNT | MNT_LOCKED | MNT_UMOUNT }; struct vfsmount { diff --git a/include/linux/nsproxy.h b/include/linux/nsproxy.h index 5a67648721c7d..dc2447f5f0924 100644 --- a/include/linux/nsproxy.h +++ b/include/linux/nsproxy.h @@ -100,6 +100,7 @@ void exit_cred_namespaces(struct task_struct *tsk); void switch_task_namespaces(struct task_struct *tsk, struct nsproxy *new); int exec_task_namespaces(void); void deactivate_nsproxy(struct nsproxy *ns); +void nsproxy_free(struct nsproxy *ns); int unshare_nsproxy_namespaces(unsigned long, struct nsproxy **, struct cred *, struct fs_struct *); int __init nsproxy_cache_init(void); diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 041f30931a908..03ddd2c6d7fa7 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -1266,8 +1266,8 @@ enum { NVME_ZRASF_ZONE_STATE_IMP_OPEN = 0x02, NVME_ZRASF_ZONE_STATE_EXP_OPEN = 0x03, NVME_ZRASF_ZONE_STATE_CLOSED = 0x04, - NVME_ZRASF_ZONE_STATE_READONLY = 0x05, - NVME_ZRASF_ZONE_STATE_FULL = 0x06, + NVME_ZRASF_ZONE_STATE_FULL = 0x05, + NVME_ZRASF_ZONE_STATE_READONLY = 0x06, NVME_ZRASF_ZONE_STATE_OFFLINE = 0x07, NVME_REPORT_ZONE_PARTIAL = 1, }; diff --git a/include/linux/pci.h b/include/linux/pci.h index de60fd2de3ef8..766d5ba0b27d6 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -650,6 +650,7 @@ struct pci_host_bridge { void (*release_fn)(struct pci_host_bridge *); int (*enable_device)(struct pci_host_bridge *bridge, struct pci_dev *dev); void (*disable_device)(struct pci_host_bridge *bridge, struct pci_dev *dev); + int (*reset_root_port)(struct pci_host_bridge *bridge, struct pci_dev *dev); void *release_data; unsigned int ignore_reset_delay:1; /* For entire hierarchy */ unsigned int no_ext_tags:1; /* No Extended Tags */ @@ -1555,6 +1556,7 @@ void set_pcie_hotplug_bridge(struct pci_dev *pdev); /* Functions for PCI Hotplug drivers to use */ unsigned int pci_rescan_bus(struct pci_bus *bus); void pci_lock_rescan_remove(void); +bool pci_trylock_rescan_remove(void); void pci_unlock_rescan_remove(void); /* Vital Product Data routines */ diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 7a5e4c3242a01..82215a7538501 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -221,6 +221,7 @@ enum power_supply_charge_behaviour { POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE, POWER_SUPPLY_CHARGE_BEHAVIOUR_INHIBIT_CHARGE_AWAKE, POWER_SUPPLY_CHARGE_BEHAVIOUR_FORCE_DISCHARGE, + POWER_SUPPLY_CHARGE_BEHAVIOUR_AUTO_DISCHARGE, }; enum power_supply_notifier_events { diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 87e353f7e0112..67a97429d8a6c 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -198,6 +198,7 @@ struct sched_ext_entity { s32 sticky_cpu; s32 holding_cpu; s32 selected_cpu; + u32 ops_qseq; /* protected by rq lock */ struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ struct list_head runnable_node; /* rq->scx.runnable_list */ diff --git a/include/linux/soc/apple/dart.h b/include/linux/soc/apple/dart.h index 61ec51a342bde..19562922d9a32 100644 --- a/include/linux/soc/apple/dart.h +++ b/include/linux/soc/apple/dart.h @@ -4,8 +4,132 @@ #include #include +#include +#include struct device; +struct device_node; + +/* Retained internal-display ownership is qualified only with a built-in + * DART provider. Other consumers, including modular DART, keep the APIs below. + */ +struct device_node; + +#if IS_ENABLED(CONFIG_APPLE_DART_J713_DCP) +/* J713 internal DCP only: requires validated, borrowed SID23 tables. */ +int apple_dart_dcp_check(struct device *dev); +/* A borrowed RTKit payload must be fully writable, not inherited firmware TEXT. */ +int apple_dart_dcp_check_buffer(struct device *dev, dma_addr_t iova, size_t size); +/* Retained CPU/DVA allocation: even failed publication pins pages until reboot. */ +void *apple_dart_dcp_alloc(struct device *dev, size_t size, dma_addr_t *dva); + +/* Qualified noncoherent J713 SID23/SID16 pair; no CPU access after publication. */ +int apple_dart_dcp_shared_alloc(struct device *dev, struct device *piodma, + size_t size, u64 *phys, u64 *dva); +int apple_dart_dcp_shared_map(struct device *dev, struct device *piodma, + u64 phys, u64 dva, size_t size); +/* SID23 only, never mapped in SID16 afterwards (J713 external DCP). */ +int apple_dart_dcp_shared_alloc_link(struct device *dev, size_t size, u64 *phys, u64 *dva); +/* An existing backing in SID0/SID16 of an external display DART, at its own DVA. */ +int apple_dart_dcp_dispext_map(struct device *piodma, unsigned int sid, u64 phys, size_t size, + u64 *dva); +/* Whether an external display DART stream is adopted and not retired. */ +int apple_dart_dcp_dispext_stream_ok(struct device *piodma, unsigned int sid); +/* A host-written surface in SID0 of an external display DART (Normal-NC CPU view). */ +void *apple_dart_dcp_dispext_scanout_alloc(struct device *piodma, size_t size, u64 *dva); +/* An adopted external display DART's state after its power-up; reads only. */ +int apple_dart_dcp_dispext_state(struct device_node *consumer); + +/* + * Remappable host windows in SID0 of an external display DART: one + * whole arena slice each, claimed until reboot, mapped and unmapped by the + * host only (every change flushed and read back). Pages are the caller's. + */ +int apple_dart_dcp_dispext_window_claim(struct device *piodma, u64 *dva); +int apple_dart_dcp_dispext_window_map(struct device *piodma, u64 dva, const phys_addr_t *phys, + unsigned int count); +int apple_dart_dcp_dispext_window_unmap(struct device *piodma, u64 dva); +/* Power confirmed on by the caller: both frontends as adopted, no fault. */ +int apple_dart_dcp_dispext_qualify(struct device *piodma); + +/* Private retained SID0 scanout, not the firmware's capped D451 allocation class. */ +void *apple_dart_dcp_scanout_alloc(struct device *dev, size_t size, dma_addr_t *dva); + +#else +static inline int apple_dart_dcp_check(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_dcp_check_buffer(struct device *dev, dma_addr_t iova, size_t size) +{ + return -EOPNOTSUPP; +} +static inline void *apple_dart_dcp_alloc(struct device *dev, size_t size, dma_addr_t *dva) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline int apple_dart_dcp_shared_alloc(struct device *dev, struct device *piodma, + size_t size, u64 *phys, u64 *dva) +{ + return -EOPNOTSUPP; +} +static inline int apple_dart_dcp_shared_map(struct device *dev, struct device *piodma, + u64 phys, u64 dva, size_t size) +{ + return -EOPNOTSUPP; +} + +static inline int apple_dart_dcp_shared_alloc_link(struct device *dev, size_t size, u64 *phys, + u64 *dva) +{ + return -EOPNOTSUPP; +} + +static inline int apple_dart_dcp_dispext_map(struct device *piodma, unsigned int sid, u64 phys, + size_t size, u64 *dva) +{ + return -EOPNOTSUPP; +} + +static inline int apple_dart_dcp_dispext_state(struct device_node *consumer) +{ + return -EOPNOTSUPP; +} + +static inline int apple_dart_dcp_dispext_window_claim(struct device *piodma, u64 *dva) +{ + return -EOPNOTSUPP; +} + +static inline int apple_dart_dcp_dispext_window_map(struct device *piodma, u64 dva, + const phys_addr_t *phys, unsigned int count) +{ + return -EOPNOTSUPP; +} + +static inline int apple_dart_dcp_dispext_window_unmap(struct device *piodma, u64 dva) +{ + return -EOPNOTSUPP; +} + +static inline int apple_dart_dcp_dispext_qualify(struct device *piodma) +{ + return -EOPNOTSUPP; +} + +static inline int apple_dart_dcp_dispext_stream_ok(struct device *piodma, unsigned int sid) +{ + return -EOPNOTSUPP; +} + +static inline void *apple_dart_dcp_dispext_scanout_alloc(struct device *piodma, size_t size, + u64 *dva) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline void *apple_dart_dcp_scanout_alloc(struct device *dev, size_t size, dma_addr_t *dva) +{ + return ERR_PTR(-EOPNOTSUPP); +} +#endif /* * The tunneled PCIe IOMMU shares the port clock. Stop issuing commands @@ -16,11 +140,31 @@ int apple_dart_quiesce_commands(struct device *dev); int apple_dart_resume_commands(struct device *dev); int apple_dart_save_tunnel_state(struct device *dev); int apple_dart_restore_tunnel_state(struct device *dev); +/* Registered J713 managed radio provider, with no owned valid root or fault. */ +int apple_dart_radio_check_idle(struct device *dev); +/* A firmware-owned buffer's address for a host copy; Linux's own mappings refused. */ +int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, phys_addr_t *phys); #else static inline int apple_dart_quiesce_commands(struct device *dev) { return 0; } static inline int apple_dart_resume_commands(struct device *dev) { return 0; } static inline int apple_dart_save_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } static inline int apple_dart_restore_tunnel_state(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_radio_check_idle(struct device *dev) { return -EOPNOTSUPP; } +static inline int apple_dart_fw_remote(struct device *dev, dma_addr_t iova, bool write, + phys_addr_t *phys) +{ + return -EOPNOTSUPP; +} +#endif + +/* + * A DART marked apple,manual-availability answers only once its client (a + * PCIe root complex) is up. The client makes it available, and it probes then. + */ +#if IS_ENABLED(CONFIG_APPLE_DART) +void apple_dart_make_available(struct device_node *np); +#else +static inline void apple_dart_make_available(struct device_node *np) { } #endif #endif diff --git a/include/linux/soc/apple/display.h b/include/linux/soc/apple/display.h new file mode 100644 index 0000000000000..8978d95d40484 --- /dev/null +++ b/include/linux/soc/apple/display.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#ifndef _LINUX_SOC_APPLE_DISPLAY_H_ +#define _LINUX_SOC_APPLE_DISPLAY_H_ + +struct device; + +/* Initial native display admission, before starting retained G15 firmware. */ +int apple_drm_t6030_link_device(struct device *consumer); + +#endif diff --git a/include/linux/soc/apple/dp-tunnel.h b/include/linux/soc/apple/dp-tunnel.h index 7bd546080f25b..540d48fe8e0dc 100644 --- a/include/linux/soc/apple/dp-tunnel.h +++ b/include/linux/soc/apple/dp-tunnel.h @@ -44,6 +44,16 @@ int apple_dcp_tb_dp_tunnel(struct device_node *connector_np, unsigned int dpin, */ int apple_atc_dp_tunnel_rate(struct phy *phy, unsigned int dpin, u8 rate); +/* T6030's external DPTX port 0 sources feed DP IN0/1 through PCLK1/2. */ +static inline int apple_dp_tunnel_t6030_pclk(unsigned int source, unsigned int dpin) +{ + return dpin < 2 && (source == 0 || source == 2) ? (int)dpin : -EOPNOTSUPP; +} + +/* T6030: keep the source owning the pixel clock across rate changes and stop. */ +int apple_atc_dp_tunnel_rate_source(struct phy *phy, unsigned int source, + unsigned int dpin, u8 rate); + /* * ATC PHY: wake the DP clock path of a PHY in Thunderbolt/USB4 mode ahead of * apple_atc_dp_tunnel_rate(), before DCP is told about the display. @@ -94,6 +104,40 @@ static inline bool apple_dp_tunnel_mux_matches(struct device_node *dcp, return matches; } +/* T6030 boot trees without DPIN names use the same three-output crossbar. + * Explicit DPIN/USB4 names remain authoritative, including invalid wiring. */ +static inline bool apple_dp_tunnel_t6030_legacy_mux(struct device_node *dcp, + unsigned int port) +{ + struct of_phandle_args base; + char name[24]; + u32 source; + bool matches; + int index, i; + + if (!of_machine_is_compatible("apple,t6030") || port >= 4 || + of_property_read_u32_index(dcp, "apple,typec-mux-indices", port, &source) || + apple_dp_tunnel_t6030_pclk(source, 0) < 0) + return false; + for (i = 0; i < 2; i++) { + snprintf(name, sizeof(name), "typec%u-dpin%u", port, i); + if (of_property_match_string(dcp, "mux-control-names", name) >= 0) + return false; + } + snprintf(name, sizeof(name), "typec%u-usb4", port); + if (of_property_match_string(dcp, "mux-control-names", name) >= 0) + return false; + snprintf(name, sizeof(name), "typec%u", port); + index = of_property_match_string(dcp, "mux-control-names", name); + if (index < 0 || of_parse_phandle_with_args(dcp, "mux-controls", + "#mux-control-cells", index, &base)) + return false; + matches = base.args_count == 1 && base.args[0] == 0 && + of_device_is_compatible(base.np, "apple,t6020-display-crossbar"); + of_node_put(base.np); + return matches; +} + static inline bool apple_dp_tunnel_route_wired(struct device_node *route, unsigned int endpoints) { @@ -122,16 +166,18 @@ static inline bool apple_dp_tunnel_route_wired(struct device_node *route, } if (!named[0] && !named[1]) { snprintf(name, sizeof(name), "typec%u-usb4", index); - legacy = apple_dp_tunnel_mux_matches(dcp, name, base.np, 1); + legacy = apple_dp_tunnel_mux_matches(dcp, name, base.np, 1) || + apple_dp_tunnel_t6030_legacy_mux(dcp, index); } of_node_put(base.np); return wired && apple_dp_tunnel_wiring_dual(endpoints, named[0], named[1], legacy); } /* - * Physical DT wiring only: no DCP registration, firmware, status or board list. + * DT wiring only: no DCP registration, firmware or display status. * Two candidates alone (e.g. j314) are single-stream. Named DPIN0/1 controls, * or old-DTB typecN-usb4 cell 1, must share the DP-alt crossbar (cell 0). + * T6030 old trees may omit DPIN names on their three-output T602X crossbar. * Consumers qualify their SoC flow separately and never select these muxes. */ static inline bool apple_dp_tunnel_dual_stream(struct device_node *connector_np) diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h index 1ba27a92e2b14..ee59cf3279a69 100644 --- a/include/linux/soc/apple/rtkit.h +++ b/include/linux/soc/apple/rtkit.h @@ -161,6 +161,12 @@ int apple_rtkit_reinit(struct apple_rtkit *rtk); */ int apple_rtkit_boot(struct apple_rtkit *rtk); +/* + * apple_rtkit_boot with AP readiness signalled before the IOP power ack is + * awaited, for coprocessors that wait for it (T6050 ANS). + */ +int apple_rtkit_boot_early_ap(struct apple_rtkit *rtk); + /* * Mark an instance created with apple_rtkit_init() as running without a * handshake, see apple_rtkit_init_adopted(). diff --git a/include/linux/usb/tipd.h b/include/linux/usb/tipd.h new file mode 100644 index 0000000000000..18e961ad53abb --- /dev/null +++ b/include/linux/usb/tipd.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_USB_TIPD_H +#define __LINUX_USB_TIPD_H + +#include +#include + +struct device_node; + +#if IS_REACHABLE(CONFIG_TYPEC_SN201202X) +int sn201202x_simulate_disconnect(struct device_node *np, u8 seconds); +#else +static inline int sn201202x_simulate_disconnect(struct device_node *np, u8 seconds) +{ + return -ENODEV; +} +#endif + +#endif /* __LINUX_USB_TIPD_H */ diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index c381b916c1b54..d6466f96cdd00 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -111,48 +111,38 @@ static inline int __virtio_net_hdr_to_skb(struct sk_buff *skb, p_off = nh_min_len + thlen; if (!pskb_may_pull(skb, p_off)) return -EINVAL; - } else { + } else if (gso_type) { /* gso packets without NEEDS_CSUM do not set transport_offset. * probe and drop if does not match one of the above types. */ - if (gso_type && skb->network_header) { - struct flow_keys_basic keys; - - if (!skb->protocol) { - __be16 protocol = dev_parse_header_protocol(skb); - - if (!protocol) - virtio_net_hdr_set_proto(skb, hdr); - else if (!virtio_net_hdr_match_proto(protocol, - hdr_gso_type)) - return -EINVAL; - else - skb->protocol = protocol; - } + struct flow_keys_basic keys; + + if (!skb->protocol) { + skb->protocol = dev_parse_header_protocol(skb); + if (!skb->protocol) + virtio_net_hdr_set_proto(skb, hdr); + } retry: - if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys, - NULL, 0, 0, 0, - 0)) { - /* UFO does not specify ipv4 or 6: try both */ - if (gso_type & SKB_GSO_UDP && - skb->protocol == htons(ETH_P_IP)) { - skb->protocol = htons(ETH_P_IPV6); - goto retry; - } - return -EINVAL; + if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys, + NULL, 0, 0, 0, + 0)) { + /* UFO does not specify ipv4 or 6: try both */ + if (gso_type & SKB_GSO_UDP && + skb->protocol == htons(ETH_P_IP)) { + skb->protocol = htons(ETH_P_IPV6); + goto retry; } + return -EINVAL; + } - p_off = keys.control.thoff + thlen; - if (!pskb_may_pull(skb, p_off) || - keys.basic.ip_proto != ip_proto) - return -EINVAL; + p_off = keys.control.thoff + thlen; + if (!pskb_may_pull(skb, p_off) || + keys.basic.ip_proto != ip_proto || + !virtio_net_hdr_match_proto(keys.basic.n_proto, + hdr_gso_type)) + return -EINVAL; - skb_set_transport_header(skb, keys.control.thoff); - } else if (gso_type) { - p_off = nh_min_len + thlen; - if (!pskb_may_pull(skb, p_off)) - return -EINVAL; - } + skb_set_transport_header(skb, keys.control.thoff); } if (hdr_gso_type != VIRTIO_NET_HDR_GSO_NONE) { diff --git a/include/net/amt.h b/include/net/amt.h index a0255491f5b05..2846dde0cadce 100644 --- a/include/net/amt.h +++ b/include/net/amt.h @@ -231,10 +231,6 @@ struct amt_relay_headers { }; } __packed; -struct amt_skb_cb { - struct amt_tunnel_list *tunnel; -}; - struct amt_tunnel_list { struct list_head list; /* Protect All resources under an amt_tunne_list */ diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index 25322491721a9..353f11051ed25 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -663,6 +663,8 @@ struct hci_dev { struct bt_codec *codec, __u8 *vnd_len, __u8 **vnd_data); u8 (*classify_pkt_type)(struct hci_dev *hdev, struct sk_buff *skb); + /* Called in command-sync context before enabling duplicate-filtered scanning. */ + int (*reset_dup_filter)(struct hci_dev *hdev); }; #define hci_set_quirk(hdev, nr) set_bit((nr), (hdev)->quirk_flags) diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h index 102c278e35844..9dc5fd4cb8c81 100644 --- a/include/net/bluetooth/rfcomm.h +++ b/include/net/bluetooth/rfcomm.h @@ -207,6 +207,7 @@ struct rfcomm_dlc { #define RFCOMM_AUTH_REJECT 7 #define RFCOMM_DEFER_SETUP 8 #define RFCOMM_ENC_DROP 9 +#define RFCOMM_CLOSED 10 /* Scheduling flags and events */ #define RFCOMM_SCHED_WAKEUP 31 diff --git a/include/net/can.h b/include/net/can.h index 6db9e826f0e05..2b8af2598732c 100644 --- a/include/net/can.h +++ b/include/net/can.h @@ -17,12 +17,14 @@ * @can_framelen: cached echo CAN frame length for bql * @can_gw_hops: can-gw CAN frame time-to-live counter * @can_ext_flags: CAN skb extensions flags + * @can_skb_uid: CAN skb UID for raw_rcv and isotp echo handling */ struct can_skb_ext { int can_iif; u16 can_framelen; u8 can_gw_hops; u8 can_ext_flags; + u32 can_skb_uid; }; #endif /* _NET_CAN_H */ diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index f5abf1db7558b..7354ba079363a 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -3722,6 +3722,7 @@ enum wiphy_params_flags { /* The per TXQ device queue limit in airtime */ #define IEEE80211_DEFAULT_AQL_TXQ_LIMIT_L 5000 #define IEEE80211_DEFAULT_AQL_TXQ_LIMIT_H 12000 +#define IEEE80211_DEFAULT_AQL_TXQ_LIMIT_MC 50000 /* The per interface airtime threshold to switch to lower queue limit */ #define IEEE80211_AQL_THRESHOLD 24000 diff --git a/include/net/ip.h b/include/net/ip.h index 6f602df72ee62..d07c2c5730240 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -74,6 +74,13 @@ static inline unsigned int ip_hdrlen(const struct sk_buff *skb) return ip_hdr(skb)->ihl * 4; } +static inline int ip_check_csum_start(const struct sk_buff *skb) +{ + if (unlikely(skb->csum_start < skb->network_header + ip_hdrlen(skb))) + return -EINVAL; + return 0; +} + struct ipcm_cookie { struct sockcm_cookie sockc; __be32 addr; diff --git a/include/net/mac80211.h b/include/net/mac80211.h index 36055d40172a1..7b610dfee2867 100644 --- a/include/net/mac80211.h +++ b/include/net/mac80211.h @@ -1317,6 +1317,7 @@ ieee80211_rate_get_vht_nss(const struct ieee80211_tx_rate *rate) * @status_data: internal data for TX status handling, assigned privately, * see also &enum ieee80211_status_data for the internal documentation * @status_data_idr: indicates status data is IDR allocated ID for ack frame + * @tx_time_mc: TX time estimate is for a multicast frame, used internally * @tx_time_est: TX time estimate in units of 4us, used internally * @control: union part for control data * @control.rates: TX rates array to try @@ -1354,8 +1355,8 @@ struct ieee80211_tx_info { status_data_idr:1, status_data:13, hw_queue:4, + tx_time_mc:1, tx_time_est:10; - /* 1 free bit */ union { struct { @@ -6876,6 +6877,17 @@ void ieee80211_sta_register_airtime(struct ieee80211_sta *pubsta, u8 tid, bool ieee80211_txq_airtime_check(struct ieee80211_hw *hw, struct ieee80211_txq *txq); +/** + * ieee80211_txq_aql_pending - get pending AQL airtime for a txq + * + * @hw: pointer obtained from ieee80211_alloc_hw() + * @txq: pointer obtained from station or virtual interface + * + * Return: pending airtime (in usec) for the given txq. + */ +u32 ieee80211_txq_aql_pending(struct ieee80211_hw *hw, + struct ieee80211_txq *txq); + /** * ieee80211_iter_keys - iterate keys programmed into the device * @hw: pointer obtained from ieee80211_alloc_hw() diff --git a/include/net/strparser.h b/include/net/strparser.h index 0ed73e364faa2..881b7ae344986 100644 --- a/include/net/strparser.h +++ b/include/net/strparser.h @@ -91,11 +91,11 @@ static inline struct strp_msg *strp_msg(struct sk_buff *skb) struct strparser { struct sock *sk; - u32 stopped : 1; - u32 paused : 1; - u32 aborted : 1; - u32 interrupted : 1; - u32 unrecov_intr : 1; + u8 stopped; + u8 paused; + u8 aborted; + u8 interrupted; + u8 unrecov_intr; struct sk_buff **skb_nextp; struct sk_buff *skb_head; diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index c5404fbe3f7ef..3d8c8a50fd312 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -2077,8 +2077,9 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr) static void io_rings_free(struct io_ring_ctx *ctx) { - io_free_region(ctx->user, &ctx->sq_region); - io_free_region(ctx->user, &ctx->ring_region); + /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ + io_free_region(NULL, &ctx->sq_region); + io_free_region(NULL, &ctx->ring_region); ctx->rings = NULL; RCU_INIT_POINTER(ctx->rings_rcu, NULL); ctx->sq_sqes = NULL; @@ -2742,7 +2743,8 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, rd.user_addr = p->cq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &ctx->ring_region, &rd, IORING_OFF_CQ_RING); + /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ + ret = io_create_region(NULL, &ctx->ring_region, &rd, IORING_OFF_CQ_RING); if (ret) return ret; ctx->rings = rings = io_region_get_ptr(&ctx->ring_region); @@ -2756,7 +2758,7 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx, rd.user_addr = p->sq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &ctx->sq_region, &rd, IORING_OFF_SQES); + ret = io_create_region(NULL, &ctx->sq_region, &rd, IORING_OFF_SQES); if (ret) { io_rings_free(ctx); return ret; @@ -2967,7 +2969,8 @@ static void io_ctx_restriction_clone(struct io_ring_ctx *ctx, if (dst->op_registered) ctx->int_flags |= IO_RING_F_OP_RESTRICTED; if (dst->reg_registered) - ctx->int_flags |= IO_RING_F_REG_RESTRICTED; + ctx->int_flags |= IO_RING_F_REG_RESTRICTED | + IO_RING_F_RESTRICT_INHERITED; } static __cold int io_uring_create(struct io_ctx_config *config) diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 1cf5be62bb656..5065a3eff8094 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -458,7 +458,7 @@ static int io_remove_buffers_legacy(struct io_ring_ctx *ctx, static void io_put_bl(struct io_ring_ctx *ctx, struct io_buffer_list *bl) { if (bl->flags & IOBL_BUF_RING) - io_free_region(ctx->user, &bl->region); + io_free_region(NULL, &bl->region); else io_remove_buffers_legacy(ctx, bl, -1U); @@ -675,7 +675,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = reg.ring_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &bl->region, &rd, mmap_offset); + ret = io_create_region(NULL, &bl->region, &rd, mmap_offset); if (ret) goto fail; br = io_region_get_ptr(&bl->region); @@ -708,7 +708,7 @@ int io_register_pbuf_ring(struct io_ring_ctx *ctx, void __user *arg) if (!ret) return 0; fail: - io_free_region(ctx->user, &bl->region); + io_free_region(NULL, &bl->region); kfree(bl); return ret; } diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 653715e940ba0..7e83c48ad8d96 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -16,8 +16,10 @@ #include "zcrx.h" static bool io_mem_alloc_compound(struct page **pages, int nr_pages, - size_t size, gfp_t gfp) + size_t size, gfp_t gfp, + struct user_struct *user) { + unsigned long nr_compound, extra; struct page *page; int i, order; @@ -27,9 +29,22 @@ static bool io_mem_alloc_compound(struct page **pages, int nr_pages, else if (order) gfp |= __GFP_COMP; + /* + * get_order() rounds a non power of two size up, so the allocation + * can hold more pages than the region exposes. Account those too, + * and leave the compound allocation alone if they do not fit. + */ + nr_compound = 1UL << order; + extra = nr_compound - nr_pages; + if (extra && user && __io_account_mem(user, extra)) + return false; + page = alloc_pages(gfp, order); - if (!page) + if (!page) { + if (extra && user) + __io_unaccount_mem(user, extra); return false; + } for (i = 0; i < nr_pages; i++) pages[i] = page + i; @@ -105,8 +120,15 @@ void io_free_region(struct user_struct *user, struct io_mapped_region *mr) } if ((mr->flags & IO_REGION_F_VMAP) && mr->ptr) vunmap(mr->ptr); - if (mr->nr_pages && user) - __io_unaccount_mem(user, mr->nr_pages); + if (mr->nr_pages && user) { + unsigned long nr_accounted = mr->nr_pages; + + /* a compound region was accounted for the whole allocation */ + if (mr->flags & IO_REGION_F_SINGLE_REF) + nr_accounted = 1UL << get_order(io_region_size(mr)); + + __io_unaccount_mem(user, nr_accounted); + } memset(mr, 0, sizeof(*mr)); } @@ -151,7 +173,8 @@ static int io_region_pin_pages(struct io_mapped_region *mr, static int io_region_allocate_pages(struct io_mapped_region *mr, struct io_uring_region_desc *reg, - unsigned long mmap_offset) + unsigned long mmap_offset, + struct user_struct *user) { gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN; size_t size = io_region_size(mr); @@ -162,7 +185,7 @@ static int io_region_allocate_pages(struct io_mapped_region *mr, if (!pages) return -ENOMEM; - if (io_mem_alloc_compound(pages, mr->nr_pages, size, gfp)) { + if (io_mem_alloc_compound(pages, mr->nr_pages, size, gfp, user)) { mr->flags |= IO_REGION_F_SINGLE_REF; goto done; } @@ -181,7 +204,7 @@ static int io_region_allocate_pages(struct io_mapped_region *mr, return 0; } -int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, +int io_create_region(struct user_struct *user, struct io_mapped_region *mr, struct io_uring_region_desc *reg, unsigned long mmap_offset) { @@ -207,8 +230,8 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, return -EOVERFLOW; nr_pages = reg->size >> PAGE_SHIFT; - if (ctx->user) { - ret = __io_account_mem(ctx->user, nr_pages); + if (user) { + ret = __io_account_mem(user, nr_pages); if (ret) return ret; } @@ -217,7 +240,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, if (reg->flags & IORING_MEM_REGION_TYPE_USER) ret = io_region_pin_pages(mr, reg); else - ret = io_region_allocate_pages(mr, reg, mmap_offset); + ret = io_region_allocate_pages(mr, reg, mmap_offset, user); if (ret) goto out_free; @@ -226,7 +249,7 @@ int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, goto out_free; return 0; out_free: - io_free_region(ctx->user, mr); + io_free_region(user, mr); return ret; } diff --git a/io_uring/memmap.h b/io_uring/memmap.h index f4cfbb6b9a1f4..0714bb5a9616e 100644 --- a/io_uring/memmap.h +++ b/io_uring/memmap.h @@ -18,7 +18,7 @@ unsigned long io_uring_get_unmapped_area(struct file *file, unsigned long addr, int io_uring_mmap(struct file *file, struct vm_area_struct *vma); void io_free_region(struct user_struct *user, struct io_mapped_region *mr); -int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr, +int io_create_region(struct user_struct *user, struct io_mapped_region *mr, struct io_uring_region_desc *reg, unsigned long mmap_offset); diff --git a/io_uring/net.c b/io_uring/net.c index e6847fb084abc..37fc7baa55b27 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -1772,7 +1773,8 @@ void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req case AF_INET6: { struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss; - if (conn->addr_len < (int)sizeof(*sin6)) + /* The IPv6 stack accepts SIN6_LEN_RFC2133; port and addr are in it. */ + if (conn->addr_len < SIN6_LEN_RFC2133) break; bctx->connect.port = sin6->sin6_port; memcpy(bctx->connect.v6_addr, &sin6->sin6_addr, diff --git a/io_uring/register.c b/io_uring/register.c index 02bc103bcc9d5..dab3eb4258dff 100644 --- a/io_uring/register.c +++ b/io_uring/register.c @@ -480,8 +480,9 @@ struct io_ring_ctx_rings { static void io_register_free_rings(struct io_ring_ctx *ctx, struct io_ring_ctx_rings *r) { - io_free_region(ctx->user, &r->sq_region); - io_free_region(ctx->user, &r->ring_region); + /* ring memory is not charged to RLIMIT_MEMLOCK, hence no user */ + io_free_region(NULL, &r->sq_region); + io_free_region(NULL, &r->ring_region); } #define swap_old(ctx, o, n, field) \ @@ -529,7 +530,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = p->cq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING); + ret = io_create_region(NULL, &n.ring_region, &rd, IORING_OFF_CQ_RING); if (ret) return ret; @@ -559,7 +560,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg) rd.user_addr = p->sq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } - ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES); + ret = io_create_region(NULL, &n.sq_region, &rd, IORING_OFF_SQES); if (ret) { io_register_free_rings(ctx, &n); return ret; @@ -730,7 +731,8 @@ static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg) !(ctx->flags & IORING_SETUP_R_DISABLED)) return -EINVAL; - ret = io_create_region(ctx, ®ion, &rd, IORING_MAP_OFF_PARAM_REGION); + ret = io_create_region(ctx->user, ®ion, &rd, + IORING_MAP_OFF_PARAM_REGION); if (ret) return ret; if (copy_to_user(rd_uptr, &rd, sizeof(rd))) { @@ -764,7 +766,14 @@ static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, if (ctx->submitter_task && ctx->submitter_task != current) return -EEXIST; - if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) && !(ctx->flags & IORING_SETUP_R_DISABLED)) { + /* + * A disabled ring is exempt from its own register restrictions, so + * that the creator can set them up and then enable the ring. That + * does not apply to restrictions inherited from the task. + */ + if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) && + (!(ctx->flags & IORING_SETUP_R_DISABLED) || + (ctx->int_flags & IO_RING_F_RESTRICT_INHERITED))) { opcode = array_index_nospec(opcode, IORING_REGISTER_LAST); if (!test_bit(opcode, ctx->restrictions.register_op)) return -EACCES; diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c index 1b6832f6e10fa..0a9e8a58b99a1 100644 --- a/io_uring/zcrx.c +++ b/io_uring/zcrx.c @@ -398,7 +398,7 @@ static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx, mmap_offset = IORING_MAP_OFF_ZCRX_REGION; mmap_offset += (u64)id << IORING_OFF_ZCRX_SHIFT; - ret = io_create_region(ctx, &ifq->rq_region, rd, mmap_offset); + ret = io_create_region(ctx->user, &ifq->rq_region, rd, mmap_offset); if (ret < 0) return ret; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index c43f15c9917c1..501f12de533f4 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -151,6 +151,12 @@ static cpumask_var_t isolated_cpus; /* CSCB */ */ static bool update_housekeeping; /* RWCS */ +/* + * Set if "cpuset_v2_mode" mount option is used + * Cached at bind time and not lock protected; accessed via {READ,WRITE}_ONCE + */ +static bool cpuset_v2_mode; + /* * Copy of isolated_cpus to be passed to housekeeping_update() */ @@ -403,8 +409,7 @@ static inline bool cpuset_v2(void) */ static inline bool is_in_v2_mode(void) { - return cpuset_v2() || - (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE); + return cpuset_v2() || READ_ONCE(cpuset_v2_mode); } /** @@ -478,10 +483,26 @@ static void guarantee_active_cpus(struct task_struct *tsk, rcu_read_lock(); cs = task_cs(tsk); - while (!cpumask_intersects(cs->effective_cpus, pmask)) + while (!cpumask_intersects(cs->effective_cpus, pmask)) { cs = parent_cs(cs); - + if (unlikely(!cs)) { + /* + * The top cpuset doesn't have any active cpu as a + * consequence of a race between its caller and the cpu + * hotplug operation where cpu_active_mask is updated + * asynchronously before cpuset_handle_hotplug() is + * being called to adjust the effective_cpus of the + * affected cpusets. But we know the top cpuset's + * effective_cpus is on its way to be identical to + * cpu_active_mask minus the exclusive CPUs dedicated + * to other valid cpuset partitions. Just pass back + * the filtered cpu_active_mask in this case. + */ + goto out_unlock; + } + } cpumask_and(pmask, pmask, cs->effective_cpus); +out_unlock: rcu_read_unlock(); } @@ -3586,6 +3607,8 @@ static void cpuset_bind(struct cgroup_subsys_state *root_css) mutex_lock(&cpuset_mutex); spin_lock_irq(&callback_lock); + WRITE_ONCE(cpuset_v2_mode, + !!(cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE)); if (is_in_v2_mode()) { cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask); cpumask_copy(top_cpuset.effective_xcpus, cpu_possible_mask); @@ -3902,15 +3925,16 @@ static void cpuset_handle_hotplug(void) static cpumask_t new_cpus; static nodemask_t new_mems; bool cpus_updated, mems_updated; - bool on_dfl = is_in_v2_mode(); + bool on_dfl; struct tmpmasks tmp, *ptmp = NULL; - if (on_dfl && !alloc_tmpmasks(&tmp)) - ptmp = &tmp; - lockdep_assert_cpus_held(); mutex_lock(&cpuset_mutex); + on_dfl = is_in_v2_mode(); + if (on_dfl && !alloc_tmpmasks(&tmp)) + ptmp = &tmp; + /* fetch the available cpus/mems and find out which changed how */ cpumask_copy(&new_cpus, cpu_active_mask); new_mems = node_states[N_MEMORY]; diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index bd53931337533..08ded023328f6 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -1680,10 +1680,10 @@ phys_addr_t default_swiotlb_base(void) phys_addr_t default_swiotlb_limit(void) { #ifdef CONFIG_SWIOTLB_DYNAMIC - return io_tlb_default_mem.phys_limit; -#else - return io_tlb_default_mem.defpool.end - 1; + if (io_tlb_default_mem.can_grow) + return io_tlb_default_mem.phys_limit; #endif + return io_tlb_default_mem.defpool.end - 1; } #ifdef CONFIG_DEBUG_FS diff --git a/kernel/fork.c b/kernel/fork.c index f9fd9c9188639..f26f22e67643b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -3360,8 +3360,9 @@ int ksys_unshare(unsigned long unshare_flags) perf_event_namespaces(current); bad_unshare_cleanup_nsproxy: + /* never installed, so no active references to drop */ if (new_nsproxy) - put_nsproxy(new_nsproxy); + nsproxy_free(new_nsproxy); bad_unshare_cleanup_cred: if (new_cred) put_cred(new_cred); diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index d9d3d5973bf52..3fb1595a4a593 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c @@ -61,7 +61,7 @@ static inline struct nsproxy *create_nsproxy(void) return nsproxy; } -static inline void nsproxy_free(struct nsproxy *ns) +void nsproxy_free(struct nsproxy *ns) { put_mnt_ns(ns->mnt_ns); put_uts_ns(ns->uts_ns); diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c index cd7cbd1c99a26..a9f33e2b17ace 100644 --- a/kernel/sched/ext/ext.c +++ b/kernel/sched/ext/ext.c @@ -1953,8 +1953,14 @@ static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags, if (unlikely(!SCX_HAS_OP(sch, enqueue))) goto global; - /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */ - qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT; + /* + * DSQ bypass didn't trigger, enqueue on the BPF scheduler. Wrap the + * per-task qseq counter where the QSEQ field wraps and skip 0, which is + * what scx_bpf_dsq_insert() records for a task in NONE or DISPATCHING. + */ + p->scx.ops_qseq = ((p->scx.ops_qseq + 1) & + (SCX_OPSS_QSEQ_MASK >> SCX_OPSS_QSEQ_SHIFT)) ?: 1; + qseq = (unsigned long)p->scx.ops_qseq << SCX_OPSS_QSEQ_SHIFT; WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE); atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq); @@ -2044,6 +2050,13 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ int sticky_cpu = p->scx.sticky_cpu; u64 enq_flags = core_enq_flags | rq->scx.extra_enq_flags; + /* + * An SCX-internal migration ends on arrival. Clear sticky_cpu so @p can + * leave custody when inserted into the destination DSQ. + */ + if (sticky_cpu >= 0) + p->scx.sticky_cpu = -1; + /* * SCX_RQ_IN_WAKEUP promises a task_woken_scx() call once this enqueue * returns. Only the core's wakeup path delivers one. The flags stashed @@ -2082,9 +2095,6 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ dl_server_start(&rq->ext_server); do_enqueue_task(rq, p, enq_flags, sticky_cpu); - - if (sticky_cpu >= 0) - p->scx.sticky_cpu = -1; out: rq->scx.flags &= ~SCX_RQ_IN_WAKEUP; @@ -6690,10 +6700,9 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s, seq_buf_init(&ns, buf, avail); dump_newline(&ns); - dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu", + dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ksync=%lu", cpu, rq->scx.nr_running, rq->scx.flags, - rq->scx.cpu_released, rq->scx.ops_qseq, - rq->scx.kick_sync); + rq->scx.cpu_released, rq->scx.kick_sync); dump_line(&ns, " curr=%s[%d] class=%ps", rq->curr->comm, rq->curr->pid, rq->curr->sched_class); diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h index b295991b9f10c..3cd223c6d7639 100644 --- a/kernel/sched/ext/internal.h +++ b/kernel/sched/ext/internal.h @@ -1508,6 +1508,11 @@ enum scx_ops_state { * dequeue/requeue, the dispatcher can tell whether it still has a claim * on the task being dispatched. * + * QSEQ is generated from the per-task p->scx.ops_qseq counter so that + * it doesn't repeat across QUEUED instances of the same task even if + * the task moves between rqs. 0 is never used as a valid QSEQ since + * NONE and DISPATCHING map to this value. + * * As some 32bit archs can't do 64bit store_release/load_acquire, * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on * 32bit machines. The dispatch race window QSEQ protects is very narrow diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index f76e6f7f4a014..a6c6f8165eecc 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -795,7 +795,6 @@ struct scx_rq { struct scx_dispatch_q local_dsq; struct list_head runnable_list; /* runnable tasks on this rq */ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ - unsigned long ops_qseq; u64 extra_enq_flags; /* see move_task_to_local_dsq() */ u32 nr_running; u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */ diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 53608ac01fab1..cb7055aee3afc 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -2244,7 +2244,7 @@ static bool is_chained_work(struct workqueue_struct *wq) * Return %true iff I'm a worker executing a work item on @wq. If * I'm @worker, it's safe to dereference it without locking. */ - return worker && worker->current_pwq->wq == wq; + return worker && worker->current_pwq && worker->current_pwq->wq == wq; } /* diff --git a/lib/dim/dim.c b/lib/dim/dim.c index 97c3d084ebf02..7f3eebae73cb9 100644 --- a/lib/dim/dim.c +++ b/lib/dim/dim.c @@ -69,11 +69,15 @@ bool dim_calc_stats(const struct dim_sample *start, if (!delta_us) return false; - curr_stats->ppms = DIV_ROUND_UP(npkts * USEC_PER_MSEC, delta_us); - curr_stats->bpms = DIV_ROUND_UP(nbytes * USEC_PER_MSEC, delta_us); + /* u32 * USEC_PER_MSEC overflows a 32-bit long */ + curr_stats->ppms = DIV_ROUND_UP_ULL((u64)npkts * USEC_PER_MSEC, + delta_us); + curr_stats->bpms = DIV_ROUND_UP_ULL((u64)nbytes * USEC_PER_MSEC, + delta_us); curr_stats->epms = DIV_ROUND_UP(DIM_NEVENTS * USEC_PER_MSEC, delta_us); - curr_stats->cpms = DIV_ROUND_UP(ncomps * USEC_PER_MSEC, delta_us); + curr_stats->cpms = DIV_ROUND_UP_ULL((u64)ncomps * USEC_PER_MSEC, + delta_us); if (curr_stats->epms != 0) curr_stats->cpe_ratio = DIV_ROUND_DOWN_ULL( curr_stats->cpms * 100, curr_stats->epms); diff --git a/mm/mremap.c b/mm/mremap.c index 8e3d0e4b107d0..15dc2e38cb4e0 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -1035,6 +1035,7 @@ static void vrm_stat_account(struct vma_remap_struct *vrm, } static bool __check_map_count_against_split(struct mm_struct *mm, + bool pre_split, bool before_unmaps) { const int sys_map_count = get_sysctl_max_map_count(); @@ -1086,26 +1087,42 @@ static bool __check_map_count_against_split(struct mm_struct *mm, * Therefore we must check to ensure we have headroom of 2 additional * VMAs. */ - return map_count + 2 <= sys_map_count; + map_count += 2; + + /* If pre-split, the -1 observed above doesn't apply. */ + if (pre_split) + map_count++; + + return map_count <= sys_map_count; +} + +static bool needs_pre_split(struct vma_remap_struct *vrm) +{ + /* + * An MREMAP_DONTUNMAP of a mlock()'d VMA needs to unlock the + * source VMA, so split in this case. + */ + return (vrm->flags & MREMAP_DONTUNMAP) && + vma_test(vrm->vma, VMA_LOCKED_BIT); } /* Do we violate the map count limit if we split VMAs when moving the VMA? */ -static bool check_map_count_against_split(void) +static bool check_map_count_against_split(struct vma_remap_struct *vrm) { return __check_map_count_against_split(current->mm, - /*before_unmaps=*/false); + needs_pre_split(vrm), /*before_unmaps=*/false); } /* Do we violate the map count limit if we split VMAs prior to early unmaps? */ -static bool check_map_count_against_split_early(void) +static bool check_map_count_against_split_early(struct vma_remap_struct *vrm) { return __check_map_count_against_split(current->mm, - /*before_unmaps=*/true); + vrm->flags & MREMAP_DONTUNMAP, /*before_unmaps=*/true); } /* - * Perform checks before attempting to write a VMA prior to it being - * moved. + * Perform checks and preparation before attempting to write a VMA prior to it + * being moved. */ static unsigned long prep_move_vma(struct vma_remap_struct *vrm) { @@ -1114,19 +1131,17 @@ static unsigned long prep_move_vma(struct vma_remap_struct *vrm) unsigned long old_addr = vrm->addr; unsigned long old_len = vrm->old_len; vm_flags_t dummy = vma->vm_flags; + const bool split_before = vma->vm_start != old_addr; + const bool split_after = vma->vm_end != old_addr + old_len; - /* - * We'd prefer to avoid failure later on in do_munmap: we copy a VMA, - * which may not merge, then (if MREMAP_DONTUNMAP is not set) unmap the - * source, which may split, causing a net increase of 2 mappings. - */ - if (!check_map_count_against_split()) + /* Avoid failure later on. */ + if (!check_map_count_against_split(vrm)) return -ENOMEM; if (vma->vm_ops && vma->vm_ops->may_split) { - if (vma->vm_start != old_addr) + if (split_before) err = vma->vm_ops->may_split(vma, old_addr); - if (!err && vma->vm_end != old_addr + old_len) + if (!err && split_after) err = vma->vm_ops->may_split(vma, old_addr + old_len); if (err) return err; @@ -1144,6 +1159,21 @@ static unsigned long prep_move_vma(struct vma_remap_struct *vrm) if (err) return err; + /* + * To account mlock()'d pages correctly in the MREMAP_DONTUNMAP + * case perform any split ahead of time for an mlock()'d VMA. + */ + if (needs_pre_split(vrm)) { + VMA_ITERATOR(vmi, vma->vm_mm, old_addr); + + if (split_before) + err = split_vma(&vmi, vma, old_addr, 1); + if (!err && split_after) + err = split_vma(&vmi, vma, old_addr + old_len, 0); + vrm->vmi_needs_invalidate = true; + return err; + } + return 0; } @@ -1269,7 +1299,8 @@ static int copy_vma_and_data(struct vma_remap_struct *vrm, PAGETABLE_MOVE(pmc, NULL, NULL, vrm->addr, vrm->new_addr, vrm->old_len); new_vma = copy_vma(&vma, vrm->new_addr, vrm->new_len, pos, - &pmc.need_rmap_locks); + &pmc.need_rmap_locks, + vrm->flags & MREMAP_DONTUNMAP); if (!new_vma) { vrm_uncharge(vrm); *new_vma_ptr = NULL; @@ -1329,6 +1360,9 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, unsigned long old_start = vma->vm_start; unsigned long old_end = vma->vm_end; + /* Self-merge is disallowed. */ + VM_WARN_ON_ONCE(new_vma == vma); + /* We always clear VM_LOCKED[ONFAULT] on the old VMA. */ vm_flags_clear(vma, VM_LOCKED_MASK); @@ -1336,8 +1370,11 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, * anon_vma links of the old vma is no longer needed after its page * table has been moved. */ - if (new_vma != vma && start == old_start && end == old_end) { - const pgoff_t pgoff_unfaulted = vma->vm_start >> PAGE_SHIFT; + if (start == old_start && end == old_end) { + const struct vm_page_offset pos_unfaulted = { + .index = vma->vm_start >> PAGE_SHIFT, + .offset = vma->vm_start & ~PAGE_MASK, + }; unlink_anon_vmas(vma); /* @@ -1346,7 +1383,7 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, * vma->vm_start >> PAGE_SHIFT. */ if (vma_is_anonymous(vma) && !vma->vm_file) - vma->vm_pgoff = pgoff_unfaulted; + vma_set_page_offset(vma, pos_unfaulted); } } @@ -1997,7 +2034,7 @@ static unsigned long do_mremap(struct vma_remap_struct *vrm) return -EINTR; vrm->mmap_locked = true; - if (!check_map_count_against_split_early()) { + if (!check_map_count_against_split_early(vrm)) { mmap_write_unlock(mm); return -ENOMEM; } diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index 7a9eef9b4f5b7..70f1cd37f007b 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -455,12 +455,30 @@ static void kernel_pgtable_work_func(struct work_struct *work) __pagetable_free(pt); } +static void schedule_kernel_pgtable_free(void) +{ + schedule_work(&kernel_pgtable_work.work); +} + void pagetable_free_kernel(struct ptdesc *pt) { spin_lock(&kernel_pgtable_work.lock); list_add(&pt->pt_list, &kernel_pgtable_work.list); spin_unlock(&kernel_pgtable_work.lock); - schedule_work(&kernel_pgtable_work.work); + /* + * The workqueue may not exist yet while the system is booting. + * kernel_pgtable_drain_early() schedules the work once it does. + */ + if (system_state != SYSTEM_BOOTING) + schedule_kernel_pgtable_free(); +} + +static int __init kernel_pgtable_drain_early(void) +{ + /* Free the kernel page tables queued while booting. */ + schedule_kernel_pgtable_free(); + return 0; } +core_initcall(kernel_pgtable_drain_early); #endif diff --git a/mm/vma.c b/mm/vma.c index 941d9ec9f4b76..8646a0a8d91a9 100644 --- a/mm/vma.c +++ b/mm/vma.c @@ -599,8 +599,8 @@ __split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, * Split a vma into two pieces at address 'addr', a new vma is allocated * either for the first part or the tail. */ -static int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, - unsigned long addr, int new_below) +int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, + unsigned long addr, int new_below) { if (vma->vm_mm->map_count >= get_sysctl_max_map_count()) return -ENOMEM; @@ -1877,7 +1877,7 @@ static int vma_link(struct mm_struct *mm, struct vm_area_struct *vma) */ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, struct vm_page_offset pos, - bool *need_rmap_locks) + bool *need_rmap_locks, bool keep_source) { struct vm_area_struct *vma = *vmap; unsigned long vma_start = vma->vm_start; @@ -1913,6 +1913,21 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, vmg_set_page_offset(&vmg, pos); vmg.next = vma_iter_next_rewind(&vmi, NULL); + + /* + * If the original VMA is kept (MREMAP_DONTUNMAP), the source and + * destination VMA must be treated distinctly. + * + * A merge violates this, so in this case disallow a self-merge. + */ + if (!faulted_in_anon_vma && keep_source) { + if (vmg.prev == vma) + vmg.prev = NULL; + if (vmg.next == vma) + vmg.next = NULL; + faulted_in_anon_vma = true; + } + new_vma = vma_merge_copied_range(&vmg); if (new_vma) { diff --git a/mm/vma.h b/mm/vma.h index 3d9c8ee4be05d..9f9dac01dde4b 100644 --- a/mm/vma.h +++ b/mm/vma.h @@ -481,7 +481,7 @@ void unlink_file_vma_batch_add(struct unlink_vma_file_batch *vb, struct vm_area_struct *copy_vma(struct vm_area_struct **vmap, unsigned long addr, unsigned long len, struct vm_page_offset pos, - bool *need_rmap_locks); + bool *need_rmap_locks, bool keep_source); struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma); @@ -502,6 +502,11 @@ int do_brk_flags(struct vma_iterator *vmi, struct vm_area_struct *brkvma, unsigned long unmapped_area(struct vm_unmapped_area_info *info); unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info); +#ifdef CONFIG_MMU +int split_vma(struct vma_iterator *vmi, struct vm_area_struct *vma, + unsigned long addr, int new_below); +#endif + static inline bool vma_wants_manual_pte_write_upgrade(struct vm_area_struct *vma) { /* diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index aea017ee5b0a3..b19e68844e1a9 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -4113,9 +4113,11 @@ static int hci_send_cmd_sync(struct hci_dev *hdev, struct sk_buff *skb) bt_dev_dbg(hdev, "skb %p", skb); + hci_dev_lock(hdev); kfree_skb(hdev->sent_cmd); hdev->sent_cmd = skb_clone(skb, GFP_KERNEL); + hci_dev_unlock(hdev); if (!hdev->sent_cmd) { skb_queue_head(&hdev->cmd_q, skb); queue_work(hdev->workqueue, &hdev->cmd_work); @@ -4136,8 +4138,10 @@ static int hci_send_cmd_sync(struct hci_dev *hdev, struct sk_buff *skb) if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND && !hci_dev_test_and_set_flag(hdev, HCI_CMD_PENDING)) { + hci_dev_lock(hdev); kfree_skb(hdev->req_skb); hdev->req_skb = skb_get(hdev->sent_cmd); + hci_dev_unlock(hdev); } return err; diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index 6d56c77741e19..81068b585764e 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -1223,8 +1223,10 @@ static int hci_sock_bind(struct socket *sock, struct sockaddr_unsized *addr, */ hdev = hci_pi(sk)->hdev; if (hdev && hci_dev_test_flag(hdev, HCI_UNREGISTER)) { + write_lock(&hci_sk_list.lock); hci_pi(sk)->hdev = NULL; sk->sk_state = BT_OPEN; + write_unlock(&hci_sk_list.lock); hci_dev_put(hdev); } hdev = NULL; diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 9f4852e2dc7bc..bc576baf1322b 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -32,8 +32,10 @@ static void hci_cmd_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode, WRITE_ONCE(hdev->req_status, HCI_REQ_DONE); /* Free the request command so it is not used as response */ + hci_dev_lock(hdev); kfree_skb(hdev->req_skb); hdev->req_skb = NULL; + hci_dev_unlock(hdev); if (skb) { struct sock *sk = hci_skb_sk(skb); @@ -2300,6 +2302,15 @@ static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val, { struct hci_cp_le_set_scan_enable cp; + /* Firmware duplicate caches may need an explicit reset between scans. */ + if (val && filter_dup != LE_SCAN_FILTER_DUP_DISABLE && + !hci_dev_test_flag(hdev, HCI_MESH) && hdev->reset_dup_filter) { + int err = hdev->reset_dup_filter(hdev); + + if (err) + return err; + } + if (use_ext_scan(hdev)) return hci_le_set_ext_scan_enable_sync(hdev, val, filter_dup); @@ -3060,15 +3071,21 @@ static int hci_le_set_ext_scan_param_sync(struct hci_dev *hdev, u8 type, */ if (hci_dev_test_flag(hdev, HCI_PA_SYNC)) { struct hci_cp_le_add_to_accept_list *sent; + bdaddr_t bdaddr; + hci_dev_lock(hdev); sent = hci_sent_cmd_data(hdev, HCI_OP_LE_ADD_TO_ACCEPT_LIST); + if (sent) + bacpy(&bdaddr, &sent->bdaddr); + hci_dev_unlock(hdev); + if (sent) { struct hci_conn *conn; rcu_read_lock(); conn = hci_conn_hash_lookup_ba(hdev, PA_LINK, - &sent->bdaddr); + &bdaddr); if (conn) { struct bt_iso_qos *qos = &conn->iso_qos; diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index 7657c2a0abbf7..d6ac5b1f49bc1 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -61,6 +61,7 @@ enum { BT_SK_BIG_SYNC, BT_SK_PA_SYNC, BT_SK_KILLED, + BT_SK_CONNECTING, }; struct iso_pinfo { @@ -350,6 +351,9 @@ static int __iso_chan_add(struct iso_conn *conn, struct sock *sk, return -EBUSY; } + if (iso_pi(sk)->conn) + return -EISCONN; + if (!conn->hcon) { BT_ERR("conn->hcon missing"); return -EIO; @@ -410,6 +414,11 @@ static int iso_connect_bis(struct sock *sk) hci_dev_lock(hdev); lock_sock(sk); + if (iso_pi(sk)->conn) { + err = -EISCONN; + goto unlock; + } + if (!bis_capable(hdev)) { err = -EOPNOTSUPP; goto unlock; @@ -562,6 +571,13 @@ static int iso_connect_cis(struct sock *sk) lockdep_assert_held(&hcon->hdev->lock); + /* The socket lock keeps the current attachment and its hcon stable. */ + if (iso_pi(sk)->conn && iso_pi(sk)->conn->hcon != hcon) { + hci_conn_drop(hcon); + err = -EISCONN; + goto unlock; + } + conn = iso_conn_add(hcon); if (!conn) { hci_conn_drop(hcon); @@ -1269,17 +1285,26 @@ static int iso_sock_connect(struct socket *sock, struct sockaddr_unsized *addr, addr->sa_family != AF_BLUETOOTH) return -EINVAL; - if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) - return -EBADFD; + lock_sock(sk); - if (sk->sk_type != SOCK_SEQPACKET) - return -EINVAL; + if ((sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) || + test_bit(BT_SK_CONNECTING, &iso_pi(sk)->flags)) { + err = -EBADFD; + goto done; + } + + if (sk->sk_type != SOCK_SEQPACKET) { + err = -EINVAL; + goto done; + } /* Check if the address type is of LE type */ - if (!bdaddr_type_is_le(sa->iso_bdaddr_type)) - return -EINVAL; + if (!bdaddr_type_is_le(sa->iso_bdaddr_type)) { + err = -EINVAL; + goto done; + } - lock_sock(sk); + set_bit(BT_SK_CONNECTING, &iso_pi(sk)->flags); bacpy(&iso_pi(sk)->dst, &sa->iso_bdaddr); iso_pi(sk)->dst_type = sa->iso_bdaddr_type; @@ -1291,16 +1316,18 @@ static int iso_sock_connect(struct socket *sock, struct sockaddr_unsized *addr, else err = iso_connect_bis(sk); - if (err) - return err; - lock_sock(sk); + clear_bit(BT_SK_CONNECTING, &iso_pi(sk)->flags); + if (err) + goto done; + if (!test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags)) { err = bt_sock_wait_state(sk, BT_CONNECTED, sock_sndtimeo(sk, flags & O_NONBLOCK)); } +done: release_sock(sk); return err; } @@ -1340,6 +1367,11 @@ static int iso_listen_bis(struct sock *sk) hci_dev_lock(hdev); lock_sock(sk); + if (sk->sk_state != BT_BOUND || iso_pi(sk)->conn) { + err = -EBADFD; + goto unlock; + } + /* Fail if user set invalid QoS */ if (iso_pi(sk)->qos_user_set && !check_bcast_qos(&iso_pi(sk)->qos)) { iso_pi(sk)->qos = default_qos; diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 2a177cd17a9ef..7928d9e45f98a 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -1489,7 +1489,7 @@ static void cmd_complete_rsp(struct mgmt_pending_cmd *cmd, void *data) return; } - cmd_status_rsp(cmd, data); + cmd_status_rsp(cmd, &match->mgmt_status); } static int generic_cmd_complete(struct mgmt_pending_cmd *cmd, u8 status) diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c index 3f757198051a2..7a7791fe56b0e 100644 --- a/net/bluetooth/rfcomm/core.c +++ b/net/bluetooth/rfcomm/core.c @@ -62,10 +62,9 @@ static void rfcomm_make_uih(struct sk_buff *skb, u8 addr); static void rfcomm_process_connect(struct rfcomm_session *s); -static struct rfcomm_session *rfcomm_session_create(bdaddr_t *src, - bdaddr_t *dst, - u8 sec_level, - int *err); +static struct socket *rfcomm_session_connect(bdaddr_t *src, bdaddr_t *dst, + u8 sec_level, int *err); +static struct rfcomm_session *rfcomm_session_add(struct socket *sock, int state); static struct rfcomm_session *rfcomm_session_get(bdaddr_t *src, bdaddr_t *dst); static struct rfcomm_session *rfcomm_session_del(struct rfcomm_session *s); @@ -365,28 +364,17 @@ static int rfcomm_check_channel(u8 channel) return channel < 1 || channel > 30; } -static int __rfcomm_dlc_open(struct rfcomm_dlc *d, bdaddr_t *src, bdaddr_t *dst, u8 channel) +static int __rfcomm_dlc_open(struct rfcomm_dlc *d, struct rfcomm_session *s, + u8 channel) { - struct rfcomm_session *s; - int err = 0; u8 dlci; - BT_DBG("dlc %p state %ld %pMR -> %pMR channel %d", - d, d->state, src, dst, channel); - - if (rfcomm_check_channel(channel)) - return -EINVAL; + BT_DBG("dlc %p state %ld session %p channel %d", + d, d->state, s, channel); if (d->state != BT_OPEN && d->state != BT_CLOSED) return 0; - s = rfcomm_session_get(src, dst); - if (!s) { - s = rfcomm_session_create(src, dst, d->sec_level, &err); - if (!s) - return err; - } - dlci = __dlci(__session_dir(s), channel); /* Check if DLCI already exists */ @@ -421,14 +409,84 @@ static int __rfcomm_dlc_open(struct rfcomm_dlc *d, bdaddr_t *src, bdaddr_t *dst, int rfcomm_dlc_open(struct rfcomm_dlc *d, bdaddr_t *src, bdaddr_t *dst, u8 channel) { - int r; + struct rfcomm_session *s; + struct socket *sock; + int err; + + BT_DBG("dlc %p state %ld %pMR -> %pMR channel %d", + d, d->state, src, dst, channel); + + if (rfcomm_check_channel(channel)) + return -EINVAL; rfcomm_lock(); - r = __rfcomm_dlc_open(d, src, dst, channel); + clear_bit(RFCOMM_CLOSED, &d->flags); + /* Do not page the remote device for a DLC that cannot be opened + * anyway. __rfcomm_dlc_open() looks at the state again once the + * lock has been re-acquired below. + */ + if (d->state != BT_OPEN && d->state != BT_CLOSED) { + rfcomm_unlock(); + return 0; + } + + s = rfcomm_session_get(src, dst); + if (s) { + err = __rfcomm_dlc_open(d, s, channel); + rfcomm_unlock(); + return err; + } rfcomm_unlock(); - return r; + + /* There is no session for this pair yet. kernel_connect() ends up in + * l2cap_chan_connect(), which takes hdev->lock, and the HCI event + * path takes rfcomm_mutex while holding hdev->lock, so the socket has + * to be connected with rfcomm_mutex released. + */ + sock = rfcomm_session_connect(src, dst, d->sec_level, &err); + if (!sock) + return err; + + rfcomm_lock(); + + /* The DLC may have been closed while the socket was connecting. It + * was not on a session, so rfcomm_dlc_close() could only mark it; + * attaching it now would leave it with no owner. + */ + if (test_bit(RFCOMM_CLOSED, &d->flags)) { + rfcomm_unlock(); + sock_release(sock); + return -ECONNRESET; + } + + /* Another opener may have added a session for the same pair in the + * meantime; that one is used and this socket is dropped. + */ + s = rfcomm_session_get(src, dst); + if (!s) { + s = rfcomm_session_add(sock, BT_BOUND); + if (s) { + s->initiator = 1; + sock = NULL; + } + } + + err = s ? __rfcomm_dlc_open(d, s, channel) : -ENOMEM; + + rfcomm_unlock(); + + if (sock) + sock_release(sock); + + /* Over an existing ACL link the connection can complete before the + * session reaches the list, and that wakeup is then lost, so let + * krfcommd look at the socket state now. + */ + rfcomm_schedule(); + + return err; } static void __rfcomm_dlc_disconn(struct rfcomm_dlc *d) @@ -508,8 +566,14 @@ int rfcomm_dlc_close(struct rfcomm_dlc *d, int err) rfcomm_lock(); s = d->session; - if (!s) + if (!s) { + /* Not on a session yet: rfcomm_dlc_open() may be connecting a + * socket for it with rfcomm_mutex released. Leave a mark so + * that it does not attach the DLC once it holds the lock again. + */ + set_bit(RFCOMM_CLOSED, &d->flags); goto no_session; + } /* after waiting on the mutex check the session still exists * then check the dlc still exists @@ -757,12 +821,12 @@ static struct rfcomm_session *rfcomm_session_close(struct rfcomm_session *s, return rfcomm_session_del(s); } -static struct rfcomm_session *rfcomm_session_create(bdaddr_t *src, - bdaddr_t *dst, - u8 sec_level, - int *err) +/* Creates the L2CAP socket a new session will run on and starts connecting + * it. Must be called with rfcomm_mutex released. + */ +static struct socket *rfcomm_session_connect(bdaddr_t *src, bdaddr_t *dst, + u8 sec_level, int *err) { - struct rfcomm_session *s = NULL; struct sockaddr_l2 addr; struct socket *sock; struct sock *sk; @@ -792,24 +856,16 @@ static struct rfcomm_session *rfcomm_session_create(bdaddr_t *src, l2cap_pi(sk)->chan->mode = L2CAP_MODE_ERTM; release_sock(sk); - s = rfcomm_session_add(sock, BT_BOUND); - if (!s) { - *err = -ENOMEM; - goto failed; - } - - s->initiator = 1; - bacpy(&addr.l2_bdaddr, dst); addr.l2_family = AF_BLUETOOTH; addr.l2_psm = cpu_to_le16(L2CAP_PSM_RFCOMM); addr.l2_cid = 0; addr.l2_bdaddr_type = BDADDR_BREDR; *err = kernel_connect(sock, (struct sockaddr_unsized *)&addr, sizeof(addr), O_NONBLOCK); - if (*err == 0 || *err == -EINPROGRESS) - return s; + if (*err && *err != -EINPROGRESS) + goto failed; - return rfcomm_session_del(s); + return sock; failed: sock_release(sock); diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c index d2855f9b3abc3..0a6ba8ca230f0 100644 --- a/net/bluetooth/rfcomm/sock.c +++ b/net/bluetooth/rfcomm/sock.c @@ -48,8 +48,11 @@ static void rfcomm_sock_kill(struct sock *sk); static void rfcomm_sk_data_ready(struct rfcomm_dlc *d, struct sk_buff *skb) { struct sock *sk = d->owner; - if (!sk) + + if (!sk) { + kfree_skb(skb); return; + } atomic_add(skb->len, &sk->sk_rmem_alloc); skb_queue_tail(&sk->sk_receive_queue, skb); diff --git a/net/bluetooth/rfcomm/tty.c b/net/bluetooth/rfcomm/tty.c index dc3cdf614defe..198c7dd1a95b4 100644 --- a/net/bluetooth/rfcomm/tty.c +++ b/net/bluetooth/rfcomm/tty.c @@ -128,7 +128,7 @@ static void rfcomm_dev_shutdown(struct tty_port *port) { struct rfcomm_dev *dev = container_of(port, struct rfcomm_dev, port); - if (dev->tty_dev->parent) + if (dev->tty_dev && dev->tty_dev->parent) device_move(dev->tty_dev, NULL, DPM_ORDER_DEV_LAST); /* close the dlc */ diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index 3d4362a09df4c..f4a20d7b5f9cb 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -84,12 +84,14 @@ static void sco_conn_free(struct kref *ref) if (conn->sk) sco_pi(conn->sk)->conn = NULL; - if (conn->hcon) { - conn->hcon->sco_data = NULL; - hci_conn_drop(conn->hcon); - } + /* hcon->sco_data is cleared and the association's reference on the + * sco_conn is dropped in sco_conn_del() under hdev->lock, and the + * hci_conn is now owned by the socket (held in __sco_chan_add() and + * dropped in sco_chan_del()/sco_sock_destruct()), so there is nothing + * left to release towards hcon here. + */ - /* Ensure no more work items will run since hci_conn has been dropped */ + /* Ensure no more work items will run before the connection is freed */ disable_delayed_work_sync(&conn->timeout_work); kfree(conn); @@ -188,25 +190,19 @@ static void sco_sock_clear_timer(struct sock *sk) } /* ---- SCO connections ---- */ -/* Consumes a reference on @hcon, which the returned sco_conn owns until it is - * freed. On failure (NULL return) the reference is left for the caller to drop. +/* Returns a new reference the caller must drop with sco_conn_put(). The + * hcon->sco_data association holds its own reference on the sco_conn for the + * connection's lifetime; it is dropped in sco_conn_del() under hdev->lock. + * @hcon is not consumed: the hci_conn reference is taken and owned by the + * socket in __sco_chan_add(). */ static struct sco_conn *sco_conn_add(struct hci_conn *hcon) { struct sco_conn *conn = hcon->sco_data; conn = sco_conn_hold_unless_zero(conn); - if (conn) { - if (!conn->hcon) { - sco_conn_lock(conn); - conn->hcon = hcon; - sco_conn_unlock(conn); - } else { - /* conn already owns a reference on hcon */ - hci_conn_drop(hcon); - } + if (conn) return conn; - } conn = kzalloc_obj(struct sco_conn); if (!conn) @@ -227,7 +223,10 @@ static struct sco_conn *sco_conn_add(struct hci_conn *hcon) BT_DBG("hcon %p conn %p", hcon, conn); - return conn; + /* kref_init() above set the association reference owned by + * hcon->sco_data; hand the caller its own reference. + */ + return sco_conn_hold(conn); } /* Delete channel. @@ -242,6 +241,19 @@ static void sco_chan_del(struct sock *sk, int err) BT_DBG("sk %p, conn %p, err %d", sk, conn, err); if (conn) { + struct hci_conn *hcon; + + sco_conn_lock(conn); + hcon = conn->hcon; + sco_conn_unlock(conn); + + /* Drop the socket's hci_conn reference BEFORE clearing + * conn->sk, so sco_conn_del() on another CPU cannot free + * the hci_conn while we still hold a pointer to it. + */ + if (hcon) + hci_conn_drop(hcon); + sco_conn_lock(conn); conn->sk = NULL; sco_conn_unlock(conn); @@ -266,6 +278,13 @@ static void sco_conn_del(struct hci_conn *hcon, int err) BT_DBG("hcon %p conn %p, err %d", hcon, conn, err); + /* Detach from the hci_conn and drop the association's reference. + * The caller holds hdev->lock, which serialises this against the + * read of hcon->sco_data in sco_recv_scodata(). + */ + hcon->sco_data = NULL; + sco_conn_put(conn); + sco_conn_lock(conn); sk = sco_sock_hold(conn); sco_conn_unlock(conn); @@ -290,6 +309,11 @@ static void __sco_chan_add(struct sco_conn *conn, struct sock *sk, sco_pi(sk)->conn = sco_conn_hold(conn); conn->sk = sk; + /* The socket owns an hci_conn reference for as long as it stays + * attached; it is dropped in sco_chan_del()/sco_sock_destruct(). + */ + hci_conn_hold(conn->hcon); + if (parent) bt_accept_enqueue(parent, sk, true); } @@ -371,6 +395,7 @@ static int sco_connect(struct sock *sk) if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) { release_sock(sk); sco_conn_put(conn); + hci_conn_drop(hcon); err = -EBADFD; goto unlock; } @@ -379,9 +404,13 @@ static int sco_connect(struct sock *sk) sco_conn_put(conn); if (err) { release_sock(sk); + hci_conn_drop(hcon); goto unlock; } + /* __sco_chan_add() took its own hci_conn reference; drop ours. */ + hci_conn_drop(hcon); + /* Update source addr of the socket */ bacpy(&sco_pi(sk)->src, &hcon->src); @@ -495,9 +524,25 @@ static struct sock *sco_get_sock_listen(bdaddr_t *src) static void sco_sock_destruct(struct sock *sk) { + struct sco_conn *conn = sco_pi(sk)->conn; + BT_DBG("sk %p", sk); - sco_conn_put(sco_pi(sk)->conn); + /* If the channel was not already torn down via sco_chan_del(), drop + * the socket's own references here. + */ + if (conn) { + struct hci_conn *hcon; + + sco_conn_lock(conn); + hcon = conn->hcon; + sco_conn_unlock(conn); + + if (hcon) + hci_conn_drop(hcon); + sco_pi(sk)->conn = NULL; + sco_conn_put(conn); + } skb_queue_purge(&sk->sk_receive_queue); skb_queue_purge(&sk->sk_write_queue); @@ -1511,12 +1556,10 @@ static void sco_connect_cfm(struct hci_conn *hcon, __u8 status) if (!status) { struct sco_conn *conn; - conn = sco_conn_add(hci_conn_hold(hcon)); + conn = sco_conn_add(hcon); if (conn) { sco_conn_ready(conn); sco_conn_put(conn); - } else { - hci_conn_drop(hcon); } } else sco_conn_del(hcon, bt_to_errno(status)); diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index 2191d35d9fa07..79b6fe54a7cbe 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -81,6 +81,10 @@ __br_multicast_add_group(struct net_bridge_mcast *brmctx, bool blocked); static void br_multicast_find_del_pg(struct net_bridge *br, struct net_bridge_port_group *pg); +static void __br_multicast_del_pg(struct net_bridge_mdb_entry *mp, + struct net_bridge_port_group *pg, + struct net_bridge_port_group __rcu **pp, + bool sg_del_exclude_ports); static void __br_multicast_stop(struct net_bridge_mcast *brmctx); static int br_mc_disabled_update(struct net_device *dev, bool value, @@ -458,7 +462,7 @@ static void br_multicast_sg_del_exclude_ports(struct net_bridge_mdb_entry *sgmp) for (pp = &sgmp->ports; (p = mlock_dereference(*pp, sgmp->br)) != NULL;) { if (!(p->flags & MDB_PG_FLAGS_PERMANENT)) - br_multicast_del_pg(sgmp, p, pp); + __br_multicast_del_pg(sgmp, p, pp, false); else pp = &p->next; } @@ -799,9 +803,10 @@ static void br_multicast_destroy_port_group(struct net_bridge_mcast_gc *gc) kfree_rcu(pg, rcu); } -void br_multicast_del_pg(struct net_bridge_mdb_entry *mp, - struct net_bridge_port_group *pg, - struct net_bridge_port_group __rcu **pp) +static void __br_multicast_del_pg(struct net_bridge_mdb_entry *mp, + struct net_bridge_port_group *pg, + struct net_bridge_port_group __rcu **pp, + bool sg_del_exclude_ports) { struct net_bridge *br = pg->key.port->br; struct net_bridge_group_src *ent; @@ -816,7 +821,8 @@ void br_multicast_del_pg(struct net_bridge_mdb_entry *mp, if (!br_multicast_is_star_g(&mp->addr)) { rhashtable_remove_fast(&br->sg_port_tbl, &pg->rhnode, br_sg_port_rht_params); - br_multicast_sg_del_exclude_ports(mp); + if (sg_del_exclude_ports) + br_multicast_sg_del_exclude_ports(mp); } else { br_multicast_star_g_handle_mode(pg, MCAST_INCLUDE); } @@ -828,6 +834,13 @@ void br_multicast_del_pg(struct net_bridge_mdb_entry *mp, mod_timer(&mp->timer, jiffies); } +void br_multicast_del_pg(struct net_bridge_mdb_entry *mp, + struct net_bridge_port_group *pg, + struct net_bridge_port_group __rcu **pp) +{ + __br_multicast_del_pg(mp, pg, pp, true); +} + static void br_multicast_find_del_pg(struct net_bridge *br, struct net_bridge_port_group *pg) { diff --git a/net/bridge/netfilter/nf_conntrack_bridge.c b/net/bridge/netfilter/nf_conntrack_bridge.c index 7ecb8a26bfa3a..b5444335b86f2 100644 --- a/net/bridge/netfilter/nf_conntrack_bridge.c +++ b/net/bridge/netfilter/nf_conntrack_bridge.c @@ -39,9 +39,13 @@ static int nf_br_ip_fragment(struct net *net, struct sock *sk, int err = 0; /* for offloaded checksums cleanup checksum before fragmentation */ - if (skb->ip_summed == CHECKSUM_PARTIAL && - (err = skb_checksum_help(skb))) - goto blackhole; + if (skb->ip_summed == CHECKSUM_PARTIAL) { + err = ip_check_csum_start(skb); + if (!err) + err = skb_checksum_help(skb); + if (err) + goto blackhole; + } iph = ip_hdr(skb); diff --git a/net/can/af_can.c b/net/can/af_can.c index 7bc86b176b4de..34fe3b28d576f 100644 --- a/net/can/af_can.c +++ b/net/can/af_can.c @@ -641,13 +641,10 @@ static int can_rcv_filter(struct can_dev_rcv_lists *dev_rcv_lists, struct sk_buf return matches; } -void can_set_skb_uid(struct sk_buff *skb) +void can_set_skb_uid(struct can_skb_ext *csx) { - /* create non-zero unique skb identifier together with *skb */ - while (!(skb->hash)) - skb->hash = atomic_inc_return(&skbcounter); - - skb->sw_hash = 1; + while (!(csx->can_skb_uid)) + csx->can_skb_uid = atomic_inc_return(&skbcounter); } EXPORT_SYMBOL(can_set_skb_uid); @@ -662,8 +659,6 @@ static void can_receive(struct sk_buff *skb, struct net_device *dev) atomic_long_inc(&pkg_stats->rx_frames); atomic_long_inc(&pkg_stats->rx_frames_delta); - can_set_skb_uid(skb); - rcu_read_lock(); /* deliver the packet to sockets listening on all devices */ @@ -687,8 +682,10 @@ static void can_receive(struct sk_buff *skb, struct net_device *dev) static int can_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { + struct can_skb_ext *csx = can_skb_ext_find(skb); + if (unlikely(dev->type != ARPHRD_CAN || !can_get_ml_priv(dev) || - !can_skb_ext_find(skb) || !can_is_can_skb(skb))) { + !csx || !can_is_can_skb(skb))) { pr_warn_once("PF_CAN: dropped non conform CAN skbuff: dev type %d, len %d\n", dev->type, skb->len); @@ -696,6 +693,14 @@ static int can_rcv(struct sk_buff *skb, struct net_device *dev, return NET_RX_DROP; } + /* create unshared CAN skb_extension for netem/mirred skb clones */ + csx = skb_ext_add(skb, SKB_EXT_CAN); + if (unlikely(!csx)) { + kfree_skb_reason(skb, SKB_DROP_REASON_NOMEM); + return NET_RX_DROP; + } + + can_set_skb_uid(csx); can_receive(skb, dev); return NET_RX_SUCCESS; } @@ -703,8 +708,10 @@ static int can_rcv(struct sk_buff *skb, struct net_device *dev, static int canfd_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { + struct can_skb_ext *csx = can_skb_ext_find(skb); + if (unlikely(dev->type != ARPHRD_CAN || !can_get_ml_priv(dev) || - !can_skb_ext_find(skb) || !can_is_canfd_skb(skb))) { + !csx || !can_is_canfd_skb(skb))) { pr_warn_once("PF_CAN: dropped non conform CAN FD skbuff: dev type %d, len %d\n", dev->type, skb->len); @@ -712,6 +719,14 @@ static int canfd_rcv(struct sk_buff *skb, struct net_device *dev, return NET_RX_DROP; } + /* create unshared CAN skb_extension for netem/mirred skb clones */ + csx = skb_ext_add(skb, SKB_EXT_CAN); + if (unlikely(!csx)) { + kfree_skb_reason(skb, SKB_DROP_REASON_NOMEM); + return NET_RX_DROP; + } + + can_set_skb_uid(csx); can_receive(skb, dev); return NET_RX_SUCCESS; } @@ -719,8 +734,10 @@ static int canfd_rcv(struct sk_buff *skb, struct net_device *dev, static int canxl_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { + struct can_skb_ext *csx = can_skb_ext_find(skb); + if (unlikely(dev->type != ARPHRD_CAN || !can_get_ml_priv(dev) || - !can_skb_ext_find(skb) || !can_is_canxl_skb(skb))) { + !csx || !can_is_canxl_skb(skb))) { pr_warn_once("PF_CAN: dropped non conform CAN XL skbuff: dev type %d, len %d\n", dev->type, skb->len); @@ -728,6 +745,14 @@ static int canxl_rcv(struct sk_buff *skb, struct net_device *dev, return NET_RX_DROP; } + /* create unshared CAN skb_extension for netem/mirred skb clones */ + csx = skb_ext_add(skb, SKB_EXT_CAN); + if (unlikely(!csx)) { + kfree_skb_reason(skb, SKB_DROP_REASON_NOMEM); + return NET_RX_DROP; + } + + can_set_skb_uid(csx); can_receive(skb, dev); return NET_RX_SUCCESS; } diff --git a/net/can/gw.c b/net/can/gw.c index 0ec99f68aa452..5793cb2420ed3 100644 --- a/net/can/gw.c +++ b/net/can/gw.c @@ -527,6 +527,9 @@ static void can_can_gw_rcv(struct sk_buff *skb, void *data) /* put the incremented hop counter in the cloned skb */ ncsx->can_gw_hops = csx->can_gw_hops + 1; + /* force a new CAN UID generation for the routed frame */ + ncsx->can_skb_uid = 0; + /* first processing of this CAN frame -> adjust to private hop limit */ if (gwj->limit_hops && ncsx->can_gw_hops == 1) ncsx->can_gw_hops = max_hops - gwj->limit_hops + 1; diff --git a/net/can/isotp.c b/net/can/isotp.c index 155530aedce2e..f5dc9d04bd686 100644 --- a/net/can/isotp.c +++ b/net/can/isotp.c @@ -891,7 +891,7 @@ static void isotp_send_cframe(struct isotp_sock *so) csx->can_iif = dev->ifindex; /* set uid in tx skb to identify CF echo frames */ - can_set_skb_uid(skb); + can_set_skb_uid(csx); cf = (struct canfd_frame *)skb->data; skb_put_zero(skb, so->ll.mtu); @@ -917,7 +917,7 @@ static void isotp_send_cframe(struct isotp_sock *so) pr_notice_once("can-isotp: cfecho is %08X != 0\n", old_cfecho); /* set consecutive frame echo tag */ - WRITE_ONCE(so->cfecho, skb->hash); + WRITE_ONCE(so->cfecho, csx->can_skb_uid); /* send frame with local echo enabled */ can_send_ret = can_send(skb, 1); @@ -969,18 +969,22 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data) { struct sock *sk = (struct sock *)data; struct isotp_sock *so = isotp_sk(sk); + struct can_skb_ext *csx = can_skb_ext_find(skb); /* only handle my own local echo CF/SF skb's (no FF!) */ if (skb->sk != sk) return; + if (WARN_ON_ONCE(!csx)) + return; + /* unlike isotp_rcv_fc()/isotp_rcv_cf(), not already under so->rx_lock * (no isotp_rcv() caller here), so take it ourselves */ spin_lock(&so->rx_lock); /* so->cfecho may since belong to a new transfer; recheck under lock */ - if (READ_ONCE(so->cfecho) != skb->hash) + if (READ_ONCE(so->cfecho) != csx->can_skb_uid) goto out_unlock; /* cancel local echo timeout */ @@ -1222,7 +1226,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) csx->can_iif = dev->ifindex; /* set uid in tx skb to identify CF echo frames */ - can_set_skb_uid(skb); + can_set_skb_uid(csx); so->tx.len = size; so->tx.idx = 0; @@ -1261,7 +1265,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) cf->data[ae] |= size; /* set CF echo tag for isotp_rcv_echo() (SF-mode) */ - WRITE_ONCE(so->cfecho, skb->hash); + WRITE_ONCE(so->cfecho, csx->can_skb_uid); } else { /* send first frame */ @@ -1278,7 +1282,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) so->txfc.bs = 0; /* set CF echo tag for isotp_rcv_echo() (CF-mode) */ - WRITE_ONCE(so->cfecho, skb->hash); + WRITE_ONCE(so->cfecho, csx->can_skb_uid); } else { /* standard flow control check */ new_state = ISOTP_WAIT_FIRST_FC; diff --git a/net/can/raw.c b/net/can/raw.c index 82d9c0499c95f..0acd4f6c6dd67 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -77,7 +77,7 @@ MODULE_ALIAS("can-proto-1"); struct uniqframe { const struct sk_buff *skb; - u32 hash; + u32 can_skb_uid; unsigned int join_rx_count; }; @@ -133,12 +133,16 @@ static void raw_rcv(struct sk_buff *oskb, void *data) enum skb_drop_reason reason; struct sockaddr_can *addr; struct sk_buff *skb; + struct can_skb_ext *csx = can_skb_ext_find(oskb); unsigned int *pflags; /* check the received tx sock reference */ if (!ro->recv_own_msgs && oskb->sk == sk) return; + if (WARN_ON_ONCE(!csx)) + return; + /* make sure to not pass oversized frames to the socket */ if (!ro->fd_frames && can_is_canfd_skb(oskb)) return; @@ -165,7 +169,7 @@ static void raw_rcv(struct sk_buff *oskb, void *data) /* eliminate multiple filter matches for the same skb */ if (this_cpu_ptr(ro->uniq)->skb == oskb && - this_cpu_ptr(ro->uniq)->hash == oskb->hash) { + this_cpu_ptr(ro->uniq)->can_skb_uid == csx->can_skb_uid) { if (!ro->join_filters) return; @@ -175,7 +179,7 @@ static void raw_rcv(struct sk_buff *oskb, void *data) return; } else { this_cpu_ptr(ro->uniq)->skb = oskb; - this_cpu_ptr(ro->uniq)->hash = oskb->hash; + this_cpu_ptr(ro->uniq)->can_skb_uid = csx->can_skb_uid; this_cpu_ptr(ro->uniq)->join_rx_count = 1; /* drop first frame to check all enabled filters? */ if (ro->join_filters && ro->count > 1) diff --git a/net/core/dev.c b/net/core/dev.c index 6f75d1d1af32c..1a06976e19563 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10356,13 +10356,15 @@ EXPORT_SYMBOL_GPL(dev_xdp_prog_count); u8 dev_xdp_sb_prog_count(struct net_device *dev) { + struct bpf_prog *prog; u8 count = 0; int i; - for (i = 0; i < __MAX_XDP_MODE; i++) - if (dev->xdp_state[i].prog && - !dev->xdp_state[i].prog->aux->xdp_has_frags) + for (i = 0; i < __MAX_XDP_MODE; i++) { + prog = dev_xdp_prog(dev, i); + if (prog && !prog->aux->xdp_has_frags) count++; + } return count; } @@ -10742,11 +10744,12 @@ static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog, bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode); err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL, xdp_link->flags, new_prog); + if (!err) + old_prog = xchg(&link->prog, new_prog); netdev_unlock_ops(xdp_link->dev); if (err) goto out_unlock; - old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); out_unlock: diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 8aa4f9b4df810..1be6c739e3c71 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -1023,6 +1023,8 @@ u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx, result = bpf_prog_run_pin_on_cpu(prog, ctx); + /* bpf_flow_keys offsets are u16: do not let @hlen be truncated. */ + hlen = min_t(int, hlen, U16_MAX); flow_keys->nhoff = clamp_t(u16, flow_keys->nhoff, nhoff, hlen); flow_keys->thoff = clamp_t(u16, flow_keys->thoff, flow_keys->nhoff, hlen); @@ -1071,6 +1073,7 @@ bool __skb_flow_dissect(const struct net *net, int mpls_lse = 0; int num_hdrs = 0; u8 ip_proto = 0; + u32 thoff; bool ret; if (!data) { @@ -1692,7 +1695,13 @@ bool __skb_flow_dissect(const struct net *net, ret = true; out: - key_control->thoff = min_t(u16, nhoff, skb ? skb->len : hlen); + thoff = min_t(u32, nhoff, skb ? skb->len : hlen); + if (unlikely(thoff > U16_MAX)) { + /* Cannot be represented in key_control->thoff. */ + thoff = U16_MAX; + ret = false; + } + key_control->thoff = thoff; key_basic->n_proto = proto; key_basic->ip_proto = ip_proto; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index a5e46d1007070..0aa0f1a377e0e 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3639,8 +3639,12 @@ __wsum skb_copy_and_csum_bits(const struct sk_buff *skb, int offset, pos = copy; } - if (!skb_frags_readable(skb)) + if (!skb_frags_readable(skb)) { + /* Don't hand the caller a buffer with stale bytes in it. */ + if (len > 0) + memset(to, 0, len); return 0; + } for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { int end; diff --git a/net/devlink/core.c b/net/devlink/core.c index fe9f6a0a67d5b..42c16940e4c43 100644 --- a/net/devlink/core.c +++ b/net/devlink/core.c @@ -96,13 +96,19 @@ static void devlink_rel_nested_in_notify_work(struct work_struct *work) return; reschedule_work: - schedule_delayed_work(&rel->nested_in.notify_work, 1); + /* The work may have been queued again meanwhile, which took its own + * reference. Drop ours in that case. + */ + if (!schedule_delayed_work(&rel->nested_in.notify_work, 1)) + __devlink_rel_put(rel); } static void devlink_rel_nested_in_notify_work_schedule(struct devlink_rel *rel) { __devlink_rel_get(rel); - schedule_delayed_work(&rel->nested_in.notify_work, 0); + /* The pending work holds a reference already, drop the new one. */ + if (!schedule_delayed_work(&rel->nested_in.notify_work, 0)) + __devlink_rel_put(rel); } static struct devlink_rel *devlink_rel_alloc(void) diff --git a/net/ipv4/cipso_ipv4.c b/net/ipv4/cipso_ipv4.c index a05aa075de1a5..1aacbbeffbc64 100644 --- a/net/ipv4/cipso_ipv4.c +++ b/net/ipv4/cipso_ipv4.c @@ -2287,6 +2287,14 @@ int cipso_v4_skbuff_delattr(struct sk_buff *skb) new_hdr_len - new_hdr_len_actual); opt->optlen -= hdr_len_delta; + if (opt->srr > opt->cipso) + opt->srr -= cipso_len; + if (opt->rr > opt->cipso) + opt->rr -= cipso_len; + if (opt->ts > opt->cipso) + opt->ts -= cipso_len; + if (opt->router_alert > opt->cipso) + opt->router_alert -= cipso_len; opt->cipso = 0; opt->is_changed = 1; if (hdr_len_delta != 0) { diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 0c25f6dfb8a6f..92cfc838e5d3b 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -542,6 +542,9 @@ size_t fib_nlmsg_size(struct fib_info *fi) struct fib_nh_common *nhc = fib_info_nhc(fi, i); size_t nhsize; + if (!nhc) + break; + nhsize = fib_nexthop_nlmsg_size(nhc, nhs != 1); if (nhs != 1) diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index 6badad29593b7..58be0752b0350 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c @@ -1079,8 +1079,6 @@ void fib_alias_hw_flags_set(struct net *net, const struct fib_rt_info *fri) err = fib_dump_info(skb, 0, 0, RTM_NEWROUTE, fri, 0); if (err < 0) { - /* -EMSGSIZE implies BUG in fib_nlmsg_size() */ - WARN_ON(err == -EMSGSIZE); kfree_skb(skb); goto errout; } diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index a24cc8ee11d3e..eaa6fabda347e 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -771,9 +771,13 @@ int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, int err = 0; /* for offloaded checksums cleanup checksum before fragmentation */ - if (skb->ip_summed == CHECKSUM_PARTIAL && - (err = skb_checksum_help(skb))) - goto fail; + if (skb->ip_summed == CHECKSUM_PARTIAL) { + err = ip_check_csum_start(skb); + if (!err) + err = skb_checksum_help(skb); + if (err) + goto fail; + } /* * Point into the IP datagram header. diff --git a/net/ipv4/route.c b/net/ipv4/route.c index f07231447883f..196ca7e8acff0 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -1080,6 +1080,8 @@ static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu) for (nhsel = 0; nhsel < fib_info_num_path(res.fi); nhsel++) { nhc = fib_info_nhc(res.fi, nhsel); + if (!nhc) + break; update_or_create_fnhe(nhc, fl4->daddr, 0, mtu, lock, jiffies + net->ipv4.ip_rt_mtu_expires); } @@ -3163,6 +3165,9 @@ int fib_dump_info_fnhe(struct sk_buff *skb, struct netlink_callback *cb, struct fnhe_hash_bucket *bucket; int err; + if (!nhc) + break; + if (nhc->nhc_flags & RTNH_F_DEAD) continue; diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index ce0a111f9eb73..19663713ef92a 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1908,6 +1908,8 @@ static bool can_map_frag(const skb_frag_t *frag) if (skb_frag_size(frag) != PAGE_SIZE || skb_frag_off(frag)) return false; + if (skb_frag_is_net_iov(frag)) + return false; page = skb_frag_page(frag); diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 59afce196079a..a50dccfbf4205 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -908,6 +908,15 @@ static struct sock *__udp4_lib_err_encap(struct net *net, return sk; } +static void udp_err_update_exception(struct net *net, struct sk_buff *skb, + int type, int code, u32 info) +{ + if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED) + ipv4_update_pmtu(skb, net, info, 0, IPPROTO_UDP); + else if (type == ICMP_REDIRECT) + ipv4_redirect(skb, net, 0, IPPROTO_UDP); +} + /* * This routine is called by the ICMP module when it gets some * sort of error condition. If err < 0 then the socket should @@ -931,6 +940,8 @@ int udp_err(struct sk_buff *skb, u32 info) int harderr; int err; + udp_err_update_exception(net, skb, type, code, info); + uh = (struct udphdr *)(skb->data + (iph->ihl << 2)); sk = __udp4_lib_lookup(net, iph->daddr, uh->dest, iph->saddr, uh->source, skb->dev->ifindex, diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index e6913a7f68811..6d7c547232a2d 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -923,7 +923,6 @@ static netdev_tx_t ip6erspan_tunnel_xmit(struct sk_buff *skb, { struct ip_tunnel_info *tun_info = NULL; struct ip6_tnl *t = netdev_priv(dev); - struct dst_entry *dst = skb_dst(skb); IP_TUNNEL_DECLARE_FLAGS(flags) = { }; bool truncate = false; int encap_limit = -1; @@ -1059,12 +1058,6 @@ static netdev_tx_t ip6erspan_tunnel_xmit(struct sk_buff *skb, gre_build_header(skb, 8, flags, proto, 0, htonl(atomic_fetch_inc(&t->o_seqno))); - /* TooBig packet may have updated dst->dev's mtu */ - if (!t->parms.collect_md && dst) { - mtu = READ_ONCE(dst_dev(dst)->mtu); - if (dst_mtu(dst) > mtu) - dst->ops->update_pmtu(dst, NULL, skb, mtu, false); - } err = ip6_tnl_xmit(skb, dev, dsfield, &fl6, encap_limit, &mtu, NEXTHDR_GRE); if (err != 0) { diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 8fddcc631fcb6..5d80a025c3e6d 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -3261,7 +3261,7 @@ void ip6_redirect_no_header(struct sk_buff *skb, struct net *net, int oif) void ip6_sk_redirect(struct sk_buff *skb, struct sock *sk) { - ip6_redirect(skb, sock_net(sk), sk->sk_bound_dev_if, + ip6_redirect(skb, sock_net(sk), skb->dev->ifindex, READ_ONCE(sk->sk_mark), sk_uid(sk)); } @@ -6474,8 +6474,6 @@ void fib6_info_hw_flags_set(struct net *net, struct fib6_info *f6i, err = rt6_fill_node(net, skb, f6i, NULL, NULL, NULL, 0, RTM_NEWROUTE, 0, 0, 0); if (err < 0) { - /* -EMSGSIZE implies BUG in rt6_nlmsg_size() */ - WARN_ON(err == -EMSGSIZE); kfree_skb(skb); goto errout; } diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 15e032194eccc..b73cf02ac1b6e 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -690,6 +690,17 @@ static struct sock *__udp6_lib_err_encap(struct net *net, return sk; } +static void udpv6_err_update_exception(struct net *net, struct sk_buff *skb, + u8 type, __be32 info) +{ + if (type == ICMPV6_PKT_TOOBIG) + ip6_update_pmtu(skb, net, info, skb->dev->ifindex, 0, + sock_net_uid(net, NULL)); + else if (type == NDISC_REDIRECT) + ip6_redirect(skb, net, skb->dev->ifindex, 0, + sock_net_uid(net, NULL)); +} + static int udpv6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, u8 type, u8 code, int offset, __be32 info) { @@ -703,6 +714,8 @@ static int udpv6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, int harderr; int err; + udpv6_err_update_exception(net, skb, type, info); + daddr = seg6_get_daddr(skb, opt) ? : &hdr->daddr; saddr = &hdr->saddr; sk = __udp6_lib_lookup(net, daddr, uh->dest, saddr, uh->source, diff --git a/net/mac80211/debugfs.c b/net/mac80211/debugfs.c index f36cc25903050..3a671225c1277 100644 --- a/net/mac80211/debugfs.c +++ b/net/mac80211/debugfs.c @@ -210,11 +210,13 @@ static ssize_t aql_pending_read(struct file *file, "VI %u us\n" "BE %u us\n" "BK %u us\n" + "MC %u us\n" "total %u us\n", atomic_read(&local->aql_ac_pending_airtime[IEEE80211_AC_VO]), atomic_read(&local->aql_ac_pending_airtime[IEEE80211_AC_VI]), atomic_read(&local->aql_ac_pending_airtime[IEEE80211_AC_BE]), atomic_read(&local->aql_ac_pending_airtime[IEEE80211_AC_BK]), + atomic_read(&local->aql_mc_pending_airtime), atomic_read(&local->aql_total_pending_airtime)); return simple_read_from_buffer(user_buf, count, ppos, buf, len); @@ -239,7 +241,8 @@ static ssize_t aql_txq_limit_read(struct file *file, "VO %u %u\n" "VI %u %u\n" "BE %u %u\n" - "BK %u %u\n", + "BK %u %u\n" + "MC %u\n", local->aql_txq_limit_low[IEEE80211_AC_VO], local->aql_txq_limit_high[IEEE80211_AC_VO], local->aql_txq_limit_low[IEEE80211_AC_VI], @@ -247,7 +250,8 @@ static ssize_t aql_txq_limit_read(struct file *file, local->aql_txq_limit_low[IEEE80211_AC_BE], local->aql_txq_limit_high[IEEE80211_AC_BE], local->aql_txq_limit_low[IEEE80211_AC_BK], - local->aql_txq_limit_high[IEEE80211_AC_BK]); + local->aql_txq_limit_high[IEEE80211_AC_BK], + local->aql_txq_limit_mc); return simple_read_from_buffer(user_buf, count, ppos, buf, len); } @@ -273,6 +277,11 @@ static ssize_t aql_txq_limit_write(struct file *file, else buf[count] = '\0'; + if (sscanf(buf, "mcast %u", &q_limit_low) == 1) { + local->aql_txq_limit_mc = q_limit_low; + return count; + } + if (sscanf(buf, "%u %u %u", &ac, &q_limit_low, &q_limit_high) != 3) return -EINVAL; diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 011ec7fbb7118..2854dcc86b21b 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -1464,10 +1464,12 @@ struct ieee80211_local { spinlock_t handle_wake_tx_queue_lock; u16 airtime_flags; + u32 aql_txq_limit_mc; u32 aql_txq_limit_low[IEEE80211_NUM_ACS]; u32 aql_txq_limit_high[IEEE80211_NUM_ACS]; u32 aql_threshold; atomic_t aql_total_pending_airtime; + atomic_t aql_mc_pending_airtime; atomic_t aql_ac_pending_airtime[IEEE80211_NUM_ACS]; const struct ieee80211_ops *ops; diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 8edfef2b57f80..d5ef44f2b3f2d 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -988,6 +988,7 @@ struct ieee80211_hw *ieee80211_alloc_hw_nm(size_t priv_data_len, spin_lock_init(&local->rx_path_lock); spin_lock_init(&local->queue_stop_reason_lock); + local->aql_txq_limit_mc = IEEE80211_DEFAULT_AQL_TXQ_LIMIT_MC; for (i = 0; i < IEEE80211_NUM_ACS; i++) { INIT_LIST_HEAD(&local->active_txqs[i]); spin_lock_init(&local->active_txq_lock[i]); diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index 01d70f71c43ec..ce7a3f839ce2e 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -2493,13 +2493,28 @@ EXPORT_SYMBOL(ieee80211_sta_recalc_aggregates); void ieee80211_sta_update_pending_airtime(struct ieee80211_local *local, struct sta_info *sta, u8 ac, - u16 tx_airtime, bool tx_completed) + u16 tx_airtime, bool tx_completed, + bool mcast) { int tx_pending; if (!wiphy_ext_feature_isset(local->hw.wiphy, NL80211_EXT_FEATURE_AQL)) return; + if (mcast) { + if (!tx_completed) { + atomic_add(tx_airtime, &local->aql_mc_pending_airtime); + return; + } + + tx_pending = atomic_sub_return(tx_airtime, + &local->aql_mc_pending_airtime); + if (tx_pending < 0) + atomic_cmpxchg(&local->aql_mc_pending_airtime, + tx_pending, 0); + return; + } + if (!tx_completed) { if (sta) atomic_add(tx_airtime, diff --git a/net/mac80211/sta_info.h b/net/mac80211/sta_info.h index 5da3142d8516e..ee0d32877c5b4 100644 --- a/net/mac80211/sta_info.h +++ b/net/mac80211/sta_info.h @@ -147,7 +147,8 @@ struct airtime_info { void ieee80211_sta_update_pending_airtime(struct ieee80211_local *local, struct sta_info *sta, u8 ac, - u16 tx_airtime, bool tx_completed); + u16 tx_airtime, bool tx_completed, + bool mcast); struct sta_info; diff --git a/net/mac80211/status.c b/net/mac80211/status.c index e10e5bcc279b6..a9da9fe87cf15 100644 --- a/net/mac80211/status.c +++ b/net/mac80211/status.c @@ -774,7 +774,7 @@ static void ieee80211_report_used_skb(struct ieee80211_local *local, ieee80211_sta_update_pending_airtime(local, sta, skb_get_queue_mapping(skb), tx_time_est, - true); + true, info->tx_time_mc); rcu_read_unlock(); } @@ -1255,10 +1255,11 @@ void ieee80211_tx_status_ext(struct ieee80211_hw *hw, /* Do this here to avoid the expensive lookup of the sta * in ieee80211_report_used_skb(). */ + bool mcast = IEEE80211_SKB_CB(skb)->tx_time_mc; ieee80211_sta_update_pending_airtime(local, sta, skb_get_queue_mapping(skb), tx_time_est, - true); + true, mcast); ieee80211_info_set_tx_time_est(IEEE80211_SKB_CB(skb), 0); } diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index 65c5a6519b895..3e5b1c05f4a3e 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -4083,20 +4083,20 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw, encap_out: info->control.vif = vif; - if (tx.sta && - wiphy_ext_feature_isset(local->hw.wiphy, NL80211_EXT_FEATURE_AQL)) { - bool ampdu = txq->ac != IEEE80211_AC_VO; + if (wiphy_ext_feature_isset(local->hw.wiphy, NL80211_EXT_FEATURE_AQL)) { + bool ampdu = txq->sta && txq->ac != IEEE80211_AC_VO; u32 airtime; airtime = ieee80211_calc_expected_tx_airtime(hw, vif, txq->sta, skb->len, ampdu); - if (airtime) { - airtime = ieee80211_info_set_tx_time_est(info, airtime); - ieee80211_sta_update_pending_airtime(local, tx.sta, - txq->ac, - airtime, - false); - } + if (!airtime) + return skb; + + airtime = ieee80211_info_set_tx_time_est(info, airtime); + info->tx_time_mc = !tx.sta; + ieee80211_sta_update_pending_airtime(local, tx.sta, txq->ac, + airtime, false, + info->tx_time_mc); } return skb; @@ -4148,6 +4148,7 @@ struct ieee80211_txq *ieee80211_next_txq(struct ieee80211_hw *hw, u8 ac) struct ieee80211_txq *ret = NULL; struct txq_info *txqi = NULL, *head = NULL; bool found_eligible_txq = false; + bool aql_check; spin_lock_bh(&local->active_txq_lock[ac]); @@ -4171,26 +4172,28 @@ struct ieee80211_txq *ieee80211_next_txq(struct ieee80211_hw *hw, u8 ac) if (!head) head = txqi; + aql_check = ieee80211_txq_airtime_check(hw, &txqi->txq); + if (aql_check) + found_eligible_txq = true; + if (txqi->txq.sta) { struct sta_info *sta = container_of(txqi->txq.sta, struct sta_info, sta); - bool aql_check = ieee80211_txq_airtime_check(hw, &txqi->txq); - s32 deficit = ieee80211_sta_deficit(sta, txqi->txq.ac); - if (aql_check) - found_eligible_txq = true; - - if (deficit < 0) + if (ieee80211_sta_deficit(sta, txqi->txq.ac) < 0) { sta->airtime[txqi->txq.ac].deficit += sta->airtime_weight; - if (deficit < 0 || !aql_check) { - list_move_tail(&txqi->schedule_order, - &local->active_txqs[txqi->txq.ac]); - goto begin; + aql_check = false; } } + if (!aql_check) { + list_move_tail(&txqi->schedule_order, + &local->active_txqs[txqi->txq.ac]); + goto begin; + } + if (txqi->schedule_round == local->schedule_round[ac]) goto out; @@ -4257,7 +4260,8 @@ bool ieee80211_txq_airtime_check(struct ieee80211_hw *hw, return true; if (!txq->sta) - return true; + return atomic_read(&local->aql_mc_pending_airtime) < + local->aql_txq_limit_mc; if (unlikely(txq->tid == IEEE80211_NUM_TIDS)) return true; @@ -4277,6 +4281,24 @@ bool ieee80211_txq_airtime_check(struct ieee80211_hw *hw, } EXPORT_SYMBOL(ieee80211_txq_airtime_check); +u32 ieee80211_txq_aql_pending(struct ieee80211_hw *hw, + struct ieee80211_txq *txq) +{ + struct ieee80211_local *local = hw_to_local(hw); + struct sta_info *sta; + + if (unlikely(txq->tid == IEEE80211_NUM_TIDS)) + return 0; + + if (!txq->sta) + return atomic_read(&local->aql_mc_pending_airtime); + + sta = container_of(txq->sta, struct sta_info, sta); + + return atomic_read(&sta->airtime[txq->ac].aql_tx_pending); +} +EXPORT_SYMBOL(ieee80211_txq_aql_pending); + static bool ieee80211_txq_schedule_airtime_check(struct ieee80211_local *local, u8 ac) { @@ -4306,15 +4328,15 @@ bool ieee80211_txq_may_transmit(struct ieee80211_hw *hw, spin_lock_bh(&local->active_txq_lock[ac]); - if (!txqi->txq.sta) - goto out; - if (list_empty(&txqi->schedule_order)) goto out; if (!ieee80211_txq_schedule_airtime_check(local, ac)) goto out; + if (!txqi->txq.sta) + goto out; + list_for_each_entry_safe(iter, tmp, &local->active_txqs[ac], schedule_order) { if (iter == txqi) diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c index ec5e7716df8b5..8a5240a9be222 100644 --- a/net/netfilter/nfnetlink_queue.c +++ b/net/netfilter/nfnetlink_queue.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -667,6 +668,12 @@ static int nfqnl_put_bridge(struct nf_queue_entry *entry, struct sk_buff *skb) static int nf_queue_checksum_help(struct sk_buff *entskb) { + if (entskb->protocol == htons(ETH_P_IP)) { + if (!pskb_network_may_pull(entskb, sizeof(struct iphdr)) || + ip_check_csum_start(entskb)) + return -EINVAL; + } + if (skb_csum_is_sctp(entskb)) return skb_crc32c_csum_help(entskb); diff --git a/net/netfilter/xt_CHECKSUM.c b/net/netfilter/xt_CHECKSUM.c index 9d99f5a3d1764..1fb0f8118404d 100644 --- a/net/netfilter/xt_CHECKSUM.c +++ b/net/netfilter/xt_CHECKSUM.c @@ -15,6 +15,7 @@ #include #include +#include MODULE_LICENSE("GPL"); MODULE_AUTHOR("Michael S. Tsirkin "); @@ -25,8 +26,11 @@ MODULE_ALIAS("ip6t_CHECKSUM"); static unsigned int checksum_tg(struct sk_buff *skb, const struct xt_action_param *par) { - if (skb->ip_summed == CHECKSUM_PARTIAL && !skb_is_gso(skb)) + if (skb->ip_summed == CHECKSUM_PARTIAL && !skb_is_gso(skb)) { + if (xt_family(par) == NFPROTO_IPV4 && ip_check_csum_start(skb)) + return NF_DROP; skb_checksum_help(skb); + } return XT_CONTINUE; } diff --git a/net/openvswitch/actions.c b/net/openvswitch/actions.c index dc5ff859f1145..27152bca8a8ce 100644 --- a/net/openvswitch/actions.c +++ b/net/openvswitch/actions.c @@ -358,7 +358,12 @@ static void set_ip_addr(struct sk_buff *skb, struct iphdr *nh, static void update_ipv6_checksum(struct sk_buff *skb, u8 l4_proto, __be32 addr[4], const __be32 new_addr[4]) { - int transport_len = skb->len - skb_transport_offset(skb); + int transport_len; + + if (l4_proto == NEXTHDR_FRAGMENT) + return; + + transport_len = skb->len - skb_transport_offset(skb); if (l4_proto == NEXTHDR_TCP) { if (likely(transport_len >= sizeof(struct tcphdr))) diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index b601e4859de92..21bc56e796277 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -1529,6 +1529,12 @@ static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb) return -ENODEV; } + /* + * Not needed for safety. Stops every spin_unlock_bh() in + * ovs_flow_stats_get() from running the softirq backlog. + */ + local_bh_disable(); + ti = rcu_dereference(dp->table.ti); for (;;) { struct sw_flow *flow; @@ -1549,6 +1555,8 @@ static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb) cb->args[0] = bucket; cb->args[1] = obj; } + + local_bh_enable(); rcu_read_unlock(); return skb->len; } diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 8865c1a426679..52ae3d875e971 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1911,7 +1911,8 @@ static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev, spkt = &PACKET_SKB_CB(skb)->sa.pkt; - skb_push(skb, skb->data - skb_mac_header(skb)); + if (dev_has_header(dev)) + skb_push(skb, skb->data - skb_mac_header(skb)); /* * The SOCK_PACKET socket receives _all_ frames. @@ -1942,8 +1943,7 @@ static void packet_parse_headers(struct sk_buff *skb, struct socket *sock) /* On TX skb->data is the L2 header; anchor it for all socket types. */ skb_reset_mac_header(skb); - if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) && - sock->type == SOCK_RAW) + if (!skb->protocol && sock->type == SOCK_RAW) skb->protocol = dev_parse_header_protocol(skb); skb_probe_transport_header(skb); @@ -2613,7 +2613,8 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, struct page *page; int err; - skb->protocol = proto; + if (proto != htons(ETH_P_ALL)) + skb->protocol = proto; skb->dev = dev; skb->priority = sockc->priority; skb->mark = sockc->mark; @@ -2677,8 +2678,6 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, if (unlikely(!skb->len)) return -EINVAL; - packet_parse_headers(skb, sock); - return tp_len; } @@ -2918,9 +2917,13 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) tp_len = -EINVAL; goto tpacket_error; } - virtio_net_hdr_set_proto(skb, &vnet_hdr); } + packet_parse_headers(skb, po->sk.sk_socket); + + if (has_vnet_hdr) + virtio_net_hdr_set_proto(skb, &vnet_hdr); + uarg = kmalloc(sizeof(*uarg), GFP_KERNEL); if (unlikely(!uarg)) { if (likely(len_sum > 0)) @@ -3121,7 +3124,8 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) goto out_free; } - skb->protocol = proto; + if (proto != htons(ETH_P_ALL)) + skb->protocol = proto; skb->dev = dev; skb->priority = sockc.priority; skb->mark = sockc.mark; @@ -3130,16 +3134,18 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) if (unlikely(extra_len == 4)) skb->no_fcs = 1; - packet_parse_headers(skb, sock); - if (vnet_hdr_sz) { err = virtio_net_hdr_to_skb(skb, &vnet_hdr, vio_le()); if (err) goto out_free; len += vnet_hdr_sz; - virtio_net_hdr_set_proto(skb, &vnet_hdr); } + packet_parse_headers(skb, sock); + + if (vnet_hdr_sz) + virtio_net_hdr_set_proto(skb, &vnet_hdr); + err = packet_xmit(po, skb); if (unlikely(err != 0)) { diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index 0f211f030fd9a..dca812a752698 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -434,6 +434,12 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, if (handle && (!fold || nhandle == (handle & ~0x7F00))) nhandle |= handle & 0x7F00; + if (fold && !tb[TCA_ROUTE4_TO] && !tb[TCA_ROUTE4_FROM] && + !tb[TCA_ROUTE4_IIF] && nhandle != fold->handle) { + NL_SET_ERR_MSG(extack, "Routing attributes required"); + return -EINVAL; + } + if (handle && !fold && nhandle != handle) { NL_SET_ERR_MSG_FMT(extack, "Handle mismatch constructed: %x (expected: %x)", diff --git a/net/sched/em_text.c b/net/sched/em_text.c index 4132f8c3c5fc9..bc45edb8c03bf 100644 --- a/net/sched/em_text.c +++ b/net/sched/em_text.c @@ -58,6 +58,9 @@ static int em_text_change(struct net *net, void *data, int len, if (len < sizeof(*conf) || len < (sizeof(*conf) + conf->pattern_len)) return -EINVAL; + if (!memchr(conf->algo, '\0', sizeof(conf->algo))) + return -EINVAL; + if (conf->from_layer > conf->to_layer) return -EINVAL; diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 4652fd90d9a6c..e35692dd8d303 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -7569,11 +7569,7 @@ static int sctp_getsockopt_pr_streamstatus(struct sock *sk, int len, /* Not allocated yet, means all stats are 0 */ params.sprstat_abandoned_unsent = 0; params.sprstat_abandoned_sent = 0; - retval = 0; - goto out; - } - - if (policy == SCTP_PR_SCTP_ALL) { + } else if (policy == SCTP_PR_SCTP_ALL) { params.sprstat_abandoned_unsent = 0; params.sprstat_abandoned_sent = 0; for (policy = 0; policy <= SCTP_PR_INDEX(MAX); policy++) { diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index e9f93b3ab435b..8b3ee70f8433f 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -2784,6 +2784,7 @@ int smc_getname(struct socket *sock, struct sockaddr *addr, int peer) { struct smc_sock *smc; + int rc = -EBADF; if (peer && (sock->sk->sk_state != SMC_ACTIVE) && (sock->sk->sk_state != SMC_APPCLOSEWAIT1)) @@ -2791,7 +2792,11 @@ int smc_getname(struct socket *sock, struct sockaddr *addr, smc = smc_sk(sock->sk); - return smc->clcsock->ops->getname(smc->clcsock, addr, peer); + mutex_lock(&smc->clcsock_release_lock); + if (smc->clcsock) + rc = smc->clcsock->ops->getname(smc->clcsock, addr, peer); + mutex_unlock(&smc->clcsock_release_lock); + return rc; } int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) diff --git a/net/strparser/strparser.c b/net/strparser/strparser.c index a23f4b4dfc673..d5b17d099a0e7 100644 --- a/net/strparser/strparser.c +++ b/net/strparser/strparser.c @@ -505,12 +505,16 @@ void strp_unpause(struct strparser *strp) EXPORT_SYMBOL_GPL(strp_unpause); /* strp must already be stopped so that strp_recv will no longer be called. - * Note that strp_done is not called with the lower socket held. + * Note that strp_done must not be called with the lower socket held. */ void strp_done(struct strparser *strp) { WARN_ON(!strp->stopped); + lock_sock(strp->sk); + /* sync with pending strp_recv */ + release_sock(strp->sk); + cancel_delayed_work_sync(&strp->msg_timer_work); cancel_work_sync(&strp->work); diff --git a/net/tipc/node.c b/net/tipc/node.c index 683a136e53efc..5a7487dab31da 100644 --- a/net/tipc/node.c +++ b/net/tipc/node.c @@ -2419,8 +2419,11 @@ static struct tipc_node *tipc_node_find_by_name(struct net *net, } } tipc_node_read_unlock(n); - if (found_node) + if (found_node) { + if (!kref_get_unless_zero(&found_node->kref)) + found_node = NULL; break; + } } rcu_read_unlock(); @@ -2505,6 +2508,7 @@ int tipc_nl_node_set_link(struct sk_buff *skb, struct genl_info *info) tipc_node_read_unlock(node); tipc_bearer_xmit(net, bearer_id, &xmitq, &node->links[bearer_id].maddr, NULL); + tipc_node_put(node); return res; } @@ -2556,12 +2560,14 @@ int tipc_nl_node_get_link(struct sk_buff *skb, struct genl_info *info) link = node->links[bearer_id].link; if (!link) { tipc_node_read_unlock(node); + tipc_node_put(node); err = -EINVAL; goto err_free; } err = __tipc_nl_add_link(net, &msg, link, 0); tipc_node_read_unlock(node); + tipc_node_put(node); if (err) goto err_free; } @@ -2632,11 +2638,13 @@ int tipc_nl_node_reset_link_stats(struct sk_buff *skb, struct genl_info *info) if (!link) { spin_unlock_bh(&le->lock); tipc_node_read_unlock(node); + tipc_node_put(node); return -EINVAL; } tipc_link_reset_stats(link); spin_unlock_bh(&le->lock); tipc_node_read_unlock(node); + tipc_node_put(node); return 0; } diff --git a/net/tipc/topsrv.c b/net/tipc/topsrv.c index af530c9ed8402..82e9f44e2fe46 100644 --- a/net/tipc/topsrv.c +++ b/net/tipc/topsrv.c @@ -55,7 +55,7 @@ /** * struct tipc_topsrv - TIPC server structure * @conn_idr: identifier set of connection - * @idr_lock: protect the connection identifier set + * @idr_lock: protect the connection identifier set and listener * @idr_in_use: amount of allocated identifier entry * @net: network namespace instance * @awork: accept work item @@ -218,6 +218,10 @@ static struct tipc_conn *tipc_conn_lookup(struct tipc_topsrv *s, int conid) struct tipc_conn *con; spin_lock_bh(&s->idr_lock); + if (!s->listener) { + spin_unlock_bh(&s->idr_lock); + return NULL; + } con = idr_find(&s->conn_idr, conid); if (!connected(con) || !kref_get_unless_zero(&con->kref)) con = NULL; @@ -301,10 +305,20 @@ static void tipc_conn_send_to_sock(struct tipc_conn *con) static void tipc_conn_send_work(struct work_struct *work) { struct tipc_conn *con = container_of(work, struct tipc_conn, swork); + struct tipc_topsrv *srv; + + srv = con->server; + spin_lock_bh(&srv->idr_lock); + if (!srv->listener) { + spin_unlock_bh(&srv->idr_lock); + goto out; + } + spin_unlock_bh(&srv->idr_lock); if (connected(con)) tipc_conn_send_to_sock(con); +out: conn_put(con); } @@ -334,8 +348,14 @@ void tipc_topsrv_queue_evt(struct net *net, int conid, list_add_tail(&e->list, &con->outqueue); spin_unlock_bh(&con->outqueue_lock); - if (queue_work(srv->send_wq, &con->swork)) - return; + spin_lock_bh(&srv->idr_lock); + if (srv->listener) { + if (queue_work(srv->send_wq, &con->swork)) { + spin_unlock_bh(&srv->idr_lock); + return; + } + } + spin_unlock_bh(&srv->idr_lock); err: conn_put(con); } @@ -346,14 +366,20 @@ void tipc_topsrv_queue_evt(struct net *net, int conid, */ static void tipc_conn_write_space(struct sock *sk) { + struct tipc_topsrv *srv; struct tipc_conn *con; read_lock_bh(&sk->sk_callback_lock); con = sk->sk_user_data; if (connected(con)) { - conn_get(con); - if (!queue_work(con->server->send_wq, &con->swork)) - conn_put(con); + srv = con->server; + spin_lock_bh(&srv->idr_lock); + if (srv->listener) { + conn_get(con); + if (!queue_work(srv->send_wq, &con->swork)) + conn_put(con); + } + spin_unlock_bh(&srv->idr_lock); } read_unlock_bh(&sk->sk_callback_lock); } @@ -418,8 +444,17 @@ static int tipc_conn_rcv_from_sock(struct tipc_conn *con) static void tipc_conn_recv_work(struct work_struct *work) { struct tipc_conn *con = container_of(work, struct tipc_conn, rwork); + struct tipc_topsrv *srv; int count = 0; + srv = con->server; + spin_lock_bh(&srv->idr_lock); + if (!srv->listener) { + spin_unlock_bh(&srv->idr_lock); + goto out; + } + spin_unlock_bh(&srv->idr_lock); + while (connected(con)) { if (tipc_conn_rcv_from_sock(con)) break; @@ -430,6 +465,7 @@ static void tipc_conn_recv_work(struct work_struct *work) count = 0; } } +out: conn_put(con); } @@ -438,6 +474,7 @@ static void tipc_conn_recv_work(struct work_struct *work) */ static void tipc_conn_data_ready(struct sock *sk) { + struct tipc_topsrv *srv; struct tipc_conn *con; trace_sk_data_ready(sk); @@ -445,9 +482,14 @@ static void tipc_conn_data_ready(struct sock *sk) read_lock_bh(&sk->sk_callback_lock); con = sk->sk_user_data; if (connected(con)) { - conn_get(con); - if (!queue_work(con->server->rcv_wq, &con->rwork)) - conn_put(con); + srv = con->server; + spin_lock_bh(&srv->idr_lock); + if (srv->listener) { + conn_get(con); + if (!queue_work(srv->rcv_wq, &con->rwork)) + conn_put(con); + } + spin_unlock_bh(&srv->idr_lock); } read_unlock_bh(&sk->sk_callback_lock); } @@ -503,8 +545,12 @@ static void tipc_topsrv_listener_data_ready(struct sock *sk) read_lock_bh(&sk->sk_callback_lock); srv = sk->sk_user_data; - if (srv) - queue_work(srv->rcv_wq, &srv->awork); + if (srv) { + spin_lock_bh(&srv->idr_lock); + if (srv->listener) + queue_work(srv->rcv_wq, &srv->awork); + spin_unlock_bh(&srv->idr_lock); + } read_unlock_bh(&sk->sk_callback_lock); } @@ -700,23 +746,27 @@ static void tipc_topsrv_stop(struct net *net) struct tipc_conn *con; int id; + spin_lock_bh(&srv->idr_lock); + srv->listener = NULL; + spin_unlock_bh(&srv->idr_lock); + tipc_topsrv_work_stop(srv); + spin_lock_bh(&srv->idr_lock); for (id = 0; srv->idr_in_use; id++) { con = idr_find(&srv->conn_idr, id); if (con) { - conn_get(con); spin_unlock_bh(&srv->idr_lock); tipc_conn_close(con); - conn_put(con); spin_lock_bh(&srv->idr_lock); + continue; } + spin_unlock_bh(&srv->idr_lock); + spin_lock_bh(&srv->idr_lock); } __module_get(lsock->ops->owner); __module_get(lsock->sk->sk_prot_creator->owner); - srv->listener = NULL; spin_unlock_bh(&srv->idr_lock); - tipc_topsrv_work_stop(srv); sock_release(lsock); idr_destroy(&srv->conn_idr); kfree(srv); diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index 4a1dfde8414f4..f271a095996c9 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -1021,6 +1021,8 @@ static void vsock_sk_destruct(struct sock *sk) { struct vsock_sock *vsk = vsock_sk(sk); + /* Socket buffers may remain in the queue in VMCI datagram mode */ + __skb_queue_purge(&sk->sk_receive_queue); /* Flush MSG_ZEROCOPY leftovers. */ __skb_queue_purge(&sk->sk_error_queue); diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index 8becad81279c8..1f3fd0895f7c6 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -1916,6 +1916,7 @@ int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t recv_acto struct sock *sk = sk_vsock(vsk); struct virtio_vsock_hdr *hdr; struct sk_buff *skb; + u32 bytes_read; u32 pkt_len; int off = 0; int err; @@ -1935,7 +1936,8 @@ int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t recv_acto vvs->msg_count--; pkt_len = le32_to_cpu(hdr->len); - virtio_transport_dec_rx_pkt(vvs, pkt_len, pkt_len); + bytes_read = skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset; + virtio_transport_dec_rx_pkt(vvs, bytes_read, pkt_len); spin_unlock_bh(&vvs->rx_lock); virtio_transport_send_credit_update(vsk); diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index e305ba32e356b..f1f612cd7b438 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -823,16 +823,22 @@ int xfrm_output(struct sock *sk, struct sk_buff *skb) } if (skb->ip_summed == CHECKSUM_PARTIAL) { - err = skb_checksum_help(skb); - if (err) { - XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); - kfree_skb(skb); - return err; + if (skb->protocol == htons(ETH_P_IP)) { + err = ip_check_csum_start(skb); + if (err) + goto error; } + err = skb_checksum_help(skb); + if (err) + goto error; } out: return xfrm_output2(net, sk, skb); +error: + XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTERROR); + kfree_skb(skb); + return err; } EXPORT_SYMBOL_GPL(xfrm_output); diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h index 099e64b3ce5fd..67b52855b719c 100644 --- a/rust/bindings/bindings_helper.h +++ b/rust/bindings/bindings_helper.h @@ -147,6 +147,7 @@ const u32 RUST_CONST_HELPER_APPLE_RTKIT_COPROC_PLACES_BUFFERS = APPLE_RTKIT_COPR const u64 BINDINGS_SNDRV_PCM_FMTBIT_FLOAT_LE = SNDRV_PCM_FMTBIT_FLOAT_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S32_LE = SNDRV_PCM_FMTBIT_S32_LE; const u64 BINDINGS_SNDRV_PCM_FMTBIT_S24_LE = SNDRV_PCM_FMTBIT_S24_LE; +const u64 BINDINGS_SNDRV_PCM_FMTBIT_S16_LE = SNDRV_PCM_FMTBIT_S16_LE; const u32 BINDINGS_SNDRV_PCM_SUBFMTBIT_STD = SNDRV_PCM_SUBFMTBIT_STD; const snd_ctl_elem_iface_t BINDINGS_SNDRV_CTL_ELEM_IFACE_MIXER = SNDRV_CTL_ELEM_IFACE_MIXER; diff --git a/rust/kernel/platform.rs b/rust/kernel/platform.rs index d41555a4b31d2..10ccce3c305fc 100644 --- a/rust/kernel/platform.rs +++ b/rust/kernel/platform.rs @@ -79,6 +79,7 @@ unsafe impl driver::RegistrationOps for Adapter { (*pdrv.get()).remove = Some(Self::remove_callback); (*pdrv.get()).driver.of_match_table = of_table; (*pdrv.get()).driver.acpi_match_table = acpi_table; + (*pdrv.get()).driver.suppress_bind_attrs = T::suppress_bind_attrs(); } // SAFETY: `pdrv` is guaranteed to be a valid `DriverType`. @@ -224,6 +225,12 @@ pub trait Driver { /// The table of ACPI device ids supported by the driver. const ACPI_ID_TABLE: Option> = None; + /// Suppress manual bind/unbind for hardware without a safe teardown path. + /// Evaluated before registration, including before a deferred probe. + fn suppress_bind_attrs() -> bool { + false + } + /// Platform driver probe. /// /// Called when a new platform device is added or discovered. diff --git a/rust/kernel/soc/apple/rtkit.rs b/rust/kernel/soc/apple/rtkit.rs index 5f7e572a3a462..6bacd092680b8 100644 --- a/rust/kernel/soc/apple/rtkit.rs +++ b/rust/kernel/soc/apple/rtkit.rs @@ -298,6 +298,13 @@ impl RtKit { to_result(unsafe { bindings::apple_rtkit_wake(self.rtk) }) } + /// Boots (wakes up) the RTKit coprocessor with the AP powered before + /// the IOP's power acknowledgment, for firmware that waits for it. + pub fn wake_early_ap(self: Pin<&mut Self>) -> Result { + // SAFETY: `rtk` is valid per the type invariant. + to_result(unsafe { bindings::apple_rtkit_wake_early_ap(self.rtk) }) + } + /// Waits for the RTKit coprocessor to finish booting. pub fn boot(self: Pin<&mut Self>) -> Result { // SAFETY: `rtk` is valid per the type invariant. diff --git a/security/keys/key.c b/security/keys/key.c index b34a64d81d47a..a438c4508595d 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -840,9 +840,6 @@ static key_ref_t __key_create_or_update(key_ref_t keyring_ref, key_check(keyring); - if (!(flags & KEY_ALLOC_BYPASS_RESTRICTION)) - restrict_link = keyring->restrict_link; - key_ref = ERR_PTR(-ENOTDIR); if (keyring->type != &key_type_keyring) goto error_put_type; @@ -880,6 +877,9 @@ static key_ref_t __key_create_or_update(key_ref_t keyring_ref, goto error_link_end; } + if (!(flags & KEY_ALLOC_BYPASS_RESTRICTION)) + restrict_link = keyring->restrict_link; + if (restrict_link && restrict_link->check) { ret = restrict_link->check(keyring, index_key.type, &prep.payload, restrict_link->key); diff --git a/security/keys/persistent.c b/security/keys/persistent.c index 97af230aa4b22..bdab995393261 100644 --- a/security/keys/persistent.c +++ b/security/keys/persistent.c @@ -33,25 +33,31 @@ static int key_create_persistent_register(struct user_namespace *ns) } /* - * Create the persistent keyring for the specified user. + * Get or create the persistent keyring for the specified user. * * Called with the namespace's sem locked for writing. + * + * Returns a boolean indicating whether the keyring already existed, + * or a negative error. On success, *persistent_ref holds a reference + * to the keyring. */ -static key_ref_t key_create_persistent(struct user_namespace *ns, kuid_t uid, - struct keyring_index_key *index_key) +static int key_get_or_create_persistent(struct user_namespace *ns, kuid_t uid, + struct keyring_index_key *index_key, + key_ref_t *persistent_ref) { struct key *persistent; - key_ref_t reg_ref, persistent_ref; + key_ref_t reg_ref; if (!ns->persistent_keyring_register) { - long err = key_create_persistent_register(ns); + int err = key_create_persistent_register(ns); + if (err < 0) - return ERR_PTR(err); + return err; } else { reg_ref = make_key_ref(ns->persistent_keyring_register, true); - persistent_ref = find_key_to_update(reg_ref, index_key); - if (persistent_ref) - return persistent_ref; + *persistent_ref = find_key_to_update(reg_ref, index_key); + if (*persistent_ref) + return 1; } persistent = keyring_alloc(index_key->description, @@ -61,9 +67,10 @@ static key_ref_t key_create_persistent(struct user_namespace *ns, kuid_t uid, KEY_ALLOC_NOT_IN_QUOTA, NULL, ns->persistent_keyring_register); if (IS_ERR(persistent)) - return ERR_CAST(persistent); + return PTR_ERR(persistent); - return make_key_ref(persistent, true); + *persistent_ref = make_key_ref(persistent, true); + return 0; } /* @@ -78,6 +85,7 @@ static long key_get_persistent(struct user_namespace *ns, kuid_t uid, key_ref_t reg_ref, persistent_ref; char buf[32]; long ret; + bool created = false; /* Look in the register if it exists */ memset(&index_key, 0, sizeof(index_key)); @@ -100,23 +108,24 @@ static long key_get_persistent(struct user_namespace *ns, kuid_t uid, * also need to create the register. */ down_write(&ns->keyring_sem); - persistent_ref = key_create_persistent(ns, uid, &index_key); + ret = key_get_or_create_persistent(ns, uid, &index_key, + &persistent_ref); up_write(&ns->keyring_sem); - if (!IS_ERR(persistent_ref)) - goto found; - - return PTR_ERR(persistent_ref); + if (ret < 0) + return ret; + created = ret == 0; found: + persistent = key_ref_to_ptr(persistent_ref); ret = key_task_permission(persistent_ref, current_cred(), KEY_NEED_LINK); - if (ret == 0) { - persistent = key_ref_to_ptr(persistent_ref); + if (ret == 0) ret = key_link(key_ref_to_ptr(dest_ref), persistent); - if (ret == 0) { - key_set_timeout(persistent, persistent_keyring_expiry); - ret = persistent->serial; - } - } + + if (ret == 0 || created) + key_set_timeout(persistent, persistent_keyring_expiry); + + if (ret == 0) + ret = persistent->serial; key_ref_put(persistent_ref); return ret; diff --git a/security/selinux/avc.c b/security/selinux/avc.c index 221c41978dc50..436b9af7b92c1 100644 --- a/security/selinux/avc.c +++ b/security/selinux/avc.c @@ -579,7 +579,7 @@ static int avc_latest_notif_update(u32 seqno, int is_insert) } } else { if (seqno > selinux_avc.avc_cache.latest_notif) - selinux_avc.avc_cache.latest_notif = seqno; + WRITE_ONCE(selinux_avc.avc_cache.latest_notif, seqno); } spin_unlock_irqrestore(¬if_lock, flag); @@ -1207,5 +1207,6 @@ int avc_has_perm(u32 ssid, u32 tsid, u16 tclass, u32 avc_policy_seqno(void) { - return selinux_avc.avc_cache.latest_notif; + /* Writers hold notif_lock; readers only sample the sequence number. */ + return READ_ONCE(selinux_avc.avc_cache.latest_notif); } diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 5832131f29c47..cf7297072ec03 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -688,8 +688,12 @@ static int selinux_set_mnt_opts(struct super_block *sb, * will be used for both mounts) */ if ((sbsec->flags & SE_SBINITIALIZED) && (sb->s_type->fs_flags & FS_BINARY_MOUNTDATA) - && !opts) + && !opts) { + if ((kern_flags & SECURITY_LSM_NATIVE_LABELS) && + sbsec->behavior == SECURITY_FS_USE_NATIVE) + *set_kern_flags |= SECURITY_LSM_NATIVE_LABELS; goto out; + } root_isec = backing_inode_security_novalidate(root); @@ -733,6 +737,9 @@ static int selinux_set_mnt_opts(struct super_block *sb, /* previously mounted with options, but not on this attempt? */ if ((sbsec->flags & SE_MNTMASK) && !opts) goto out_double_mount; + if ((kern_flags & SECURITY_LSM_NATIVE_LABELS) && + sbsec->behavior == SECURITY_FS_USE_NATIVE) + *set_kern_flags |= SECURITY_LSM_NATIVE_LABELS; rc = 0; goto out; } diff --git a/sound/hda/codecs/side-codecs/tas2781_hda.c b/sound/hda/codecs/side-codecs/tas2781_hda.c index b22f93424c621..e0da299df780c 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda.c @@ -139,9 +139,21 @@ static void tas2781_apply_calib(struct tasdevice_priv *p) */ crc = crc32(~0, data, 84) ^ ~0; if (crc != tmp_val[21]) { - cali_data->total_sz = 0; - dev_err(p->dev, "%s: V1 CRC error\n", __func__); - return; + /* + * Some Lenovo BIOSes (e.g. Legion Pro 7 16IRX9H, + * SSID 17aa:38cd) store valid V1 calibration data + * but leave TimeStamp and CRC zeroed. + */ + if (!tmp_val[20] && !tmp_val[21] && + tmp_val[0] && tmp_val[1]) { + dev_warn(p->dev, + "%s: V1 data without CRC, accepting\n", + __func__); + } else { + cali_data->total_sz = 0; + dev_err(p->dev, "%s: V1 CRC error\n", __func__); + return; + } } for (j = p->ndev - 1; j >= 0; j--) { diff --git a/sound/pci/ctxfi/ctdaio.c b/sound/pci/ctxfi/ctdaio.c index 69aacd06716ce..59ed96fb938a2 100644 --- a/sound/pci/ctxfi/ctdaio.c +++ b/sound/pci/ctxfi/ctdaio.c @@ -221,14 +221,18 @@ static int dao_set_right_input(struct dao *dao, struct rsc *input) static int dao_clear_input(struct dao *dao, unsigned int start, unsigned int end) { + struct imapper *entry; unsigned int i; - if (!dao->imappers[start]) + entry = dao->imappers[start]; + if (!entry) return 0; + for (i = start; i < end; i++) { dao->mgr->imap_delete(dao->mgr, dao->imappers[i]); dao->imappers[i] = NULL; } + kfree(entry); return 0; } diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index 79858fc948065..7eaf15a27b7b9 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -759,6 +759,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_BOARD_NAME, "Herbag_MDU"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "MDC"), + DMI_MATCH(DMI_BOARD_NAME, "Herbag2_MDU"), + } + }, { .driver_data = &acp6x_card, .matches = { diff --git a/sound/soc/apple/macaudio.c b/sound/soc/apple/macaudio.c index 095a0391ea411..bba20decc6cdd 100644 --- a/sound/soc/apple/macaudio.c +++ b/sound/soc/apple/macaudio.c @@ -89,6 +89,12 @@ struct macaudio_platform_cfg { bool stereo; int amp_gain; int safe_vol; + /* + * Ceiling in dB for the speaker volume controls while speakersafetyd + * holds them unlocked. 0 means no ceiling (0 dB). For bring-up of + * machines whose speaker profile is not validated yet. + */ + int max_vol; /* * The speaker codec's output widget, "OUT" (the TAS and SSM * amplifiers) unless set. @@ -191,6 +197,10 @@ static int please_blow_up_my_speakers; module_param(please_blow_up_my_speakers, int, 0644); MODULE_PARM_DESC(please_blow_up_my_speakers, "Allow unsafe or untested operating configurations"); +static int speaker_max_db; +module_param(speaker_max_db, int, 0444); +MODULE_PARM_DESC(speaker_max_db, "Lower the unlocked speaker volume ceiling to this many dB (<= 0; can only lower it)"); + SND_SOC_DAILINK_DEFS(primary, DAILINK_COMP_ARRAY(COMP_CPU("mca-pcm-0")), // CPU DAILINK_COMP_ARRAY(COMP_DUMMY()), // CODEC @@ -281,6 +291,17 @@ static struct macaudio_link_props macaudio_fe_link_props[] = { } }; +/* Unlocked ceiling in dB: the lower of the machine's and the module's, <= 0 */ +static int macaudio_max_vol(struct macaudio_snd_data *ma) +{ + int cap = min(ma->cfg->max_vol, 0); + + if (speaker_max_db < cap) + cap = max(speaker_max_db, -100); + + return cap; +} + static void macaudio_vlimit_unlock(struct macaudio_snd_data *ma, bool unlock) { int ret, max; @@ -297,25 +318,25 @@ static void macaudio_vlimit_unlock(struct macaudio_snd_data *ma, bool unlock) return; case AMP_TAS5770: if (unlock) - max = TAS5770_0DB; + max = TAS5770_DB(macaudio_max_vol(ma)); else max = 1; //TAS5770_DB(ma->cfg->safe_vol); break; case AMP_SN012776: if (unlock) - max = SN012776_0DB; + max = SN012776_DB(macaudio_max_vol(ma)); else max = 1; //SN012776_DB(ma->cfg->safe_vol); break; case AMP_SSM3515: if (unlock) - max = SSM3515_0DB; + max = SSM3515_DB(macaudio_max_vol(ma)); else max = SSM3515_DB(ma->cfg->safe_vol); break; case AMP_MAX98360A: if (unlock) - max = MAX98360A_0DB; + max = MAX98360A_DB(macaudio_max_vol(ma)); else max = MAX98360A_DB(ma->cfg->safe_vol); break; @@ -325,6 +346,9 @@ static void macaudio_vlimit_unlock(struct macaudio_snd_data *ma, bool unlock) if (ret < 0) dev_err(ma->card.dev, "Failed to %slock volume %s: %d\n", unlock ? "un" : "", name, ret); + else if (unlock && macaudio_max_vol(ma) < 0) + dev_info(ma->card.dev, "Speaker volume ceiling %d dB\n", + macaudio_max_vol(ma)); /* * With a software gain the limit is the whole protection: refuse to * play rather than run the amplifier at full scale unlimited. @@ -1837,6 +1861,38 @@ struct macaudio_platform_cfg macaudio_j493_cfg = { true, AMP_SN012776, SPKR_2W, true, 15, -20, }; +/* + * MacBook Pro (14-inch, M5 Max), AID40. Same amps and layout as j314/j316 + * (6x SN012776 on i2c1/i2c3 at 0x38-0x3d, 2W+1T per side, sense slots in + * ADT order), new chassis and acoustic ID: the speaker thermal model is + * not validated. As on j314/j316, the amp gain is the ADT value (15) and + * the unlocked volume has no ceiling: speakersafetyd's j714 profile, with + * thermal limits more conservative than j316's, protects the speakers. + */ +struct macaudio_platform_cfg macaudio_j714_cfg = { + /* + * Speakers on with speakersafetyd's j714 profile (bring-up values: + * J716-conservative thermal limits). + */ + .enable_speakers = true, + .amp = AMP_SN012776, + .speakers = SPKR_2W1T, + .stereo = true, + .amp_gain = 15, + /* + * Locked (no speakersafetyd holding the controls): -30 dB. Unlocked, + * with the daemon modelling the voice coils: no ceiling, as j314/j316. + */ + .safe_vol = -30, + .max_vol = 0, + /* + * The AOP's audio streams (t8140_aop_audio, T6050 variant) instead + * of the MCA: the jack's serializer runs 2 x 32-bit frames. + */ + .fe_dai_names = { "j714-pcm-0", "j714-pcm-1", "j714-pcm-2" }, + .primary_bclk_ratio = 64, +}; + struct macaudio_platform_cfg macaudio_fallback_cfg = { false, AMP_NONE, SPKR_NONE, false, 0, 0, }; @@ -1898,6 +1954,8 @@ static const struct of_device_id macaudio_snd_device_id[] = { /* j615 AID21 sn012776 15 2x 2W+1T Compat: apple,j415-macaudio */ /* j700 AID44 max98360a - 2x 1W (AOP-fed, software gain) */ { .compatible = "apple,j700-macaudio", .data = &macaudio_j700_cfg }, + /* j714 AID40 sn012776 10 2x 2W+1T speakers off by default */ + { .compatible = "apple,j714-macaudio", .data = &macaudio_j714_cfg }, /* Fallback, jack only */ { .compatible = "apple,macaudio"}, { } diff --git a/sound/soc/apple/t8140_aop_audio.rs b/sound/soc/apple/t8140_aop_audio.rs index f077ff4df6e6c..b51402002a4c9 100644 --- a/sound/soc/apple/t8140_aop_audio.rs +++ b/sound/soc/apple/t8140_aop_audio.rs @@ -121,6 +121,216 @@ const AUDIO_DEV_CIN: u32 = from_fourcc(b"cin "); /// `lpai` it is the machine's only `iop-audio-s` stream: its data comes back /// over leap-s ADMAC RX0, not the AOP source ring. const AUDIO_DEV_HPAI: u32 = from_fourcc(b"hpai"); +/// The PDM microphone interface behind the array (J714c and the T604x +/// family): configured by the host before the array runs. +const AUDIO_DEV_PDM0: u32 = from_fourcc(b"pdm0"); +/// PDM interface configuration, property 200 of pdm0 (the T604x flow of +/// aurora-silicon's aop_audio, which follows macOS): 632 bytes. +const PDM_CONFIG_LEN: usize = 632; +/// High-frequency decimator configuration, property 212 of pdm0: a header +/// of 16 bytes and 1023 eight-byte coefficient slots. +const DECIM_HF_NUM_COEFFS: usize = 1023; +const DECIM_HF_LATENCY: u32 = 36; +const DECIM_HF_RATIOS: [u8; 3] = [10, 5, 3]; +const DECIM_HF_FILTER_LENGTHS: u32 = 0xd94960; +/// The microphone channel masks on lpai (property 300): three microphones. +const LPAI_CHANNELS_J714: u32 = 7; +const DECIM_HF_COEFFICIENTS: [u8; 3088] = [ + 0x6b, 0xae, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x70, 0x95, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x47, 0xf1, 0xfb, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01, 0x72, 0xf6, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x6b, 0x6f, 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0x46, 0xd3, 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc2, 0x82, 0xc2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbd, 0xb2, 0x9e, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x4c, 0xa8, 0x6f, 0xff, 0xff, 0xff, 0xff, 0xff, 0x99, 0x69, 0x36, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x1d, 0xa3, 0xf6, 0xfe, 0xff, 0xff, 0xff, 0xff, 0x4c, 0x45, 0xb7, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x64, 0xec, 0x82, 0xfe, 0xff, 0xff, 0xff, 0xff, 0x0c, 0x9d, 0x67, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x79, 0xd7, 0x75, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xfc, 0xb5, 0xbe, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x7c, 0x3e, 0x51, 0xff, 0xff, 0xff, 0xff, 0xff, 0x9b, 0xf3, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xc9, 0x1d, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x41, 0xf0, 0x02, 0x00, 0x00, 0x00, 0x00, + 0xed, 0x80, 0x9b, 0x04, 0x00, 0x00, 0x00, 0x00, 0x43, 0x9c, 0x45, 0x06, 0x00, 0x00, 0x00, 0x00, + 0xaa, 0x3b, 0xb3, 0x07, 0x00, 0x00, 0x00, 0x00, 0x4e, 0xfe, 0x9d, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x18, 0x76, 0xbb, 0x08, 0x00, 0x00, 0x00, 0x00, 0x48, 0xc4, 0xc6, 0x07, 0x00, 0x00, 0x00, 0x00, + 0x8b, 0x1a, 0x8c, 0x05, 0x00, 0x00, 0x00, 0x00, 0x48, 0xe3, 0xf4, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x89, 0xfc, 0x12, 0xfd, 0xff, 0xff, 0xff, 0xff, 0x5f, 0x39, 0x29, 0xf7, 0xff, 0xff, 0xff, 0xff, + 0x6d, 0x6a, 0xaf, 0xf0, 0xff, 0xff, 0xff, 0xff, 0x64, 0x7b, 0x50, 0xea, 0xff, 0xff, 0xff, 0xff, + 0xef, 0xc6, 0xe1, 0xe4, 0xff, 0xff, 0xff, 0xff, 0xfb, 0x8c, 0x53, 0xe1, 0xff, 0xff, 0xff, 0xff, + 0x0d, 0x64, 0x9b, 0xe0, 0xff, 0xff, 0xff, 0xff, 0x09, 0x6d, 0x9a, 0xe3, 0xff, 0xff, 0xff, 0xff, + 0x19, 0xe0, 0x01, 0xeb, 0xff, 0xff, 0xff, 0xff, 0x17, 0x15, 0x39, 0xf7, 0xff, 0xff, 0xff, 0xff, + 0x19, 0x59, 0x48, 0x08, 0x00, 0x00, 0x00, 0x00, 0xe2, 0xa1, 0xcb, 0x1d, 0x00, 0x00, 0x00, 0x00, + 0x64, 0x81, 0xef, 0x36, 0x00, 0x00, 0x00, 0x00, 0xe3, 0xa9, 0x79, 0x52, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0x02, 0xdd, 0x6e, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xdf, 0x57, 0x8a, 0x00, 0x00, 0x00, 0x00, + 0xfb, 0x9f, 0x19, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x47, 0xeb, 0x6b, 0xb7, 0x00, 0x00, 0x00, 0x00, + 0x9d, 0x23, 0xdb, 0xc5, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x9d, 0x59, 0xcd, 0x00, 0x00, 0x00, 0x00, + 0x6f, 0x9d, 0x59, 0xcd, 0x00, 0x00, 0x00, 0x00, 0x9d, 0x23, 0xdb, 0xc5, 0x00, 0x00, 0x00, 0x00, + 0x47, 0xeb, 0x6b, 0xb7, 0x00, 0x00, 0x00, 0x00, 0xfb, 0x9f, 0x19, 0xa3, 0x00, 0x00, 0x00, 0x00, + 0x4a, 0xdf, 0x57, 0x8a, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x02, 0xdd, 0x6e, 0x00, 0x00, 0x00, 0x00, + 0xe3, 0xa9, 0x79, 0x52, 0x00, 0x00, 0x00, 0x00, 0x64, 0x81, 0xef, 0x36, 0x00, 0x00, 0x00, 0x00, + 0xe2, 0xa1, 0xcb, 0x1d, 0x00, 0x00, 0x00, 0x00, 0x19, 0x59, 0x48, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x17, 0x15, 0x39, 0xf7, 0xff, 0xff, 0xff, 0xff, 0x19, 0xe0, 0x01, 0xeb, 0xff, 0xff, 0xff, 0xff, + 0x09, 0x6d, 0x9a, 0xe3, 0xff, 0xff, 0xff, 0xff, 0x0d, 0x64, 0x9b, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xfb, 0x8c, 0x53, 0xe1, 0xff, 0xff, 0xff, 0xff, 0xef, 0xc6, 0xe1, 0xe4, 0xff, 0xff, 0xff, 0xff, + 0x64, 0x7b, 0x50, 0xea, 0xff, 0xff, 0xff, 0xff, 0x6d, 0x6a, 0xaf, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0x5f, 0x39, 0x29, 0xf7, 0xff, 0xff, 0xff, 0xff, 0x89, 0xfc, 0x12, 0xfd, 0xff, 0xff, 0xff, 0xff, + 0x48, 0xe3, 0xf4, 0x01, 0x00, 0x00, 0x00, 0x00, 0x8b, 0x1a, 0x8c, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x48, 0xc4, 0xc6, 0x07, 0x00, 0x00, 0x00, 0x00, 0x18, 0x76, 0xbb, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x4e, 0xfe, 0x9d, 0x08, 0x00, 0x00, 0x00, 0x00, 0xaa, 0x3b, 0xb3, 0x07, 0x00, 0x00, 0x00, 0x00, + 0x43, 0x9c, 0x45, 0x06, 0x00, 0x00, 0x00, 0x00, 0xed, 0x80, 0x9b, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x1e, 0x41, 0xf0, 0x02, 0x00, 0x00, 0x00, 0x00, 0xc9, 0x1d, 0x70, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x9b, 0xf3, 0x36, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x3e, 0x51, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xfc, 0xb5, 0xbe, 0xfe, 0xff, 0xff, 0xff, 0xff, 0x79, 0xd7, 0x75, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x0c, 0x9d, 0x67, 0xfe, 0xff, 0xff, 0xff, 0xff, 0x64, 0xec, 0x82, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x4c, 0x45, 0xb7, 0xfe, 0xff, 0xff, 0xff, 0xff, 0x1d, 0xa3, 0xf6, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x99, 0x69, 0x36, 0xff, 0xff, 0xff, 0xff, 0xff, 0x4c, 0xa8, 0x6f, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xbd, 0xb2, 0x9e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc2, 0x82, 0xc2, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x46, 0xd3, 0xdb, 0xff, 0xff, 0xff, 0xff, 0xff, 0x6b, 0x6f, 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x01, 0x72, 0xf6, 0xff, 0xff, 0xff, 0xff, 0xff, 0x47, 0xf1, 0xfb, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x70, 0x95, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0x6b, 0xae, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x8f, 0xc2, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfa, 0x98, 0xf5, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x9f, 0x52, 0xe2, 0xff, 0xff, 0xff, 0xff, 0xff, 0x35, 0x21, 0xc1, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x25, 0xe4, 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcd, 0xec, 0x7b, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x6b, 0x33, 0x94, 0xff, 0xff, 0xff, 0xff, 0xff, 0x4c, 0xd8, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x9a, 0x64, 0x14, 0x01, 0x00, 0x00, 0x00, 0x00, 0x75, 0x22, 0x92, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x84, 0x8e, 0x2e, 0x04, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x9d, 0x36, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x68, 0x66, 0xc2, 0x04, 0x00, 0x00, 0x00, 0x00, 0xbf, 0x3c, 0x0b, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x18, 0xf7, 0xe4, 0xfc, 0xff, 0xff, 0xff, 0xff, 0x1b, 0x0e, 0x2e, 0xf6, 0xff, 0xff, 0xff, 0xff, + 0x72, 0xea, 0xf7, 0xef, 0xff, 0xff, 0xff, 0xff, 0x41, 0xcb, 0x2e, 0xed, 0xff, 0xff, 0xff, 0xff, + 0x17, 0x9c, 0xae, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xb1, 0xb5, 0xf1, 0xfb, 0xff, 0xff, 0xff, 0xff, + 0xe1, 0x3a, 0xca, 0x0d, 0x00, 0x00, 0x00, 0x00, 0xe4, 0xf4, 0xbe, 0x21, 0x00, 0x00, 0x00, 0x00, + 0x7e, 0x72, 0x94, 0x30, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x8a, 0x37, 0x32, 0x00, 0x00, 0x00, 0x00, + 0x75, 0x57, 0xb8, 0x20, 0x00, 0x00, 0x00, 0x00, 0xd1, 0x52, 0x79, 0xfb, 0xff, 0xff, 0xff, 0xff, + 0x8e, 0x8f, 0x60, 0xc9, 0xff, 0xff, 0xff, 0xff, 0x32, 0x60, 0xeb, 0x98, 0xff, 0xff, 0xff, 0xff, + 0x65, 0x2d, 0x8f, 0x7d, 0xff, 0xff, 0xff, 0xff, 0x67, 0xdd, 0xb5, 0x8a, 0xff, 0xff, 0xff, 0xff, + 0x26, 0x98, 0x96, 0xcd, 0xff, 0xff, 0xff, 0xff, 0x1d, 0xe5, 0xcf, 0x47, 0x00, 0x00, 0x00, 0x00, + 0xe1, 0x3c, 0xb2, 0xec, 0x00, 0x00, 0x00, 0x00, 0x66, 0x20, 0x81, 0xa2, 0x01, 0x00, 0x00, 0x00, + 0x2b, 0xb4, 0xbe, 0x47, 0x02, 0x00, 0x00, 0x00, 0x99, 0x16, 0x3e, 0xbb, 0x02, 0x00, 0x00, 0x00, + 0xd9, 0x88, 0xb5, 0xe4, 0x02, 0x00, 0x00, 0x00, 0x99, 0x16, 0x3e, 0xbb, 0x02, 0x00, 0x00, 0x00, + 0x2b, 0xb4, 0xbe, 0x47, 0x02, 0x00, 0x00, 0x00, 0x66, 0x20, 0x81, 0xa2, 0x01, 0x00, 0x00, 0x00, + 0xe1, 0x3c, 0xb2, 0xec, 0x00, 0x00, 0x00, 0x00, 0x1d, 0xe5, 0xcf, 0x47, 0x00, 0x00, 0x00, 0x00, + 0x26, 0x98, 0x96, 0xcd, 0xff, 0xff, 0xff, 0xff, 0x67, 0xdd, 0xb5, 0x8a, 0xff, 0xff, 0xff, 0xff, + 0x65, 0x2d, 0x8f, 0x7d, 0xff, 0xff, 0xff, 0xff, 0x32, 0x60, 0xeb, 0x98, 0xff, 0xff, 0xff, 0xff, + 0x8e, 0x8f, 0x60, 0xc9, 0xff, 0xff, 0xff, 0xff, 0xd1, 0x52, 0x79, 0xfb, 0xff, 0xff, 0xff, 0xff, + 0x75, 0x57, 0xb8, 0x20, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x8a, 0x37, 0x32, 0x00, 0x00, 0x00, 0x00, + 0x7e, 0x72, 0x94, 0x30, 0x00, 0x00, 0x00, 0x00, 0xe4, 0xf4, 0xbe, 0x21, 0x00, 0x00, 0x00, 0x00, + 0xe1, 0x3a, 0xca, 0x0d, 0x00, 0x00, 0x00, 0x00, 0xb1, 0xb5, 0xf1, 0xfb, 0xff, 0xff, 0xff, 0xff, + 0x17, 0x9c, 0xae, 0xf0, 0xff, 0xff, 0xff, 0xff, 0x41, 0xcb, 0x2e, 0xed, 0xff, 0xff, 0xff, 0xff, + 0x72, 0xea, 0xf7, 0xef, 0xff, 0xff, 0xff, 0xff, 0x1b, 0x0e, 0x2e, 0xf6, 0xff, 0xff, 0xff, 0xff, + 0x18, 0xf7, 0xe4, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x3c, 0x0b, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x68, 0x66, 0xc2, 0x04, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x9d, 0x36, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x84, 0x8e, 0x2e, 0x04, 0x00, 0x00, 0x00, 0x00, 0x75, 0x22, 0x92, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x9a, 0x64, 0x14, 0x01, 0x00, 0x00, 0x00, 0x00, 0x4c, 0xd8, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0x33, 0x94, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcd, 0xec, 0x7b, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x25, 0xe4, 0x97, 0xff, 0xff, 0xff, 0xff, 0xff, 0x35, 0x21, 0xc1, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x9f, 0x52, 0xe2, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfa, 0x98, 0xf5, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x8f, 0xc2, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0x83, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x79, 0x48, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0x19, 0x6e, 0xf9, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xb5, 0xec, 0xf4, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbb, 0xbd, 0xf2, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x9a, 0xd2, 0xf5, 0xff, 0xff, 0xff, 0xff, 0xff, 0x70, 0x86, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x31, 0xdf, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd4, 0xed, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xe0, 0x97, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0xa4, 0xfa, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xc4, 0x9b, 0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xc0, 0xf6, 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0xad, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, 0xcc, 0x77, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xf9, 0x42, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0xc9, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xfc, 0xbf, 0xf2, 0xff, 0xff, 0xff, 0xff, 0xff, 0x94, 0x67, 0xd6, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x91, 0x9d, 0xd7, 0xff, 0xff, 0xff, 0xff, 0xff, 0xbe, 0x71, 0xfc, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xf5, 0xde, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0xf3, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x61, 0xa5, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xd1, 0xdc, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xaf, 0x6c, 0xa9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa8, 0x0e, 0xb7, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x65, 0x2b, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0xed, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0x92, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6c, 0x3f, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xd8, 0x9f, 0xa9, 0xff, 0xff, 0xff, 0xff, 0xff, 0x3d, 0x9f, 0x5c, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x87, 0xfc, 0x8f, 0xff, 0xff, 0xff, 0xff, 0xff, 0x99, 0xe7, 0x2d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x8a, 0x5a, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x57, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xcd, 0x5a, 0x1e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0xd4, 0x43, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xe0, 0x19, 0xe9, 0xfe, 0xff, 0xff, 0xff, 0xff, 0x68, 0xa7, 0x71, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x73, 0xe3, 0x8a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x69, 0x3a, 0x57, 0x01, 0x00, 0x00, 0x00, 0x00, + 0xba, 0xa4, 0x19, 0x01, 0x00, 0x00, 0x00, 0x00, 0x96, 0x6b, 0xdf, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x35, 0x3d, 0x93, 0xfe, 0xff, 0xff, 0xff, 0xff, 0x8d, 0xf7, 0x51, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x6e, 0x64, 0x7a, 0xff, 0xff, 0xff, 0xff, 0xff, 0x41, 0x7b, 0x3f, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x27, 0xc5, 0x2f, 0x02, 0x00, 0x00, 0x00, 0x00, 0xb8, 0xbe, 0x5e, 0x01, 0x00, 0x00, 0x00, 0x00, + 0xc2, 0x04, 0x44, 0xff, 0xff, 0xff, 0xff, 0xff, 0x62, 0xfc, 0x83, 0xfd, 0xff, 0xff, 0xff, 0xff, + 0xbd, 0x4e, 0xac, 0xfd, 0xff, 0xff, 0xff, 0xff, 0x5f, 0x43, 0xd9, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xe9, 0x98, 0x6d, 0x02, 0x00, 0x00, 0x00, 0x00, 0xd8, 0xdc, 0x3e, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x68, 0xbb, 0x61, 0x01, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x25, 0x1c, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x67, 0x17, 0x11, 0xfc, 0xff, 0xff, 0xff, 0xff, 0x11, 0xe7, 0x25, 0xfd, 0xff, 0xff, 0xff, 0xff, + 0xc8, 0xb5, 0xca, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed, 0xa6, 0x2c, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x8a, 0xb3, 0x5f, 0x04, 0x00, 0x00, 0x00, 0x00, 0x79, 0x42, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x00, 0x3c, 0xfc, 0xff, 0xff, 0xff, 0xff, 0x3e, 0x3b, 0x50, 0xfa, 0xff, 0xff, 0xff, 0xff, + 0x0e, 0x98, 0x07, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xb5, 0xe7, 0x8e, 0x02, 0x00, 0x00, 0x00, 0x00, + 0xcc, 0xd9, 0x7b, 0x06, 0x00, 0x00, 0x00, 0x00, 0x35, 0xf4, 0x4c, 0x05, 0x00, 0x00, 0x00, 0x00, + 0xfd, 0x2b, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0x44, 0xf5, 0x8a, 0xf9, 0xff, 0xff, 0xff, 0xff, + 0x55, 0x2b, 0x7b, 0xf8, 0xff, 0xff, 0xff, 0xff, 0x21, 0x8f, 0xb0, 0xfd, 0xff, 0xff, 0xff, 0xff, + 0x4b, 0x6c, 0x5a, 0x05, 0x00, 0x00, 0x00, 0x00, 0x36, 0x50, 0x35, 0x09, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xd8, 0xa1, 0x05, 0x00, 0x00, 0x00, 0x00, 0xd5, 0x27, 0xf7, 0xfc, 0xff, 0xff, 0xff, 0xff, + 0xac, 0x6c, 0x13, 0xf6, 0xff, 0xff, 0xff, 0xff, 0xaa, 0xdd, 0xf2, 0xf6, 0xff, 0xff, 0xff, 0xff, + 0x99, 0x6d, 0x89, 0xff, 0xff, 0xff, 0xff, 0xff, 0x68, 0x1a, 0x44, 0x09, 0x00, 0x00, 0x00, 0x00, + 0x44, 0x7e, 0x06, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x80, 0x2e, 0xe4, 0x04, 0x00, 0x00, 0x00, 0x00, + 0xd8, 0x30, 0x0a, 0xf9, 0xff, 0xff, 0xff, 0xff, 0x43, 0xe2, 0x0e, 0xf2, 0xff, 0xff, 0xff, 0xff, + 0x75, 0x3f, 0x39, 0xf6, 0xff, 0xff, 0xff, 0xff, 0xb9, 0xab, 0xf0, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x01, 0xe6, 0x34, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x5d, 0xd6, 0x71, 0x0e, 0x00, 0x00, 0x00, 0x00, + 0x52, 0xee, 0x96, 0x02, 0x00, 0x00, 0x00, 0x00, 0x51, 0x29, 0xa6, 0xf3, 0xff, 0xff, 0xff, 0xff, + 0x09, 0x4e, 0xe9, 0xed, 0xff, 0xff, 0xff, 0xff, 0x06, 0xef, 0xe0, 0xf6, 0xff, 0xff, 0xff, 0xff, + 0x88, 0x68, 0x24, 0x08, 0x00, 0x00, 0x00, 0x00, 0xb2, 0x30, 0xdf, 0x13, 0x00, 0x00, 0x00, 0x00, + 0x2e, 0xa3, 0xdb, 0x0f, 0x00, 0x00, 0x00, 0x00, 0xde, 0xdb, 0x51, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x93, 0xba, 0xef, 0xec, 0xff, 0xff, 0xff, 0xff, 0x0b, 0xb4, 0x38, 0xea, 0xff, 0xff, 0xff, 0xff, + 0xcd, 0x51, 0x75, 0xf9, 0xff, 0xff, 0xff, 0xff, 0x00, 0xbf, 0x2f, 0x0f, 0x00, 0x00, 0x00, 0x00, + 0x1c, 0x05, 0xc2, 0x19, 0x00, 0x00, 0x00, 0x00, 0x20, 0xed, 0xa0, 0x0f, 0x00, 0x00, 0x00, 0x00, + 0x3a, 0x63, 0xd8, 0xf7, 0xff, 0xff, 0xff, 0xff, 0x4b, 0xe2, 0x44, 0xe5, 0xff, 0xff, 0xff, 0xff, + 0x56, 0x89, 0xa8, 0xe7, 0xff, 0xff, 0xff, 0xff, 0x0e, 0x62, 0x60, 0xfe, 0xff, 0xff, 0xff, 0xff, + 0x85, 0x0c, 0xe2, 0x17, 0x00, 0x00, 0x00, 0x00, 0x91, 0x22, 0x3a, 0x1f, 0x00, 0x00, 0x00, 0x00, + 0xe7, 0x04, 0x34, 0x0d, 0x00, 0x00, 0x00, 0x00, 0xc6, 0xac, 0x28, 0xef, 0xff, 0xff, 0xff, 0xff, + 0x2e, 0xba, 0x30, 0xdd, 0xff, 0xff, 0xff, 0xff, 0x4c, 0x07, 0xdc, 0xe6, 0xff, 0xff, 0xff, 0xff, + 0x6d, 0x3e, 0xd5, 0x05, 0x00, 0x00, 0x00, 0x00, 0x5b, 0x0f, 0xd5, 0x21, 0x00, 0x00, 0x00, 0x00, + 0xbf, 0x57, 0x9f, 0x23, 0x00, 0x00, 0x00, 0x00, 0xae, 0x0a, 0x38, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x58, 0xe7, 0x7c, 0xe4, 0xff, 0xff, 0xff, 0xff, 0xe9, 0x39, 0x50, 0xd5, 0xff, 0xff, 0xff, 0xff, + 0xb2, 0x32, 0x4d, 0xe8, 0xff, 0xff, 0xff, 0xff, 0x2c, 0x38, 0xc6, 0x0f, 0x00, 0x00, 0x00, 0x00, + 0x50, 0xcc, 0x83, 0x2c, 0x00, 0x00, 0x00, 0x00, 0x69, 0xb0, 0x69, 0x26, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x91, 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x66, 0x3b, 0xd8, 0xff, 0xff, 0xff, 0xff, + 0x7d, 0xf0, 0x2a, 0xce, 0xff, 0xff, 0xff, 0xff, 0xb2, 0x0b, 0x2f, 0xec, 0xff, 0xff, 0xff, 0xff, + 0x2d, 0xad, 0xea, 0x1b, 0x00, 0x00, 0x00, 0x00, 0xee, 0xb9, 0x70, 0x37, 0x00, 0x00, 0x00, 0x00, + 0x74, 0xdc, 0x58, 0x27, 0x00, 0x00, 0x00, 0x00, 0xc5, 0xc2, 0x7c, 0xf6, 0xff, 0xff, 0xff, 0xff, + 0x0b, 0x05, 0xd6, 0xca, 0xff, 0xff, 0xff, 0xff, 0xc6, 0x46, 0x04, 0xc8, 0xff, 0xff, 0xff, 0xff, + 0x7e, 0x32, 0x57, 0xf2, 0xff, 0xff, 0xff, 0xff, 0x29, 0xa3, 0xd4, 0x29, 0x00, 0x00, 0x00, 0x00, + 0x62, 0x4d, 0x57, 0x42, 0x00, 0x00, 0x00, 0x00, 0x2c, 0x2a, 0x9a, 0x26, 0x00, 0x00, 0x00, 0x00, + 0x9a, 0x8d, 0x6a, 0xea, 0xff, 0xff, 0xff, 0xff, 0x1c, 0xe2, 0x9c, 0xbc, 0xff, 0xff, 0xff, 0xff, + 0xb5, 0x9c, 0xa8, 0xc2, 0xff, 0xff, 0xff, 0xff, 0x04, 0x55, 0x2d, 0xfa, 0xff, 0xff, 0xff, 0xff, + 0xd5, 0x68, 0x10, 0x39, 0x00, 0x00, 0x00, 0x00, 0x17, 0xa8, 0x64, 0x4d, 0x00, 0x00, 0x00, 0x00, + 0xc2, 0xca, 0xef, 0x24, 0x00, 0x00, 0x00, 0x00, 0xb2, 0xab, 0x16, 0xdd, 0xff, 0xff, 0xff, 0xff, + 0xec, 0x0e, 0x8c, 0xad, 0xff, 0xff, 0xff, 0xff, 0x63, 0x20, 0x34, 0xbd, 0xff, 0xff, 0xff, 0xff, + 0x62, 0xc0, 0x93, 0x02, 0x00, 0x00, 0x00, 0x00, 0xf6, 0x38, 0x40, 0x49, 0x00, 0x00, 0x00, 0x00, + 0xc2, 0x28, 0x75, 0x59, 0x00, 0x00, 0x00, 0x00, 0xf2, 0x57, 0xe8, 0x23, 0x00, 0x00, 0x00, 0x00, + 0x82, 0xda, 0x84, 0xcf, 0xff, 0xff, 0xff, 0xff, 0x2f, 0xeb, 0x20, 0x9d, 0xff, 0xff, 0xff, 0xff, + 0xb9, 0x6b, 0xc7, 0xb5, 0xff, 0xff, 0xff, 0xff, 0xb0, 0x48, 0xa8, 0x09, 0x00, 0x00, 0x00, 0x00, + 0x9c, 0x37, 0x15, 0x5a, 0x00, 0x00, 0x00, 0x00, 0x74, 0x23, 0x55, 0x68, 0x00, 0x00, 0x00, 0x00, + 0x8b, 0xd6, 0x4a, 0x26, 0x00, 0x00, 0x00, 0x00, 0x1e, 0xe6, 0x59, 0xc3, 0xff, 0xff, 0xff, 0xff, + 0x3d, 0x3f, 0x56, 0x8a, 0xff, 0xff, 0xff, 0xff, 0xa4, 0xcf, 0x12, 0xa9, 0xff, 0xff, 0xff, 0xff, + 0x3a, 0xc5, 0x14, 0x0c, 0x00, 0x00, 0x00, 0x00, 0xae, 0x04, 0xd9, 0x6a, 0x00, 0x00, 0x00, 0x00, + 0x27, 0xe7, 0xf0, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x83, 0x35, 0xfc, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x7c, 0xbf, 0xfb, 0xbb, 0xff, 0xff, 0xff, 0xff, 0x63, 0xba, 0x2d, 0x74, 0xff, 0xff, 0xff, 0xff, + 0x97, 0xe2, 0xab, 0x91, 0xff, 0xff, 0xff, 0xff, 0x57, 0xa5, 0x55, 0x03, 0x00, 0x00, 0x00, 0x00, + 0xf0, 0x0b, 0xae, 0x78, 0x00, 0x00, 0x00, 0x00, 0xa0, 0x97, 0xf1, 0x9a, 0x00, 0x00, 0x00, 0x00, + 0xc8, 0x79, 0xb8, 0x4c, 0x00, 0x00, 0x00, 0x00, 0xfb, 0x9a, 0xb7, 0xc1, 0xff, 0xff, 0xff, 0xff, + 0xe3, 0xf0, 0x39, 0x5c, 0xff, 0xff, 0xff, 0xff, 0xcb, 0x28, 0xc4, 0x67, 0xff, 0xff, 0xff, 0xff, + 0x79, 0xe2, 0xe5, 0xe1, 0xff, 0xff, 0xff, 0xff, 0x8c, 0x7a, 0x04, 0x79, 0x00, 0x00, 0x00, 0x00, + 0x84, 0xdc, 0x42, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x87, 0xa2, 0x1c, 0x88, 0x00, 0x00, 0x00, 0x00, + 0x26, 0x4e, 0xaa, 0xe9, 0xff, 0xff, 0xff, 0xff, 0x45, 0x8e, 0x53, 0x50, 0xff, 0xff, 0xff, 0xff, + 0xad, 0x81, 0xc2, 0x25, 0xff, 0xff, 0xff, 0xff, 0x16, 0x3c, 0x3e, 0x8d, 0xff, 0xff, 0xff, 0xff, + 0x5f, 0xde, 0x95, 0x49, 0x00, 0x00, 0x00, 0x00, 0x91, 0xc6, 0x7f, 0xe2, 0x00, 0x00, 0x00, 0x00, + 0xe8, 0x96, 0x73, 0xf1, 0x00, 0x00, 0x00, 0x00, 0xcf, 0x44, 0x66, 0x64, 0x00, 0x00, 0x00, 0x00, + 0xb6, 0xf3, 0xb5, 0x8b, 0xff, 0xff, 0xff, 0xff, 0x2e, 0xbd, 0x92, 0xea, 0xfe, 0xff, 0xff, 0xff, + 0x06, 0x5a, 0xb8, 0xe7, 0xfe, 0xff, 0xff, 0xff, 0x2d, 0xca, 0x07, 0x8f, 0xff, 0xff, 0xff, 0xff, + 0xca, 0x3a, 0xeb, 0x88, 0x00, 0x00, 0x00, 0x00, 0xab, 0xac, 0x3a, 0x4b, 0x01, 0x00, 0x00, 0x00, + 0x4f, 0x55, 0xde, 0x65, 0x01, 0x00, 0x00, 0x00, 0xa5, 0x44, 0x63, 0xbf, 0x00, 0x00, 0x00, 0x00, + 0xc6, 0x47, 0x94, 0xa1, 0xff, 0xff, 0xff, 0xff, 0xb0, 0x5c, 0xad, 0x92, 0xfe, 0xff, 0xff, 0xff, + 0x62, 0x2e, 0x32, 0x13, 0xfe, 0xff, 0xff, 0xff, 0xe9, 0xae, 0xe5, 0x63, 0xfe, 0xff, 0xff, 0xff, + 0xdd, 0x67, 0xd8, 0x6e, 0xff, 0xff, 0xff, 0xff, 0xbb, 0xab, 0x7c, 0xd9, 0x00, 0x00, 0x00, 0x00, + 0x69, 0x27, 0x26, 0x31, 0x02, 0x00, 0x00, 0x00, 0x43, 0x5c, 0xb1, 0x19, 0x03, 0x00, 0x00, 0x00, + 0x49, 0xaf, 0xf9, 0x69, 0x03, 0x00, 0x00, 0x00, 0x2d, 0x04, 0x67, 0x2e, 0x03, 0x00, 0x00, 0x00, + 0xee, 0x25, 0x09, 0x97, 0x02, 0x00, 0x00, 0x00, 0xea, 0x83, 0xfb, 0xdd, 0x01, 0x00, 0x00, 0x00, + 0xda, 0x7a, 0x51, 0x32, 0x01, 0x00, 0x00, 0x00, 0x1f, 0xcb, 0x0c, 0xae, 0x00, 0x00, 0x00, 0x00, + 0xba, 0xe8, 0xe9, 0x56, 0x00, 0x00, 0x00, 0x00, 0xb6, 0xe4, 0x85, 0x25, 0x00, 0x00, 0x00, 0x00, + 0xbd, 0xc0, 0x9e, 0x0d, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x2a, 0xf5, 0x03, 0x00, 0x00, 0x00, 0x00, + 0xb1, 0xdb, 0xd5, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd2, 0x5f, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, +]; + /// kIOReturnBusy from the firmware: the device is already attached. const AUDIO_RET_BUSY: u32 = 0xe00002d5; const SPKR_POWER_STATE_PW0: u32 = from_fourcc(b"pw0 "); @@ -153,7 +363,6 @@ const REPORT_MIN_LEN: usize = 0x68; /// The high-quality microphone array: 48 kHz stereo float32 over the LEAP /// ADMAC. const HPAI_RATE: u32 = 48000; -const HPAI_CHANNELS: u32 = 2; /// Firmware power requests carry a timestamp in the 24 MHz always-on /// timebase; the architected counter runs at a different rate (1 GHz on @@ -556,7 +765,9 @@ struct SndSocT8140AopData { dev: ARef, adata: Arc, service: EPICService, - ring: Arc, + ring: Option>, + /// What this SoC leaves to the AP, see [`HwConfig`]. + hw: &'static HwConfig, ring_bytes: Atomic, sequence: Atomic, /// The low-power microphone PCM holds a reference on `lpai_run`. @@ -592,6 +803,28 @@ struct SndSocT8140AopData { spkr_want_run: AtomicFlag, /// A microphone-array stream has been triggered and not yet stopped. hpai_want_run: AtomicFlag, + /// The sense capture (T6050) reads the drain's ring: the bytes the drain + /// has produced into it (monotonic), the capture's origin at its start, + /// and the capture substream while it runs. + sense_ring_pos: core::sync::atomic::AtomicU64, + /// The count at the drain's last arm: the DMA starts over at the ring's + /// first byte, so a count maps to `(count - base) % ring`. + sense_arm_base: core::sync::atomic::AtomicU64, + sense_start_pos: core::sync::atomic::AtomicU64, + sense_sub: core::sync::atomic::AtomicPtr, + /// The sense RX channel, requested on first use and kept: the capture + /// and the drain take turns on it. + sense_chan: core::sync::atomic::AtomicPtr, + /// The driver's drain of the sense RX: its coherent buffer (allocated + /// with the channel, kept) and whether its cyclic transfer runs. + /// Changed under `spkr_lock`. + sense_drain_cpu: core::sync::atomic::AtomicPtr, + sense_drain_dma: core::sync::atomic::AtomicU64, + sense_drain_running: AtomicFlag, + /// Orders the speaker service's `pwrd` and `pw0` between the start work + /// and the prepare and shutdown paths. + #[pin] + spkr_lock: Mutex<()>, /// "Speaker Playback Volume" (REG_SPKR_VOLUME), 0..=SPKR_VOLUME_MAX. spkr_volume: Atomic, /// The sleeping firmware starts queued from the atomic triggers. @@ -634,15 +867,15 @@ impl WorkItem for SndSocT8140AopData { } } -/// The speaker stream start (pwrd + unmute), queued by the back-end DAI's -/// trigger START: those are sleeping firmware calls, and the DMA must already -/// be running when the LEAP starts consuming. +/// The speaker stream start (pwrd + unmute) or stop (mute + pw0), queued by +/// the back-end DAI's trigger: those are sleeping firmware calls, and the DMA +/// must already be running when the LEAP starts consuming. impl WorkItem for SndSocT8140AopData { type Pointer = Arc; fn run(this: Arc) { if let Err(e) = this.spkr_go() { - dev_err!(this.dev, "spkr: start failed: {:?}\n", e); + dev_err!(this.dev, "spkr: start or stop failed: {:?}\n", e); } } } @@ -864,8 +1097,15 @@ impl SndSocT8140AopData { /// Property 202 with the service's sequence and a 24 MHz timestamp; /// `pw0 ` is confirmed through property 200 like the native host does. fn service_set_power(&self, svc: &Service, state: u32) -> Result<()> { + self.service_set_power_param(svc, state, 0) + } + + /// As [`Self::service_set_power`], with the request's parameter word: + /// 1 on the array's `pwrd` where the host configures the PDM + /// microphones (the T604x flow sends it for the microphone power). + fn service_set_power_param(&self, svc: &Service, state: u32, param: u32) -> Result<()> { let sequence = svc.sequence.fetch_add(1, Relaxed).wrapping_add(1); - let req = PowerRequest::new(svc.dev_id, sequence, state, 0)?; + let req = PowerRequest::new(svc.dev_id, sequence, state, param)?; let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( svc.dev_id, PROP_POWER_REQUEST, @@ -903,6 +1143,9 @@ impl SndSocT8140AopData { /// power up fails the stream; recovering a domain from that state is the /// power-domain driver's business, not a consumer's. fn power_up(&self, name: &str, pds: &[&Option], attempts: u32) -> Result<()> { + if !self.hw.ap_power_domains { + return Ok(()); + } for (i, slot) in pds.iter().enumerate() { let Some(pd) = slot else { dev_err!(self.dev, "{}: power domain not attached\n", name); @@ -967,6 +1210,9 @@ impl SndSocT8140AopData { /// microphone is attached for the firmware's lifetime, so hold it in /// `runn` around `f`, once its first report has arrived. fn with_fabric_raised(&self, f: impl FnOnce() -> Result) -> Result { + if !self.hw.lpai { + return f(); + } self.lpai_run_get()?; let mut polls = 0; while !self.reported.load(Relaxed) && polls < FABRIC_RAISE_POLLS { @@ -989,10 +1235,14 @@ impl SndSocT8140AopData { /// `pwrd`: the firmware starts the serializer clocks and the stream. fn service_run(&self, svc: &Service) -> Result<()> { + self.service_run_param(svc, 0) + } + + fn service_run_param(&self, svc: &Service, param: u32) -> Result<()> { if svc.running.load(Relaxed) { return Ok(()); } - self.service_set_power(svc, SPKR_POWER_STATE_PWRD)?; + self.service_set_power_param(svc, SPKR_POWER_STATE_PWRD, param)?; svc.running.store(true, Relaxed); Ok(()) } @@ -1061,9 +1311,94 @@ impl SndSocT8140AopData { } fn hpai_startup(&self) -> Result<()> { + if self.hw.hpai_pdm { + self.hpai_pdm_setup(); + } self.service_startup(&self.hpai, &self.hpai_domains()) } + /// A property set with the payload assembled on the heap: the same + /// record as [`AudioSetDeviceProp`], for payloads too big for the stack. + fn set_prop_bytes(&self, dev_id: u32, modifier: u32, data: &[u8]) -> Result { + let mut msg: KVec = KVec::with_capacity(52 + data.len(), GFP_KERNEL)?; + msg.extend_from_slice(&0u32.to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(&0xffff_ffffu32.to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(&CALLTYPE_AUDIO_SET_PROP.to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(&[0u8; 20], GFP_KERNEL)?; + msg.extend_from_slice(&(data.len() as u64 + 0x30).to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(&dev_id.to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(&modifier.to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(&(data.len() as u32).to_le_bytes(), GFP_KERNEL)?; + msg.extend_from_slice(data, GFP_KERNEL)?; + self.adata + .epic_call(&self.service, EPIC_SUBTYPE_WRAPPED_CALL, &msg) + } + + /// The array's microphone setup macOS sends on this firmware family: + /// attach pdm0 and lpai, the lpai channel masks, the PDM interface + /// configuration and the high-frequency decimator. Each answer is + /// reported; a refusal does not stop the array, so the next one is seen. + fn hpai_pdm_setup(&self) { + for (dev, name) in [(AUDIO_DEV_PDM0, "pdm0"), (AUDIO_DEV_LPAI, "lpai"), (from_fourcc(b"lilb"), "lilb")] { + if let Err(e) = self.attach_device(dev, name) { + dev_warn!(self.dev, "{}: attach failed: {:?}\n", name, e); + } + } + let ctl = ChannelControl { + supported: LPAI_CHANNELS_J714, + enabled: LPAI_CHANNELS_J714, + history: LPAI_CHANNELS_J714, + }; + match self.epic_wrapped_call(&AudioSetDeviceProp::new( + AUDIO_DEV_LPAI, + PROP_CHANNEL_CONTROL, + ctl, + )) { + Ok(ret) => dev_info!(self.dev, "lpai channel masks: {:#x}\n", ret), + Err(e) => dev_warn!(self.dev, "lpai channel masks: {:?}\n", e), + } + // PDM interface configuration (pdm0 property 200). + let mut cfg = [0u8; PDM_CONFIG_LEN]; + let mut w = |off: usize, v: u32| cfg[off..off + 4].copy_from_slice(&v.to_le_bytes()); + w(0, 3); // bytes per sample + w(4, from_fourcc(b"pll ")); // clock source + w(8, 2_400_000); // PDM clock + w(12, 24_000_000); // PDMC clock + w(16, 24_000_000); // slow clock + w(20, 24_000_000); // fast clock + w(24, 256); // channel polarity select + // 28: unk1 (u8) 0; 29..33: channel phase select (u32) 0; 33: unk2 (u8) 0; + // 34..530: the low-frequency decimator, unused with the HF one. + w(530, 1); // mic turn-on time valid + w(534, 20); // mic turn-on time, ms + // 538..554: zero + w(554, 1); // mic settle time valid + w(558, 50); // mic settle time, ms + // 562..632: zero + match self.set_prop_bytes(AUDIO_DEV_PDM0, 200, &cfg) { + Ok(ret) => dev_info!(self.dev, "pdm0 config: {:#x}\n", ret), + Err(e) => dev_warn!(self.dev, "pdm0 config: {:?}\n", e), + } + // High-frequency decimator (pdm0 property 212). + let decim: Result> = (|| { + let mut d: KVec = KVec::with_capacity(16 + DECIM_HF_NUM_COEFFS * 8, GFP_KERNEL)?; + d.extend_from_slice(&DECIM_HF_LATENCY.to_le_bytes(), GFP_KERNEL)?; + d.extend_from_slice(&DECIM_HF_RATIOS, GFP_KERNEL)?; + d.push(0, GFP_KERNEL)?; + d.extend_from_slice(&DECIM_HF_FILTER_LENGTHS.to_le_bytes(), GFP_KERNEL)?; + d.extend_from_slice(&(DECIM_HF_NUM_COEFFS as u32).to_le_bytes(), GFP_KERNEL)?; + d.extend_from_slice(&DECIM_HF_COEFFICIENTS, GFP_KERNEL)?; + while d.len() < 16 + DECIM_HF_NUM_COEFFS * 8 { + d.push(0, GFP_KERNEL)?; + } + Ok(d) + })(); + match decim.and_then(|d| self.set_prop_bytes(AUDIO_DEV_PDM0, 212, &d)) { + Ok(ret) => dev_info!(self.dev, "pdm0 hf decimator: {:#x}\n", ret), + Err(e) => dev_warn!(self.dev, "pdm0 hf decimator: {:?}\n", e), + } + } + /// The array's start (`pwrd`), from the system workqueue right after /// trigger START. Started earlier, from prepare, the LEAP produces into /// a ring the ADMAC is not servicing yet and the capture is noise from @@ -1072,7 +1407,7 @@ impl SndSocT8140AopData { if !self.hpai_want_run.load(Relaxed) || self.hpai.running.load(Relaxed) { return Ok(()); } - self.service_run(&self.hpai) + self.service_run_param(&self.hpai, if self.hw.hpai_pdm { 1 } else { 0 }) } /// Called from sleeping close paths: a queued start must finish before @@ -1138,6 +1473,9 @@ impl SndSocT8140AopData { /// Output mute (property 700) is independent of the amplifier's SD_MODE; /// the amplifier stays off across a failure here. fn spkr_set_mute(&self, muted: bool) -> Result<()> { + if !self.hw.spkr_output_mute { + return Ok(()); + } let ret = self.epic_wrapped_call(&AudioSetDeviceProp::new( AUDIO_DEV_SPKR, PROP_OUTPUT_MUTE, @@ -1155,6 +1493,7 @@ impl SndSocT8140AopData { } fn spkr_startup(&self) -> Result<()> { + let _guard = self.spkr_lock.lock(); self.service_startup(&self.spkr, &self.spkr_domains()) } @@ -1164,33 +1503,81 @@ impl SndSocT8140AopData { /// unmute once the DMA has restarted. fn spkr_prepare(&self) -> Result<()> { self.flush_spkr_start(); - if !self.spkr.running.load(Relaxed) { - return Ok(()); + let _guard = self.spkr_lock.lock(); + self.spkr_retire(); + if !self.hw.spkr_leap { + // An MCA serializer stream (T6050): the service runs from here, + // before ASoC unmutes the amplifiers, so they see their TDM + // clock when they come out of shutdown; faulted on a missing + // clock, most of them stay down for the whole stream. The DMA + // follows at trigger START. + self.spkr_start()?; } - self.spkr_set_mute(true)?; - self.service_set_power(&self.spkr, SPKR_POWER_STATE_PW0)?; - self.spkr.running.store(false, Relaxed); Ok(()) } - /// Speaker stream start, from the system workqueue right after trigger - /// START: the DMA must be running before pwrd starts the LEAP consuming - /// (issued from prepare, before the DMA, the wire stays silent), then the - /// firmware unmute, the value observed on the bus once the amplifier - /// output is enabled. - fn spkr_go(&self) -> Result<()> { - if !self.spkr_want_run.load(Relaxed) || self.spkr.running.load(Relaxed) { - return Ok(()); + /// Drain armed, `pwrd`, unmute. Under `spkr_lock`. + fn spkr_start(&self) -> Result<()> { + if self.hw.sense_channels > 0 { + self.sense_drain_arm()?; } self.service_run(&self.spkr)?; + if !self.hw.spkr_leap { + // Let the serializer clock settle before ASoC unmutes the + // amplifiers right after this prepare. + kernel::time::delay::fsleep(kernel::time::Delta::from_millis(20)); + } if let Err(e) = self.spkr_set_mute(false) { - let _ = self.service_set_power(&self.spkr, SPKR_POWER_STATE_PW0); - self.spkr.running.store(false, Relaxed); + self.spkr_retire(); return Err(e); } Ok(()) } + /// Speaker stream start or stop, from the system workqueue right after + /// the trigger: the DMA must be running before pwrd starts the LEAP + /// consuming (issued from prepare, before the DMA, the wire stays silent), + /// then the firmware unmute, the value observed on the bus once the + /// amplifier output is enabled. + /// + /// The service follows playback alone. With a V/I sense RX (T6050) the + /// firmware stops the stream within about 0.2 s of that RX going + /// undrained, and does not deliver on it again until the service is + /// restarted, even to a restarted DMA. So the driver's own drain DMA runs + /// on the RX for the whole run and the sense capture reads its ring. The + /// service never stops under a running playback: the amplifiers keep + /// their clock through a daemon restart, which they do not survive + /// otherwise. + fn spkr_go(&self) -> Result<()> { + let _guard = self.spkr_lock.lock(); + if self.spkr.running.load(Relaxed) { + if !self.spkr_want_run.load(Relaxed) { + self.spkr_retire(); + } + return Ok(()); + } + if !self.spkr_want_run.load(Relaxed) { + return Ok(()); + } + self.spkr_start() + } + + /// End the speaker service's run: mute, pw0, the drain off. A failed pw0 + /// still ends the run, so the next trigger sends a fresh pwrd. Under + /// `spkr_lock`. + fn spkr_retire(&self) { + if self.spkr.running.load(Relaxed) { + if let Err(e) = self.spkr_set_mute(true) { + dev_err!(self.dev, "spkr: mute failed: {:?}\n", e); + } + if let Err(e) = self.service_set_power(&self.spkr, SPKR_POWER_STATE_PW0) { + dev_err!(self.dev, "spkr: pw0 failed: {:?}\n", e); + } + self.spkr.running.store(false, Relaxed); + } + self.sense_drain_disarm(); + } + /// Called from sleeping prepare/shutdown paths while ALSA serializes /// this stream: a queued start must finish before the mute and pw0. fn flush_spkr_start(&self) { @@ -1204,13 +1591,163 @@ impl SndSocT8140AopData { /// already off (codec trigger STOP) and a queued start was flushed. fn spkr_shutdown(&self) { self.flush_spkr_start(); + let _guard = self.spkr_lock.lock(); if self.spkr.running.load(Relaxed) { if let Err(e) = self.spkr_set_mute(true) { dev_err!(self.dev, "spkr: mute failed: {:?}\n", e); } } self.service_shutdown(&self.spkr, &self.spkr_domains()); + self.sense_drain_disarm(); + } + + // ---- the sense RX drain (T6050): a cyclic transfer into the driver's + // ring for the whole run of the speaker service, never stopped by the + // capture, which reads the ring (`asoc_pcm_copy`, `asoc_pcm_pointer`). ---- + + /// The sense RX channel and the drain's buffer, requested and allocated + /// on first use and kept for the driver's life. + fn sense_chan(&self) -> Result<*mut bindings::dma_chan> { + let chan = self.sense_chan.load(core::sync::atomic::Ordering::Relaxed); + if !chan.is_null() { + return Ok(chan); + } + // SAFETY: the platform device is live and the name is NUL-terminated. + let chan = from_err_ptr(unsafe { + bindings::dma_request_chan(self.dev.as_raw(), c"spkr-sense".as_char_ptr()) + })?; + // SAFETY: `chan` is a live channel of the ADMAC, whose device does + // the allocation; freed in `teardown` after the channel is released. + let dev = unsafe { (*(*chan).device).dev }; + let mut dma: bindings::dma_addr_t = 0; + // SAFETY: as above. + let cpu = unsafe { + bindings::dma_alloc_attrs(dev, SENSE_DRAIN_BYTES, &mut dma, bindings::GFP_KERNEL, 0) + }; + if cpu.is_null() { + // SAFETY: requested above, unused. + unsafe { bindings::dma_release_channel(chan) }; + return Err(ENOMEM); + } + self.sense_drain_cpu.store(cpu, core::sync::atomic::Ordering::Relaxed); + self.sense_drain_dma.store(dma, core::sync::atomic::Ordering::Relaxed); + self.sense_chan.store(chan, core::sync::atomic::Ordering::Relaxed); + Ok(chan) + } + + /// Arm the drain: a cyclic transfer into its buffer. Under `spkr_lock`, + /// with the capture not owning the channel. + fn sense_drain_arm(&self) -> Result<()> { + if self.sense_drain_running.load(Relaxed) { + return Ok(()); + } + let chan = self.sense_chan()?; + // SAFETY: `chan` is live and the buffer is the one allocated with it; + // `self` outlives the transfer, disarmed before the driver data goes. + let param = ptr::from_ref(self).cast_mut().cast::(); + self.sense_arm_base.store(self.sense_ring_pos.load(core::sync::atomic::Ordering::Relaxed), core::sync::atomic::Ordering::Relaxed); + if let Err(e) = unsafe { sense_drain_submit(chan, self.sense_drain_dma.load(core::sync::atomic::Ordering::Relaxed), param) } { + dev_err!(self.dev, "sense drain: start failed: {:?}\n", e); + return Err(e); + } + self.sense_drain_running.store(true, Relaxed); + Ok(()) } + + /// Disarm the drain and wait for the channel to stop. Under `spkr_lock`. + fn sense_drain_disarm(&self) { + if !self.sense_drain_running.load(Relaxed) { + return; + } + let chan = self.sense_chan.load(core::sync::atomic::Ordering::Relaxed); + // SAFETY: `chan` is the live channel the drain runs on. + unsafe { + let ops = &*(*chan).device; + if let Some(terminate) = ops.device_terminate_all { + terminate(chan); + } + if let Some(sync) = ops.device_synchronize { + sync(chan); + } + } + self.sense_drain_running.store(false, Relaxed); + } + +} + +/// A period of the drain landed in the ring: count it, and wake a running +/// sense capture. DMA completion context. +/// +/// # Safety +/// +/// `param` is the driver data the drain was armed with, alive until the +/// drain is disarmed and synchronized. +unsafe extern "C" fn sense_dma_period(param: *mut core::ffi::c_void) { + // SAFETY: by the contract above. + let data = unsafe { &*param.cast::() }; + data.sense_ring_pos.fetch_add(SENSE_DRAIN_PERIOD as u64, core::sync::atomic::Ordering::Relaxed); + let sub = data.sense_sub.load(core::sync::atomic::Ordering::Relaxed); + if !sub.is_null() { + // SAFETY: a substream of this component's PCM, which outlives the + // drain; the core checks its runtime under the stream lock. + unsafe { bindings::snd_pcm_period_elapsed(sub) }; + } +} + +/// Configure the sense channel (16-bit words, one burst of four) and submit +/// the drain's cyclic transfer, with `param` for its period callback. +/// +/// # Safety +/// +/// `chan` is a live, requested channel of the ADMAC, `dma` the address of a +/// buffer of `SENSE_DRAIN_BYTES` on its device, `param` the driver data. +unsafe fn sense_drain_submit( + chan: *mut bindings::dma_chan, + dma: bindings::dma_addr_t, + param: *mut core::ffi::c_void, +) -> Result<()> { + let mut cfg = bindings::dma_slave_config::default(); + cfg.direction = bindings::dma_transfer_direction_DMA_DEV_TO_MEM; + cfg.src_addr_width = bindings::dma_slave_buswidth_DMA_SLAVE_BUSWIDTH_2_BYTES; + cfg.src_port_window_size = 4; + // SAFETY: by the caller's contract. + let ops = unsafe { &*(*chan).device }; + let config = ops.device_config.ok_or(ENOSYS)?; + let prep = ops.device_prep_dma_cyclic.ok_or(ENOSYS)?; + let issue = ops.device_issue_pending.ok_or(ENOSYS)?; + // SAFETY: the channel's own ops, with a live channel and configuration. + let rc = unsafe { config(chan, &mut cfg) }; + if rc < 0 { + return Err(Error::from_errno(rc)); + } + // SAFETY: as above. + let desc = unsafe { + prep( + chan, + dma, + SENSE_DRAIN_BYTES, + SENSE_DRAIN_PERIOD, + bindings::dma_transfer_direction_DMA_DEV_TO_MEM, + 0, + ) + }; + if desc.is_null() { + return Err(EIO); + } + // SAFETY: a descriptor the channel just prepared, not submitted yet. + unsafe { + (*desc).callback = Some(sense_dma_period); + (*desc).callback_param = param; + } + // SAFETY: as above. + let submit = unsafe { (*desc).tx_submit }.ok_or(ENOSYS)?; + // SAFETY: as above. + if unsafe { submit(desc) } < 0 { + return Err(EIO); + } + // SAFETY: as above. + unsafe { issue(chan) }; + Ok(()) } /// Copy `len` bytes from the source ring (absolute producer offset `from`) @@ -1340,7 +1877,7 @@ impl ReportListener for SndSocT8140AopData { let dma_area = unsafe { (*(*st.substream).runtime).dma_area }; let buffer_bytes = st.buffer_bytes; let new_ptr = copy_span( - &self.ring, + self.ring.as_ref().ok_or(ENODEV)?, dma_area, buffer_bytes, ring_bytes, @@ -1716,8 +2253,8 @@ unsafe extern "C" fn hpai_pcm_open(substream: *mut bindings::snd_pcm_substream) hw.rates = bindings::SNDRV_PCM_RATE_48000; hw.rate_min = HPAI_RATE; hw.rate_max = HPAI_RATE; - hw.channels_min = HPAI_CHANNELS; - hw.channels_max = HPAI_CHANNELS; + hw.channels_min = data.hw.hpai_channels; + hw.channels_max = data.hw.hpai_channels; let ret = pcm_open_dmaengine( substream, chan, @@ -1932,6 +2469,13 @@ const FE_DMA_NAMES: [[Option<&core::ffi::CStr>; 2]; FE_COUNT] = [ /// The front-end that plays the internal speaker (j700-pcm-1, the card's /// "Secondary" PCM): its data goes through `spkr_copy` below. const FE_SPKR: usize = 1; +/// The front-end that captures the speaker tap or, on T6050, the amplifiers' +/// V/I sense return (j7xx-pcm-2, the card's "Speaker Sense" PCM). +const FE_SENSE: usize = 2; +/// The driver's sense RX drain ring (T6050): 256 frames of 12 x 16-bit words +/// per period, 128 periods (0.68 s). The sense capture reads it. +const SENSE_DRAIN_PERIOD: usize = 256 * 12 * 2; +const SENSE_DRAIN_BYTES: usize = SENSE_DRAIN_PERIOD * 128; /// Digital ceiling of the speaker path, as a Q31 gain applied to every /// sample userspace writes: 0.5, the amplifier's full scale. @@ -2103,15 +2647,68 @@ unsafe extern "C" fn asoc_pcm_open( }; // SAFETY: the substream is live for the duration of the op. let playback = unsafe { substream_is_playback(substream) }; - let Some(name) = FE_DMA_NAMES[fe][usize::from(!playback)] else { + let Some(name) = data.hw.fe_dma_names[fe][usize::from(!playback)] else { return ENODEV.to_errno(); }; - let leap = fe == FE_SPKR; + let leap = fe == FE_SPKR && data.hw.spkr_leap; + // The sense return goes through `asoc_pcm_copy`, which re-aligns its + // words: readers must not map the ring (T6050). + let sense_copy = !playback && fe == FE_SENSE && data.hw.sense_channels > 0; if leap { if let Err(e) = data.power_up_raising(data.spkr.name, &data.spkr_domains()) { return e.to_errno(); } } + if !data.hw.ap_power_domains && !sense_copy { + // The firmware owns the audio fabric, the ADMAC's power with it: + // attach the stream's service and request pw0 before the first + // touch of the channel's registers. Not for the sense capture: it + // has no channel of its own, and a pw0 would stop a running + // speaker service under it. + let svc = if fe == 0 { + data.jack_service(playback) + } else { + &data.spkr + }; + let _guard = (fe != 0).then(|| data.spkr_lock.lock()); + if let Err(e) = data.service_startup(svc, &[]) { + return e.to_errno(); + } + } + if sense_copy { + // The capture reads the drain's ring: no DMA of its own, a vmalloc + // buffer that sizes the stream, its words copied from the ring. + let mut hw = dmaengine_hw(); + hw.info = bindings::SNDRV_PCM_INFO_INTERLEAVED | bindings::SNDRV_PCM_INFO_BATCH; + hw.formats = SENSE_FORMATS; + hw.rates = bindings::SNDRV_PCM_RATE_48000; + hw.rate_min = 48000; + hw.rate_max = 48000; + hw.channels_min = data.hw.sense_channels; + hw.channels_max = data.hw.sense_channels; + // Periods of at least one drain period (the pointer advances by + // one), buffers up to half the ring. + hw.buffer_bytes_max = SENSE_DRAIN_BYTES / 2; + hw.period_bytes_min = SENSE_DRAIN_PERIOD; + hw.period_bytes_max = SENSE_DRAIN_BYTES / 8; + hw.periods_max = (SENSE_DRAIN_BYTES / 2 / SENSE_DRAIN_PERIOD) as u32; + hw.fifo_size = 0; + // SAFETY: the substream's runtime exists for the op's duration; a + // vmalloc buffer needs no device, and is set once per substream. + unsafe { + (*(*substream).runtime).hw = hw; + if (*substream).dma_buffer.dev.type_ != bindings::SNDRV_DMA_TYPE_UNKNOWN as i32 { + return 0; + } + return bindings::snd_pcm_set_managed_buffer( + substream, + bindings::SNDRV_DMA_TYPE_VMALLOC as i32, + ptr::null_mut(), + 0, + 0, + ); + } + } // SAFETY: the platform device is live and the channel name is // NUL-terminated. let chan = match from_err_ptr(unsafe { @@ -2134,7 +2731,7 @@ unsafe extern "C" fn asoc_pcm_open( }; let mut hw = dmaengine_hw(); let mut formats = u64::MAX; - if fe == FE_SPKR { + if leap { // S32 in, converted by `asoc_pcm_copy` through the ceiling into the // float32 the LEAP consumes: no mmap, and a bounded queue. hw.info = bindings::SNDRV_PCM_INFO_INTERLEAVED; @@ -2172,11 +2769,16 @@ unsafe extern "C" fn asoc_pcm_close( if unsafe { rtd_is_be(rtd) } { return 0; } + // SAFETY: as for `rtd_is_be`. + let fe = unsafe { fe_index(rtd) }; + if fe == Some(FE_SENSE) && data.hw.sense_channels > 0 { + // No DMA of its own to close. + return 0; + } // SAFETY: the substream is live and its dmaengine runtime was set up by // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. let ret = unsafe { bindings::snd_dmaengine_pcm_close_release_chan(substream) }; - // SAFETY: as for `rtd_is_be`. - if unsafe { fe_index(rtd) } == Some(FE_SPKR) { + if fe == Some(FE_SPKR) { data.power_down(&data.spkr_domains()); } ret @@ -2187,13 +2789,22 @@ unsafe extern "C" fn asoc_pcm_close( /// Called by ASoC on this driver's registered component, with live /// arguments belonging to one of its links. unsafe extern "C" fn asoc_pcm_hw_params( - _component: *mut bindings::snd_soc_component, + component: *mut bindings::snd_soc_component, substream: *mut bindings::snd_pcm_substream, params: *mut bindings::snd_pcm_hw_params, ) -> i32 { // SAFETY: ASoC hands its ops a live substream whose private data is the // `snd_soc_pcm_runtime` of one of this component's links. - if unsafe { rtd_is_be(substream_rtd(substream)) } { + let rtd = unsafe { substream_rtd(substream) }; + // SAFETY: as above. + if unsafe { rtd_is_be(rtd) } { + return 0; + } + // SAFETY: the component is ours and live for the call; rtd as above. + if unsafe { component_data(component) }.hw.sense_channels > 0 + && unsafe { fe_index(rtd) } == Some(FE_SENSE) + { + // The sense capture has no DMA of its own. return 0; } // SAFETY: the substream is live and its dmaengine runtime was set up by @@ -2245,7 +2856,15 @@ unsafe extern "C" fn asoc_pcm_prepare( // `snd_soc_pcm_runtime` of one of this component's links. let rtd = unsafe { substream_rtd(substream) }; // SAFETY: as above. - if unsafe { rtd_is_be(rtd) } || unsafe { fe_index(rtd) } != Some(FE_SPKR) { + if unsafe { rtd_is_be(rtd) } { + return 0; + } + // SAFETY: as above. + let fe = unsafe { fe_index(rtd) }; + if fe == Some(FE_SENSE) { + return 0; + } + if fe != Some(FE_SPKR) { return 0; } // SAFETY: ALSA calls this op with a live substream; its runtime and DMA @@ -2328,6 +2947,64 @@ unsafe extern "C" fn asoc_pcm_copy( if !playback { // SAFETY: ALSA hands us a live iov_iter for the whole call. let iov = unsafe { kernel::iov::IovIterDest::from_raw(iter) }; + // SAFETY: the component is ours and live for the call. + let data = unsafe { component_data(component) }; + // SAFETY: rtd is live for the call. + if data.hw.sense_channels > 0 && unsafe { fe_index(rtd) } == Some(FE_SENSE) { + // J714c: the serializer samples the amplifiers' sense words one + // BCLK early (the TAS2764 puts SDOUT one clock after the frame + // edge, and offset 0 mutes the right-side amplifiers on this + // board), so each 16-bit word arrives as its own bits 14..0 + // followed by the next word's bit 15. Re-slice the stream one bit + // later: word k = (w[k] << 1) | (w[k+1] >> 15), the successor + // taken from the ring, which wraps. The last word of a read takes + // its bit 0 from the slot after the read, which the DMA may not + // have written yet: one LSB per read, below the sense noise. + let ring_base = data.sense_drain_cpu.load(core::sync::atomic::Ordering::Relaxed).cast::().cast_const(); + if bytes & 1 != 0 || ring_base.is_null() { + return EINVAL.to_errno(); + } + // The read's place in the ring: the capture's origin plus the + // frames the application has taken so far (`pos` is that count + // within its own buffer). + // SAFETY: the substream's runtime and control exist for the op. + let appl = unsafe { (*(*(*substream).runtime).control).appl_ptr } as u64; + let frame_bytes = u64::from(data.hw.sense_channels) * 2; + let ring_bytes = SENSE_DRAIN_BYTES; + let pos = ((data.sense_start_pos.load(core::sync::atomic::Ordering::Relaxed) + appl * frame_bytes) + .wrapping_sub(data.sense_arm_base.load(core::sync::atomic::Ordering::Relaxed)) + % ring_bytes as u64) as usize; + // SAFETY: the ring is SENSE_DRAIN_BYTES long for the driver's life. + let ring = unsafe { slice::from_raw_parts(ring_base, ring_bytes) }; + let word_at = |p: usize| -> u16 { + let p = p % ring_bytes; + u16::from_le_bytes([ring[p], ring[p + 1]]) + }; + let late = data.hw.sense_late_current; + let frame = frame_bytes as usize; + let mut chunk = [0u8; 512]; + let mut off = 0; + while off < bytes { + let n = core::cmp::min(chunk.len(), bytes - off) & !1; + for i in (0..n).step_by(2) { + let mut p = pos + off + i; + // A late pair's voltage is taken from the frame before. + let ch = (p % frame) / 2; + if ch & 1 == 1 && late & (1 << (ch / 2)) != 0 { + p += ring_bytes - frame; + } + let w = word_at(p); + let next = word_at(p + 2); + let v = (w << 1) | (next >> 15); + chunk[i..i + 2].copy_from_slice(&v.to_le_bytes()); + } + if iov.copy_to_iter(&chunk[..n]) != n { + return EFAULT.to_errno(); + } + off += n; + } + return 0; + } return if iov.copy_to_iter(dst) == bytes { 0 } else { @@ -2336,8 +3013,10 @@ unsafe extern "C" fn asoc_pcm_copy( } // SAFETY: as above. let iov = unsafe { kernel::iov::IovIterSource::from_raw(iter) }; - // SAFETY: as for `rtd_is_be`. - if unsafe { fe_index(rtd) } != Some(FE_SPKR) { + // SAFETY: the component is ours and live for the call. + let leap_spkr = unsafe { fe_index(rtd) } == Some(FE_SPKR) + && unsafe { component_data(component) }.hw.spkr_leap; + if !leap_spkr { return if iov.copy_from_iter(dst) == bytes { 0 } else { @@ -2375,15 +3054,38 @@ unsafe extern "C" fn asoc_pcm_copy( /// Called by ASoC on this driver's registered component, with live /// arguments belonging to one of its links. unsafe extern "C" fn asoc_pcm_trigger( - _component: *mut bindings::snd_soc_component, + component: *mut bindings::snd_soc_component, substream: *mut bindings::snd_pcm_substream, cmd: i32, ) -> i32 { // SAFETY: ASoC hands its ops a live substream whose private data is the // `snd_soc_pcm_runtime` of one of this component's links. - if unsafe { rtd_is_be(substream_rtd(substream)) } { + let rtd = unsafe { substream_rtd(substream) }; + // SAFETY: as above. + if unsafe { rtd_is_be(rtd) } { return 0; } + // SAFETY: the component is ours and live for the call; rtd as above. + let data = unsafe { component_data(component) }; + if data.hw.sense_channels > 0 && unsafe { fe_index(rtd) } == Some(FE_SENSE) { + // The capture reads the drain's ring from here on, or stops. + return match cmd as u32 { + bindings::SNDRV_PCM_TRIGGER_START + | bindings::SNDRV_PCM_TRIGGER_RESUME + | bindings::SNDRV_PCM_TRIGGER_PAUSE_RELEASE => { + data.sense_start_pos.store(data.sense_ring_pos.load(core::sync::atomic::Ordering::Relaxed), core::sync::atomic::Ordering::Relaxed); + data.sense_sub.store(substream, core::sync::atomic::Ordering::Relaxed); + 0 + } + bindings::SNDRV_PCM_TRIGGER_STOP + | bindings::SNDRV_PCM_TRIGGER_SUSPEND + | bindings::SNDRV_PCM_TRIGGER_PAUSE_PUSH => { + data.sense_sub.store(ptr::null_mut(), core::sync::atomic::Ordering::Relaxed); + 0 + } + _ => EINVAL.to_errno(), + }; + } // SAFETY: the substream is live and its dmaengine runtime was set up by // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. unsafe { bindings::snd_dmaengine_pcm_trigger(substream, cmd) } @@ -2398,12 +3100,21 @@ unsafe extern "C" fn asoc_pcm_trigger( /// Called by ASoC on this driver's registered component, with live /// arguments belonging to one of its links. unsafe extern "C" fn asoc_pcm_sync_stop( - _component: *mut bindings::snd_soc_component, + component: *mut bindings::snd_soc_component, substream: *mut bindings::snd_pcm_substream, ) -> i32 { // SAFETY: ASoC hands its ops a live substream whose private data is the // `snd_soc_pcm_runtime` of one of this component's links. - if unsafe { rtd_is_be(substream_rtd(substream)) } { + let rtd = unsafe { substream_rtd(substream) }; + // SAFETY: as above. + if unsafe { rtd_is_be(rtd) } { + return 0; + } + // SAFETY: the component is ours and live for the call; rtd as above. + if unsafe { component_data(component) }.hw.sense_channels > 0 + && unsafe { fe_index(rtd) } == Some(FE_SENSE) + { + // No DMA of its own to synchronize. return 0; } // SAFETY: the substream is live and its dmaengine runtime was set up by @@ -2416,15 +3127,31 @@ unsafe extern "C" fn asoc_pcm_sync_stop( /// Called by ASoC on this driver's registered component, with live /// arguments belonging to one of its links. unsafe extern "C" fn asoc_pcm_pointer( - _component: *mut bindings::snd_soc_component, + component: *mut bindings::snd_soc_component, substream: *mut bindings::snd_pcm_substream, ) -> bindings::snd_pcm_uframes_t { // SAFETY: ASoC hands its ops a live substream whose private data is the // `snd_soc_pcm_runtime` of one of this component's links. - if unsafe { rtd_is_be(substream_rtd(substream)) } { + let rtd = unsafe { substream_rtd(substream) }; + // SAFETY: as above. + if unsafe { rtd_is_be(rtd) } { // -ENOTSUPP as the pointer callback returns it (mca does the same). return (-i64::from(bindings::ENOTSUPP)) as bindings::snd_pcm_uframes_t; } + // SAFETY: the component is ours and live for the call; rtd as above. + let data = unsafe { component_data(component) }; + if data.hw.sense_channels > 0 && unsafe { fe_index(rtd) } == Some(FE_SENSE) { + // Frames the drain has produced since the capture's start, in the + // capture's buffer. + let produced = data.sense_ring_pos.load(core::sync::atomic::Ordering::Relaxed).wrapping_sub(data.sense_start_pos.load(core::sync::atomic::Ordering::Relaxed)); + let frame_bytes = u64::from(data.hw.sense_channels) * 2; + // SAFETY: the substream's runtime exists for the op's duration. + let buffer = unsafe { (*(*substream).runtime).buffer_size } as u64; + if buffer == 0 { + return 0; + } + return ((produced / frame_bytes) % buffer) as bindings::snd_pcm_uframes_t; + } // SAFETY: the substream is live and its dmaengine runtime was set up by // `snd_dmaengine_pcm_open` in `asoc_pcm_open`. unsafe { bindings::snd_dmaengine_pcm_pointer(substream) } @@ -2584,6 +3311,9 @@ unsafe extern "C" fn spkr_dai_startup( // SAFETY: the substream is live for the duration of the op. let ret = if unsafe { substream_is_playback(substream) } { data.spkr_startup() + } else if data.hw.sense_channels > 0 { + // The V/I sense rides the speaker service's own RX channel. + Ok(()) } else { data.sense_startup() }; @@ -2609,6 +3339,8 @@ unsafe extern "C" fn spkr_dai_prepare( // SAFETY: the substream is live for the duration of the op. let ret = if unsafe { substream_is_playback(substream) } { data.spkr_prepare() + } else if data.hw.sense_channels > 0 { + Ok(()) } else { data.sense_prepare() }; @@ -2618,8 +3350,8 @@ unsafe extern "C" fn spkr_dai_prepare( } } -/// Atomic context: queue the sleeping start, or note the stop (playback -/// only; the tap side needs nothing here). +/// Atomic context: queue the sleeping start or stop of the speaker service +/// with the playback trigger. The capture side needs nothing here. /// /// # Safety /// @@ -2630,25 +3362,28 @@ unsafe extern "C" fn spkr_dai_trigger( cmd: i32, dai: *mut bindings::snd_soc_dai, ) -> i32 { + // SAFETY: ASoC calls DAI ops with a DAI of our registered component. + let context = unsafe { dai_context(dai) }; // SAFETY: the substream is live for the duration of the op. if !unsafe { substream_is_playback(substream) } { + // The capture's DMA is ALSA's; the service follows playback. return 0; } - // SAFETY: ASoC calls DAI ops with a DAI of our registered component. - let context = unsafe { dai_context(dai) }; + let flag = &context.data.spkr_want_run; match cmd as u32 { bindings::SNDRV_PCM_TRIGGER_START | bindings::SNDRV_PCM_TRIGGER_RESUME | bindings::SNDRV_PCM_TRIGGER_PAUSE_RELEASE => { - context.data.spkr_want_run.store(true, Relaxed); - // Err = already queued: the pending run will see want_run. + flag.store(true, Relaxed); + // Err = already queued: the pending run will see the flag. let _ = workqueue::system_dfl().enqueue::<_, SPKR_START_WORK>(context.data.clone()); 0 } bindings::SNDRV_PCM_TRIGGER_STOP | bindings::SNDRV_PCM_TRIGGER_SUSPEND | bindings::SNDRV_PCM_TRIGGER_PAUSE_PUSH => { - context.data.spkr_want_run.store(false, Relaxed); + flag.store(false, Relaxed); + let _ = workqueue::system_dfl().enqueue::<_, SPKR_START_WORK>(context.data.clone()); 0 } _ => EINVAL.to_errno(), @@ -2668,7 +3403,7 @@ unsafe extern "C" fn spkr_dai_shutdown( // SAFETY: the substream is live for the duration of the op. if unsafe { substream_is_playback(substream) } { data.spkr_shutdown(); - } else { + } else if data.hw.sense_channels == 0 { data.sense_shutdown(); } } @@ -2729,6 +3464,8 @@ fn pcm_stream( /// of each 32-bit DMA word: S24_LE. const OUT_FORMATS: u64 = bindings::BINDINGS_SNDRV_PCM_FMTBIT_S24_LE; const S32_FORMATS: u64 = bindings::BINDINGS_SNDRV_PCM_FMTBIT_S32_LE; +/// The V/I sense return: 16-bit slots. +const SENSE_FORMATS: u64 = bindings::BINDINGS_SNDRV_PCM_FMTBIT_S16_LE; /// The DAI table: [0] the cout back-end, [1..=3] the front-end PCMs, [4] the /// speaker back-end. The order is ABI: `sound-dai = <&aop_audio N>` in the @@ -2737,7 +3474,17 @@ const S32_FORMATS: u64 = bindings::BINDINGS_SNDRV_PCM_FMTBIT_S32_LE; /// renumbers every DAI after it and silently repoints the existing links. const DAI_COUNT: usize = 2 + FE_COUNT; -fn dai_drivers(ops: &DaiOps) -> [bindings::snd_soc_dai_driver; DAI_COUNT] { +/// The speaker back-end's capture: the SpeakerTap words (T8140) or the +/// amplifiers' V/I sense in 16-bit slots (T6050). +fn sense_stream(name: &'static core::ffi::CStr, hw: &HwConfig) -> bindings::snd_soc_pcm_stream { + if hw.sense_channels > 0 { + pcm_stream(name, SENSE_FORMATS, hw.sense_channels) + } else { + pcm_stream(name, S32_FORMATS, 2) + } +} + +fn dai_drivers(ops: &DaiOps, hw: &HwConfig) -> [bindings::snd_soc_dai_driver; DAI_COUNT] { [ bindings::snd_soc_dai_driver { name: c"cout".as_char_ptr(), @@ -2749,7 +3496,7 @@ fn dai_drivers(ops: &DaiOps) -> [bindings::snd_soc_dai_driver; DAI_COUNT] { ..Default::default() }, bindings::snd_soc_dai_driver { - name: c"j700-pcm-0".as_char_ptr(), + name: hw.fe_dai_names[0].as_char_ptr(), id: FE_DAI_ID_BASE, ops: &ops.fe, playback: pcm_stream(c"PCM0 TX", OUT_FORMATS, 2), @@ -2758,19 +3505,23 @@ fn dai_drivers(ops: &DaiOps) -> [bindings::snd_soc_dai_driver; DAI_COUNT] { ..Default::default() }, bindings::snd_soc_dai_driver { - name: c"j700-pcm-1".as_char_ptr(), + name: hw.fe_dai_names[1].as_char_ptr(), id: FE_DAI_ID_BASE + 1, ops: &ops.fe, // the speaker front-end: S32 in, converted through the ceiling - playback: pcm_stream(c"PCM1 TX", S32_FORMATS, 2), + playback: pcm_stream( + c"PCM1 TX", + if hw.spkr_leap { S32_FORMATS } else { OUT_FORMATS }, + hw.spkr_channels, + ), ..Default::default() }, bindings::snd_soc_dai_driver { - name: c"j700-pcm-2".as_char_ptr(), + name: hw.fe_dai_names[2].as_char_ptr(), id: FE_DAI_ID_BASE + 2, ops: &ops.fe, // the speaker tap front-end: the words on the speaker wire - capture: pcm_stream(c"PCM2 RX", S32_FORMATS, 2), + capture: sense_stream(c"PCM2 RX", hw), ..Default::default() }, // The internal speaker back-end: LEAP TX0 under the AOP spkr service, @@ -2780,8 +3531,12 @@ fn dai_drivers(ops: &DaiOps) -> [bindings::snd_soc_dai_driver; DAI_COUNT] { name: c"spkr".as_char_ptr(), id: BE_DAI_ID_SPKR, ops: &ops.spkr, - playback: pcm_stream(c"SPKR TX", S32_FORMATS, 2), - capture: pcm_stream(c"SPKR RX", S32_FORMATS, 2), + playback: pcm_stream( + c"SPKR TX", + if hw.spkr_leap { S32_FORMATS } else { OUT_FORMATS }, + hw.spkr_channels, + ), + capture: sense_stream(c"SPKR RX", hw), ..Default::default() }, ] @@ -2806,7 +3561,7 @@ const _: () = assert!(mem::offset_of!(AsocContext, component) == 0); impl AsocContext { fn new(data: Arc) -> Result> { let ops = KBox::new(DaiOps::new(), GFP_KERNEL)?; - let dais = KBox::new(Opaque::new(dai_drivers(&ops)), GFP_KERNEL)?; + let dais = KBox::new(Opaque::new(dai_drivers(&ops, data.hw)), GFP_KERNEL)?; let volume = KBox::new( Opaque::new(bindings::soc_mixer_control { min: 0, @@ -2849,7 +3604,11 @@ impl AsocContext { trigger: Some(asoc_pcm_trigger), sync_stop: Some(asoc_pcm_sync_stop), pointer: Some(asoc_pcm_pointer), - copy: Some(asoc_pcm_copy), + copy: if data.hw.spkr_leap || data.hw.sense_channels > 0 { + Some(asoc_pcm_copy) + } else { + None + }, ..Default::default() }; Ok(KBox::new( @@ -2931,14 +3690,19 @@ impl SndSocT8140AopDriver { closed: AtomicFlag::new(false), }; - // The low-power microphone: its ring is reported to us by the - // firmware for the AOP's lifetime. - data.attach_device(AUDIO_DEV_LPAI, "lpai")?; - data.set_channel_control()?; - data.read_geometry()?; - let listener: Arc = data.clone(); - data.adata - .add_report_listener(data.service, EPIC_SUBTYPE_PRODUCER_REPORT, listener)?; + // Without a microphone of the AOP's own there is no card of our + // own: the back-ends and front-ends below belong to the machine card. + if data.hw.lpai || data.hw.hpai_channels > 0 { + if data.hw.lpai { + // The low-power microphone: its ring is reported to us by the + // firmware for the AOP's lifetime. + data.attach_device(AUDIO_DEV_LPAI, "lpai")?; + data.set_channel_control()?; + data.read_geometry()?; + let listener: Arc = data.clone(); + data.adata + .add_report_listener(data.service, EPIC_SUBTYPE_PRODUCER_REPORT, listener)?; + } // SAFETY: FFI call with the live device and the module; `this.card` receives the new card. let ret = unsafe { @@ -2980,8 +3744,13 @@ impl SndSocT8140AopDriver { return Err(e); } - this.new_pcm(0, c"lpai", c"Low-Power Microphone", &Self::LPAI_OPS)?; - this.new_pcm(1, c"hpai", c"High-Quality Microphone", &Self::HPAI_OPS)?; + if data.hw.lpai { + this.new_pcm(0, c"lpai", c"Low-Power Microphone", &Self::LPAI_OPS)?; + } + if data.hw.hpai_channels > 0 { + let dev = if data.hw.lpai { 1 } else { 0 }; + this.new_pcm(dev, c"hpai", c"High-Quality Microphone", &Self::HPAI_OPS)?; + } // Both microphones' jack: unplugged while a lid is closed. let mut jack = ptr::null_mut(); @@ -3016,6 +3785,8 @@ impl SndSocT8140AopDriver { return Err(Error::from_errno(ret)); } + } + // The ASoC side: the jack (cout) and speaker back-ends, the // front-end PCMs and the speaker volume control. // SAFETY: the descriptor, the DAI table and the controls are heap @@ -3093,7 +3864,7 @@ impl SndSocT8140AopDriver { } // SAFETY: as above. copy_str(unsafe { &mut (*pcm).name }, name.to_bytes_with_nul())?; - if device == 0 { + if ptr::eq(ops, &Self::LPAI_OPS) { // The low-power microphone's buffer is filled by the driver from // the source ring; the array's is set up on its DMA device at // open. @@ -3147,6 +3918,26 @@ impl SndSocT8140AopDriver { ) }; } + { + let _guard = self.data.spkr_lock.lock(); + self.data.sense_drain_disarm(); + } + let chan = self.data.sense_chan.swap(ptr::null_mut(), core::sync::atomic::Ordering::Relaxed); + if !chan.is_null() { + // SAFETY: requested and the buffer allocated in `sense_chan`; no + // stream uses either now. + unsafe { + let dev = (*(*chan).device).dev; + bindings::dma_release_channel(chan); + bindings::dma_free_attrs( + dev, + SENSE_DRAIN_BYTES, + self.data.sense_drain_cpu.load(core::sync::atomic::Ordering::Relaxed), + self.data.sense_drain_dma.load(core::sync::atomic::Ordering::Relaxed), + 0, + ); + } + } self.data.flush_spkr_start(); self.data.flush_hpai_start(); if !self.card.is_null() { @@ -3184,8 +3975,12 @@ unsafe extern "C" fn hpai_pcm_hw_params( if rc < 0 { return rc; } - // One stereo frame per burst. - cfg.src_port_window_size = HPAI_CHANNELS; + // One frame per burst where the ADMAC can signal it: its frame-size + // field encodes 1, 2 or 4 words. A three-microphone frame (J714c) has + // no encoding; its words go unframed. + // SAFETY: ALSA calls this op with a live substream of the HPAI PCM. + let channels = unsafe { pcm_data(substream) }.hw.hpai_channels; + cfg.src_port_window_size = if matches!(channels, 1 | 2 | 4) { channels } else { 1 }; // SAFETY: `chan` is the live channel behind this substream's dmaengine runtime. unsafe { match (*(*chan).device).device_config { @@ -3195,23 +3990,109 @@ unsafe extern "C" fn hpai_pcm_hw_params( } } +/// What the SoC leaves to the AP around the AOP's audio services. +struct HwConfig { + /// The low-power microphone: its ring is bound through the setup port + /// and read here (T8140). On T6050 it is exclave-owned: no ring, no + /// card of our own, and no raising the fabric through it. + lpai: bool, + /// The serializer leaves are AP PMGR domains named on the audio node + /// (T8140). On T6050 they are the firmware's (no_ps): nothing to power, + /// and each stream's service is attached before its DMA channel. + ap_power_domains: bool, + /// The speaker back-end feeds the LEAP: float32 through the software + /// ceiling for SD_MODE amplifiers (T8140). Otherwise it is an MCA + /// serializer stream to TDM amplifiers, like the jack's. + spkr_leap: bool, + /// Speaker playback channels. + spkr_channels: u32, + /// The speaker service has an output mute (property 700) the host + /// toggles around pwrd (T8140). T6050's rejects it as not found + /// (0xe00002f0): the stream runs unmuted from pwrd and the amplifiers' + /// own shutdown does the muting. + spkr_output_mute: bool, + /// The speaker back-end's capture: the tap service on its own channel + /// (0, T8140) or the speaker service's own V/I sense return with this + /// many 16-bit channels (T6050). + sense_channels: u32, + /// Amplifier pairs (bit n: sense channels 2n current, 2n+1 voltage) whose + /// current word arrives one frame after the voltage it belongs to; the + /// copy delays those voltages by a frame so each V/I pair is one instant. + sense_late_current: u32, + /// The microphone array (hpai, LEAP-S ADMAC RX0): its channel count, + /// float32 at 48 kHz; 0 without one. + hpai_channels: u32, + /// The array's PDM interface is the host's to configure (pdm0 property + /// 200 and 212, lpai channel masks), as macOS does on the T604x family + /// and J714c; T8140's firmware brings its own. + hpai_pdm: bool, + /// Front-end DAI names, the machine driver's `fe_dai_names`. + fe_dai_names: [&'static core::ffi::CStr; FE_COUNT], + /// DMA names per front-end and stream, [playback, capture]. + fe_dma_names: [[Option<&'static core::ffi::CStr>; 2]; FE_COUNT], +} + +const HW_T8140: HwConfig = HwConfig { + lpai: true, + ap_power_domains: true, + spkr_leap: true, + spkr_channels: 2, + spkr_output_mute: true, + sense_channels: 0, + sense_late_current: 0, + hpai_channels: 2, + hpai_pdm: false, + fe_dai_names: [c"j700-pcm-0", c"j700-pcm-1", c"j700-pcm-2"], + fe_dma_names: FE_DMA_NAMES, +}; + +/// T6050 (J714c): six SN012776 amplifiers on base-ns TX0 with their V/I +/// sense (12 x 16 bit) back on base-ns RX0 under the same spkr service, the +/// jack on base-ns TX2/RX2 as on T8140. The audio power domains are the +/// firmware's and the low-power microphone is exclave-owned. +const HW_T6050: HwConfig = HwConfig { + lpai: false, + ap_power_domains: false, + spkr_leap: false, + spkr_channels: 6, + spkr_output_mute: false, + sense_channels: 12, + // J714c: Left Woofer 1 (slot 0) reads its current one sample behind its + // voltage; the other five pairs are aligned (phase of V/I to 12 kHz). + sense_late_current: 1 << 0, + // ADT audio-hp-mic-proxy physical-descriptions: lpcm, float, 12 bytes + // per frame at 48 kHz, three microphones. + hpai_channels: 3, + hpai_pdm: true, + fe_dai_names: [c"j714-pcm-0", c"j714-pcm-1", c"j714-pcm-2"], + fe_dma_names: [ + [Some(c"cout"), Some(c"cin")], + [Some(c"spkr"), None], + [None, Some(c"spkr-sense")], + ], +}; + kernel::of_device_table!( OF_TABLE, MODULE_OF_TABLE, ::IdInfo, - [(of::DeviceId::new(c_str!("apple,t8140-aop-audio")), ())] + [ + (of::DeviceId::new(c_str!("apple,t8140-aop-audio")), &HW_T8140), + (of::DeviceId::new(c_str!("apple,t6050-aop-audio")), &HW_T6050), + ] ); impl platform::Driver for SndSocT8140AopDriver { - type IdInfo = (); + type IdInfo = &'static HwConfig; type Data<'bound> = SndSocT8140AopDriver; const OF_ID_TABLE: Option> = Some(&OF_TABLE); fn probe<'bound>( pdev: &'bound platform::Device>, - _info: Option<&'bound Self::IdInfo>, + info: Option<&'bound Self::IdInfo>, ) -> impl PinInit, Error> + 'bound { + let hw: &'static HwConfig = *info.ok_or(ENODEV)?; let dev = ARef::::from(pdev.as_ref()); // SAFETY: this is a service device the AOP core registered, being // probed here; the lookup defers the probe until the core has @@ -3230,11 +4111,16 @@ impl platform::Driver for SndSocT8140AopDriver { // owned by the AOP driver because the firmware binds it once per boot. // SAFETY: no DMA is in flight on this device yet. unsafe { pdev.dma_set_mask_and_coherent(DmaMask::new::<42>())? }; - let ring = adata.source_ring(pdev.as_ref(), RING_BYTES)?; + let ring = if hw.lpai { + Some(adata.source_ring(pdev.as_ref(), RING_BYTES)?) + } else { + None + }; // The serializer leaves. A missing one only disables the paths that // need it; those fail to open with ENODEV. let mut pds: [Option; 5] = [None, None, None, None, None]; + if hw.ap_power_domains { for (slot, (name, dt_name)) in pds.iter_mut().zip([ ("mca1", c"mca1"), ("mca0", c"mca0"), @@ -3248,6 +4134,7 @@ impl platform::Driver for SndSocT8140AopDriver { Err(e) => dev_warn!(dev, "{} power domain attach failed: {:?}\n", name, e), } } + } let [pd_mca1, pd_mca0, pd_tx0, pd_leap_mca, pd_rx0] = pds; let fwnode = pdev.as_ref().fwnode().ok_or(ENOENT)?; @@ -3257,6 +4144,7 @@ impl platform::Driver for SndSocT8140AopDriver { adata, service: svc, ring, + hw, ring_bytes: Atomic::new(0), sequence: Atomic::new(0), powered: AtomicFlag::new(false), @@ -3276,6 +4164,15 @@ impl platform::Driver for SndSocT8140AopDriver { pd_rx0, spkr_want_run: AtomicFlag::new(false), hpai_want_run: AtomicFlag::new(false), + sense_ring_pos: core::sync::atomic::AtomicU64::new(0), + sense_arm_base: core::sync::atomic::AtomicU64::new(0), + sense_start_pos: core::sync::atomic::AtomicU64::new(0), + sense_sub: core::sync::atomic::AtomicPtr::new(ptr::null_mut()), + sense_chan: core::sync::atomic::AtomicPtr::new(ptr::null_mut()), + sense_drain_cpu: core::sync::atomic::AtomicPtr::new(ptr::null_mut()), + sense_drain_dma: core::sync::atomic::AtomicU64::new(0), + sense_drain_running: AtomicFlag::new(false), + spkr_lock <- new_mutex!(()), spkr_volume: Atomic::new(SPKR_VOLUME_MAX), spkr_start_work <- new_work!("SndSocT8140AopData::spkr_start_work"), hpai_start_work <- new_work!("SndSocT8140AopData::hpai_start_work"), diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig index e4d139108d720..fbd8b94453bfe 100644 --- a/sound/soc/codecs/Kconfig +++ b/sound/soc/codecs/Kconfig @@ -990,6 +990,21 @@ config SND_SOC_CS42L42_SDW help Enable support for Cirrus Logic CS42L42 codec with Soundwire control +config SND_SOC_CS42L42_TEST + bool "KUnit test for Cirrus Logic cs42l42 driver" if !KUNIT_ALL_TESTS + depends on SND_SOC_CS42L42_CORE=y && SND_SOC=y && KUNIT=y + default KUNIT_ALL_TESTS + help + This builds KUnit tests for the clock switching that the + Cirrus Logic cs42l42 codec driver does when a stream is + muted or unmuted, including PCM resume refusal and prepare recovery. + These tests use built-in ALSA/ASoC entry points. + + For more information on KUnit and unit tests in general, + please refer to the KUnit documentation in + Documentation/dev-tools/kunit/. + If in doubt, say "N". + config SND_SOC_CS42L43 tristate "Cirrus Logic CS42L43 CODEC" depends on MFD_CS42L43 @@ -1037,6 +1052,21 @@ config SND_SOC_CS42L84 select REGMAP select REGMAP_I2C +config SND_SOC_CS42L84_TEST + bool "KUnit test for Cirrus Logic cs42l84 driver" if !KUNIT_ALL_TESTS + depends on SND_SOC_CS42L84 && (KUNIT=y || KUNIT=SND_SOC_CS42L84) + default KUNIT_ALL_TESTS + help + This builds KUnit tests for the clock switching that the + Cirrus Logic cs42l84 codec driver does when a stream is + muted or unmuted. The tests are built into the cs42l84 + driver and run when it is loaded. + + For more information on KUnit and unit tests in general, + please refer to the KUnit documentation in + Documentation/dev-tools/kunit/. + If in doubt, say "N". + config SND_SOC_CS4234 tristate "Cirrus Logic CS4234 CODEC" depends on I2C diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile index aa0396e5b575b..1ed18ec77cd15 100644 --- a/sound/soc/codecs/Makefile +++ b/sound/soc/codecs/Makefile @@ -86,6 +86,7 @@ snd-soc-cs40l50-y := cs40l50-codec.o snd-soc-cs42l42-y := cs42l42.o snd-soc-cs42l42-i2c-y := cs42l42-i2c.o snd-soc-cs42l42-sdw-y := cs42l42-sdw.o +snd-soc-cs42l42-test-y := cs42l42-test.o snd-soc-cs42l43-y := cs42l43.o cs42l43-jack.o snd-soc-cs42l43-sdw-y := cs42l43-sdw.o snd-soc-cs42l51-y := cs42l51.o @@ -527,6 +528,7 @@ obj-$(CONFIG_SND_SOC_CS40L50) += snd-soc-cs40l50.o obj-$(CONFIG_SND_SOC_CS42L42_CORE) += snd-soc-cs42l42.o obj-$(CONFIG_SND_SOC_CS42L42) += snd-soc-cs42l42-i2c.o obj-$(CONFIG_SND_SOC_CS42L42_SDW) += snd-soc-cs42l42-sdw.o +obj-$(CONFIG_SND_SOC_CS42L42_TEST) += snd-soc-cs42l42-test.o obj-$(CONFIG_SND_SOC_CS42L43) += snd-soc-cs42l43.o obj-$(CONFIG_SND_SOC_CS42L43_SDW) += snd-soc-cs42l43-sdw.o obj-$(CONFIG_SND_SOC_CS42L51) += snd-soc-cs42l51.o diff --git a/sound/soc/codecs/adau1372.c b/sound/soc/codecs/adau1372.c index cc174ec3a1f82..c3d3be898ba6c 100644 --- a/sound/soc/codecs/adau1372.c +++ b/sound/soc/codecs/adau1372.c @@ -798,7 +798,7 @@ static int adau1372_set_power(struct adau1372 *adau1372, bool enable) if (ret) return ret; if (adau1372->pd_gpio) - gpiod_set_value(adau1372->pd_gpio, 0); + gpiod_set_value_cansleep(adau1372->pd_gpio, 0); if (adau1372->switch_mode) adau1372->switch_mode(adau1372->dev); @@ -818,8 +818,7 @@ static int adau1372_set_power(struct adau1372 *adau1372, bool enable) ADAU1372_CLK_CTRL_PLL_EN, 0); regcache_cache_only(adau1372->regmap, true); - if (adau1372->pd_gpio) - gpiod_set_value(adau1372->pd_gpio, 1); + gpiod_set_value_cansleep(adau1372->pd_gpio, 1); clk_disable_unprepare(adau1372->mclk); return ret; } @@ -836,7 +835,7 @@ static int adau1372_set_power(struct adau1372 *adau1372, bool enable) * map. No need to do any register writes to manually * turn things off. */ - gpiod_set_value(adau1372->pd_gpio, 1); + gpiod_set_value_cansleep(adau1372->pd_gpio, 1); regcache_mark_dirty(adau1372->regmap); } else { regmap_update_bits(adau1372->regmap, ADAU1372_REG_CLK_CTRL, @@ -1003,6 +1002,17 @@ int adau1372_probe(struct device *dev, struct regmap *regmap, regmap_write(regmap, ADAU1372_REG_OP_STAGE_MUTE, 0x0); + /* + * Set sane default values for the Output ASRC and DAC input muxes, + * since the power-on reset defaults are invalid "Reserved" states. + */ + regmap_write(regmap, ADAU1372_REG_ASRCO_SOURCE_0_1, + 0x54); /* Decimator0, Decimator1 */ + regmap_write(regmap, ADAU1372_REG_ASRCO_SOURCE_2_3, + 0x76); /* Decimator2, Decimator3 */ + regmap_write(regmap, ADAU1372_REG_DAC_SOURCE, + 0xdc); /* Input ASRC0, Input ASRC1 */ + regmap_write(regmap, 0x7, 0x01); /* CLOCK OUT */ return devm_snd_soc_register_component(dev, &adau1372_driver, &adau1372_dai_driver, 1); diff --git a/sound/soc/codecs/cs42l42-sdw.c b/sound/soc/codecs/cs42l42-sdw.c index ad1256910a180..d0621c56581d2 100644 --- a/sound/soc/codecs/cs42l42-sdw.c +++ b/sound/soc/codecs/cs42l42-sdw.c @@ -98,6 +98,7 @@ static int cs42l42_sdw_dai_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(dai->component); + int ret; dev_dbg(dai->dev, "dai_prepare: sclk=%u rate=%u\n", cs42l42->sclk, cs42l42->sample_rate); @@ -110,7 +111,11 @@ static int cs42l42_sdw_dai_prepare(struct snd_pcm_substream *substream, * an unsupported SWIRE_CLK and sample_rate combination. */ - return cs42l42_pll_config(dai->component, cs42l42->sclk, cs42l42->sample_rate); + ret = cs42l42_pll_config(dai->component, cs42l42->sclk, cs42l42->sample_rate); + if (ret) + return ret; + + return cs42l42_dai_prepare(substream, dai); } static int cs42l42_sdw_dai_hw_free(struct snd_pcm_substream *substream, diff --git a/sound/soc/codecs/cs42l42-test.c b/sound/soc/codecs/cs42l42-test.c new file mode 100644 index 0000000000000..4ce8219c29336 --- /dev/null +++ b/sound/soc/codecs/cs42l42-test.c @@ -0,0 +1,886 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// KUnit tests for the clock switching in cs42l42_mute_stream(). +// +// The codec runs from its internal oscillator while no stream is active. +// Unmuting the first stream moves it onto SCLK, through the PLL if the +// selected clock configuration needs one. Once the codec has left the +// oscillator it needs a working clock to answer on I2C, so these tests +// check when that switch happens and when it must not happen. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cs42l42.h" + +#define CS42L42_TEST_MAX_WRITES 64 + +#define CS42L42_TEST_HP_MUTE_MASK (CS42L42_HP_ANA_AMUTE_MASK | \ + CS42L42_HP_ANA_BMUTE_MASK) + +/* PLL_LOCK_STATUS bit polled by cs42l42_mute_stream() */ +#define CS42L42_TEST_PLL_LOCKED BIT(0) + +struct cs42l42_test_write { + unsigned int reg; + unsigned int val; +}; + +struct cs42l42_test_priv { + struct kunit *test; + + struct cs42l42_private cs42l42; + struct snd_soc_component component; + struct snd_soc_dai dai; + struct snd_soc_card soc_card; + struct snd_card card; + + /* Register contents seen by the driver */ + u8 *regs; + + /* Every register write made by the driver, in order */ + struct cs42l42_test_write writes[CS42L42_TEST_MAX_WRITES]; + unsigned int num_writes; + + unsigned int num_lock_reads; + bool pll_locks; +}; + +struct cs42l42_test_clk_param { + unsigned int sclk; + unsigned int rate; + bool pll_divout_step; +}; + +struct cs42l42_test_power_param { + unsigned int state; + const char *name; +}; + +/* Clock configurations that run the codec from the PLL */ +static const struct cs42l42_test_clk_param cs42l42_test_pll_clks[] = { + { .sclk = 1536000, .rate = 48000, .pll_divout_step = true }, + { .sclk = 3072000, .rate = 48000, .pll_divout_step = false }, +}; + +static const struct cs42l42_test_power_param cs42l42_test_low_power[] = { + { .state = SNDRV_CTL_POWER_D1, .name = "D1" }, + { .state = SNDRV_CTL_POWER_D2, .name = "D2" }, + { .state = SNDRV_CTL_POWER_D3hot, .name = "D3hot" }, + { .state = SNDRV_CTL_POWER_D3cold, .name = "D3cold" }, +}; + +static int cs42l42_test_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + struct cs42l42_test_priv *priv = context; + + if (reg > CS42L42_MAX_REGISTER) + return -EINVAL; + + if (reg == CS42L42_PLL_LOCK_STATUS) { + priv->num_lock_reads++; + *val = priv->pll_locks ? CS42L42_TEST_PLL_LOCKED : 0; + return 0; + } + + *val = priv->regs[reg]; + + return 0; +} + +static int cs42l42_test_reg_write(void *context, unsigned int reg, unsigned int val) +{ + struct cs42l42_test_priv *priv = context; + + if (reg > CS42L42_MAX_REGISTER) + return -EINVAL; + + if (priv->num_writes < ARRAY_SIZE(priv->writes)) { + priv->writes[priv->num_writes].reg = reg; + priv->writes[priv->num_writes].val = val; + priv->num_writes++; + } else { + KUNIT_FAIL(priv->test, "write log full at reg %#x", reg); + } + + priv->regs[reg] = val; + + return 0; +} + +static const struct regmap_config cs42l42_test_regmap_config = { + .reg_bits = 16, + .val_bits = 8, + .max_register = CS42L42_MAX_REGISTER, + .reg_read = cs42l42_test_reg_read, + .reg_write = cs42l42_test_reg_write, + .cache_type = REGCACHE_NONE, +}; + +static void cs42l42_test_clear_log(struct cs42l42_test_priv *priv) +{ + priv->num_writes = 0; + priv->num_lock_reads = 0; +} + +/* Index of the first write to @reg at or after index @from, or -1 */ +static int cs42l42_test_find_write(struct cs42l42_test_priv *priv, + unsigned int reg, int from) +{ + unsigned int i; + + for (i = max(from, 0); i < priv->num_writes; i++) { + if (priv->writes[i].reg == reg) + return i; + } + + return -1; +} + +/* Index of the last write to @reg, or -1 */ +static int cs42l42_test_last_write(struct cs42l42_test_priv *priv, unsigned int reg) +{ + unsigned int i; + + for (i = priv->num_writes; i > 0; i--) { + if (priv->writes[i - 1].reg == reg) + return i - 1; + } + + return -1; +} + +static unsigned int cs42l42_test_count_writes(struct cs42l42_test_priv *priv, + unsigned int reg) +{ + unsigned int i, n = 0; + + for (i = 0; i < priv->num_writes; i++) { + if (priv->writes[i].reg == reg) + n++; + } + + return n; +} + +/* Select the clock configuration the way hw_params does */ +static void cs42l42_test_set_clk(struct kunit *test, unsigned int sclk, + unsigned int rate) +{ + struct cs42l42_test_priv *priv = test->priv; + + KUNIT_ASSERT_EQ(test, cs42l42_pll_config(&priv->component, sclk, rate), 0); + cs42l42_test_clear_log(priv); +} + +static void cs42l42_test_set_power(struct cs42l42_test_priv *priv, unsigned int state) +{ + snd_power_change_state(priv->soc_card.snd_card, state); +} + +/* + * Check that the writes logged since the last clear are a complete switch + * from the oscillator to SCLK through the PLL, with the headphone unmuted + * last. The PLL output divider is only rewritten if it still holds the + * startup value from cs42l42_pll_config(), so @check_divout must be false + * when an earlier unmute has already set the final value. + */ +static void cs42l42_test_expect_switch_to_pll(struct kunit *test, + const struct cs42l42_test_clk_param *param, + bool check_divout) +{ + struct cs42l42_test_priv *priv = test->priv; + int pll_start, divout, mclk_sel, sclk, hp; + + pll_start = cs42l42_test_find_write(priv, CS42L42_PLL_CTL1, 0); + KUNIT_ASSERT_GE(test, pll_start, 0); + KUNIT_EXPECT_TRUE(test, priv->writes[pll_start].val & CS42L42_PLL_START_MASK); + + divout = cs42l42_test_find_write(priv, CS42L42_PLL_CTL3, pll_start); + if (check_divout && param->pll_divout_step) + KUNIT_EXPECT_GT(test, divout, pll_start); + else if (check_divout) + KUNIT_EXPECT_LT(test, divout, 0); + + /* The PLL output must only be selected once the PLL has locked */ + KUNIT_EXPECT_GT(test, priv->num_lock_reads, 0); + mclk_sel = cs42l42_test_find_write(priv, CS42L42_MCLK_SRC_SEL, pll_start); + KUNIT_ASSERT_GT(test, mclk_sel, pll_start); + KUNIT_EXPECT_EQ(test, priv->writes[mclk_sel].val & CS42L42_MCLK_SRC_SEL_MASK, + CS42L42_MCLK_SRC_SEL_MASK); + + sclk = cs42l42_test_find_write(priv, CS42L42_OSC_SWITCH, mclk_sel); + KUNIT_ASSERT_GT(test, sclk, mclk_sel); + KUNIT_EXPECT_EQ(test, priv->writes[sclk].val, CS42L42_SCLK_PRESENT_MASK); + + hp = cs42l42_test_find_write(priv, CS42L42_HP_CTL, sclk); + KUNIT_ASSERT_GT(test, hp, sclk); + KUNIT_EXPECT_EQ(test, priv->writes[hp].val & CS42L42_TEST_HP_MUTE_MASK, 0); +} + +/* + * ASoC resume sets the card to D2, unmutes the playback DAIs that were + * active at suspend and only then sets D0. That early unmute must not move + * the codec off the oscillator: the stream is still suspended and SCLK may + * not be running yet. + */ +static void cs42l42_test_unmute_below_d0_does_nothing(struct kunit *test) +{ + const struct cs42l42_test_power_param *param = test->param_value; + const struct cs42l42_test_clk_param *clk = &cs42l42_test_pll_clks[0]; + struct cs42l42_test_priv *priv = test->priv; + + if (!IS_ENABLED(CONFIG_PM)) + kunit_skip(test, "card power state needs CONFIG_PM"); + + /* A PLL configuration whose PLL would lock if it were started */ + cs42l42_test_set_clk(test, clk->sclk, clk->rate); + cs42l42_test_set_power(priv, param->state); + + KUNIT_EXPECT_EQ(test, cs42l42_mute_stream(&priv->dai, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + KUNIT_EXPECT_EQ(test, priv->num_writes, 0); + KUNIT_EXPECT_EQ(test, priv->num_lock_reads, 0); + KUNIT_EXPECT_EQ(test, priv->cs42l42.stream_use, 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L42_HP_CTL] & CS42L42_TEST_HP_MUTE_MASK, + CS42L42_TEST_HP_MUTE_MASK); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L42_OSC_SWITCH], 0); +} + +/* + * Use real ALSA RESUME/PREPARE ioctls and the ASoC trigger installed by + * soc_new_pcm(). Only the host DAI and prepare callback are synthetic: the + * host prepare supplies a working clock before calling the codec unmute. + */ +struct cs42l42_test_pcm { + struct snd_soc_pcm_runtime rtd; + struct snd_soc_dai_link link; + struct snd_soc_dai_link_ch_map map; + struct snd_soc_dai cpu; + struct snd_soc_dai_driver cpu_driver; + struct snd_soc_dai *dais[2]; + struct snd_pcm_runtime runtime; + struct snd_pcm_mmap_status status; + struct snd_pcm_mmap_control control; + struct snd_pcm_runtime capture_runtime; + struct snd_pcm_mmap_status capture_status; + struct snd_pcm_group linked_group; + struct snd_pcm_file pcm_file; + struct file file; + struct cs42l42_test_priv *priv; + unsigned int resumes; + unsigned int suspends; + unsigned int prepares; +}; + +static int cs42l42_test_cpu_trigger(struct snd_pcm_substream *substream, + int cmd, struct snd_soc_dai *dai) +{ + struct cs42l42_test_pcm *pcm = container_of(snd_soc_substream_to_rtd(substream), + struct cs42l42_test_pcm, rtd); + + if (cmd == SNDRV_PCM_TRIGGER_RESUME) + pcm->resumes++; + else if (cmd == SNDRV_PCM_TRIGGER_SUSPEND) + pcm->suspends++; + return 0; +} + +static const struct snd_soc_dai_ops cs42l42_test_cpu_ops = { + .trigger = cs42l42_test_cpu_trigger, +}; + +static int cs42l42_test_pcm_prepare(struct snd_pcm_substream *substream) +{ + struct cs42l42_test_pcm *pcm = container_of(snd_soc_substream_to_rtd(substream), + struct cs42l42_test_pcm, rtd); + int ret; + + pcm->prepares++; + pcm->priv->pll_locks = true; + ret = snd_soc_pcm_dai_prepare(substream); + if (ret) + return ret; + return cs42l42_mute_stream(&pcm->priv->dai, 0, substream->stream); +} + +static int cs42l42_test_pcm_ioctl(struct snd_pcm_substream *substream, + unsigned int cmd, void *arg) +{ + return cmd == SNDRV_PCM_IOCTL1_RESET ? 0 : -EINVAL; +} + +static void cs42l42_test_pcm_free(void *data) +{ + struct cs42l42_test_pcm *pcm = data; + int stream; + + if (pcm->rtd.pcm) { + for_each_pcm_streams(stream) { + struct snd_pcm_substream *substream; + + substream = pcm->rtd.pcm->streams[stream].substream; + if (!substream) + continue; + substream->runtime = NULL; + if (snd_pcm_stream_linked(substream)) { + list_move(&substream->link_list, &substream->self_group.substreams); + substream->group = &substream->self_group; + } + } + } + snd_card_free(pcm->rtd.card->snd_card); +} + +static struct cs42l42_test_pcm *cs42l42_test_pcm_init(struct kunit *test) +{ + struct cs42l42_test_priv *priv = test->priv; + struct cs42l42_test_pcm *pcm; + struct snd_pcm_substream *substream; + struct snd_card *card; + + pcm = kunit_kzalloc(test, sizeof(*pcm), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, pcm); + KUNIT_ASSERT_EQ(test, snd_card_new(priv->component.dev, -1, NULL, + THIS_MODULE, 0, &card), 0); + priv->soc_card.snd_card = card; + priv->soc_card.dev = priv->component.dev; + pcm->priv = priv; + pcm->rtd.dev = priv->component.dev; + pcm->rtd.card = &priv->soc_card; + KUNIT_ASSERT_EQ(test, kunit_add_action_or_reset(test, cs42l42_test_pcm_free, pcm), 0); + pcm->rtd.dai_link = &pcm->link; + pcm->rtd.dais = pcm->dais; + pcm->link.name = "cs42l42-resume-test"; + pcm->link.stream_name = pcm->link.name; + pcm->link.num_cpus = 1; + pcm->link.num_codecs = 1; + pcm->link.ch_maps = &pcm->map; + pcm->cpu_driver = cs42l42_dai; + pcm->cpu_driver.ops = &cs42l42_test_cpu_ops; + pcm->cpu.driver = &pcm->cpu_driver; + pcm->cpu.name = "clock-provider"; + pcm->cpu.dev = priv->component.dev; + pcm->cpu.component = &priv->component; + pcm->dais[0] = &pcm->cpu; + pcm->dais[1] = &priv->dai; + KUNIT_ASSERT_EQ(test, soc_new_pcm(&pcm->rtd), 0); + pcm->rtd.ops.prepare = cs42l42_test_pcm_prepare; + pcm->rtd.ops.ioctl = cs42l42_test_pcm_ioctl; + substream = pcm->rtd.pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream; + substream->private_data = &pcm->rtd; + substream->runtime = &pcm->runtime; + pcm->runtime.status = &pcm->status; + pcm->runtime.control = &pcm->control; + pcm->runtime.info = SNDRV_PCM_INFO_RESUME; + pcm->runtime.suspended_state = SNDRV_PCM_STATE_RUNNING; + pcm->runtime.period_size = 128; + mutex_init(&pcm->runtime.buffer_mutex); + init_waitqueue_head(&pcm->runtime.sleep); + init_waitqueue_head(&pcm->runtime.tsleep); + pcm->pcm_file.substream = substream; + pcm->file.private_data = &pcm->pcm_file; + return pcm; +} + +static long cs42l42_test_pcm_command(struct cs42l42_test_pcm *pcm, unsigned int cmd) +{ + return snd_pcm_f_ops[SNDRV_PCM_STREAM_PLAYBACK].unlocked_ioctl(&pcm->file, cmd, 0); +} + +static void cs42l42_test_resume_requires_prepare(struct kunit *test) +{ + const struct cs42l42_test_clk_param *clk = &cs42l42_test_pll_clks[0]; + struct cs42l42_test_priv *priv = test->priv; + struct cs42l42_test_pcm *pcm; + + if (!IS_ENABLED(CONFIG_PM)) + kunit_skip(test, "card power state needs CONFIG_PM"); + pcm = cs42l42_test_pcm_init(test); + KUNIT_ASSERT_NOT_NULL(test, pcm); + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D0); + cs42l42_test_set_clk(test, clk->sclk, clk->rate); + __snd_pcm_set_state(&pcm->runtime, SNDRV_PCM_STATE_SETUP); + KUNIT_ASSERT_EQ(test, cs42l42_test_pcm_command(pcm, SNDRV_PCM_IOCTL_PREPARE), 0); + KUNIT_EXPECT_FALSE(test, pcm->runtime.info & SNDRV_PCM_INFO_RESUME); + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D3hot); + KUNIT_ASSERT_EQ(test, cs42l42_mute_stream(&priv->dai, 1, SNDRV_PCM_STREAM_PLAYBACK), 0); + __snd_pcm_set_state(&pcm->runtime, SNDRV_PCM_STATE_SUSPENDED); + cs42l42_test_clear_log(priv); + priv->pll_locks = false; + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D2); + KUNIT_ASSERT_EQ(test, cs42l42_mute_stream(&priv->dai, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + KUNIT_EXPECT_EQ(test, priv->num_writes, 0); + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D0); + + /* ALSA rejects before the real ASoC trigger can start any DAI. */ + KUNIT_EXPECT_EQ(test, cs42l42_test_pcm_command(pcm, SNDRV_PCM_IOCTL_RESUME), -ENOSYS); + KUNIT_EXPECT_EQ(test, pcm->resumes, 0); + KUNIT_EXPECT_EQ(test, pcm->suspends, 0); + KUNIT_EXPECT_EQ(test, pcm->runtime.state, SNDRV_PCM_STATE_SUSPENDED); + KUNIT_EXPECT_EQ(test, pcm->prepares, 1); + KUNIT_EXPECT_EQ(test, priv->num_writes, 0); + + /* ALSA's documented -ENOSYS recovery is PREPARE, not a D0 callback. */ + KUNIT_ASSERT_EQ(test, cs42l42_test_pcm_command(pcm, SNDRV_PCM_IOCTL_PREPARE), 0); + KUNIT_EXPECT_EQ(test, pcm->prepares, 2); + KUNIT_EXPECT_EQ(test, pcm->runtime.state, SNDRV_PCM_STATE_PREPARED); + cs42l42_test_expect_switch_to_pll(test, clk, false); + KUNIT_EXPECT_EQ(test, priv->cs42l42.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); +} + +static void cs42l42_test_linked_resume_preflight(struct kunit *test) +{ + struct cs42l42_test_priv *priv = test->priv; + struct cs42l42_test_pcm *pcm; + struct snd_pcm_substream *playback, *capture; + + if (!IS_ENABLED(CONFIG_PM)) + kunit_skip(test, "card power state needs CONFIG_PM"); + pcm = cs42l42_test_pcm_init(test); + KUNIT_ASSERT_NOT_NULL(test, pcm); + playback = pcm->pcm_file.substream; + capture = pcm->rtd.pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream; + capture->private_data = &pcm->rtd; + capture->runtime = &pcm->capture_runtime; + pcm->capture_runtime.info = SNDRV_PCM_INFO_RESUME; + pcm->capture_runtime.status = &pcm->capture_status; + pcm->capture_runtime.suspended_state = SNDRV_PCM_STATE_RUNNING; + __snd_pcm_set_state(&pcm->capture_runtime, SNDRV_PCM_STATE_SUSPENDED); + __snd_pcm_set_state(&pcm->runtime, SNDRV_PCM_STATE_SUSPENDED); + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D0); + KUNIT_ASSERT_EQ(test, snd_soc_pcm_dai_prepare(playback), 0); + + /* Same group layout as linked ALSA streams, with capture visited first. */ + spin_lock_init(&pcm->linked_group.lock); + mutex_init(&pcm->linked_group.mutex); + INIT_LIST_HEAD(&pcm->linked_group.substreams); + refcount_set(&pcm->linked_group.refs, 2); + list_move_tail(&capture->link_list, &pcm->linked_group.substreams); + list_move_tail(&playback->link_list, &pcm->linked_group.substreams); + capture->group = &pcm->linked_group; + playback->group = &pcm->linked_group; + + KUNIT_EXPECT_EQ(test, cs42l42_test_pcm_command(pcm, SNDRV_PCM_IOCTL_RESUME), -ENOSYS); + KUNIT_EXPECT_EQ(test, pcm->resumes, 0); + KUNIT_EXPECT_EQ(test, pcm->suspends, 0); + KUNIT_EXPECT_EQ(test, pcm->runtime.state, SNDRV_PCM_STATE_SUSPENDED); + KUNIT_EXPECT_EQ(test, pcm->capture_runtime.state, SNDRV_PCM_STATE_SUSPENDED); + KUNIT_EXPECT_TRUE(test, pcm->capture_runtime.info & SNDRV_PCM_INFO_RESUME); + KUNIT_EXPECT_EQ(test, priv->num_writes, 0); +} + +static void cs42l42_test_capture_resume_remains_supported(struct kunit *test) +{ + struct cs42l42_test_priv *priv = test->priv; + struct cs42l42_test_pcm *pcm; + struct snd_pcm_substream *capture; + + if (!IS_ENABLED(CONFIG_PM)) + kunit_skip(test, "card power state needs CONFIG_PM"); + pcm = cs42l42_test_pcm_init(test); + KUNIT_ASSERT_NOT_NULL(test, pcm); + capture = pcm->rtd.pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream; + capture->private_data = &pcm->rtd; + capture->runtime = &pcm->capture_runtime; + pcm->capture_runtime.info = SNDRV_PCM_INFO_RESUME; + pcm->capture_runtime.status = &pcm->capture_status; + pcm->capture_runtime.suspended_state = SNDRV_PCM_STATE_RUNNING; + __snd_pcm_set_state(&pcm->capture_runtime, SNDRV_PCM_STATE_SUSPENDED); + pcm->pcm_file.substream = capture; + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D0); + KUNIT_ASSERT_EQ(test, snd_soc_pcm_dai_prepare(capture), 0); + KUNIT_EXPECT_EQ(test, cs42l42_test_pcm_command(pcm, SNDRV_PCM_IOCTL_RESUME), 0); + KUNIT_EXPECT_EQ(test, pcm->resumes, 1); + KUNIT_EXPECT_EQ(test, pcm->suspends, 0); + KUNIT_EXPECT_EQ(test, pcm->capture_runtime.state, SNDRV_PCM_STATE_RUNNING); + KUNIT_EXPECT_EQ(test, priv->num_writes, 0); +} + +static void cs42l42_test_prepare_resume_capability_is_per_stream(struct kunit *test) +{ + struct cs42l42_test_priv *priv = test->priv; + struct snd_pcm_runtime runtime = { + .info = SNDRV_PCM_INFO_RESUME | SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_MMAP, + }; + struct snd_pcm_substream substream = { + .stream = SNDRV_PCM_STREAM_CAPTURE, + .runtime = &runtime, + }; + + KUNIT_ASSERT_EQ(test, snd_soc_dai_prepare(&priv->dai, &substream), 0); + KUNIT_EXPECT_EQ(test, runtime.info, + SNDRV_PCM_INFO_RESUME | SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_MMAP); + substream.stream = SNDRV_PCM_STREAM_PLAYBACK; + KUNIT_ASSERT_EQ(test, snd_soc_dai_prepare(&priv->dai, &substream), 0); + KUNIT_EXPECT_EQ(test, runtime.info, SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_MMAP); + KUNIT_EXPECT_EQ(test, priv->num_writes, 0); +} + +struct cs42l42_test_admission_param { + const char *name; + const char *root; + size_t root_len; + const char *codec; + size_t codec_len; + bool soundwire; + bool expected; +}; + +#define CS42L42_TEST_M1_ROOT(board) "apple," board "\0apple,t8103\0apple,arm-platform" +#define CS42L42_TEST_ADMISSION(label, machine, device, sdw, admitted) \ + { label, machine, sizeof(machine), device, sizeof(device), sdw, admitted } + +static const struct cs42l42_test_admission_param cs42l42_test_admissions[] = { + CS42L42_TEST_ADMISSION("J274 CS42L83", CS42L42_TEST_M1_ROOT("j274"), + "cirrus,cs42l83", false, true), + CS42L42_TEST_ADMISSION("J293 CS42L83", CS42L42_TEST_M1_ROOT("j293"), + "cirrus,cs42l83", false, true), + CS42L42_TEST_ADMISSION("J313 CS42L83", CS42L42_TEST_M1_ROOT("j313"), + "cirrus,cs42l83", false, true), + CS42L42_TEST_ADMISSION("J456 CS42L83", CS42L42_TEST_M1_ROOT("j456"), + "cirrus,cs42l83", false, true), + CS42L42_TEST_ADMISSION("J457 CS42L83", CS42L42_TEST_M1_ROOT("j457"), + "cirrus,cs42l83", false, true), + CS42L42_TEST_ADMISSION("base-M1 CS42L42", "apple,t8103", + "cirrus,cs42l42", false, true), + CS42L42_TEST_ADMISSION("J700 CS42L83", "apple,j700\0apple,t8140", + "cirrus,cs42l83", false, false), + CS42L42_TEST_ADMISSION("M1 Pro", "apple,j314s\0apple,t6000", + "cirrus,cs42l83", false, false), + CS42L42_TEST_ADMISSION("M2 Air", "apple,j413\0apple,t8112", + "cirrus,cs42l83", false, false), + CS42L42_TEST_ADMISSION("other codec", "apple,t8103", + "cirrus,cs42l84", false, false), + CS42L42_TEST_ADMISSION("absent codec OF node", "apple,t8103", + NULL, false, false), + CS42L42_TEST_ADMISSION("ACPI/non-OF host", NULL, + "cirrus,cs42l42", false, false), + CS42L42_TEST_ADMISSION("CS42L42 SoundWire", "apple,t8103", + "cirrus,cs42l42", true, false), + CS42L42_TEST_ADMISSION("CS42L83 SoundWire", "apple,t8103", + "cirrus,cs42l83", true, false), + CS42L42_TEST_ADMISSION("generic shared-BE host", "test,shared-be", + "cirrus,cs42l42", false, false), + CS42L42_TEST_ADMISSION("root lookalike", "apple,t8103-other", + "cirrus,cs42l83", false, false), +}; + +static void cs42l42_test_resume_admission(struct kunit *test) +{ + const struct cs42l42_test_admission_param *param = test->param_value; + struct property root_prop = { + .name = "compatible", .value = (void *)param->root, .length = param->root_len, + }; + struct property codec_prop = { + .name = "compatible", .value = (void *)param->codec, .length = param->codec_len, + }; + struct device_node root = { .full_name = "/", .properties = &root_prop }; + struct device_node codec = { .full_name = "/codec", .properties = &codec_prop }; + bool required; + + if (!IS_ENABLED(CONFIG_OF)) + kunit_skip(test, "admission matching needs CONFIG_OF"); + required = cs42l42_resume_workaround_required(param->root ? &root : NULL, + param->codec ? &codec : NULL, + param->soundwire); + KUNIT_EXPECT_EQ(test, required, param->expected); +} + +/* Excluded hosts keep their sole D2 unmute and transparent PCM resume. */ +static void cs42l42_test_unadmitted_resume_without_prepare(struct kunit *test) +{ + struct cs42l42_test_priv *priv = test->priv; + struct cs42l42_test_pcm *pcm; + + if (!IS_ENABLED(CONFIG_PM)) + kunit_skip(test, "card power state needs CONFIG_PM"); + priv->cs42l42.resume_reprepare = false; + pcm = cs42l42_test_pcm_init(test); + KUNIT_ASSERT_NOT_NULL(test, pcm); + cs42l42_test_set_clk(test, 3072000, 48000); + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D0); + __snd_pcm_set_state(&pcm->runtime, SNDRV_PCM_STATE_SETUP); + KUNIT_ASSERT_EQ(test, cs42l42_test_pcm_command(pcm, SNDRV_PCM_IOCTL_PREPARE), 0); + KUNIT_EXPECT_TRUE(test, pcm->runtime.info & SNDRV_PCM_INFO_RESUME); + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D3hot); + KUNIT_ASSERT_EQ(test, cs42l42_mute_stream(&priv->dai, 1, SNDRV_PCM_STREAM_PLAYBACK), 0); + __snd_pcm_set_state(&pcm->runtime, SNDRV_PCM_STATE_SUSPENDED); + cs42l42_test_clear_log(priv); + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D2); + KUNIT_ASSERT_EQ(test, cs42l42_mute_stream(&priv->dai, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + cs42l42_test_expect_switch_to_pll(test, &cs42l42_test_pll_clks[1], false); + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D0); + cs42l42_test_clear_log(priv); + KUNIT_EXPECT_EQ(test, cs42l42_test_pcm_command(pcm, SNDRV_PCM_IOCTL_RESUME), 0); + KUNIT_EXPECT_EQ(test, pcm->runtime.state, SNDRV_PCM_STATE_RUNNING); + KUNIT_EXPECT_EQ(test, pcm->prepares, 1); + KUNIT_EXPECT_EQ(test, pcm->resumes, 1); + KUNIT_EXPECT_EQ(test, priv->cs42l42.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L42_HP_CTL] & CS42L42_TEST_HP_MUTE_MASK, 0); + KUNIT_EXPECT_EQ(test, priv->num_writes, 0); +} + +static void cs42l42_test_unadmitted_prepare_preserves_capabilities(struct kunit *test) +{ + struct cs42l42_test_priv *priv = test->priv; + struct snd_pcm_runtime runtime = { + .info = SNDRV_PCM_INFO_RESUME | SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_MMAP, + }; + struct snd_pcm_substream substream = { + .stream = SNDRV_PCM_STREAM_PLAYBACK, .runtime = &runtime, + }; + + priv->cs42l42.resume_reprepare = false; + KUNIT_ASSERT_EQ(test, snd_soc_dai_prepare(&priv->dai, &substream), 0); + KUNIT_EXPECT_EQ(test, runtime.info, + SNDRV_PCM_INFO_RESUME | SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_MMAP); + KUNIT_EXPECT_EQ(test, priv->num_writes, 0); +} + +/* At D0 with a locked PLL the codec moves onto the PLL output and SCLK */ +static void cs42l42_test_unmute_pll_locked(struct kunit *test) +{ + const struct cs42l42_test_clk_param *param = test->param_value; + struct cs42l42_test_priv *priv = test->priv; + + cs42l42_test_set_clk(test, param->sclk, param->rate); + + KUNIT_EXPECT_EQ(test, cs42l42_mute_stream(&priv->dai, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + cs42l42_test_expect_switch_to_pll(test, param, true); + KUNIT_EXPECT_EQ(test, priv->cs42l42.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L42_HP_CTL] & CS42L42_TEST_HP_MUTE_MASK, 0); +} + +/* + * If the PLL does not lock the codec must stay on the oscillator with the + * PLL stopped and the headphone muted, and the error must be returned. + */ +static void cs42l42_test_unmute_pll_lock_timeout(struct kunit *test) +{ + const struct cs42l42_test_clk_param *param = test->param_value; + struct cs42l42_test_priv *priv = test->priv; + int pll_stop; + + cs42l42_test_set_clk(test, param->sclk, param->rate); + priv->pll_locks = false; + + KUNIT_EXPECT_EQ(test, cs42l42_mute_stream(&priv->dai, 0, SNDRV_PCM_STREAM_PLAYBACK), + -ETIMEDOUT); + KUNIT_EXPECT_GT(test, priv->num_lock_reads, 0); + + /* The PLL was started, then stopped again */ + pll_stop = cs42l42_test_last_write(priv, CS42L42_PLL_CTL1); + KUNIT_ASSERT_GE(test, pll_stop, 0); + KUNIT_EXPECT_EQ(test, priv->writes[pll_stop].val & CS42L42_PLL_START_MASK, 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L42_PLL_CTL1] & CS42L42_PLL_START_MASK, 0); + + /* Still on the oscillator */ + KUNIT_EXPECT_EQ(test, cs42l42_test_count_writes(priv, CS42L42_MCLK_SRC_SEL), 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L42_MCLK_SRC_SEL] & CS42L42_MCLK_SRC_SEL_MASK, 0); + KUNIT_EXPECT_EQ(test, cs42l42_test_count_writes(priv, CS42L42_OSC_SWITCH), 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L42_OSC_SWITCH], 0); + + /* Headphone still muted, stream not counted as running */ + KUNIT_EXPECT_EQ(test, cs42l42_test_count_writes(priv, CS42L42_HP_CTL), 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L42_HP_CTL] & CS42L42_TEST_HP_MUTE_MASK, + CS42L42_TEST_HP_MUTE_MASK); + KUNIT_EXPECT_EQ(test, priv->cs42l42.stream_use, 0); +} + +/* + * A failed unmute leaves stream_use clear, so the next unmute retries the + * whole switch instead of assuming the codec is already on SCLK. + */ +static void cs42l42_test_unmute_after_pll_lock_timeout(struct kunit *test) +{ + const struct cs42l42_test_clk_param *param = test->param_value; + struct cs42l42_test_priv *priv = test->priv; + + cs42l42_test_set_clk(test, param->sclk, param->rate); + + priv->pll_locks = false; + KUNIT_EXPECT_LT(test, cs42l42_mute_stream(&priv->dai, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + cs42l42_test_clear_log(priv); + priv->pll_locks = true; + KUNIT_EXPECT_EQ(test, cs42l42_mute_stream(&priv->dai, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + cs42l42_test_expect_switch_to_pll(test, param, false); + KUNIT_EXPECT_EQ(test, priv->cs42l42.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); +} + +/* Muting the last stream goes back to the oscillator, then stops the PLL */ +static void cs42l42_test_mute_returns_to_osc(struct kunit *test) +{ + const struct cs42l42_test_clk_param *param = test->param_value; + struct cs42l42_test_priv *priv = test->priv; + int hp, osc, mclk_sel, pll_stop; + + cs42l42_test_set_clk(test, param->sclk, param->rate); + KUNIT_ASSERT_EQ(test, cs42l42_mute_stream(&priv->dai, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + cs42l42_test_clear_log(priv); + + KUNIT_EXPECT_EQ(test, cs42l42_mute_stream(&priv->dai, 1, SNDRV_PCM_STREAM_PLAYBACK), 0); + + hp = cs42l42_test_find_write(priv, CS42L42_HP_CTL, 0); + KUNIT_ASSERT_GE(test, hp, 0); + KUNIT_EXPECT_EQ(test, priv->writes[hp].val & CS42L42_TEST_HP_MUTE_MASK, + CS42L42_TEST_HP_MUTE_MASK); + + osc = cs42l42_test_find_write(priv, CS42L42_OSC_SWITCH, hp); + KUNIT_ASSERT_GT(test, osc, hp); + KUNIT_EXPECT_EQ(test, priv->writes[osc].val, 0); + + mclk_sel = cs42l42_test_find_write(priv, CS42L42_MCLK_SRC_SEL, osc); + KUNIT_ASSERT_GT(test, mclk_sel, osc); + KUNIT_EXPECT_EQ(test, priv->writes[mclk_sel].val & CS42L42_MCLK_SRC_SEL_MASK, 0); + + pll_stop = cs42l42_test_find_write(priv, CS42L42_PLL_CTL1, mclk_sel); + KUNIT_ASSERT_GT(test, pll_stop, mclk_sel); + KUNIT_EXPECT_EQ(test, priv->writes[pll_stop].val & CS42L42_PLL_START_MASK, 0); + + KUNIT_EXPECT_EQ(test, priv->cs42l42.stream_use, 0); +} + +/* Without the PLL the codec goes straight onto SCLK, no lock poll */ +static void cs42l42_test_unmute_no_pll(struct kunit *test) +{ + struct cs42l42_test_priv *priv = test->priv; + int sclk, hp; + + /* SCLK is used directly as MCLK */ + cs42l42_test_set_clk(test, 12288000, 48000); + priv->pll_locks = false; + + KUNIT_EXPECT_EQ(test, cs42l42_mute_stream(&priv->dai, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + KUNIT_EXPECT_EQ(test, priv->num_lock_reads, 0); + KUNIT_EXPECT_EQ(test, cs42l42_test_count_writes(priv, CS42L42_PLL_CTL1), 0); + KUNIT_EXPECT_EQ(test, cs42l42_test_count_writes(priv, CS42L42_MCLK_SRC_SEL), 0); + + sclk = cs42l42_test_find_write(priv, CS42L42_OSC_SWITCH, 0); + KUNIT_ASSERT_GE(test, sclk, 0); + KUNIT_EXPECT_EQ(test, priv->writes[sclk].val, CS42L42_SCLK_PRESENT_MASK); + + hp = cs42l42_test_find_write(priv, CS42L42_HP_CTL, sclk); + KUNIT_ASSERT_GT(test, hp, sclk); + KUNIT_EXPECT_EQ(test, priv->writes[hp].val & CS42L42_TEST_HP_MUTE_MASK, 0); + + KUNIT_EXPECT_EQ(test, priv->cs42l42.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); +} + +static int cs42l42_test_case_init(struct kunit *test) +{ + struct cs42l42_test_priv *priv; + struct regmap *regmap; + struct device *dev; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + priv->test = test; + test->priv = priv; + + priv->regs = kunit_kzalloc(test, CS42L42_MAX_REGISTER + 1, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv->regs); + + dev = kunit_device_register(test, "cs42l42_test_drv"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + regmap = devm_regmap_init(dev, NULL, priv, &cs42l42_test_regmap_config); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, regmap); + + /* Construct the minimal set of driver structs */ + priv->cs42l42.dev = dev; + priv->cs42l42.regmap = regmap; + /* Existing clock-safety cases model the admitted base-M1 I2C path. */ + priv->cs42l42.resume_reprepare = true; + + priv->component.dev = dev; + priv->component.card = &priv->soc_card; + snd_soc_component_init_regmap(&priv->component, regmap); + snd_soc_component_set_drvdata(&priv->component, &priv->cs42l42); + + priv->dai.component = &priv->component; + priv->dai.dev = dev; + priv->dai.name = cs42l42_dai.name; + priv->dai.driver = &cs42l42_dai; + + priv->soc_card.snd_card = &priv->card; +#ifdef CONFIG_PM + init_waitqueue_head(&priv->card.power_sleep); + snd_refcount_init(&priv->card.power_ref); +#endif + cs42l42_test_set_power(priv, SNDRV_CTL_POWER_D0); + + /* Idle codec: on the oscillator, PLL stopped, headphone muted */ + priv->regs[CS42L42_HP_CTL] = CS42L42_TEST_HP_MUTE_MASK; + priv->pll_locks = true; + + return 0; +} + +static void cs42l42_test_clk_desc(const struct cs42l42_test_clk_param *param, char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "SCLK %u Hz, %u Hz", param->sclk, param->rate); +} + +static void cs42l42_test_power_desc(const struct cs42l42_test_power_param *param, char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s", param->name); +} + +static void cs42l42_test_admission_desc(const struct cs42l42_test_admission_param *param, + char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s", param->name); +} + +KUNIT_ARRAY_PARAM(cs42l42_test_admissions, cs42l42_test_admissions, cs42l42_test_admission_desc); +KUNIT_ARRAY_PARAM(cs42l42_test_pll_clks, cs42l42_test_pll_clks, cs42l42_test_clk_desc); +KUNIT_ARRAY_PARAM(cs42l42_test_low_power, cs42l42_test_low_power, cs42l42_test_power_desc); + +static struct kunit_case cs42l42_test_cases[] = { + KUNIT_CASE_PARAM(cs42l42_test_resume_admission, cs42l42_test_admissions_gen_params), + KUNIT_CASE(cs42l42_test_unadmitted_resume_without_prepare), + KUNIT_CASE(cs42l42_test_unadmitted_prepare_preserves_capabilities), + KUNIT_CASE_PARAM(cs42l42_test_unmute_below_d0_does_nothing, + cs42l42_test_low_power_gen_params), + KUNIT_CASE(cs42l42_test_resume_requires_prepare), + KUNIT_CASE(cs42l42_test_linked_resume_preflight), + KUNIT_CASE(cs42l42_test_capture_resume_remains_supported), + KUNIT_CASE(cs42l42_test_prepare_resume_capability_is_per_stream), + KUNIT_CASE_PARAM(cs42l42_test_unmute_pll_locked, cs42l42_test_pll_clks_gen_params), + KUNIT_CASE_PARAM(cs42l42_test_unmute_pll_lock_timeout, cs42l42_test_pll_clks_gen_params), + KUNIT_CASE_PARAM(cs42l42_test_unmute_after_pll_lock_timeout, + cs42l42_test_pll_clks_gen_params), + KUNIT_CASE_PARAM(cs42l42_test_mute_returns_to_osc, cs42l42_test_pll_clks_gen_params), + KUNIT_CASE(cs42l42_test_unmute_no_pll), + { } /* terminator */ +}; + +static struct kunit_suite cs42l42_test_suite = { + .name = "snd-soc-cs42l42-test", + .init = cs42l42_test_case_init, + .test_cases = cs42l42_test_cases, +}; + +kunit_test_suite(cs42l42_test_suite); + +MODULE_IMPORT_NS("SND_SOC_CS42L42_CORE"); +MODULE_DESCRIPTION("KUnit test for Cirrus Logic cs42l42 codec driver"); +MODULE_LICENSE("GPL"); diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c index 08ab73ef96560..9a742905dd3af 100644 --- a/sound/soc/codecs/cs42l42.c +++ b/sound/soc/codecs/cs42l42.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1110,6 +1111,18 @@ int cs42l42_mute_stream(struct snd_soc_dai *dai, int mute, int stream) CS42L42_PLL_START_MASK, 0); } } else { + /* + * On the admitted base-M1 I2C path, ASoC resume unmutes + * playback while the card is still below D0. The PCM is + * suspended then and the host may not be driving SCLK yet, + * so moving the codec onto it would take the I2C bus down. + * Stay on the oscillator; playback prepare disables PCM RESUME + * so userspace must prepare and send another unmute at D0. + */ + if (cs42l42->resume_reprepare && + snd_power_get_state(component->card->snd_card) != SNDRV_CTL_POWER_D0) + return 0; + if (!cs42l42->stream_use) { /* SCLK must be running before codec unmute. * @@ -1139,8 +1152,17 @@ int cs42l42_mute_stream(struct snd_soc_dai *dai, int mute, int stream) (regval & 1), CS42L42_PLL_LOCK_POLL_US, CS42L42_PLL_LOCK_TIMEOUT_US); - if (ret < 0) + if (ret < 0) { + /* + * Without a locked PLL the codec has no clock + * once it leaves the oscillator, and then the + * I2C bus stops working. Stay on the oscillator. + */ dev_warn(component->dev, "PLL failed to lock: %d\n", ret); + snd_soc_component_update_bits(component, CS42L42_PLL_CTL1, + CS42L42_PLL_START_MASK, 0); + return ret; + } /* PLL must be running to drive glitchless switch logic */ snd_soc_component_update_bits(component, @@ -1168,6 +1190,23 @@ int cs42l42_mute_stream(struct snd_soc_dai *dai, int mute, int stream) } EXPORT_SYMBOL_NS_GPL(cs42l42_mute_stream, "SND_SOC_CS42L42_CORE"); +int cs42l42_dai_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) +{ + struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(dai->component); + + /* + * Only the admitted base-M1 I2C workaround drops the D2 unmute and + * needs another prepare. Its macaudio topology permits one FE per BE, + * whose runtime is shared. This is not a generic shared-BE workaround: + * generic and SoundWire users retain their existing resume capability. + */ + if (cs42l42->resume_reprepare && substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + substream->runtime->info &= ~SNDRV_PCM_INFO_RESUME; + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cs42l42_dai_prepare, "SND_SOC_CS42L42_CORE"); + #define CS42L42_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\ SNDRV_PCM_FMTBIT_S24_LE |\ SNDRV_PCM_FMTBIT_S32_LE) @@ -1179,6 +1218,7 @@ static const struct snd_soc_dai_ops cs42l42_ops = { .set_sysclk = cs42l42_set_sysclk, .set_bclk_ratio = cs42l42_set_bclk_ratio, .mute_stream = cs42l42_mute_stream, + .prepare = cs42l42_dai_prepare, }; struct snd_soc_dai_driver cs42l42_dai = { @@ -2659,12 +2699,29 @@ static int __maybe_unused cs42l42_i2c_resume(struct device *dev) return 0; } +/* No new policy for unqualified hosts, ACPI codecs or SoundWire devices. */ +bool cs42l42_resume_workaround_required(const struct device_node *root, + const struct device_node *codec, bool soundwire) +{ + if (soundwire || !root || !codec) + return false; + + return of_device_is_compatible(root, "apple,t8103") && + (of_device_is_compatible(codec, "cirrus,cs42l42") || + of_device_is_compatible(codec, "cirrus,cs42l83")); +} + int cs42l42_common_probe(struct cs42l42_private *cs42l42, const struct snd_soc_component_driver *component_drv, struct snd_soc_dai_driver *dai) { + struct device_node *root = of_find_node_by_path("/"); + bool reprepare = cs42l42_resume_workaround_required(root, cs42l42->dev->of_node, + !!cs42l42->sdw_peripheral); int ret, i; + cs42l42->resume_reprepare = reprepare; + of_node_put(root); dev_set_drvdata(cs42l42->dev, cs42l42); mutex_init(&cs42l42->irq_lock); diff --git a/sound/soc/codecs/cs42l42.h b/sound/soc/codecs/cs42l42.h index 7a9f2a5be5ef8..dd5a456fa6273 100644 --- a/sound/soc/codecs/cs42l42.h +++ b/sound/soc/codecs/cs42l42.h @@ -69,6 +69,8 @@ struct cs42l42_private { bool remote_active; unsigned int remote_saved[CS42L83_REMOTE_NUM_SAVED_REGS]; bool hp_adc_up_pending; + /* Set once at probe for the base-M1 I2C resume workaround. */ + bool resume_reprepare; bool suspended; bool sdw_waiting_first_unattach; bool init_done; @@ -86,6 +88,9 @@ int cs42l42_pll_config(struct snd_soc_component *component, unsigned int clk, unsigned int sample_rate); void cs42l42_src_config(struct snd_soc_component *component, unsigned int sample_rate); int cs42l42_mute_stream(struct snd_soc_dai *dai, int mute, int stream); +int cs42l42_dai_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai); +bool cs42l42_resume_workaround_required(const struct device_node *root, + const struct device_node *codec, bool soundwire); irqreturn_t cs42l42_irq_thread(int irq, void *data); int cs42l42_suspend(struct device *dev); int cs42l42_resume(struct device *dev); diff --git a/sound/soc/codecs/cs42l84-test.c b/sound/soc/codecs/cs42l84-test.c new file mode 100644 index 0000000000000..4d4952a33c848 --- /dev/null +++ b/sound/soc/codecs/cs42l84-test.c @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// KUnit tests for the clock switching in cs42l84_mute_stream(). +// +// This file is included at the end of cs42l84.c when +// CONFIG_SND_SOC_CS42L84_TEST is enabled, so the tests can reach the +// driver's static DAI ops and PLL configuration. +// +// The codec runs from its internal oscillator (RCO) while no stream is +// active. Unmuting the first stream moves it onto BCLK, through the PLL +// if the BCLK rate needs one. Once the codec has left the oscillator it +// needs a working clock to answer on I2C, so these tests check when that +// switch happens and when it must not happen. + +#include +#include + +#define CS42L84_TEST_MAX_WRITES 64 +#define CS42L84_TEST_NUM_REGS 0x7400 + +#define CS42L84_TEST_CCM_CTL1(src, freq) \ + (FIELD_PREP_CONST(CS42L84_CCM_CTL1_MCLK_SRC, (src)) | \ + FIELD_PREP_CONST(CS42L84_CCM_CTL1_MCLK_FREQ, (freq))) + +struct cs42l84_test_write { + unsigned int reg; + unsigned int val; +}; + +struct cs42l84_test_priv { + struct kunit *test; + + struct cs42l84_private cs42l84; + struct snd_soc_component component; + struct snd_soc_dai dai; + struct snd_soc_card soc_card; + struct snd_card card; + + /* Register contents seen by the driver */ + u8 *regs; + + /* Every register write made by the driver, in order */ + struct cs42l84_test_write writes[CS42L84_TEST_MAX_WRITES]; + unsigned int num_writes; + + unsigned int num_lock_reads; + unsigned int lock_status; +}; + +struct cs42l84_test_clk_param { + unsigned int bclk; + unsigned int rate; + unsigned int ccm_ctl1; +}; + +/* BCLK rates that run the codec from the PLL */ +static const struct cs42l84_test_clk_param cs42l84_test_pll_clks[] = { + { + .bclk = 3072000, .rate = 48000, + .ccm_ctl1 = CS42L84_TEST_CCM_CTL1(CS42L84_CCM_CTL1_MCLK_SRC_PLL, + CS42L84_CCM_CTL1_MCLK_F_12_288KHZ), + }, + { + .bclk = 2822400, .rate = 44100, + .ccm_ctl1 = CS42L84_TEST_CCM_CTL1(CS42L84_CCM_CTL1_MCLK_SRC_PLL, + CS42L84_CCM_CTL1_MCLK_F_12_288KHZ), + }, +}; + +/* BCLK rates that are used directly as MCLK */ +static const struct cs42l84_test_clk_param cs42l84_test_bclk_clks[] = { + { + .bclk = 12288000, .rate = 48000, + .ccm_ctl1 = CS42L84_TEST_CCM_CTL1(CS42L84_CCM_CTL1_MCLK_SRC_BCLK, + CS42L84_CCM_CTL1_MCLK_F_12_288KHZ), + }, + { + .bclk = 11289600, .rate = 44100, + .ccm_ctl1 = CS42L84_TEST_CCM_CTL1(CS42L84_CCM_CTL1_MCLK_SRC_BCLK, + CS42L84_CCM_CTL1_MCLK_F_12_288KHZ), + }, +}; + +static int cs42l84_test_reg_read(void *context, unsigned int reg, unsigned int *val) +{ + struct cs42l84_test_priv *priv = context; + + if (reg >= CS42L84_TEST_NUM_REGS) + return -EINVAL; + + if (reg == CS42L84_PLL_LOCK_STATUS) { + priv->num_lock_reads++; + *val = priv->lock_status; + return 0; + } + + *val = priv->regs[reg]; + + return 0; +} + +static int cs42l84_test_reg_write(void *context, unsigned int reg, unsigned int val) +{ + struct cs42l84_test_priv *priv = context; + + if (reg >= CS42L84_TEST_NUM_REGS) + return -EINVAL; + + if (priv->num_writes < ARRAY_SIZE(priv->writes)) { + priv->writes[priv->num_writes].reg = reg; + priv->writes[priv->num_writes].val = val; + priv->num_writes++; + } else { + KUNIT_FAIL(priv->test, "write log full at reg %#x", reg); + } + + priv->regs[reg] = val; + + return 0; +} + +static const struct regmap_config cs42l84_test_regmap_config = { + .reg_bits = 16, + .val_bits = 8, + .max_register = CS42L84_TEST_NUM_REGS - 1, + .reg_read = cs42l84_test_reg_read, + .reg_write = cs42l84_test_reg_write, + .cache_type = REGCACHE_NONE, +}; + +static void cs42l84_test_clear_log(struct cs42l84_test_priv *priv) +{ + priv->num_writes = 0; + priv->num_lock_reads = 0; +} + +/* Index of the first write to @reg at or after index @from, or -1 */ +static int cs42l84_test_find_write(struct cs42l84_test_priv *priv, + unsigned int reg, int from) +{ + unsigned int i; + + for (i = max(from, 0); i < priv->num_writes; i++) { + if (priv->writes[i].reg == reg) + return i; + } + + return -1; +} + +/* Index of the last write to @reg, or -1 */ +static int cs42l84_test_last_write(struct cs42l84_test_priv *priv, unsigned int reg) +{ + unsigned int i; + + for (i = priv->num_writes; i > 0; i--) { + if (priv->writes[i - 1].reg == reg) + return i - 1; + } + + return -1; +} + +static unsigned int cs42l84_test_count_writes(struct cs42l84_test_priv *priv, + unsigned int reg) +{ + unsigned int i, n = 0; + + for (i = 0; i < priv->num_writes; i++) { + if (priv->writes[i].reg == reg) + n++; + } + + return n; +} + +/* Select the clock configuration the way the machine driver and hw_params do */ +static void cs42l84_test_set_clk(struct kunit *test, const struct cs42l84_test_clk_param *clk) +{ + struct cs42l84_test_priv *priv = test->priv; + + KUNIT_ASSERT_EQ(test, snd_soc_dai_set_sysclk(&priv->dai, 0, clk->bclk, + SND_SOC_CLOCK_IN), 0); + priv->cs42l84.srate = clk->rate; + KUNIT_ASSERT_EQ(test, cs42l84_pll_config(&priv->component), 0); + cs42l84_test_clear_log(priv); +} + +static void cs42l84_test_set_power(struct cs42l84_test_priv *priv, unsigned int state) +{ + snd_power_change_state(&priv->card, state); +} + +/* Mute or unmute through the DAI ops, as the ASoC core does */ +static int cs42l84_test_mute(struct cs42l84_test_priv *priv, int mute, int stream) +{ + return snd_soc_dai_digital_mute(&priv->dai, mute, stream); +} + +/* + * Check that the writes logged since the last clear are a complete switch + * from the oscillator to the PLL output, with the DAC unmuted last. + */ +static void cs42l84_test_expect_switch_to_pll(struct kunit *test, + const struct cs42l84_test_clk_param *clk) +{ + struct cs42l84_test_priv *priv = test->priv; + int refclk, pll_start, ccm, dac; + + refclk = cs42l84_test_find_write(priv, CS42L84_CCM_CTL4, 0); + KUNIT_ASSERT_GE(test, refclk, 0); + KUNIT_EXPECT_TRUE(test, priv->writes[refclk].val & CS42L84_CCM_CTL4_REFCLK_EN); + + pll_start = cs42l84_test_find_write(priv, CS42L84_PLL_CTL1, refclk); + KUNIT_ASSERT_GT(test, pll_start, refclk); + KUNIT_EXPECT_TRUE(test, priv->writes[pll_start].val & CS42L84_PLL_CTL1_EN); + + /* The PLL output must only be selected once the PLL has locked */ + KUNIT_EXPECT_GT(test, priv->num_lock_reads, 0); + ccm = cs42l84_test_find_write(priv, CS42L84_CCM_CTL1, pll_start); + KUNIT_ASSERT_GT(test, ccm, pll_start); + KUNIT_EXPECT_EQ(test, priv->writes[ccm].val, clk->ccm_ctl1); + + dac = cs42l84_test_find_write(priv, CS42L84_DAC_CTL1, ccm); + KUNIT_ASSERT_GT(test, dac, ccm); + KUNIT_EXPECT_TRUE(test, priv->writes[dac].val & CS42L84_DAC_CTL1_UNMUTE); +} + +/* + * Follow the codec callbacks from a running stream through card suspend and + * resume. ASoC sends the only resume unmute at D2, then publishes D0. A PCM + * with SNDRV_PCM_INFO_RESUME can trigger without another prepare/unmute. + * A successful PLL lock represents clocks being available, independently of + * card power state; the direct-BCLK case retains the host's clock contract. + */ +static void cs42l84_test_resume_without_prepare(struct kunit *test) +{ + const struct cs42l84_test_clk_param *clk = test->param_value; + struct cs42l84_test_priv *priv = test->priv; + unsigned int writes; + + if (!IS_ENABLED(CONFIG_PM)) + kunit_skip(test, "card power state needs CONFIG_PM"); + + cs42l84_test_set_clk(test, clk); + KUNIT_ASSERT_EQ(test, cs42l84_test_mute(priv, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + cs42l84_test_set_power(priv, SNDRV_CTL_POWER_D3hot); + KUNIT_ASSERT_EQ(test, cs42l84_test_mute(priv, 1, SNDRV_PCM_STREAM_PLAYBACK), 0); + KUNIT_ASSERT_EQ(test, priv->cs42l84.stream_use, 0); + KUNIT_ASSERT_EQ(test, priv->regs[CS42L84_CCM_CTL1], CS42L84_CCM_CTL1_RCO); + cs42l84_test_clear_log(priv); + + cs42l84_test_set_power(priv, SNDRV_CTL_POWER_D2); + KUNIT_ASSERT_EQ(test, cs42l84_test_mute(priv, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L84_CCM_CTL1], clk->ccm_ctl1); + KUNIT_EXPECT_TRUE(test, priv->regs[CS42L84_DAC_CTL1] & CS42L84_DAC_CTL1_UNMUTE); + KUNIT_EXPECT_EQ(test, priv->cs42l84.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); + writes = priv->num_writes; + KUNIT_EXPECT_GT(test, writes, 0); + + /* D0 is a notification, not a second codec callback or clock proof. */ + cs42l84_test_set_power(priv, SNDRV_CTL_POWER_D0); + KUNIT_EXPECT_EQ(test, priv->num_writes, writes); + KUNIT_EXPECT_TRUE(test, priv->regs[CS42L84_DAC_CTL1] & CS42L84_DAC_CTL1_UNMUTE); +} + +static void cs42l84_test_resume_without_prepare_bclk(struct kunit *test) +{ + cs42l84_test_resume_without_prepare(test); +} + +/* At D0 with a locked PLL the codec moves onto the PLL output */ +static void cs42l84_test_unmute_pll_locked(struct kunit *test) +{ + const struct cs42l84_test_clk_param *clk = test->param_value; + struct cs42l84_test_priv *priv = test->priv; + + cs42l84_test_set_clk(test, clk); + + KUNIT_EXPECT_EQ(test, cs42l84_test_mute(priv, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + cs42l84_test_expect_switch_to_pll(test, clk); + KUNIT_EXPECT_EQ(test, priv->cs42l84.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L84_CCM_CTL1], clk->ccm_ctl1); + KUNIT_EXPECT_TRUE(test, priv->regs[CS42L84_DAC_CTL1] & CS42L84_DAC_CTL1_UNMUTE); +} + +/* + * The PLL lock error bit is only reported: once the lock bit is set the + * codec still moves onto the PLL output. + */ +static void cs42l84_test_unmute_pll_lock_error_bit(struct kunit *test) +{ + const struct cs42l84_test_clk_param *clk = &cs42l84_test_pll_clks[0]; + struct cs42l84_test_priv *priv = test->priv; + + cs42l84_test_set_clk(test, clk); + priv->lock_status = CS42L84_PLL_LOCK_STATUS_LOCKED | CS42L84_PLL_LOCK_STATUS_ERROR; + + KUNIT_EXPECT_EQ(test, cs42l84_test_mute(priv, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + cs42l84_test_expect_switch_to_pll(test, clk); + KUNIT_EXPECT_EQ(test, priv->cs42l84.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); +} + +/* + * If the PLL does not lock the codec must stay on the oscillator with the + * PLL and its reference clock stopped again and the DAC muted, and the + * error must be returned. + */ +static void cs42l84_test_unmute_pll_lock_timeout(struct kunit *test) +{ + const struct cs42l84_test_clk_param *clk = test->param_value; + struct cs42l84_test_priv *priv = test->priv; + int pll_stop, refclk_stop; + + cs42l84_test_set_clk(test, clk); + priv->lock_status = 0; + + KUNIT_EXPECT_EQ(test, cs42l84_test_mute(priv, 0, SNDRV_PCM_STREAM_PLAYBACK), + -ETIMEDOUT); + KUNIT_EXPECT_GT(test, priv->num_lock_reads, 0); + + /* Still on the oscillator */ + KUNIT_EXPECT_EQ(test, cs42l84_test_count_writes(priv, CS42L84_CCM_CTL1), 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L84_CCM_CTL1], CS42L84_CCM_CTL1_RCO); + + /* The PLL was started and stopped again, then its reference clock */ + pll_stop = cs42l84_test_last_write(priv, CS42L84_PLL_CTL1); + KUNIT_ASSERT_GE(test, pll_stop, 0); + KUNIT_EXPECT_EQ(test, priv->writes[pll_stop].val & CS42L84_PLL_CTL1_EN, 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L84_PLL_CTL1] & CS42L84_PLL_CTL1_EN, 0); + + refclk_stop = cs42l84_test_last_write(priv, CS42L84_CCM_CTL4); + KUNIT_ASSERT_GT(test, refclk_stop, pll_stop); + KUNIT_EXPECT_EQ(test, priv->writes[refclk_stop].val & CS42L84_CCM_CTL4_REFCLK_EN, 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L84_CCM_CTL4] & CS42L84_CCM_CTL4_REFCLK_EN, 0); + + /* DAC still muted, stream not counted as running */ + KUNIT_EXPECT_EQ(test, cs42l84_test_count_writes(priv, CS42L84_DAC_CTL1), 0); + KUNIT_EXPECT_EQ(test, priv->regs[CS42L84_DAC_CTL1] & CS42L84_DAC_CTL1_UNMUTE, 0); + KUNIT_EXPECT_EQ(test, priv->cs42l84.stream_use, 0); +} + +/* No PLL lock at D2 must not be mistaken for a usable clock at resume. */ +static void cs42l84_test_resume_without_pll_lock(struct kunit *test) +{ + struct cs42l84_test_priv *priv = test->priv; + + if (!IS_ENABLED(CONFIG_PM)) + kunit_skip(test, "card power state needs CONFIG_PM"); + + cs42l84_test_set_power(priv, SNDRV_CTL_POWER_D2); + cs42l84_test_unmute_pll_lock_timeout(test); +} + +/* + * A failed unmute leaves stream_use clear, so the next unmute retries the + * whole switch instead of assuming the codec is already on the PLL. + */ +static void cs42l84_test_unmute_after_pll_lock_timeout(struct kunit *test) +{ + const struct cs42l84_test_clk_param *clk = test->param_value; + struct cs42l84_test_priv *priv = test->priv; + + cs42l84_test_set_clk(test, clk); + + priv->lock_status = 0; + KUNIT_EXPECT_LT(test, cs42l84_test_mute(priv, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + cs42l84_test_clear_log(priv); + priv->lock_status = CS42L84_PLL_LOCK_STATUS_LOCKED; + KUNIT_EXPECT_EQ(test, cs42l84_test_mute(priv, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + cs42l84_test_expect_switch_to_pll(test, clk); + KUNIT_EXPECT_EQ(test, priv->cs42l84.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); +} + +/* + * Muting the last stream goes back to the oscillator before stopping the + * PLL and then its reference clock. + */ +static void cs42l84_test_mute_returns_to_rco(struct kunit *test) +{ + const struct cs42l84_test_clk_param *clk = test->param_value; + struct cs42l84_test_priv *priv = test->priv; + int dac, ccm, pll_stop, refclk_stop; + + cs42l84_test_set_clk(test, clk); + KUNIT_ASSERT_EQ(test, cs42l84_test_mute(priv, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + cs42l84_test_clear_log(priv); + + KUNIT_EXPECT_EQ(test, cs42l84_test_mute(priv, 1, SNDRV_PCM_STREAM_PLAYBACK), 0); + + dac = cs42l84_test_find_write(priv, CS42L84_DAC_CTL1, 0); + KUNIT_ASSERT_GE(test, dac, 0); + KUNIT_EXPECT_EQ(test, priv->writes[dac].val & CS42L84_DAC_CTL1_UNMUTE, 0); + + ccm = cs42l84_test_find_write(priv, CS42L84_CCM_CTL1, dac); + KUNIT_ASSERT_GT(test, ccm, dac); + KUNIT_EXPECT_EQ(test, priv->writes[ccm].val, CS42L84_CCM_CTL1_RCO); + + pll_stop = cs42l84_test_find_write(priv, CS42L84_PLL_CTL1, ccm); + KUNIT_ASSERT_GT(test, pll_stop, ccm); + KUNIT_EXPECT_EQ(test, priv->writes[pll_stop].val & CS42L84_PLL_CTL1_EN, 0); + + refclk_stop = cs42l84_test_find_write(priv, CS42L84_CCM_CTL4, pll_stop); + KUNIT_ASSERT_GT(test, refclk_stop, pll_stop); + KUNIT_EXPECT_EQ(test, priv->writes[refclk_stop].val & CS42L84_CCM_CTL4_REFCLK_EN, 0); + + KUNIT_EXPECT_EQ(test, priv->cs42l84.stream_use, 0); +} + +/* Without the PLL the codec goes straight onto BCLK, no lock poll */ +static void cs42l84_test_unmute_bclk(struct kunit *test) +{ + const struct cs42l84_test_clk_param *clk = test->param_value; + struct cs42l84_test_priv *priv = test->priv; + int refclk, ccm, dac; + + cs42l84_test_set_clk(test, clk); + priv->lock_status = 0; + + KUNIT_EXPECT_EQ(test, cs42l84_test_mute(priv, 0, SNDRV_PCM_STREAM_PLAYBACK), 0); + + KUNIT_EXPECT_EQ(test, priv->num_lock_reads, 0); + KUNIT_EXPECT_EQ(test, cs42l84_test_count_writes(priv, CS42L84_PLL_CTL1), 0); + + refclk = cs42l84_test_find_write(priv, CS42L84_CCM_CTL4, 0); + KUNIT_ASSERT_GE(test, refclk, 0); + KUNIT_EXPECT_TRUE(test, priv->writes[refclk].val & CS42L84_CCM_CTL4_REFCLK_EN); + + ccm = cs42l84_test_find_write(priv, CS42L84_CCM_CTL1, refclk); + KUNIT_ASSERT_GT(test, ccm, refclk); + KUNIT_EXPECT_EQ(test, priv->writes[ccm].val, clk->ccm_ctl1); + + dac = cs42l84_test_find_write(priv, CS42L84_DAC_CTL1, ccm); + KUNIT_ASSERT_GT(test, dac, ccm); + KUNIT_EXPECT_TRUE(test, priv->writes[dac].val & CS42L84_DAC_CTL1_UNMUTE); + + KUNIT_EXPECT_EQ(test, priv->cs42l84.stream_use, BIT(SNDRV_PCM_STREAM_PLAYBACK)); +} + +static int cs42l84_test_case_init(struct kunit *test) +{ + struct cs42l84_test_priv *priv; + struct regmap *regmap; + struct device *dev; + + priv = kunit_kzalloc(test, sizeof(*priv), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv); + priv->test = test; + test->priv = priv; + + priv->regs = kunit_kzalloc(test, CS42L84_TEST_NUM_REGS, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, priv->regs); + + dev = kunit_device_register(test, "cs42l84_test_drv"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + regmap = devm_regmap_init(dev, NULL, priv, &cs42l84_test_regmap_config); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, regmap); + + /* Construct the minimal set of driver structs */ + priv->cs42l84.dev = dev; + priv->cs42l84.regmap = regmap; + + priv->component.dev = dev; + priv->component.card = &priv->soc_card; + snd_soc_component_init_regmap(&priv->component, regmap); + snd_soc_component_set_drvdata(&priv->component, &priv->cs42l84); + + priv->dai.dev = dev; + priv->dai.name = cs42l84_dai.name; + priv->dai.driver = &cs42l84_dai; + priv->dai.component = &priv->component; + + priv->soc_card.snd_card = &priv->card; +#ifdef CONFIG_PM + init_waitqueue_head(&priv->card.power_sleep); + snd_refcount_init(&priv->card.power_ref); +#endif + cs42l84_test_set_power(priv, SNDRV_CTL_POWER_D0); + + /* Idle codec: on the oscillator, PLL and REFCLK off, DAC muted */ + priv->regs[CS42L84_CCM_CTL1] = CS42L84_CCM_CTL1_RCO; + priv->lock_status = CS42L84_PLL_LOCK_STATUS_LOCKED; + + return 0; +} + +static void cs42l84_test_clk_desc(const struct cs42l84_test_clk_param *param, char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "BCLK %u Hz, %u Hz", param->bclk, param->rate); +} + +KUNIT_ARRAY_PARAM(cs42l84_test_pll_clks, cs42l84_test_pll_clks, cs42l84_test_clk_desc); +KUNIT_ARRAY_PARAM(cs42l84_test_bclk_clks, cs42l84_test_bclk_clks, cs42l84_test_clk_desc); + +static struct kunit_case cs42l84_test_cases[] = { + KUNIT_CASE_PARAM(cs42l84_test_resume_without_prepare, cs42l84_test_pll_clks_gen_params), + KUNIT_CASE_PARAM(cs42l84_test_resume_without_prepare_bclk, cs42l84_test_bclk_clks_gen_params), + KUNIT_CASE_PARAM(cs42l84_test_unmute_pll_locked, cs42l84_test_pll_clks_gen_params), + KUNIT_CASE(cs42l84_test_unmute_pll_lock_error_bit), + KUNIT_CASE_PARAM(cs42l84_test_unmute_pll_lock_timeout, cs42l84_test_pll_clks_gen_params), + KUNIT_CASE_PARAM(cs42l84_test_resume_without_pll_lock, cs42l84_test_pll_clks_gen_params), + KUNIT_CASE_PARAM(cs42l84_test_unmute_after_pll_lock_timeout, + cs42l84_test_pll_clks_gen_params), + KUNIT_CASE_PARAM(cs42l84_test_mute_returns_to_rco, cs42l84_test_pll_clks_gen_params), + KUNIT_CASE_PARAM(cs42l84_test_unmute_bclk, cs42l84_test_bclk_clks_gen_params), + { } /* terminator */ +}; + +static struct kunit_suite cs42l84_test_suite = { + .name = "snd-soc-cs42l84-test", + .init = cs42l84_test_case_init, + .test_cases = cs42l84_test_cases, +}; + +kunit_test_suite(cs42l84_test_suite); diff --git a/sound/soc/codecs/cs42l84.c b/sound/soc/codecs/cs42l84.c index 48d8e95a53405..b8c85c5b96f08 100644 --- a/sound/soc/codecs/cs42l84.c +++ b/sound/soc/codecs/cs42l84.c @@ -608,6 +608,12 @@ static int cs42l84_mute_stream(struct snd_soc_dai *dai, int mute, int stream) CS42L84_CCM_CTL4_REFCLK_EN, 0); } } else { + /* + * ASoC also unmutes here at D2 during card resume. A PCM + * advertising RESUME can restart without another prepare, so + * do not discard that unmute based on the card power state. + * The PLL path below still requires lock before switching. + */ if (!cs42l84->stream_use) { /* SCLK must be running before codec unmute. * @@ -636,8 +642,24 @@ static int cs42l84_mute_stream(struct snd_soc_dai *dai, int mute, int stream) (regval & CS42L84_PLL_LOCK_STATUS_LOCKED), CS42L84_PLL_LOCK_POLL_US, CS42L84_PLL_LOCK_TIMEOUT_US); - if (ret < 0) + if (ret < 0) { + /* + * Without a locked PLL the codec has no clock + * once it leaves the oscillator, and then the + * I2C bus stops working. Stay on the oscillator + * and undo what was enabled above, in the same + * order as the mute path. + */ dev_warn(component->dev, "PLL failed to lock: %d\n", ret); + snd_soc_component_update_bits(component, + CS42L84_PLL_CTL1, + CS42L84_PLL_CTL1_EN, 0); + snd_soc_component_update_bits(component, + CS42L84_CCM_CTL4, + CS42L84_CCM_CTL4_REFCLK_EN, + 0); + return ret; + } if (regval & CS42L84_PLL_LOCK_STATUS_ERROR) dev_warn(component->dev, "PLL lock error\n"); @@ -1224,3 +1246,7 @@ MODULE_AUTHOR("Martin Povišer "); MODULE_AUTHOR("Hector Martin "); MODULE_AUTHOR("James Calligeros "); MODULE_LICENSE("GPL"); + +#if IS_ENABLED(CONFIG_SND_SOC_CS42L84_TEST) +#include "cs42l84-test.c" +#endif diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c index f80e8d4981565..525126ecb0c30 100644 --- a/sound/soc/samsung/i2s.c +++ b/sound/soc/samsung/i2s.c @@ -1518,9 +1518,16 @@ static int samsung_i2s_probe(struct platform_device *pdev) goto err_disable_pm; priv->op_clk = clk_get_parent(priv->clk_table[CLK_I2S_RCLK_SRC]); + ret = clk_prepare_enable(priv->op_clk); + if (ret < 0) { + priv->op_clk = NULL; + goto err_unregister_clk; + } return 0; +err_unregister_clk: + i2s_unregister_clock_provider(priv); err_disable_pm: pm_runtime_disable(&pdev->dev); err_del_sec: @@ -1543,6 +1550,7 @@ static void samsung_i2s_remove(struct platform_device *pdev) i2s_unregister_clock_provider(priv); i2s_delete_secondary_device(priv); + clk_disable_unprepare(priv->op_clk); clk_disable_unprepare(priv->clk); pm_runtime_put_noidle(&pdev->dev); diff --git a/sound/soc/sdw_utils/soc_sdw_utils.c b/sound/soc/sdw_utils/soc_sdw_utils.c index 5266720e58b3c..a47f34afd1d19 100644 --- a/sound/soc/sdw_utils/soc_sdw_utils.c +++ b/sound/soc/sdw_utils/soc_sdw_utils.c @@ -797,7 +797,7 @@ struct asoc_sdw_codec_info codec_info_list[] = { }, { .direction = {false, true}, - .dai_name = "cs35l56-sdw1c", + .dai_name = "cs35l56-ot25", .dai_type = SOC_SDW_DAI_TYPE_AMP, .dailink = {SOC_SDW_UNUSED_DAI_ID, SOC_SDW_AMP_IN_DAI_ID}, .rtd_init = asoc_sdw_cs_spk_feedback_rtd_init, @@ -826,7 +826,36 @@ struct asoc_sdw_codec_info codec_info_list[] = { }, { .direction = {false, true}, - .dai_name = "cs35l56-sdw1c", + .dai_name = "cs35l56-ot25", + .dai_type = SOC_SDW_DAI_TYPE_AMP, + .dailink = {SOC_SDW_UNUSED_DAI_ID, SOC_SDW_AMP_IN_DAI_ID}, + .rtd_init = asoc_sdw_cs_spk_feedback_rtd_init, + }, + }, + .dai_num = 2, + }, + { + .vendor_id = 0x01fa, + .part_id = 0x3562, + .name_prefix = "AMP", + .is_amp = true, + .dais = { + { + .direction = {true, false}, + .dai_name = "cs35l56-sdw1", + .component_name = "cs35l56", + .dai_type = SOC_SDW_DAI_TYPE_AMP, + .dailink = {SOC_SDW_AMP_OUT_DAI_ID, SOC_SDW_UNUSED_DAI_ID}, + .init = asoc_sdw_cs_amp_init, + .rtd_init = asoc_sdw_cs_spk_rtd_init, + .controls = generic_spk_controls, + .num_controls = ARRAY_SIZE(generic_spk_controls), + .widgets = generic_spk_widgets, + .num_widgets = ARRAY_SIZE(generic_spk_widgets), + }, + { + .direction = {false, true}, + .dai_name = "cs35l56-ot25", .dai_type = SOC_SDW_DAI_TYPE_AMP, .dailink = {SOC_SDW_UNUSED_DAI_ID, SOC_SDW_AMP_IN_DAI_ID}, .rtd_init = asoc_sdw_cs_spk_feedback_rtd_init, @@ -855,7 +884,7 @@ struct asoc_sdw_codec_info codec_info_list[] = { }, { .direction = {false, true}, - .dai_name = "cs35l56-sdw1c", + .dai_name = "cs35l56-ot25", .dai_type = SOC_SDW_DAI_TYPE_AMP, .dailink = {SOC_SDW_UNUSED_DAI_ID, SOC_SDW_AMP_IN_DAI_ID}, .rtd_init = asoc_sdw_cs_spk_feedback_rtd_init, diff --git a/tools/asahi/atc-usb3-tests.py b/tools/asahi/atc-usb3-tests.py new file mode 100644 index 0000000000000..5e45e11c3c46d --- /dev/null +++ b/tools/asahi/atc-usb3-tests.py @@ -0,0 +1,880 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +"""Host regression tests for the ATCPHY USB3 handoff. + +Compile the actual driver functions, defines, enums and mode table. MMIO polling +models a single completed timeout/success per invocation; no timing, electrical +behavior, firmware, real BIST sequence or kernel mutex behavior is simulated. +The actual admission helper restricts the workaround to base M1 machines; +T600x/T8112/T602x still match the T8103 PHY fallback but keep their previous +single-poll/BIST/role behavior. OF compatible lookup is a modeled boundary. +The DT-to-driver match contract is checked from the actual sources. +Hardware qualification remains necessary. +Run --mutations for sensitivity checks. +""" + +import argparse +import json +import hashlib +import os +from pathlib import Path +import re +import subprocess +import tempfile + + +def block(source, marker): + if marker.startswith("static"): + # Skip a forward declaration; the next opening brace after a prototype + # belongs to an unrelated function. + match = re.search(re.escape(marker) + r"[^;{]*\{", source) + if match is None: + raise ValueError("function definition not found: " + marker) + start = match.start() + else: + start = source.index(marker) + opening = source.index("{", start) + depth = 1 + pos = opening + 1 + while depth: + if source[pos] == "{": + depth += 1 + elif source[pos] == "}": + depth -= 1 + pos += 1 + return source[start:pos] + (";" if source[pos:pos + 1] == ";" else "") + + +PRELUDE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1U << (n)) +#define GENMASK(h,l) ((UINT32_MAX >> (31-(h))) & (UINT32_MAX << (l))) +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +#define dev_dbg(...) ((void)0) +#define lockdep_assert_held(...) ((void)0) +#define guard(which) (void) +#define IS_ENABLED(x) 0 +#define KUNIT_STATIC_STUB_REDIRECT(...) ((void)0) +#define __iomem +#define FIELD_PREP(mask,value) (((u32)(value) << __builtin_ctz(mask)) & (mask)) +#define FIELD_GET(mask,value) (((value) & (mask)) >> __builtin_ctz(mask)) +#define fallthrough __attribute__((fallthrough)) +#define WARN_ON_ONCE(cond) do { if (cond) warnings++; } while (0) +#define container_of(ptr,type,member) ((type *)((char *)(ptr) - offsetof(type,member))) +''' + +STUBS = r''' +struct atcphy_hw { enum atcphy_generation gen; + bool has_t8140_dp, dp_t8122, has_usb4, park_pipe_unlocked, park_dummy_phy; }; +struct device_node { const char *const *compatible; unsigned int count; }; +static bool of_device_is_compatible(const struct device_node *root, const char *compatible) { + if (!root) return false; + for (unsigned int i = 0; i < root->count; i++) + if (!strcasecmp(root->compatible[i], compatible)) return true; + return false; +} +/* ACTUAL_ADMISSION */ +struct reset_controller_dev { int unused; }; +struct apple_atcphy { + struct atcphy_hw *hw; + struct { unsigned char *pipehandler, *core; } regs; + void *dev; + int lock; + struct reset_controller_dev rcdev; + enum atcphy_mode mode; + enum atcphy_pipehandler_state pipe_state; + enum atcphy_mode typec_mode; + bool pipe_pending, t8103_dummy_pending, t8103_usb3_retry, t8103_usb3_workaround, mode_dirty, usb_off_pending; + bool host_active, usb3_port_off, dp_only, ss_tunables, fixed_usb2, t8140_dp_tunables, dp_tunables; +}; +struct phy { struct apple_atcphy *data; }; +struct typec_mux_dev { struct apple_atcphy *data; }; +struct typec_altmode { unsigned svid; }; +struct typec_mux_state { unsigned long mode; struct typec_altmode *alt; void *data; }; +struct enter_usb_data { u32 eudo; }; +enum phy_mode { PHY_MODE_USB_HOST, PHY_MODE_USB_DEVICE, PHY_MODE_INVALID }; +static unsigned char memory[0x18000] __attribute__((aligned(4))); +static struct atcphy_hw hw; +static struct apple_atcphy p; +static struct phy phy; +static unsigned polls, sleeps, sets, clears, muxes, bist_old, bist_new; +static unsigned configure_calls, restore_calls, unlocks, warnings; +static unsigned dummy_muxes, pending_seen_at_unlock; +static unsigned usb2_power_off_calls; +static unsigned ack_on; +static int check_ret, bist_ret, unlock_ret, configure_ret; +static bool concurrent_override_change; +static u32 readl(const void *ptr) { u32 v; memcpy(&v, ptr, sizeof(v)); return v; } +static void writel(u32 val, void *ptr) { memcpy(ptr, &val, sizeof(val)); } +static void set32(void *ptr, u32 mask) { + if (ptr == memory + PIPEHANDLER_LOCK_REQ) sets++; + writel(readl(ptr) | mask, ptr); +} +static void clear32(void *ptr, u32 mask) { + if (ptr == memory + PIPEHANDLER_LOCK_REQ) clears++; + writel(readl(ptr) & ~mask, ptr); +} +static void mask32(void *ptr, u32 mask, u32 value) { + writel((readl(ptr) & ~mask) | value, ptr); +} +static u32 poll_value(void *ptr) { + assert(ptr == memory + PIPEHANDLER_LOCK_ACK); + polls++; + if (concurrent_override_change) { + set32(memory + PIPEHANDLER_OVERRIDE, BIT(9)); + set32(memory + PIPEHANDLER_OVERRIDE_VALUES, BIT(9)); + } + return ack_on && polls >= ack_on ? PIPEHANDLER_LOCK_EN : 0; +} +#define readl_poll_timeout(addr,val,cond,delay,timeout) \ + ({ assert((delay) == 10 && (timeout) == 1000); \ + (val) = poll_value(addr); (cond) ? 0 : -ETIMEDOUT; }) +static void usleep_range(unsigned lo, unsigned hi) { + assert(lo == 5000 && hi == 10000); sleeps++; +} +static int atcphy_pipehandler_check(struct apple_atcphy *a) { (void)a; return check_ret; } +static int atcphy_pipehandler_unlock(struct apple_atcphy *a) { + unlocks++; + if (a->pipe_state == ATCPHY_PIPEHANDLER_STATE_DUMMY && a->t8103_dummy_pending) + pending_seen_at_unlock++; + clear32(a->regs.pipehandler + PIPEHANDLER_LOCK_REQ, PIPEHANDLER_LOCK_EN); + return unlock_ret; +} +static int atcphy_configure_pipehandler_usb3_host_t8103(struct apple_atcphy *a) { + (void)a; bist_old++; return bist_ret; +} +static int atcphy_configure_pipehandler_usb3_host_t8122(struct apple_atcphy *a) { + (void)a; bist_new++; return bist_ret; +} +static void atcphy_pipehandler_set_mux(struct apple_atcphy *a, u32 data, u32 clk) { + (void)a; + assert((data == PIPEHANDLER_MUX_CTRL_DATA_USB3 && clk == PIPEHANDLER_MUX_CTRL_CLK_USB3) || + (data == PIPEHANDLER_MUX_CTRL_DATA_DUMMY && clk == PIPEHANDLER_MUX_CTRL_CLK_DUMMY) || + (data == PIPEHANDLER_MUX_CTRL_DATA_USB4 && clk == PIPEHANDLER_MUX_CTRL_CLK_USB4)); + if (data == PIPEHANDLER_MUX_CTRL_DATA_DUMMY) dummy_muxes++; + muxes++; +} +static struct apple_atcphy *phy_get_drvdata(struct phy *f) { return f->data; } +static struct apple_atcphy *typec_mux_get_drvdata(struct typec_mux_dev *m) { return m->data; } +static void atcphy_restore_after_pd_off(struct apple_atcphy *a) { (void)a; restore_calls++; } +static int atcphy_usb2_power_off(struct apple_atcphy *a) { + (void)a; usb2_power_off_calls++; return 0; +} +static int atcphy_configure(struct apple_atcphy *atcphy, enum atcphy_mode mode) { + /* ACTUAL_CONFIGURE_ENTRY */ + struct apple_atcphy *a = atcphy; + configure_calls++; + assert(!a->t8103_dummy_pending); + if (!configure_ret) { a->mode = mode; a->mode_dirty = false; } + else a->mode_dirty = true; + return configure_ret; +} +static void reset(enum atcphy_generation generation, bool t8140) { + memset(memory, 0, sizeof(memory)); + hw = (struct atcphy_hw){ .gen = generation, .has_t8140_dp = t8140 }; + p = (struct apple_atcphy){ .hw = &hw, .regs = {memory, memory}, + .mode = APPLE_ATCPHY_MODE_OFF, .typec_mode = APPLE_ATCPHY_MODE_USB3, + .ss_tunables = true }; + const char *const compatibles[] = {generation == ATCPHY_GENERATION_T8103 ? "apple,t8103" : "apple,t8122"}; + const struct device_node root = {compatibles, 1}; + p.t8103_usb3_workaround = atcphy_has_t8103_usb3_workaround(&hw, &root); + phy.data = &p; + polls = sleeps = sets = clears = muxes = bist_old = bist_new = 0; + configure_calls = restore_calls = unlocks = warnings = 0; + dummy_muxes = pending_seen_at_unlock = 0; + usb2_power_off_calls = 0; + ack_on = 1; + check_ret = bist_ret = unlock_ret = configure_ret = 0; + concurrent_override_change = false; +} +static void reset_machine(enum atcphy_generation generation, const char *machine) { + reset(generation, false); + const char *const compatibles[] = {"apple,synthetic-board", machine}; + const struct device_node root = {compatibles, 2}; + p.t8103_usb3_workaround = atcphy_has_t8103_usb3_workaround(&hw, &root); +} +''' + +TESTS = r''' +static unsigned tests; +#define TEST(name) do { name(); tests++; } while (0) +static void live_retry_last(void) { + reset(ATCPHY_GENERATION_T8103,false); ack_on=10; + writel(BIT(7),memory+PIPEHANDLER_LOCK_REQ); + assert(!atcphy_pipehandler_lock_live(&p)); + assert(polls==10 && sleeps==9 && sets==10 && clears==9); + assert(readl(memory+PIPEHANDLER_LOCK_REQ)==(BIT(7)|PIPEHANDLER_LOCK_EN)); +} +static void live_retry_exhausted(void) { + reset(ATCPHY_GENERATION_T8103,false); ack_on=0; + assert(atcphy_pipehandler_lock_live(&p)==-ETIMEDOUT); + assert(polls==10 && sleeps==9 && sets==10 && clears==10); +} +static void request_is_not_ack(void) { + reset(ATCPHY_GENERATION_T8103,false); ack_on=2; + writel(PIPEHANDLER_LOCK_EN,memory+PIPEHANDLER_LOCK_REQ); + assert(!atcphy_pipehandler_lock_live(&p)); + assert(polls==2 && sleeps==1 && sets==1 && clears==1); +} +static void requested_ack_no_writes(void) { + reset(ATCPHY_GENERATION_T8103,false); + writel(PIPEHANDLER_LOCK_EN,memory+PIPEHANDLER_LOCK_REQ); + assert(!atcphy_pipehandler_lock_live(&p)); assert(polls==1 && !sets && !clears && !sleeps); +} +static void later_ack_semantics_unchanged(void) { + for(unsigned neo=0;neo<2;neo++) { + reset(ATCPHY_GENERATION_T8122,neo); ack_on=0; + writel(PIPEHANDLER_LOCK_EN,memory+PIPEHANDLER_LOCK_REQ); + assert(atcphy_pipehandler_lock_live(&p)==-ETIMEDOUT); + assert(polls==1 && !sleeps && !sets && clears==1); + } +} +static void usb4_and_dead_parking_single_poll(void) { + reset(ATCPHY_GENERATION_T8103,false); ack_on=0; + assert(!atcphy_configure_pipehandler_usb4(&p)); assert(polls==1 && !sleeps); + reset(ATCPHY_GENERATION_T8103,false); ack_on=0; + p.pipe_state=ATCPHY_PIPEHANDLER_STATE_USB4; + assert(!atcphy_park_pipehandler(&p,true)); assert(polls==1 && !sleeps); +} +static void early_host_and_device(void) { + for(unsigned host=0;host<2;host++) { + reset(ATCPHY_GENERATION_T8103,false); + assert(!atcphy_usb3_set_mode(&phy,host?PHY_MODE_USB_HOST:PHY_MODE_USB_DEVICE,0)); + assert(p.mode==APPLE_ATCPHY_MODE_USB3 && p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + assert(configure_calls==1 && muxes==1 && !polls && !bist_old && !bist_new); + assert(p.host_active==host && !p.pipe_pending); + assert(!atcphy_usb3_set_mode(&phy,host?PHY_MODE_USB_HOST:PHY_MODE_USB_DEVICE,0)); + assert(configure_calls==1 && muxes==1); + } +} +static void early_admission_and_failure(void) { + reset(ATCPHY_GENERATION_T8103,false);p.ss_tunables=false; + assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)==-EOPNOTSUPP);assert(!configure_calls&&!muxes); + reset(ATCPHY_GENERATION_T8103,false);p.dp_only=true; + assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)==-EOPNOTSUPP);assert(!configure_calls&&!muxes); + reset(ATCPHY_GENERATION_T8103,false);configure_ret=-EIO; + assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)==-EIO);assert(configure_calls==1&&!muxes); +} +static void later_off_and_bist_unchanged(void) { + for(unsigned neo=0;neo<2;neo++) { + reset(ATCPHY_GENERATION_T8122,neo); + assert(!atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0));assert(p.mode==APPLE_ATCPHY_MODE_OFF&&!configure_calls&&!muxes); + p.mode=APPLE_ATCPHY_MODE_USB3; + assert(!atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0));assert(bist_new==1&&!bist_old&&polls==1); + } +} +static void success_only_all_generations(void) { + for(unsigned gen=0;gen<2;gen++)for(unsigned mode=0;mode<2;mode++) { + reset(gen?ATCPHY_GENERATION_T8122:ATCPHY_GENERATION_T8103,false); + p.mode=mode?APPLE_ATCPHY_MODE_USB4:APPLE_ATCPHY_MODE_USB3;check_ret=-EIO; + assert(atcphy_configure_pipehandler(&p,true)==-EIO); + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY&&!muxes); + check_ret=0;assert(!atcphy_configure_pipehandler(&p,true)); + assert(p.pipe_state==atcphy_modes[p.mode].pipehandler_state&&!p.pipe_pending); + } +} +static void setup_legacy_usb3(void) { + reset(ATCPHY_GENERATION_T8103,false);p.mode=APPLE_ATCPHY_MODE_USB3;p.pipe_state=ATCPHY_PIPEHANDLER_STATE_USB3; +} +static void live_lock_failure_restores_masks(void) { + setup_legacy_usb3();ack_on=0;concurrent_override_change=true; + writel(BIT(7),memory+PIPEHANDLER_OVERRIDE); + writel(BIT(7)|PIPEHANDLER_OVERRIDE_VAL_RXDETECT0,memory+PIPEHANDLER_OVERRIDE_VALUES); + assert(atcphy_park_live_t8103_pipehandler(&p)==-ETIMEDOUT); + assert(polls==10&&sleeps==9&&!muxes&&!unlocks); + assert(readl(memory+PIPEHANDLER_OVERRIDE)==(BIT(7)|BIT(9))); + assert(readl(memory+PIPEHANDLER_OVERRIDE_VALUES)==(BIT(7)|BIT(9)|PIPEHANDLER_OVERRIDE_VAL_RXDETECT0)); + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3&&!p.t8103_dummy_pending&&!p.t8103_usb3_retry&&!p.mode_dirty); +} +static void live_park_then_pending_completion(void) { + setup_legacy_usb3();unlock_ret=-ETIMEDOUT; + assert(atcphy_park_live_t8103_pipehandler(&p)==-ETIMEDOUT); + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY&&p.t8103_dummy_pending&&p.t8103_usb3_retry); + assert(muxes==1&&polls==1&&pending_seen_at_unlock==1); + unsigned savedpolls=polls; + assert(atcphy_park_pipehandler(&p,true)==-ETIMEDOUT);assert(polls==savedpolls&&muxes==1&&unlocks==2); + unlock_ret=0;assert(!atcphy_park_pipehandler(&p,true)); + assert(!p.t8103_dummy_pending&&p.t8103_usb3_retry&&unlocks==3&&polls==savedpolls); + assert(readl(memory+PIPEHANDLER_NONSELECTED_OVERRIDE)&PIPEHANDLER_NATIVE_RESET); +} +static void legacy_live_dispatch_and_generation(void) { + setup_legacy_usb3();ack_on=0; + assert(atcphy_park_pipehandler(&p,false)==-ETIMEDOUT);assert(polls==10&&!muxes); + reset(ATCPHY_GENERATION_T8122,true);p.fixed_usb2=true;p.host_active=true;p.pipe_pending=true;p.pipe_state=ATCPHY_PIPEHANDLER_STATE_USB3;ack_on=0; + assert(atcphy_park_pipehandler(&p,false)==-ETIMEDOUT);assert(polls==1&&!sleeps&&!muxes&&p.mode_dirty); + assert(atcphy_park_live_t8103_pipehandler(&p)==-EOPNOTSUPP); +} +static void pending_blocks_lane_and_backend(void) { + setup_legacy_usb3();p.t8103_dummy_pending=true; + assert(atcphy_configure(&p,APPLE_ATCPHY_MODE_DP)==-EBUSY);assert(!configure_calls); + assert(atcphy_configure_pipehandler(&p,true)==-EBUSY);assert(!muxes); +} +static void canceled_lane_change_reconnects(void) { + setup_legacy_usb3();unlock_ret=-ETIMEDOUT; + assert(atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_DP)==-ETIMEDOUT); + assert(p.mode==APPLE_ATCPHY_MODE_USB3&&p.t8103_dummy_pending&&p.t8103_usb3_retry&&!configure_calls); + assert(!atcphy_mux_in_mode(&p,APPLE_ATCPHY_MODE_USB3));assert(!atcphy_mux_is_noop(&p,APPLE_ATCPHY_MODE_USB3)); + unlock_ret=0;assert(!atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_USB3)); + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3&&!p.t8103_dummy_pending&&!p.t8103_usb3_retry); + assert(polls==1&&dummy_muxes==1&&muxes==2&&!configure_calls&&!bist_old&&!bist_new); +} +static void role_finishes_pending(void) { + setup_legacy_usb3();unlock_ret=-ETIMEDOUT;assert(atcphy_park_live_t8103_pipehandler(&p)==-ETIMEDOUT); + assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)==-ETIMEDOUT);assert(muxes==1); + unlock_ret=0;assert(!atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)); + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3&&!p.t8103_usb3_retry&&!p.t8103_dummy_pending&&muxes==2); +} +static void target_failure_preserves_reconnect(void) { + setup_legacy_usb3();configure_ret=-EIO; + assert(atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_DP)==-EIO); + assert(p.t8103_usb3_retry&&!p.t8103_dummy_pending&&p.mode_dirty); + configure_ret=0;assert(!atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_USB3)); + assert(p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3&&!p.t8103_usb3_retry&&!p.mode_dirty); +} +static void nonusb_commit_clears_retry(void) { + setup_legacy_usb3();assert(!atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_DP)); + assert(p.mode==APPLE_ATCPHY_MODE_DP&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY&&!p.t8103_usb3_retry&&!p.t8103_dummy_pending); +} +static void dp_assignment_d_retains_usb3(void) { + setup_legacy_usb3();assert(!atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_USB3_DP)); + assert(p.mode==APPLE_ATCPHY_MODE_USB3_DP&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3&&!polls&&!muxes&&!warnings); +} +static void modern_safe_refusal_retained(void) { + setup_legacy_usb3();assert(atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_OFF)==-EBUSY);assert(!polls&&!muxes&&!configure_calls); +} +static void fixed_hub_host_dispatch(void) { + for(unsigned gen=0;gen<2;gen++) { + reset(gen?ATCPHY_GENERATION_T8122:ATCPHY_GENERATION_T8103,false);p.fixed_usb2=true;p.host_active=true; + assert(!atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_USB3)); + assert(p.mode==APPLE_ATCPHY_MODE_USB3&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); + assert(bist_new==gen&&!bist_old&&configure_calls==1&&muxes==1); + } +} +static void reset_release_retries_before_writes(void) { + setup_legacy_usb3();unlock_ret=-ETIMEDOUT;assert(atcphy_park_live_t8103_pipehandler(&p)==-ETIMEDOUT); + u32 aon=BIT(14)|PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN;writel(aon,memory+PIPEHANDLER_AON_GEN); + assert(atcphy_dwc3_reset_deassert(&p.rcdev,0)==-ETIMEDOUT);assert(readl(memory+PIPEHANDLER_AON_GEN)==aon); + unlock_ret=0;assert(!atcphy_dwc3_reset_deassert(&p.rcdev,0));assert(!p.t8103_dummy_pending&&p.t8103_usb3_retry); + assert(readl(memory+PIPEHANDLER_AON_GEN)==((aon&~PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN)|PIPEHANDLER_AON_GEN_DWC3_RESET_N)); + assert(!atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0));assert(!p.t8103_usb3_retry&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_USB3); +} +static void reset_assert_failure_and_stop(void) { + setup_legacy_usb3();unlock_ret=-ETIMEDOUT;assert(atcphy_park_live_t8103_pipehandler(&p)==-ETIMEDOUT); + u32 aon=BIT(14)|PIPEHANDLER_AON_GEN_DWC3_RESET_N;writel(aon,memory+PIPEHANDLER_AON_GEN); + assert(atcphy_dwc3_reset_assert(&p.rcdev,0)==-ETIMEDOUT);assert(readl(memory+PIPEHANDLER_AON_GEN)==aon&&!usb2_power_off_calls); + unlock_ret=0;assert(!atcphy_dwc3_reset_assert(&p.rcdev,0));assert(!p.t8103_dummy_pending&&!p.t8103_usb3_retry&&usb2_power_off_calls==1); +} +static void power_off_preserves_dp_and_finishes_pending(void) { + setup_legacy_usb3();unlock_ret=-ETIMEDOUT;assert(atcphy_park_live_t8103_pipehandler(&p)==-ETIMEDOUT);p.mode=APPLE_ATCPHY_MODE_USB3_DP; + assert(atcphy_usb3_power_off(&phy)==-ETIMEDOUT);assert(!configure_calls); + unlock_ret=0;assert(!atcphy_usb3_power_off(&phy));assert(p.mode==APPLE_ATCPHY_MODE_USB3_DP&&!configure_calls&&!p.t8103_usb3_retry&&!p.t8103_dummy_pending); +} +static void setup_known_reset_clears_obligations(void) { + setup_legacy_usb3();p.t8103_dummy_pending=p.t8103_usb3_retry=true;p.pipe_pending=true; + atcphy_setup_pipehandler(&p);assert(!p.t8103_dummy_pending&&!p.t8103_usb3_retry&&!p.pipe_pending&&p.pipe_state==ATCPHY_PIPEHANDLER_STATE_DUMMY); +} +static void target_mode_table_and_dirty(void) { + reset(ATCPHY_GENERATION_T8103,false);struct typec_mux_state state={.mode=TYPEC_DP_STATE_D};struct typec_altmode alt={.svid=USB_TYPEC_DP_SID};state.alt=&alt; + assert(atcphy_typec_target_mode(&p,&state)==APPLE_ATCPHY_MODE_USB3_DP);state.mode=TYPEC_DP_STATE_C; + assert(atcphy_typec_target_mode(&p,&state)==APPLE_ATCPHY_MODE_DP); + p.mode=APPLE_ATCPHY_MODE_USB3;p.mode_dirty=true;assert(atcphy_usb3_route(&p,true)==-EBUSY);assert(!muxes&&!configure_calls); + p.mode_dirty=false;p.usb_off_pending=true;assert(atcphy_usb3_set_mode(&phy,PHY_MODE_USB_HOST,0)==-EBUSY); +} +static void live_park_finishes_generic_pending(void) { + setup_legacy_usb3(); p.pipe_pending=true; + assert(!atcphy_park_pipehandler(&p,false)); + assert(!p.pipe_pending && !p.t8103_dummy_pending && p.t8103_usb3_retry); + assert(!atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_DP)); + assert(p.mode==APPLE_ATCPHY_MODE_DP&&!p.t8103_usb3_retry); +} +static void fixed_hub_nonusb3_commit_discharges_retry(void) { + setup_legacy_usb3();p.fixed_usb2=true;p.host_active=true; + assert(!atcphy_mux_switch(&p,APPLE_ATCPHY_MODE_DP)); + assert(!p.t8103_usb3_retry&&!p.t8103_dummy_pending&&!p.pipe_pending); + assert(atcphy_mux_in_mode(&p,APPLE_ATCPHY_MODE_DP)); + assert(atcphy_mux_is_noop(&p,APPLE_ATCPHY_MODE_DP)); +} +static void actual_dt_fallback_generation_scope(void) { + for(unsigned i=0;iusb_off_pending") + stubs = STUBS.replace("/* ACTUAL_CONFIGURE_ENTRY */", configure[guard_start:guard_end]) + stubs = stubs.replace("/* ACTUAL_ADMISSION */", + block(source, "static bool atcphy_has_t8103_usb3_workaround(")) + scope_c += "static const char *const scope_machines[] = {" + ",".join( + json.dumps(row["machine_compatible"]) for row in scope) + "};\n" + return PRELUDE + defines + "\n" + "\n".join(usb_defines) + "\n" + types + "\n" + table + stubs + functions + scope_c + TESTS + + +MUTATIONS = {'no-live-retry': ('atcphy_pipehandler_lock_live', '? 10 : 1;', '? 1 : 1;'), + 'retry-later-generations': ('atcphy_pipehandler_lock_live', '? 10 : 1;', '? 10 : 10;'), + 'sleep-after-last': ('atcphy_pipehandler_lock_live', 'if (attempt + 1 < attempts)', 'if (true)'), + 'missing-ack-check': ('atcphy_pipehandler_lock', + 'if (requested & PIPEHANDLER_LOCK_EN) {', + 'if (requested & PIPEHANDLER_LOCK_EN) return 0;\n' + '\tif (requested & PIPEHANDLER_LOCK_EN) {'), + 'usb4-live-retry': ('atcphy_configure_pipehandler_usb4', + 'atcphy_pipehandler_lock(atcphy)', + 'atcphy_pipehandler_lock_live(atcphy)'), + 'dead-park-live-retry': ('atcphy_configure_pipehandler_dummy', + 'atcphy_pipehandler_lock(atcphy)', + 'atcphy_pipehandler_lock_live(atcphy)'), + 'early-missing-tunables': ('atcphy_usb3_route', + 'if (atcphy->dp_only || !atcphy->ss_tunables)', + 'if (atcphy->dp_only)'), + 'early-missing-dp-only': ('atcphy_usb3_route', + 'if (atcphy->dp_only || !atcphy->ss_tunables)', + 'if (!atcphy->ss_tunables)'), + 'early-all-generations': ('atcphy_usb3_route', + 'atcphy->t8103_usb3_workaround &&', + 'true &&'), + 'early-ignores-typec-request': ('atcphy_usb3_route', + 'atcphy->typec_mode != APPLE_ATCPHY_MODE_USB3', + 'false'), + 'early-role-not-remembered': ('atcphy_usb3_route', + 'atcphy->t8103_usb3_retry = true;', + '(void)0;'), + 'early-configure-error-loses-obligation': ('atcphy_usb3_route', + 'atcphy->t8103_usb3_retry = true;', + 'if (atcphy->typec_mode != APPLE_ATCPHY_MODE_USB3) atcphy->t8103_usb3_retry = true;'), + 'same-mode-nonusb3-keeps-role-obligation': ('atcphy_mux_switch', + '\t\t\tatcphy->t8103_usb3_retry = false;\n\t\t}\n\t\t/* Retry a failed PIPE setup', + '\t\t\t(void)0;\n\t\t}\n\t\t/* Retry a failed PIPE setup'), + 'admission-ignores-machine': ('bool atcphy_has_t8103_usb3_workaround', + 'of_device_is_compatible(root, "apple,t8103")', + 'true'), + 'admission-ignores-generation': ('bool atcphy_has_t8103_usb3_workaround', + 'hw->gen == ATCPHY_GENERATION_T8103 &&', + 'true &&'), + 'bist-restored-t8103': ('atcphy_configure_pipehandler', + 'host && !atcphy->t8103_usb3_workaround', + 'host'), + 'bist-skipped-later': ('atcphy_configure_pipehandler', + 'host && !atcphy->t8103_usb3_workaround', + 'false'), + 'bist-skipped-fallback-machines': ('atcphy_configure_pipehandler', + 'host && !atcphy->t8103_usb3_workaround', + 'host && atcphy->hw->gen != ATCPHY_GENERATION_T8103'), + 'failed-state-published': ('atcphy_configure_pipehandler', 'if (!ret) {', 'if (true) {'), + 'live-generation-gate-removed': ('atcphy_park_live_t8103_pipehandler', + 'if (!atcphy->t8103_usb3_workaround)', + 'if (false)'), + 'live-park-dispatch-all-fallbacks': ('atcphy_park_pipehandler', + '!stopped && atcphy->t8103_usb3_workaround &&', + '!stopped && atcphy->hw->gen == ATCPHY_GENERATION_T8103 &&'), + 'mux-live-park-all-fallbacks': ('atcphy_mux_switch', + 'if (atcphy->t8103_usb3_workaround)', + 'if (atcphy->hw->gen == ATCPHY_GENERATION_T8103)'), + 'no-values-restore': ('atcphy_park_live_t8103_pipehandler', + 'mask32(pipe + PIPEHANDLER_OVERRIDE_VALUES, value_mask,\n' + '\t\t\t values & value_mask);', + '(void)values;'), + 'no-overrides-restore': ('atcphy_park_live_t8103_pipehandler', + 'mask32(pipe + PIPEHANDLER_OVERRIDE, override_mask,\n' + '\t\t\t overrides & override_mask);', + '(void)overrides;'), + 'clobber-unrelated-overrides': ('atcphy_park_live_t8103_pipehandler', + 'mask32(pipe + PIPEHANDLER_OVERRIDE, override_mask,\n' + '\t\t\t overrides & override_mask);', + 'writel(overrides, pipe + PIPEHANDLER_OVERRIDE);'), + 'no-physical-mux-state': ('atcphy_park_live_t8103_pipehandler', + 'atcphy->pipe_state = ATCPHY_PIPEHANDLER_STATE_DUMMY;', + '(void)0;'), + 'no-pending-after-move': ('atcphy_park_live_t8103_pipehandler', + 'atcphy->t8103_dummy_pending = true;', + '(void)0;'), + 'relock-pending': ('atcphy_park_live_t8103_pipehandler', + 'if (!atcphy->t8103_dummy_pending)', + 'if (true)'), + 'no-reconnect-obligation': ('atcphy_park_live_t8103_pipehandler', + 'atcphy->t8103_usb3_retry = true;', + '(void)0;'), + 'pending-cleared-before-ack': ('atcphy_park_live_t8103_pipehandler', + 'ret = atcphy_pipehandler_unlock(atcphy);', + 'atcphy->t8103_dummy_pending = false;\n' + '\tret = atcphy_pipehandler_unlock(atcphy);'), + 'park-bypasses-pending': ('atcphy_park_pipehandler', + 'if (atcphy->t8103_dummy_pending ||', + 'if (false ||'), + 'lanes-ignore-pending': ('atcphy_configure', 'if (atcphy->t8103_dummy_pending)', 'if (false)'), + 'backend-ignores-pending': ('atcphy_configure_pipehandler', + 'if (atcphy->t8103_dummy_pending)', + 'if (false)'), + 'role-ignores-pending': ('atcphy_usb3_route', 'if (atcphy->t8103_dummy_pending)', 'if (false)'), + 'mux-ignores-pending': ('atcphy_mux_switch', 'if (atcphy->t8103_dummy_pending)', 'if (false)'), + 'canceled-not-reconnected': ('atcphy_mux_switch', + 'if (atcphy_modes[target_mode].pipehandler_state == ATCPHY_PIPEHANDLER_STATE_USB3)\n\t\t\t\treturn atcphy_configure_pipehandler', + 'if (false)\n\t\t\t\treturn atcphy_configure_pipehandler'), + 'retry-cleared-at-unlock': ('atcphy_park_live_t8103_pipehandler', + 'atcphy->t8103_dummy_pending = false;', + 'atcphy->t8103_dummy_pending = false; atcphy->t8103_usb3_retry = ' + 'false;'), + 'retry-not-cleared-after-usb3': ('atcphy_configure_pipehandler', + 'atcphy->t8103_usb3_retry = false;', + '(void)0;'), + 'retry-not-cleared-after-nonusb3': ('atcphy_mux_switch', + 'return atcphy_configure_pipehandler(atcphy, false);\n' + '\t\tatcphy->t8103_usb3_retry = false;', + 'return atcphy_configure_pipehandler(atcphy, false);\n' + '\t\t(void)0;'), + 'usb3-dp-wrongly-parked': ('atcphy_mux_switch', + 'atcphy_modes[target_mode].pipehandler_state == ' + 'ATCPHY_PIPEHANDLER_STATE_DUMMY) {', + 'true) {'), + 'reset-release-ignores-pending': ('atcphy_dwc3_reset_deassert', + 'if (atcphy->t8103_dummy_pending)', + 'if (false)'), + 'reset-release-permanent-ebusy': ('atcphy_dwc3_reset_deassert', + 'if (atcphy->t8103_dummy_pending) {', + 'if (atcphy->t8103_dummy_pending) return -EBUSY;\n' + '\tif (atcphy->t8103_dummy_pending) {'), + 'reset-release-writes-early': ('atcphy_dwc3_reset_deassert', + 'if (atcphy->t8103_dummy_pending) {', + 'clear32(atcphy->regs.pipehandler + PIPEHANDLER_AON_GEN, ' + 'PIPEHANDLER_AON_GEN_DWC3_FORCE_CLAMP_EN);\n' + '\tif (atcphy->t8103_dummy_pending) {'), + 'reset-no-retry-clear': ('atcphy_dwc3_reset_assert', + 'atcphy->t8103_usb3_retry = false;', + '(void)0;'), + 'poweroff-no-retry-clear': ('atcphy_usb3_power_off', + 'atcphy->t8103_usb3_retry = false;', + '(void)0;'), + 'known-reset-no-retry-clear': ('void atcphy_setup_pipehandler', + 'atcphy->t8103_usb3_retry = false;', + '(void)0;'), + 'live-completion-leaves-generic-pending': ('atcphy_park_live_t8103_pipehandler', + 'atcphy->pipe_pending = false;', + '(void)0;'), + 'fixedhub-nonusb3-keeps-retry': ('atcphy_mux_switch', + 'if (!ret && pipe_state != ATCPHY_PIPEHANDLER_STATE_USB3)\n' + '\t\t\tatcphy->t8103_usb3_retry = false;', + '(void)0;'), + 'outer-noop-ignores-new-obligations': ('bool atcphy_mux_in_mode', + 'atcphy->t8103_dummy_pending || atcphy->t8103_usb3_retry', + 'false')} + + +def run(code, cc, directory, name): + cfile = directory / (name + ".c") + binary = directory / name + cfile.write_text(code) + build = subprocess.run([cc, "-std=gnu11", "-O1", "-g", "-fsanitize=address,undefined", + "-fno-omit-frame-pointer", "-Wall", "-Wextra", + "-Wno-unused-parameter", "-Wno-unused-function", + str(cfile), "-o", str(binary)], text=True, capture_output=True) + if build.returncode: + raise RuntimeError("compile failed for " + name + ":\n" + build.stderr) + env = dict(os.environ, ASAN_OPTIONS="detect_leaks=1:abort_on_error=1") + return subprocess.run([str(binary)], text=True, capture_output=True, env=env) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, + default=Path(__file__).resolve().parents[2] / "drivers/phy/apple/atc.c") + parser.add_argument("--cc", default="clang") + parser.add_argument("--mutations", action="store_true") + parser.add_argument("--out", type=Path, help="retain generated C, binaries and receipt") + args = parser.parse_args() + code = harness(args.source.read_text()) + results = {} + context = tempfile.TemporaryDirectory(prefix="atc-usb3-tests-") if args.out is None else None + directory = Path(context.name) if context else args.out + directory.mkdir(parents=True, exist_ok=True) + result = run(code, args.cc, directory, "actual") + (directory / "actual.stdout").write_text(result.stdout) + (directory / "actual.stderr").write_text(result.stderr) + if result.returncode: + raise SystemExit(result.stderr or result.stdout) + print(result.stdout, end="") + if args.mutations: + for name, mutation in MUTATIONS.items(): + function, before, after = mutation + declaration = function if " " in function else "int " + function + old_scope = block(code, "static " + declaration + "(") + if old_scope.count(before) != 1: + raise RuntimeError(name + ": expected one mutation site") + changed = old_scope.replace(before, after, 1) + mutated = code.replace(old_scope, changed, 1) + result = run(mutated, args.cc, directory, name) + (directory / (name+".stdout")).write_text(result.stdout) + (directory / (name+".stderr")).write_text(result.stderr) + if result.returncode == 0: + raise RuntimeError("mutation survived: " + name) + if "Assertion" not in result.stderr and "runtime error:" not in result.stderr: + raise RuntimeError("mutant failed without an assertion or sanitizer diagnostic: " + name) + results[name] = {"class": "KILLED", "returncode": result.returncode} + print(json.dumps({"mutations": results}, sort_keys=True)) + after = run(code, args.cc, directory, "restored-control") + (directory/"restored-control.stdout").write_text(after.stdout) + (directory/"restored-control.stderr").write_text(after.stderr) + if after.returncode: + raise RuntimeError("post-mutation control failed: " + after.stderr) + receipt = {"source_sha256": hashlib.sha256(args.source.read_bytes()).hexdigest(), + "harness_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "generated_sha256": hashlib.sha256(code.encode()).hexdigest(), + "generation_scope": generation_scope(args.source.read_text(),Path(__file__).resolve().parents[2]), + "control_stdout": (directory/"actual.stdout").read_text(), + "after_control_stdout": after.stdout, + "mutations": results, + "scope": "actual extracted driver bodies; MMIO, polling, BIST, lane configuration and power restoration boundary models"} + (directory/"receipt.json").write_text(json.dumps(receipt, indent=2)+"\n") + if context: context.cleanup() + + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/avd-codec-tail-tests.py b/tools/asahi/avd-codec-tail-tests.py new file mode 100755 index 0000000000000..31616efc2651c --- /dev/null +++ b/tools/asahi/avd-codec-tail-tests.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise the actual AVD capability-tail block under ASan/UBSan. + +Firmware loading, MMIO and device logging are not exercised. The firmware and +variant structures, kernel BIT helper and bounded strnstr are host stand-ins. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +SOURCE = "drivers/media/platform/apple/avd/avd-drv.c" +HEADER = "drivers/media/platform/apple/avd/avd.h" + + +def read(path, ref): + if ref: + return subprocess.check_output( + ["git", "-C", str(ROOT), "show", f"{ref}:{path}"], text=True + ) + return (ROOT / path).read_text() + + +def build_source(body, defines): + return r''' +#include +#include +#include +#define BIT(n) (1u << (n)) +#define dev_info(...) ((void)0) +''' + defines + r''' +struct firmware { const unsigned char *data; size_t size; }; +struct variant { unsigned int capabilities; }; +struct avd_dev { + struct firmware *fw; + const struct variant *variant; + unsigned int capabilities; +}; +static char *strnstr(const char *s, const char *needle, size_t n) +{ + size_t len = strlen(needle); + for (size_t i = 0; i + len <= n && s[i]; i++) + if (!memcmp(s + i, needle, len)) + return (char *)s + i; + return NULL; +} +static unsigned int probe_tail(struct avd_dev *avd) +{ + const char *codec_info = NULL; +''' + body + r''' + return avd->capabilities; +} +int main(int argc, char **argv) +{ + if (argc != 5) + return 2; + size_t size = strtoul(argv[1], NULL, 0); + unsigned char *data = calloc(size ? size : 1, 1); + if (!data) + return 2; + struct firmware fw = {data, size}; + struct variant variant = {strtoul(argv[3], NULL, 0)}; + struct avd_dev avd = {&fw, &variant, 0}; + size_t len = strlen(argv[2]); + if (size >= AVD_CODEC_INFO_MAX_SIZE && len <= AVD_CODEC_INFO_MAX_SIZE) + memcpy(data + size - AVD_CODEC_INFO_MAX_SIZE, argv[2], len); + unsigned int actual = probe_tail(&avd); + unsigned int expected = strtoul(argv[4], NULL, 0); + printf("size=%zu variant=%u capabilities=%u expected=%u\n", + size, variant.capabilities, actual, expected); + free(data); + return actual == expected ? 0 : 1; +} +''' + + +def run_variant(out, name, body, defines, cases): + dest = out / name + dest.mkdir() + source = dest / "actual-tail.c" + source.write_text(build_source(body, defines)) + cmd = shlex.split(os.environ.get("CC", "cc")) + [ + # Match scripts/Makefile.warn for the existing firmware u8*/char* use. + "-std=gnu11", "-g", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-pointer-sign", + "-fsanitize=address,undefined", "-fno-omit-frame-pointer", + str(source), "-o", str(dest / "actual-tail"), + ] + build = subprocess.run(cmd, capture_output=True, text=True) + (dest / "build.log").write_text(build.stdout + build.stderr) + rows = [] + if build.returncode == 0: + for index, (size, tail, mask, expected) in enumerate(cases): + result = subprocess.run( + [str(dest / "actual-tail"), str(size), tail, str(mask), str(expected)], + capture_output=True, text=True, + ) + (dest / f"case-{index:02d}.log").write_text(result.stdout + result.stderr) + rows.append({"size": size, "tail": tail, "variant": mask, + "expected": expected, "returncode": result.returncode, + "sanitizer_diagnostic": bool(re.search( + r"AddressSanitizer|runtime error:", result.stderr))}) + return {"name": name, "build_command": cmd, + "build_returncode": build.returncode, "cases": rows, + "failed_cases": sum(x["returncode"] != 0 or x["sanitizer_diagnostic"] + for x in rows)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ref", help="extract production code from this Git ref") + parser.add_argument("--output", type=Path, required=True, + help="new evidence directory (must not already exist)") + parser.add_argument("--mutations", action="store_true") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + source, header = read(SOURCE, args.ref), read(HEADER, args.ref) + start = source.index("\t/* Determine firmware and HW variant capabilities */") + end = source.index("\n\tret = dma_set_mask_and_coherent", start) + body = source[start:end] + wanted = ["AVD_CODEC_INFO_MAX_SIZE", "AVD_CAPABILITY_AV1", "AVD_CAPABILITY_H264", + "AVD_CAPABILITY_HEVC", "AVD_CAPABILITY_VP9"] + defines = "" + for macro in wanted: + matches = re.findall(r"^#define\s+" + macro + r"\s+[^\n]+$", + source + "\n" + header, re.M) + if len(matches) != 1: + raise ValueError(f"expected exactly one definition of {macro}") + defines += matches[0] + "\n" + # Codec bits are read from the header above; the expected masks below track + # the current public ABI (HEVC/H264/VP9/AV1 = bits 0/1/2/3). + all_codecs = "CODECS:av1 h264 h265 vp9" + cases = [(n, "legacy firmware", 15, 15) for n in (0, 1, 7, 63, 64, 128)] + cases += [(n, all_codecs, 15, 15) for n in (64, 128)] + cases += [(64, "CODECS:" + codec, 15, bit) + for codec, bit in (("av1", 8), ("h264", 2), ("h265", 1), ("vp9", 4))] + cases += [(128, all_codecs, 3, 3), (128, all_codecs, 0, 0), + (64, "CODECS:vvc", 15, 0), (64, "CODECS:", 15, 0), + (64, "CODECS:" + "x" * 57, 15, 0), + (64, "CODECSx:av1", 5, 5), (128, "legacy", 0, 0), + (128, "legacy", 5, 5)] + variants = [("control-before", body)] + if args.mutations: + anchor = "\tif (avd->fw->size >= AVD_CODEC_INFO_MAX_SIZE)\n" + if body.count(anchor) != 1: + raise ValueError("size-guard mutation did not match exactly once") + variants += [("no-size-guard", body.replace(anchor, "")), + ("no-variant-intersection", body.replace( + "\t\tunsigned int variant_caps = avd->variant->capabilities;\n", "" + ).replace("caps |= variant_caps & ", "caps |= ")), + ("no-legacy-fallback", body.replace( + "avd->capabilities = avd->variant->capabilities;", + "avd->capabilities = 0;"))] + variants.append(("control-after", body)) + results = [run_variant(args.output, name, text, defines, cases) + for name, text in variants] + ok = True + for result in results: + is_control = result["name"].startswith("control-") + valid = result["build_returncode"] == 0 and len(result["cases"]) == len(cases) + valid &= (result["failed_cases"] == 0) if is_control else (result["failed_cases"] > 0) + result["verdict"] = ("PASS" if is_control else "KILLED") if valid else "FAIL" + ok &= valid + compiler = subprocess.check_output( + shlex.split(os.environ.get("CC", "cc")) + ["--version"], text=True + ).splitlines()[0] + receipt = {"ref": args.ref, "compiler": compiler, + "source_sha256": hashlib.sha256(source.encode()).hexdigest(), + "body_sha256": hashlib.sha256(body.encode()).hexdigest(), + "defines_sha256": hashlib.sha256(defines.encode()).hexdigest(), + "cases_per_variant": len(cases), "results": results, "passed": ok} + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps({"passed": ok, "cases": len(cases), + "variants": [{k: x[k] for k in ("name", "failed_cases", "verdict")} + for x in results]})) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/asahi/dcp14-panel-tests.py b/tools/asahi/dcp14-panel-tests.py index 3d0d98a887566..0abcd1e89d3f9 100755 --- a/tools/asahi/dcp14-panel-tests.py +++ b/tools/asahi/dcp14-panel-tests.py @@ -114,10 +114,32 @@ height=2234;assert(dcp_v14_geometry(&m)==0&&m.panel_height==2234); height=2160;notch=0;assert(dcp_v14_geometry(&m)==-EINVAL); notch=74;width=3024;stride=width*4;height=1890;assert(dcp_v14_geometry(&m)==-EINVAL); - machine="apple,j514c";assert(IS_ERR(iomfb_v14_7_board(&dev))); + /* J514C is now admitted explicitly; its dimensions come from simplefb. */ + machine="apple,j514c"; + const struct dcp_v14_board *c14=iomfb_v14_7_board(&dev); + assert(!IS_ERR(c14)&&c14==&dcp_v14_board_j514c&&c14!=c); + assert(!strcmp(c14->name,"J514C")&&!strcmp(c14->debugfs,"dcp-j514c")); + assert(!strcmp(c14->firmware_uuid,"F493F03E-CC40-3D89-9A08-5F259BA2B246")); + assert(!strcmp(c14->handoff,"apple,t6031-handoff")&&c14->promotion&&c14->log_bringup); + assert(!c14->panel_width&&!c14->panel_height&&!c14->ctm_set&&!c14->ctm_get); + image_uuid=c14->firmware_uuid;assert(probe_gate(&dev,c14)==0); + image_uuid="90F849E1-B422-367E-B389-50246F8DEC47";assert(probe_gate(&dev,c14)==-ENODEV); + image_uuid=NULL;assert(probe_gate(&dev,c14)==-ENODEV); + image_uuid=c14->firmware_uuid;marker_value=0;assert(probe_gate(&dev,c14)==-ENODEV); + marker_value=2;assert(probe_gate(&dev,c14)==-ENODEV);marker_value=1; + struct apple_dcp_v14 m14={.dev=&dev,.board=c14}; + width=3024;height=1890;stride=width*4;notch=74; + assert(dcp_v14_geometry(&m14)==0&&m14.panel_width==3024&&m14.panel_height==1964); + assert(m14.notch_rows==74); + stride++;assert(dcp_v14_geometry(&m14)==-EINVAL);stride--; + notch=MAX_NOTCH_HEIGHT+1;assert(dcp_v14_geometry(&m14)==-EINVAL);notch=74; + width=0;stride=0;assert(dcp_v14_geometry(&m14)==-EINVAL); + dcp.compatible="apple,t8122-dcp";assert(IS_ERR(iomfb_v14_7_board(&dev))); + dcp.compatible="apple,t6031-dcp";machine="apple,j514s";assert(IS_ERR(iomfb_v14_7_board(&dev))); + machine="apple,j613";assert(IS_ERR(iomfb_v14_7_board(&dev))); machine="apple,j516m";assert(IS_ERR(iomfb_v14_7_board(&dev))); dcp.compatible="apple,t6030-dcp";machine="apple,j516s";assert(iomfb_v14_7_board(&dev)==&dcp_v14_board_t6030); - puts("current J615/J613/T6030/J516C admission + hidden/full notch, wrong geometry controls: PASS"); + puts("current J615/J613/T6030/J516C/J514C admission + hidden/full notch, wrong geometry controls: PASS"); #endif } ''' diff --git a/tools/asahi/g15-input-error-tests.py b/tools/asahi/g15-input-error-tests.py new file mode 100644 index 0000000000000..47971fccfa710 --- /dev/null +++ b/tools/asahi/g15-input-error-tests.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Actual G15 job dependency completion and G17 status/policy bodies on host services. + +DMA fence lookup, engine submission, queue-fence ordering and kernel mutexes are +models. No kernel scheduling, GPU access or firmware execution is exercised. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[2] + + +def extract(text, marker): + start = text.index(marker) + opening = text.index("{", start) + depth, end = 1, opening + 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PRELUDE = r''' +#![allow(dead_code)] +use std::sync::{Arc,Mutex as StdMutex,MutexGuard,atomic::{AtomicBool,AtomicI32,AtomicU32,Ordering}}; +#[derive(Copy,Clone,Debug,PartialEq,Eq)] struct Error(i32); +impl Error { fn from_errno(n:i32)->Self {assert!(n<0);Self(n)} fn to_errno(self)->i32{self.0} } +type Result = std::result::Result; +const EIO:Error=Error(-5);const ENODEV:Error=Error(-19);const ETIMEDOUT:Error=Error(-110); +const ECANCELED:Error=Error(-125);const EINVAL:Error=Error(-22);const ENOMEM:Error=Error(-12); +macro_rules! pr_warn { ($($arg:tt)*)=>{{ let _=format_args!($($arg)*); }} } +struct Mutex(StdMutex);impl Mutex{fn lock(&self)->MutexGuard<'_,T>{self.0.lock().unwrap()}} +struct Mirror{word:*mut i32} +struct VmStatus{error:AtomicI32,reported_error:AtomicI32,mirror:Mutex>} +impl VmStatus { + fn new(word:*mut i32)->Self{Self{error:AtomicI32::new(0),reported_error:AtomicI32::new(0),mirror:Mutex(StdMutex::new(Some(Mirror{word})))}} +__STATUS__ +} +mod g17 {use super::*;__POLICY__} +mod bindings {pub unsafe fn dma_fence_get_status(p:*const i32)->i32{unsafe{*p}}} +struct Fence(i32);impl Fence{fn raw(&self)->*const i32{&self.0}} +struct Vm{status:VmStatus} +struct Progress{signaled:AtomicBool} +struct Guard{quarantines:AtomicU32}impl Guard{fn quarantine(&self){self.quarantines.fetch_add(1,Ordering::SeqCst);}} +struct JobFence{vm:Arc,signals:AtomicU32,result:AtomicI32,saw_sticky:AtomicI32,saw_report:AtomicI32,saw_mirror:AtomicI32} +impl JobFence{ + fn signal(&self,result:Result){ + self.saw_sticky.store(self.vm.status.get(),Ordering::SeqCst); + self.saw_report.store(self.vm.status.reported_error(),Ordering::SeqCst); + let mirror=self.vm.status.mirror.lock(); + self.saw_mirror.store(unsafe{AtomicI32::from_ptr(mirror.as_ref().unwrap().word)}.load(Ordering::SeqCst),Ordering::SeqCst); + self.result.store(result.err().map_or(0,|e|e.to_errno()),Ordering::SeqCst); + self.signals.fetch_add(1,Ordering::SeqCst); + } +} +struct Shared{crashed:AtomicBool,submits:AtomicU32,submit_result:Result} +impl Shared{fn submit(&self,_:Arc)->Result{self.submits.fetch_add(1,Ordering::SeqCst);self.submit_result}} +struct Job{started:AtomicBool,vm:Arc,progress:Progress,_guard:Guard,fence:JobFence} +impl Job {__JOB__} +fn make_job(own:Option)->(Arc,Box){ + let mut mirror=Box::new(AtomicI32::new(0));let vm=Arc::new(Vm{status:VmStatus::new((&mut *mirror as *mut AtomicI32).cast())}); + if let Some(error)=own{vm.status.record(error);} + let job=Arc::new(Job{started:AtomicBool::new(false),vm:vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm,signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + (job,mirror) +} +fn shared(crashed:bool,result:Result)->Arc{Arc::new(Shared{crashed:AtomicBool::new(crashed),submits:AtomicU32::new(0),submit_result:result})} +fn n(n:&AtomicI32)->i32{n.load(Ordering::SeqCst)} +fn main(){ + let (unrelated,other_mirror)=make_job(None); + for input in [ECANCELED,EIO,ENODEV,ETIMEDOUT,EINVAL,ENOMEM]{ + let (job,mirror)=make_job(None);let device=shared(false,Ok(())); + job.start(&device,&[Fence(1),Fence(input.to_errno()),Fence(-1)]); + let expected=g17::inherited_error(input).to_errno(); + assert_eq!(device.submits.load(Ordering::SeqCst),0,"errored input must never submit"); + assert_eq!(job.fence.signals.load(Ordering::SeqCst),1,"dependency completion must signal once"); + assert_eq!(n(&job.fence.result),expected,"inherited error mapping"); + assert_eq!(n(&job.fence.saw_report),expected,"report must precede fence signal"); + assert_eq!(n(&job.fence.saw_mirror),expected,"mirror must precede fence signal"); + assert_eq!(n(&job.fence.saw_sticky),0,"healthy dependency consumer admission stays open"); + assert_eq!(job.vm.status.get(),0,"healthy dependency consumer admission stays open"); + assert_eq!(mirror.load(Ordering::SeqCst),expected); + assert_eq!(job._guard.quarantines.load(Ordering::SeqCst),0,"unpublished job must not quarantine"); + job.finish_dependency(EIO);job.complete(Err(EIO)); + assert_eq!(job.fence.signals.load(Ordering::SeqCst),1,"dependency completion must signal once"); + assert_eq!(job.vm.status.get(),0,"duplicate completion must not poison VM"); + // A later job on the same VM is admitted despite the reported client error. + let later=Arc::new(Job{started:AtomicBool::new(false),vm:job.vm.clone(),progress:Progress{signaled:AtomicBool::new(false)},_guard:Guard{quarantines:AtomicU32::new(0)},fence:JobFence{vm:job.vm.clone(),signals:AtomicU32::new(0),result:AtomicI32::new(0),saw_sticky:AtomicI32::new(0),saw_report:AtomicI32::new(0),saw_mirror:AtomicI32::new(0)}}); + later.start(&device,&[Fence(1)]);assert_eq!(device.submits.load(Ordering::SeqCst),1,"successful dependencies after report must submit"); + } + for own in [EIO,EINVAL]{for input in [ECANCELED,EIO,ENODEV,ETIMEDOUT,EINVAL]{ + let(job,_mirror)=make_job(Some(own));let device=shared(false,Ok(()));job.start(&device,&[Fence(input.to_errno())]); + assert_eq!(device.submits.load(Ordering::SeqCst),0);assert_eq!(n(&job.fence.result),input.to_errno(),"own-failed VM must retain original dependency errno"); + assert_eq!(job.vm.status.get(),own.to_errno(),"own-failed VM sticky error must remain"); + assert_eq!(n(&job.fence.saw_report),own.to_errno());assert_eq!(n(&job.fence.saw_mirror),own.to_errno()); + assert_eq!(job._guard.quarantines.load(Ordering::SeqCst),0); + }} + for inputs in [vec![],vec![Fence(1)],vec![Fence(1),Fence(1)]]{ + let(job,_mirror)=make_job(None);let device=shared(false,Ok(()));job.start(&device,&inputs); + assert_eq!(device.submits.load(Ordering::SeqCst),1,"successful dependencies must submit");assert_eq!(job.fence.signals.load(Ordering::SeqCst),0);assert_eq!(job.vm.status.get(),0); + } + // Preserve device-crash and submit-refusal sticky semantics. + for (crashed,result) in [(true,Ok(())),(false,Err(ENODEV)),(false,Err(EINVAL))]{ + let(job,_mirror)=make_job(None);let device=shared(crashed,result);job.start(&device,&[Fence(1)]); + let expected=if crashed||result==Err(ENODEV){EIO}else{EINVAL};assert_eq!(job.vm.status.get(),expected.to_errno());assert_eq!(n(&job.fence.saw_mirror),expected.to_errno()); + } + // An engine-owned fault still quarantines before completion. + let(job,_mirror)=make_job(None);job.complete(Err(EIO));assert_eq!(job._guard.quarantines.load(Ordering::SeqCst),1);assert_eq!(job.vm.status.get(),EIO.to_errno()); + assert_eq!(unrelated.vm.status.get(),0,"unrelated VM untouched");assert_eq!(unrelated.vm.status.reported_error(),0);assert_eq!(other_mirror.load(Ordering::SeqCst),0);assert_eq!(unrelated.fence.signals.load(Ordering::SeqCst),0); + println!("PASS: actual Job::start/completion and VmStatus/policy: healthy/failed VMs, mirror ordering, one-shot completion, successful input, device faults and isolation"); +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--mutations", action="store_true") + parser.add_argument("--ref", help="Read job.rs at a public git revision for a removed-fix control") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + job_path = "drivers/gpu/drm/asahi/g15/job.rs" + job = (subprocess.check_output(["git", "show", args.ref + ":" + job_path], cwd=ROOT, text=True) + if args.ref else (ROOT / job_path).read_text()) + current = (ROOT / job_path).read_text() + status = (ROOT / "drivers/gpu/drm/asahi/g17/status.rs").read_text() + policy = (ROOT / "drivers/gpu/drm/asahi/g17/dependency.rs").read_text() + methods = [extract(job, " pub(crate) fn complete("), extract(job, " fn finish("), + extract(job, " fn start(self: &Arc")] + # The old control needs the current helper for the duplicate-completion test; + # it still runs its actual unmodified start, which reproduces the lost input error. + methods.append(extract(job if "fn finish_dependency(" in job else current, " fn finish_dependency(")) + bodies = "\n".join(methods) + code = PRELUDE.replace("__JOB__", bodies).replace("__STATUS__", "\n".join( + extract(status, marker) for marker in [" pub(crate) fn get(", " pub(crate) fn record(", + " pub(crate) fn reported_error(", " pub(crate) fn report_failure("])).replace( + "__POLICY__", extract(policy, "pub(crate) fn inherited_error(")) + variants = {"control": code} + expected = {} + if args.mutations: + changes = { + "healthy-errors-submit": ("if let Some(status) = failed {", "if let Some(status) = failed.filter(|_| self.vm.status.get() != 0) {", "errored input must never submit"), + "foreign-device-error-unmapped": ("crate::g17::inherited_error(error)", "error", "inherited error mapping"), + "dependency-records-sticky": ("self.finish_dependency(crate::g17::inherited_error(error));", "self.finish(Err(crate::g17::inherited_error(error)), false);", "healthy dependency consumer admission stays open"), + "report-after-signal": ("self.vm.status.report_failure(error);\n self.fence.signal(Err(error));", "self.fence.signal(Err(error));\n self.vm.status.report_failure(error);", "report must precede fence signal"), + "dependency-quarantines": (" fn finish_dependency(&self, error: Error) {", " fn finish_dependency(&self, error: Error) {\n self._guard.quarantine();", "unpublished job must not quarantine"), + "own-error-remapped": ("self.finish(Err(error), false);\n } else", "self.finish(Err(crate::g17::inherited_error(error)), false);\n } else", "own-failed VM must retain original dependency errno"), + "success-refused": ("if let Some(status) = failed {", "if let Some(status) = failed.or(Some(-125)) {", "successful dependencies after report must submit"), + "dependency-completes-twice": (" fn finish_dependency(&self, error: Error) {\n if self.progress.signaled.swap(true, Ordering::AcqRel) {\n return;\n }", " fn finish_dependency(&self, error: Error) {", "dependency completion must signal once"), + "mirror-not-published": ("unsafe { AtomicI32::from_ptr(mirror.word) }.store(error, Ordering::SeqCst);", "let _ = mirror;", "mirror must precede fence signal"), + } + for name, (old, new, message) in changes.items(): + if code.count(old) != 1: + raise RuntimeError(f"{name}: unique mutation anchor count {code.count(old)}") + variants[name] = code.replace(old, new) + expected[name] = message + variants["control-after"] = code + receipt = {"scope": __doc__, "ref": args.ref, "sources": {name: hashlib.sha256(text.encode()).hexdigest() + for name, text in [(job_path, job), ("g17/status.rs", status), ("g17/dependency.rs", policy)]}, "variants": {}} + for name, source in variants.items(): + path = args.output / (name + ".rs"); path.write_text(source) + binary = args.output / name + built = subprocess.run([os.environ.get("RUSTC", "rustc"), "--edition=2021", str(path), "-o", str(binary)], capture_output=True, text=True) + (args.output / (name + ".build.log")).write_text(built.stdout + built.stderr) + if built.returncode: + raise RuntimeError(f"{name}: NOBUILD, not a runtime kill: {built.stderr}") + ran = subprocess.run([str(binary.resolve())], capture_output=True, text=True, timeout=30) + receipt["variants"][name] = {"exit": ran.returncode, "stdout": ran.stdout, "stderr": ran.stderr, + "intended_failure": name in expected and expected[name] in ran.stderr} + if name.startswith("control") and not args.ref and ran.returncode: + raise RuntimeError(ran.stdout + ran.stderr) + if name in expected and (ran.returncode == 0 or expected[name] not in ran.stderr): + raise RuntimeError(f"{name}: did not die at intended runtime assertion: {ran.stderr}") + (args.output / "result.json").write_text(json.dumps(receipt, indent=2) + "\n") + if args.ref: + for name in ["control", "control-after"]: + result = receipt["variants"][name] + if result["exit"] == 0 or "errored input must never submit" not in result["stderr"]: + raise RuntimeError("removed-fix control did not reproduce the intended lost-input failure") + print(json.dumps(receipt, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/g15-maintained-handoff-tests.py b/tools/asahi/g15-maintained-handoff-tests.py new file mode 100644 index 0000000000000..4dd2961d3a8ac --- /dev/null +++ b/tools/asahi/g15-maintained-handoff-tests.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Actual native G15 ADT parser/admission and RTKit constructor on public synthetic inputs. + +Uses the production ADT byte parser, performance parser, region admission, +T6030 firmware rows and RTKit constructor. Resource/owner lookup and RTKit init +are models. It does not read firmware, a device or private machine data. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +G15 = ROOT / 'drivers/gpu/drm/asahi/g15' + + +def item(s, marker): + a = s.index(marker); i = s.index('{', a) + 1; depth = 1 + while depth: + depth += (s[i] == '{') - (s[i] == '}'); i += 1 + return s[a:i] + + +PREFIX = r''' +#![allow(dead_code, unused_variables)] +use std::{cell::Cell, ffi::{c_void, CStr}, ops::Range, sync::Arc}; +type Result = std::result::Result; +const EINVAL: i32 = -22; +macro_rules! dev_err { ($($x:tt)*) => {{}}; } +#[macro_export] macro_rules! c_str { ($s:literal) => { match CStr::from_bytes_with_nul(concat!($s, "\0").as_bytes()) { Ok(s) => s, Err(_) => panic!("bad C string") } }; } +extern crate self as kernel; +pub mod str { pub use std::ffi::CStr; } +mod device { pub struct Device; impl Device { pub fn as_raw(&self) -> *mut c_void { 1usize as *mut c_void } } use std::ffi::c_void; } +const PERF_STATES: usize = 14; +const CLUSTERS: usize = 4; +const REGION_NAMES: [&str; 6] = ["ttbs", "tables", "handoff", "shared-l2", "text", "data"]; +const BASE: [u64; 6] = [0x100000000, 0x100100000, 0x100200000, 0x100300000, 0x200000000, 0x20005c000]; +const SIZES: [u64; 6] = [0x4000, 0x40000, 0x4000, 0x4000, 0x5c000, 0x114000]; +thread_local! { + static OWNERS: Cell = const { Cell::new(15) }; + static OWNER_CALLS: Cell = const { Cell::new(0) }; + static RAM_OK: Cell = const { Cell::new(true) }; + static TEXT_RAM: Cell = const { Cell::new(false) }; + static INIT_ERROR: Cell = const { Cell::new(0) }; + static COOKIE: Cell<*const Shared> = const { Cell::new(core::ptr::null()) }; +} +mod glue { + use super::*; + pub fn reserved_uat(index: u32, start: u64, size: u64) -> bool { + OWNER_CALLS.set(OWNER_CALLS.get() + 1); + index < 4 && BASE[index as usize] == start && SIZES[index as usize] == size + && OWNERS.get() & (1 << index) != 0 + } + pub fn ram_disjoint(start: u64, _size: u64) -> bool { + start >= BASE[4] && !TEXT_RAM.get() + } + pub fn ram_within(start: u64, _size: u64) -> bool { start < BASE[4] && RAM_OK.get() } +} +#[derive(Clone)] +struct Node { name: &'static str, props: Vec<(&'static str, Vec)>, kids: Vec } +impl Node { + fn encode(&self) -> Vec { + let mut out = Vec::new(); + let mut props = self.props.clone(); + props.push(("name", [self.name.as_bytes(), &[0]].concat())); + out.extend_from_slice(&(props.len() as u32).to_le_bytes()); + out.extend_from_slice(&(self.kids.len() as u32).to_le_bytes()); + for (name, data) in props { + let mut n = [0u8; 32]; n[..name.len()].copy_from_slice(name.as_bytes()); + out.extend_from_slice(&n); out.extend_from_slice(&(data.len() as u32).to_le_bytes()); + out.extend_from_slice(&data); while out.len() % 4 != 0 { out.push(0); } + } + for child in &self.kids { out.extend_from_slice(&child.encode()); } + out + } +} +fn fixture(kind: u8) -> Vec { + let mut sgx = Node { name: "sgx", props: vec![("compatible", b"gpu,t6030\0".to_vec())], kids: vec![] }; + for (i, (b, s)) in [("gpu-region-base", "gpu-region-size"), ("gfx-shared-region-base", "gfx-shared-region-size"), ("gfx-handoff-base", "gfx-handoff-size"), ("gfx-shared-l2-region-base", "gfx-shared-l2-region-size")].iter().enumerate() { + sgx.props.push((b, BASE[i].to_le_bytes().to_vec())); sgx.props.push((s, SIZES[i].to_le_bytes().to_vec())); + } + for (name, value) in [("perf-state-count", 14u32), ("gpu-num-perf-states", 13), ("perf-state-table-count", 1), ("gpu-power-sample-period", 8), ("gpu-pwr-sample-period-aic-clks", 200000)] { sgx.props.push((name, value.to_le_bytes().to_vec())); } + for (name, mv) in [("perf-states", 600u32), ("perf-states-sram", 700)] { + let mut table = vec![]; + for i in 0..14u32 { table.extend_from_slice(&(i * 100000000).to_le_bytes()); table.extend_from_slice(&mv.to_le_bytes()); } + sgx.props.push((name, table)); + } + let mut segments = vec![]; + for i in 0..2 { + segments.extend_from_slice(&BASE[4+i].to_le_bytes()); + segments.extend_from_slice(&(hw::UPPER + if i == 0 { 0 } else { SIZES[4] } + if kind == 3 { 0x4000 } else { 0 }).to_le_bytes()); + segments.extend_from_slice(&BASE[4+i].to_le_bytes()); + segments.extend_from_slice(&(SIZES[4+i] as u32).to_le_bytes()); + segments.extend_from_slice(&(if i == 0 { 1u32 } else { 0 }).to_le_bytes()); + } + let mut nub_segments = segments.clone(); if kind == 2 { nub_segments[0] ^= 1; } + let nub = Node { name: "iop-gfx-nub", props: vec![("segment-ranges", nub_segments), ("pre-loaded", (if kind == 1 { 0u32 } else { 1 }).to_le_bytes().to_vec())], kids: vec![] }; + let asc = Node { name: "gfx-asc", props: vec![("segment-ranges", segments), ("segment-names", b"__TEXT;__DATA\0".to_vec())], kids: vec![nub] }; + let chosen = Node { name: "chosen", props: vec![("dram-base", (if kind == 4 { u64::MAX } else { 0x100000000u64 }).to_le_bytes().to_vec()), ("dram-size", 0x200000000u64.to_le_bytes().to_vec())], kids: vec![] }; + Node { name: "root", props: vec![], kids: vec![Node { name: "arm-io", props: vec![], kids: vec![sgx, asc] }, chosen] }.encode() +} +struct Shared; +struct Rtkit { rtk: *mut c_void } +static RTKIT_OPS: u8 = 0; +fn from_err_ptr(p: *mut c_void) -> Result<*mut c_void> { if (p as isize) < 0 { Err(p as isize as i32) } else { Ok(p) } } +mod bindings { + use super::*; + pub unsafe fn apple_rtkit_init(_dev: *mut c_void, cookie: *mut c_void, name: *const core::ffi::c_char, index: u32, _ops: *const u8) -> *mut c_void { + assert!(name.is_null(), "maintained DT mailbox must be selected by index"); + assert_eq!(index, 0); + COOKIE.set(cookie.cast()); + let e = INIT_ERROR.get(); if e == 0 { 1usize as *mut c_void } else { e as isize as *mut c_void } + } +} +''' + +CASES = r''' +fn main() { + let bytes = fixture(0); let adt = Adt { data: &bytes }; let dev = device::Device; + OWNERS.set(15); OWNER_CALLS.set(0); RAM_OK.set(true); TEXT_RAM.set(false); + let parsed = parse(&dev, &adt).expect("maintained reserved-RAM ADT admission"); + assert_eq!(OWNER_CALLS.get(), 4, "all four owner reservations must be checked"); assert_eq!(parsed.ttbs, BASE[0]..BASE[0]+SIZES[0]); + assert_eq!(parsed.perf_mhz[13], 1300); assert_eq!(parsed.cluster_mv[13][0], 600); + for missing in 0..4 { + OWNERS.set(15 & !(1 << missing)); OWNER_CALLS.set(0); + assert!(parse(&dev, &adt).is_err(), "every maintained UAT owner is mandatory"); + assert_eq!(OWNER_CALLS.get(), missing + 1); + } + OWNERS.set(15); RAM_OK.set(false); assert!(parse(&dev, &adt).is_err()); RAM_OK.set(true); + TEXT_RAM.set(true); assert!(parse(&dev, &adt).is_err()); TEXT_RAM.set(false); + for kind in 1..=4 { + OWNER_CALLS.set(0); let bytes = fixture(kind); + assert!(parse(&dev, &Adt { data: &bytes }).is_err()); assert_eq!(OWNER_CALLS.get(), 0); + } + assert_eq!(BUILDS.len(), 2, "maintained B0 firmware row must not require an absent profile symbol"); + assert_eq!(BUILDS[1].machine.unwrap().to_bytes(), b"apple,j516s"); + assert_eq!((BUILDS[1].text_size, BUILDS[1].data_size, BUILDS[1].marker), (0x5c000, 0x114000, 0x58e88)); + assert_eq!(BUILDS[1].text_sha256, [0xda,0x26,0xec,0xca,0xb1,0xf8,0x59,0x8d,0xa8,0x0c,0x85,0x5e,0xca,0x91,0x65,0x20,0x40,0x6d,0x1d,0x06,0x70,0xf1,0xd4,0x70,0xe7,0x9b,0xb9,0xb0,0x22,0x09,0xdb,0x53]); + for error in [0, -517, -12] { + INIT_ERROR.set(error); let shared = Arc::new(Shared); let result = Rtkit::new(&dev, &shared); + assert_eq!(result.is_ok(), error == 0); + assert_eq!(Arc::strong_count(&shared), if error == 0 { 2 } else { 1 }); + if error == 0 { unsafe { drop(Arc::from_raw(COOKIE.get())); } } + } + println!("PASS actual ADT/owner/performance parser, B0 identity row, unnamed RTKit mailbox and cookie lifetime"); +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__); p.add_argument('--output', type=Path); p.add_argument('--mutations', action='store_true'); a=p.parse_args() + out=(a.output or Path(tempfile.mkdtemp(prefix='g15-maintained-handoff-'))).resolve(); out.mkdir(parents=True, exist_ok=True) + adt=(G15/'adt.rs').read_text(); boot=(G15/'boot.rs').read_text(); hw=(G15/'hw.rs').read_text() + helpers='\n'.join(item(adt,m) for m in ["pub(crate) struct GpuAdt", "struct Adt<'a>", 'fn le32(', 'fn le64(', 'fn strncmp_eq(', "impl Adt<'_>", 'fn collect_perf(', 'fn parse(', 'fn admit_regions(']) + constants='\n'.join(re.search(r'pub\(crate\) const '+n+r':[^;]+;', hw).group(0) for n in ['PAGE','PA_MASK','TTBS_SIZE','TABLES_SIZE','HANDOFF_SIZE','SHARED_L2_SIZE','UPPER']) + start=hw.index(' firmware: &['); start=hw.index('[',start); end=start+1; depth=1 + while depth: depth+=(hw[end]=='[')-(hw[end]==']'); end+=1 + firmware='static BUILDS: &[FirmwareBuild] = &'+hw[start:end]+';\n'+item(hw,'pub(crate) struct FirmwareBuild') + constructor=item(boot,' fn new(dev: &device::Device, shared: &Arc)') + source=PREFIX+'\nmod hw {\n'+constants+'\n}\n'+helpers+'\n'+firmware+'\nimpl Rtkit {\n'+constructor+'\n}\n'+CASES + variants={'actual':source} + if a.mutations: + variants['skip-owner-check']=source.replace('if !glue::reserved_uat(i as u32, r.start, r.end - r.start) {','if false {') + variants['skip-fourth-owner']=source.replace('region[..4].iter().enumerate()', 'region[..3].iter().enumerate()') + variants['drop-reserved-ram']=source.replace('Some(dram_base..dram_base + dram_size), true)?;', 'Some(dram_base..dram_base + dram_size), false)?;') + variants['require-obsolete-b0-profile']=source.replace(' FirmwareBuild {\n name: "J516S B0', ' #[cfg(CONFIG_SILICONFOREST_NATIVE_GPU)]\n FirmwareBuild {\n name: "J516S B0') + variants['named-mailbox']=source.replace('cookie, core::ptr::null(), 0, &RTKIT_OPS', 'cookie, c"mbox".as_ptr(), 0, &RTKIT_OPS') + receipt={'scope':__doc__,'sources':{p:hashlib.sha256(t.encode()).hexdigest() for p,t in [('adt.rs',adt),('boot.rs',boot),('hw.rs',hw)]},'variants':{}} + for name,text in variants.items(): + if name!='actual' and text==source: raise SystemExit('NOT_APPLIED '+name) + src=out/(name+'.rs');src.write_text(text); binary=out/name + subprocess.run([os.environ.get('HOSTRUSTC','rustc'),'--edition=2021',str(src),'-o',str(binary)],check=True) + r=subprocess.run([str(binary)],capture_output=True,text=True) + receipt['variants'][name]={'exit':r.returncode,'stdout':r.stdout,'stderr':r.stderr} + if name=='actual' and r.returncode: raise SystemExit(r.stderr) + if name!='actual' and not r.returncode: raise SystemExit('SURVIVED '+name) + if name!='actual': + expected={'skip-owner-check':'all four owner reservations must be checked','skip-fourth-owner':'all four owner reservations must be checked','drop-reserved-ram':'maintained reserved-RAM ADT admission','require-obsolete-b0-profile':'maintained B0 firmware row must not require an absent profile symbol','named-mailbox':'maintained DT mailbox must be selected by index'}[name] + receipt['variants'][name]['intended_failure']=expected in r.stderr + if expected not in r.stderr: raise SystemExit('INVALID unrelated failure '+name+'\n'+r.stderr) + (out/'result.json').write_text(json.dumps(receipt,indent=2)+'\n');print(json.dumps(receipt,indent=2)) +if __name__=='__main__': main() diff --git a/tools/asahi/g15-pb-growth-tests.py b/tools/asahi/g15-pb-growth-tests.py new file mode 100644 index 0000000000000..7cdedc9f0cc95 --- /dev/null +++ b/tools/asahi/g15-pb-growth-tests.py @@ -0,0 +1,1573 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the G15 PB growth answer and its on-demand backing, actual bodies, against a model. + +The harness extracts from drivers/gpu/drm/asahi/g15/pools.rs Firmware::answer_growth(), +answer_request(), map_pb(), running_vm() and back_pb(), Pools::soft_max() and backed(), +PbChunk and Pools::new's PB backing statements; pb_max_blocks() from params.rs; the PB sizes, +tuning and PB leaf word from hw.rs; Prot::from_pte() and is_cached_noncoherent() from +pgtable.rs; and the parameter's default from asahi_core.rs. It includes pools/growth.rs, +fw/buffer.rs and fw/channels.rs whole. The bodies run against a model of the firmware objects, +the GEM backing allocation, the shared layer's map_at() and append_driver_mappings(), and the +DEVCTL wait, which logs every allocation, alias map, append, unmap, free, block record, +barrier, producer store, reply and log line in order. Allocation, alias map and append +failures can be injected. Source checks: publish_prefix_into() starts with the PB catch-up, +and pools.rs asserts the PB leaf GPU-only at compile time. + +What it shows: +- a poll pass answers only the requests queued when it starts, in order; one the reply's wait + queues goes to the next pass. A request that cannot be processed does not end the pass; +- from the driver's initial size to its maximum, every block record is written once at its + offset, each producer store follows its records and a barrier, each reply follows its producer + store and a barrier, and every producer is in 1..=4095, ending at 4095; +- at the maximum two refusals are answered; further requests get no reply, count as bad and + warn once until the ledger is made again, as init() does; +- the soft maximum is the parameter clamped to 64..=4095 and then to the ledger's + initial..=max; growth answers above it are logged at info level, and the one that reaches it + exactly is not; +- invalid requests (type, slot, stale counter, epoch, no live render) get no reply, and requests + rejected so allocate nothing; +- the backing: the initial blocks only when the GPU starts; chunks of at most 4 MiB, allocated + and held by the running VM before any record or producer names their blocks, never past the + maximum, at most growth::max_chunks(); a failed allocation, alias map or append posts and + answers nothing, leaves the request counter and the VM's count alone and drops what is not + held, and the next request is answered; a failed reply keeps everything; +- a VM's count always equals the aliases it holds, contiguous from the heap's start; a VM that + runs after another grew the backing maps it first (at publication, and again before it grows); + a partial catch-up counts only held aliases; +- the one-VM rule: a catch-up with a flight of another VM, or a count past the chunks, and a + grant with flights of two VMs, are refused (EPROTO) before anything is mapped or posted; +- the ledger reset of Pools::init() re-posts from the backing without allocating; +- the PB leaf decodes as GPU-only, not cached-noncoherent: dropping an alias is a plain unmap; +- Ledger::new() is Ledger::new() as of dba2a110de00 (the carried upstream constructor), capped at + 4095, over every growth and fixed input in 0..=4200 and at the u32 extremes. The driver's + values are unchanged. + +Not shown: kernel, firmware or GPU execution, timing, or what the firmware does with a reply, a +refusal or an unanswered request. The model's VMs have no open/closed state: that a closed VM +whose jobs still run keeps its aliases comes from the VM lifetime source (Job and VmJobGuard +hold the VM), not from these tests. The ledger reset is not a recovery. The shared layer's +mapping and its rollback are modelled, not run. Running only running_vm()'s or only map_pb()'s +one-VM check on a grant is not distinguishable: each covers it. + +With --mutations, each mutation below must fail its intended test with the expected message, and +the control (a driver maximum of the whole 4096-block heap) must pass, held by the ledger's cap. +A run that hangs (60 s) counts as no test failed. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import resource +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +G15 = "drivers/gpu/drm/asahi/g15" +POOLS = f"{G15}/pools.rs" +GROWTH = f"{G15}/pools/growth.rs" +BUFFER = f"{G15}/fw/buffer.rs" +CHANNELS = f"{G15}/fw/channels.rs" +INITDATA = f"{G15}/fw/initdata.rs" +PARAMS = f"{G15}/params.rs" +HW = f"{G15}/hw.rs" +CORE = "drivers/gpu/drm/asahi/asahi_core.rs" +ENGINE = f"{G15}/engine.rs" +PGTABLE = "drivers/gpu/drm/asahi/pgtable.rs" +OLD_REF = "dba2a110de00743eeecd41edf23e54ede4491d5e" + +# Ledger::new() at OLD_REF, the tip of the four PR363 carries (upstream 027bd7623f37), verbatim: +# the constructor before the cap below K. +OLD_LEDGER_NEW = ''' pub(crate) const fn new(grow: bool, blocks: u32, max_blocks: u32) -> Self { + let (initial, max) = if grow { + let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS); + (nb, clamp(max_blocks, nb, MAX_BLOCKS)) + } else { + (MAX_BLOCKS, MAX_BLOCKS) + }; + Self { + initial, + max, + posted: initial, + next_request: 0, + refusal_logged: false, + refusals: 0, + } + }''' + +# Pools::new's PB backing, run verbatim by the model. +POOLS_NEW_BACKING = (' // Backing for the initial blocks; growth adds the rest.\n', ' chunks.push(') +# The compile-time proof that a PB alias drop is a plain unmap. +PB_LEAF_ASSERT = 'static_assert!(!mmu::Prot::from_pte(pte::LEAF_CACHED).is_cached_noncoherent());' + +# The two statements of Pools::init() that start a new ledger. +LEDGER_RESET = ['self.ledger = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS);', + 'self.exhausted_logged = false;'] + +MUTATIONS = { + # Answer until the queue is empty, including requests queued by the replies' waits. + 'drain-until-empty': (POOLS, 'for _ in 0..queued {', 'while !self.events.grow.is_empty() {'), + # Leave the last request queued before the pass for later. + 'skip-last-queued': (POOLS, 'for _ in 0..queued {', 'for _ in 0..queued.saturating_sub(1) {'), + # End the pass at a request that cannot be processed. + 'return-after-request-error': (POOLS, ' self.pools.stats.bad += 1;\n dev_err!(', + ' self.pools.stats.bad += 1;\n return Ok(());\n' + ' dev_err!('), + # Answer three refusals at the maximum. + 'three-refusals': (GROWTH, 'pub(crate) const MAX_REFUSALS: u32 = 2;', 'pub(crate) const MAX_REFUSALS: u32 = 3;'), + # Answer refusals for ever. + 'never-exhausted': (GROWTH, 'self.refusals >= MAX_REFUSALS', 'false && self.refusals >= MAX_REFUSALS'), + # Warn for every unanswered request. + 'warn-every-time': (POOLS, 'if !self.pools.exhausted_logged {', 'if true {'), + # Store the producer without the barrier after the block records. + 'no-record-barrier': (POOLS, ' // Records before their producer.\n crate::mem::sync();\n', + ' // Records before their producer.\n'), + # Reply without the barrier after the producer store. + 'no-reply-barrier': (POOLS, ' // Producer before the reply.\n crate::mem::sync();\n', + ' // Producer before the reply.\n'), + # Log the answer that reaches the soft maximum exactly. + 'soft-boundary-inclusive': (POOLS, 'let past_soft = n != 0 && ledger.posted > soft;', + 'let past_soft = n != 0 && ledger.posted >= soft;'), + # Give the firmware the parameter without the ledger's bounds. + 'soft-max-unclamped': (POOLS, 'super::params::pb_max_blocks().clamp(self.ledger.initial, self.ledger.max)', + 'super::params::pb_max_blocks()'), + # Ledger::new() without the cap below K. + 'uncapped-ledger': (GROWTH, 'let nb = clamp(blocks, MIN_BLOCKS, MAX_POSTED);\n' + ' (nb, clamp(max_blocks, nb, MAX_POSTED))\n' + ' } else {\n (MAX_POSTED, MAX_POSTED)', + 'let nb = clamp(blocks, MIN_BLOCKS, MAX_BLOCKS);\n' + ' (nb, clamp(max_blocks, nb, MAX_BLOCKS))\n' + ' } else {\n (MAX_BLOCKS, MAX_BLOCKS)'), + # Post blocks without backing them first. + 'grant-without-backing': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', + ' let (first, n) = self.pools.ledger.accept(request);'), + # Back the blocks only after the request is accepted. + 'back-after-accept': (POOLS, ' self.back_pb(end, ctx)?;\n let (first, n) = self.pools.ledger.accept(request);', + ' let (first, n) = self.pools.ledger.accept(request);\n self.back_pb(end, ctx)?;'), + # Count a catch-up alias before the VM holds it. + 'catch-up-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n mapped += 1;\n' + ' vm.pb_chunks.store(mapped, Ordering::Relaxed);', + ' mapped += 1;\n vm.pb_chunks.store(mapped, Ordering::Relaxed);\n' + ' vm.vm.append_driver_mappings(aliases)?;'), + # Count a growth chunk's alias before the VM holds it. + 'growth-cursor-before-append': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.', + ' vm.pb_chunks.store(self.pools.chunks.len() + 1, Ordering::Relaxed);\n' + ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.'), + # Keep a chunk whose alias the VM never held. + 'keep-unappended-chunk': (POOLS, ' vm.vm.append_driver_mappings(aliases)?;\n // Within the capacity checked above.\n' + ' self.pools\n .chunks\n .push_within_capacity(chunk)\n' + ' .map_err(|_| EIO)?;', + ' // Within the capacity checked above.\n self.pools\n .chunks\n' + ' .push_within_capacity(chunk)\n .map_err(|_| EIO)?;\n' + ' vm.vm.append_driver_mappings(aliases)?;'), + # Grow without catching the running VM up first. + 'no-growth-catch-up': (POOLS, ' let vm = self.running_vm(ctx)?;\n self.map_pb(&vm)?;\n', + ' let vm = self.running_vm(ctx)?;\n'), + # Catch up without the one-VM check. + 'no-activation-guard': (POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), + # Neither the catch-up's nor the grant's one-VM check (each alone covers a grant). + 'no-same-vm-checks': [(POOLS, '.any(|f| !core::ptr::eq(&*f.job.vm, vm))', '.any(|_| false)'), + (POOLS, 'if vm.slot != ctx || flights.iter().any(|f| !Arc::ptr_eq(&f.job.vm, &vm)) {', + 'if vm.slot != ctx {')], + # Back the whole heap when the GPU starts. + 'initial-backs-whole-heap': (POOLS, 'chunks.push(PbChunk::new(dev, 0, ledger.initial)?, GFP_KERNEL)?;', + 'chunks.push(PbChunk::new(dev, 0, buffer::MAX_BLOCKS)?, GFP_KERNEL)?;'), + # Growth chunks of 8 MiB. + 'chunks-above-4mib': (GROWTH, 'pub(crate) const CHUNK_BLOCKS: u32 = 32;', 'pub(crate) const CHUNK_BLOCKS: u32 = 64;'), + # A firmware-visible cached PB leaf (its drop would ask the firmware to flush). + 'pb-leaf-fw-visible': (HW, 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c83;', + 'pub(crate) const LEAF_CACHED: u64 = 0x00c0000000000c03;'), +} + +# The test each mutation must fail, and a piece of its failure message. +INTENDED = { + 'drain-until-empty': ('queued_snapshot_keeps_order', 'unbounded answering in one poll pass'), + 'skip-last-queued': ('queued_snapshot_keeps_order', 'replies in the first pass'), + 'return-after-request-error': ('unprocessed_request_does_not_end_the_pass', 'requests taken in the pass'), + 'three-refusals': ('default_growth_to_exhaustion', 'want Some(Log("warn"'), + 'never-exhausted': ('default_growth_to_exhaustion', 'growth did not end'), + 'warn-every-time': ('default_growth_to_exhaustion', 'later request after exhaustion'), + 'no-record-barrier': ('default_growth_to_exhaustion', 'got Some(Producer(1034)), want Some(Sync)'), + 'no-reply-barrier': ('default_growth_to_exhaustion', + 'got Some(Reply([7, 1, 0, 5, 0, 256, 3, 1, 0, 0, 0, 0, 0, 0])), want Some(Sync)'), + 'soft-boundary-inclusive': ('soft_max_boundaries', 'got Some(Log("info", "")), want Some(Log("dbg", ""))'), + 'soft-max-unclamped': ('soft_max_bounds', 'soft_max, param 0'), + 'uncapped-ledger': ('constructor_is_the_old_one_capped', 'new(true, '), + 'grant-without-backing': ('growth_backs_before_posting', 'record of block'), + 'back-after-accept': ('allocation_failure_posts_nothing', 'ledger after a backing failure'), + 'catch-up-cursor-before-append': ('partial_catch_up_counts_only_held_aliases', 'count after a failed append'), + 'growth-cursor-before-append': ('append_failure_posts_nothing', 'count differs from held aliases'), + 'keep-unappended-chunk': ('append_failure_posts_nothing', 'chunks after a backing failure'), + 'no-growth-catch-up': ('growth_catches_the_running_vm_up_first', 'running VM not caught up'), + 'no-activation-guard': ('catch_up_refuses_flights_of_another_vm', 'catch-up with a flight of another VM'), + 'no-same-vm-checks': ('grant_refuses_flights_of_two_vms', 'grant with flights of two VMs'), + 'initial-backs-whole-heap': ('startup_backs_the_initial_blocks', 'startup allocation'), + 'chunks-above-4mib': ('growth_backs_before_posting', 'a growth chunk above 4 MiB'), + 'pb-leaf-fw-visible': ('pb_leaf_is_gpu_only', 'PB leaf not GPU-only'), +} +assert set(INTENDED) == set(MUTATIONS) + +# Changes that must pass: the ledger's cap holds a driver maximum of the whole heap. +CONTROLS = { + 'tuning-whole-heap': (HW, 'pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS - 1;', + 'pub(crate) const PB_MAX_BLOCKS: u32 = super::sizes::PB_BLOCKS;'), +} + + +def item(text, start): + a = text.index(start) + b = text.index("{", a) + 1 + depth = 1 + while depth: + depth += (text[b] == "{") - (text[b] == "}") + b += 1 + return text[a:b] + + +def consts(text, names): + out = [] + for name in names: + m = re.search(rf'pub\(crate\) const {name}: (\w+) = ([^;]+);', text) + if not m: + raise SystemExit(f'{name} not found') + out.append(f'pub const {name}: {m.group(1)} = {m.group(2)};') + return '\n'.join(out) + + +PREFIX = r''' +#![allow(dead_code, unused_imports, unused_macros, unused_variables, unreachable_code, non_upper_case_globals)] +extern crate self as kernel; +use std::cell::{Cell, RefCell}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Error(i32); +impl Error { + pub fn to_errno(&self) -> i32 { + self.0 + } +} +pub type Result = core::result::Result; +pub const EINVAL: Error = Error(-22); +pub const ENODEV: Error = Error(-19); +pub const EIO: Error = Error(-5); +pub const ENOMEM: Error = Error(-12); +pub const ENOSPC: Error = Error(-28); +pub const EBUSY: Error = Error(-16); +pub const EPROTO: Error = Error(-71); +pub mod prelude { + pub use crate::{static_assert, Error, Result, EBUSY, EINVAL, EIO, ENODEV, ENOMEM, ENOSPC, KVec, GFP_KERNEL}; +} +pub type ARef = std::sync::Arc; +#[derive(Copy, Clone)] +pub struct Flags; +pub const GFP_KERNEL: Flags = Flags; +/// A kernel vector: pushes past the reserved capacity fail only through push_within_capacity. +pub struct KVec(pub Vec); +impl KVec { + pub fn with_capacity(n: usize, _: Flags) -> Result { + Ok(KVec(Vec::with_capacity(n))) + } + pub fn push(&mut self, value: T, _: Flags) -> Result { + self.0.push(value); + Ok(()) + } + pub fn push_within_capacity(&mut self, value: T) -> core::result::Result<(), T> { + if self.0.len() == self.0.capacity() { + return Err(value); + } + self.0.push(value); + Ok(()) + } + pub fn capacity(&self) -> usize { + self.0.capacity() + } + pub fn len(&self) -> usize { + self.0.len() + } + pub fn get(&self, i: usize) -> Option<&T> { + self.0.get(i) + } + pub fn last(&self) -> Option<&T> { + self.0.last() + } +} +#[macro_export] +macro_rules! static_assert { + ($e:expr) => { + const _: () = assert!($e); + }; +} + +/// One firmware-visible access, barrier or log line, in program order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Ev { + Record(usize, u64), + Sync, + Producer(u32), + Reply([u32; 14]), + Log(&'static str, String), + /// Backing object `id` of this many bytes allocated, or freed (its last reference gone). + Alloc(u32, usize), + Free(u32), + /// A PB alias of object `id` at GVA in the VM with this context, made, held by the VM + /// (appended to its driver mappings), or unmapped. + Map(u32, u64, usize, u32), + Append(u32, u64), + Unmap(u32, u64), +} +thread_local! { + pub static LOG: RefCell> = RefCell::new(Vec::new()); + pub static PARAM: Cell = Cell::new(PARAM_DEFAULT); + /// Failures to inject: the next this many allocations, alias maps, appends fail. + pub static FAIL_ALLOC: Cell = Cell::new(0); + pub static FAIL_MAP: Cell = Cell::new(0); + pub static MAP_OK_BEFORE_FAIL: Cell = Cell::new(0); + pub static FAIL_APPEND: Cell = Cell::new(0); + pub static NEXT_ID: Cell = Cell::new(1); +} +/// Takes one injected failure of `which`. +pub fn fail(which: &'static std::thread::LocalKey>) -> bool { + which.with(|c| { + let n = c.get(); + if n != 0 { + c.set(n - 1); + } + n != 0 + }) +} +pub fn ev(e: Ev) { + LOG.with(|l| l.borrow_mut().push(e)); +} +pub fn take() -> Vec { + LOG.with(|l| std::mem::take(&mut *l.borrow_mut())) +} +macro_rules! dev_log { + ($lvl:expr, $dev:expr, $($arg:tt)*) => {{ + let _ = $dev; + crate::ev(crate::Ev::Log($lvl, format!($($arg)*))); + }}; +} +macro_rules! dev_err { ($($t:tt)*) => { dev_log!("err", $($t)*) }; } +macro_rules! dev_warn { ($($t:tt)*) => { dev_log!("warn", $($t)*) }; } +macro_rules! dev_info { ($($t:tt)*) => { dev_log!("info", $($t)*) }; } +macro_rules! dev_dbg { ($($t:tt)*) => { dev_log!("dbg", $($t)*) }; } + +pub mod mem { + pub fn sync() { + crate::ev(crate::Ev::Sync); + } +} + +/// A boot parameter: read-only after load. +pub mod module_parameters { + pub struct Param; + impl Param { + pub fn value(&self) -> &'static u32 { + Box::leak(Box::new(crate::PARAM.with(|p| p.get()))) + } + } + pub static g15_pb_max_blocks: Param = Param; +} + +pub mod driver { + pub struct AsahiDevice; +} + +/// GEM kernel objects: freed when the last reference (chunk or alias) goes. +pub mod gem { + pub struct Object { + pub id: u32, + pub size: usize, + } + impl Drop for Object { + fn drop(&mut self) { + crate::ev(crate::Ev::Free(self.id)); + } + } + pub struct ObjectRef { + pub gem: crate::ARef, + } +} + +/// The shared layer's VM: kernel aliases at fixed addresses, held by the VM once appended. +pub mod mmu { + use crate::prelude::*; + pub struct Prot(pub u64); + impl Prot { + pub const fn from_pte(pte: u64) -> Self { + Prot(pte) + } + } + pub struct KernelMapping { + pub slot: u32, + pub gva: u64, + pub size: usize, + pub prot: u64, + pub gem: crate::ARef, + } + impl Drop for KernelMapping { + fn drop(&mut self) { + crate::ev(crate::Ev::Unmap(self.slot, self.gva)); + } + } + pub struct Vm { + pub slot: u32, + pub held: std::cell::RefCell>, + } + impl Vm { + pub fn new(slot: u32) -> Self { + Vm { slot, held: Default::default() } + } + pub fn map_at( + &self, + addr: u64, + size: usize, + gem: &crate::ARef, + prot: Prot, + guard: bool, + ) -> Result { + assert!(!guard); + // The shared layer maps whatever size it is given: the whole object, exactly. + assert_eq!(size, gem.size, "alias size differs from its object"); + let skip = crate::MAP_OK_BEFORE_FAIL.with(|c| { + let n = c.get(); + c.set(n.saturating_sub(1)); + n != 0 + }); + if !skip && crate::fail(&crate::FAIL_MAP) { + return Err(ENOMEM); + } + // Leaves already in use: EBUSY (prepare_map). + let end = addr + size as u64; + if self.held.borrow().iter().any(|m| m.gva < end && addr < m.gva + m.size as u64) { + return Err(EBUSY); + } + crate::ev(crate::Ev::Map(self.slot, addr, size, gem.id)); + Ok(KernelMapping { slot: self.slot, gva: addr, size, prot: prot.0, gem: gem.clone() }) + } + pub fn append_driver_mappings(&self, mappings: KVec) -> Result { + if crate::fail(&crate::FAIL_APPEND) { + return Err(ENOMEM); // the mappings drop here + } + for m in mappings.0 { + crate::ev(crate::Ev::Append(self.slot, m.gva)); + self.held.borrow_mut().push(m); + } + Ok(()) + } + } +} +''' + +MODEL = r''' + pub struct Ctl { + pub epoch: u64, + } + impl Ctl { + pub fn r32(&self, offset: usize) -> Result { + if offset == control::HALT_COUNT { + Ok(self.epoch as u32) + } else if offset == control::HALT_COUNT + 4 { + Ok((self.epoch >> 32) as u32) + } else { + panic!("unexpected control read {offset:#x}") + } + } + } + pub struct Init { + pub control: Option>, + } + pub struct VmState { + pub slot: u32, + pub vm: mmu::Vm, + pub pb_chunks: AtomicUsize, + } + impl VmState { + pub fn new(slot: u32) -> Arc { + Arc::new(VmState { slot, vm: mmu::Vm::new(slot), pb_chunks: AtomicUsize::new(0) }) + } + } + pub struct Job { + pub vm: Arc, + } + pub struct Flight { + pub job: Job, + } + pub struct Engine { + pub flights: Vec, + } + pub struct Receipts { + pub renders: u32, + } + /// A firmware object: the block list, block_ctl or the heap. + pub struct FwObject { + pub gva: Option, + pub size: usize, + /// Writes that fail from now on. + pub failures: Cell, + } + impl FwObject { + pub fn w64(&self, offset: usize, value: u64) -> Result { + assert!(offset % 8 == 0 && offset + 8 <= self.size, "record write at {offset:#x}"); + if self.failures.get() != 0 { + self.failures.set(self.failures.get() - 1); + return Err(EIO); + } + crate::ev(crate::Ev::Record(offset, value)); + Ok(()) + } + pub fn w32(&self, offset: usize, value: u32) -> Result { + assert_eq!(offset, block_ctl::PRODUCER); + assert!(offset + 4 <= self.size); + crate::ev(crate::Ev::Producer(value)); + Ok(()) + } + } + fn gva(object: &FwObject) -> Result { + object.gva.ok_or(EINVAL) + } + #[derive(Default)] + pub struct PbStats { + pub requests: u64, + pub grown: u64, + pub refused: u64, + pub bad: u64, + } + pub struct Live { + pub block_ctl: FwObject, + } + pub struct Pools { + pub ledger: Ledger, + pub exhausted_logged: bool, + pub stats: PbStats, + pub pb_blocks: FwObject, + pub live: Live, + pub drm: ARef, + pub chunks: KVec, + } + impl Pools { + pub fn chunk_count(&self) -> usize { + self.chunks.len() + } + pub fn chunk_capacity(&self) -> usize { + self.chunks.capacity() + } + } + pub struct Queue(pub std::collections::VecDeque<[u32; 14]>); + impl Queue { + pub fn len(&self) -> usize { + self.0.len() + } + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + pub fn remove(&mut self, i: usize) -> core::result::Result<[u32; 14], ()> { + assert_eq!(i, 0); + self.0.pop_front().ok_or(()) + } + } + pub struct Events { + pub grow: Queue, + } + /// The firmware side: its requests, and whether the next one arrives during a reply. + pub struct FwModel { + pub ctx: u32, + pub epoch: u64, + pub next_counter: u32, + pub ask_after_reply: bool, + pub reply_fails: bool, + pub replies_this_pass: u32, + } + impl FwModel { + pub fn request(&mut self) -> [u32; 14] { + let mut q = [0u32; 14]; + q[buffer::grow_request::TYPE] = 7; + q[buffer::grow_request::CONTEXT] = self.ctx; + q[buffer::grow_request::SLOT] = 0; + q[buffer::grow_request::COUNTER] = self.next_counter; + q[buffer::grow_request::POSITION] = 0x100 + self.next_counter; + q[buffer::grow_request::EPOCH] = self.epoch as u32; + q[buffer::grow_request::EPOCH + 1] = (self.epoch >> 32) as u32; + self.next_counter += 1; + q + } + } + pub struct Firmware { + pub events: Events, + pub pools: Pools, + pub init: Init, + pub engine: Engine, + pub renders: Vec, + pub fw: FwModel, + } + pub const HEAP: u64 = super::hw::client::PB_HEAP; + impl Firmware { + /// The VM with context 5 runs: one flight, its PB backing mapped (map_pb, as at the + /// start of its prefix's publication). + pub fn new() -> Self { + let vm = VmState::new(5); + let mut f = Self::without_flights(); + f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + f.map_pb(&vm).unwrap(); + crate::take(); + f + } + /// Pools::new's PB backing: the initial chunk, room for every chunk the ledger needs. + pub fn without_flights() -> Self { + let drm = ARef::new(AsahiDevice); + let (ledger, chunks) = pools_new_backing(&drm).unwrap(); + Firmware { + events: Events { grow: Queue(Default::default()) }, + pools: Pools { + ledger, + exhausted_logged: false, + stats: PbStats::default(), + drm, + chunks, + pb_blocks: FwObject { + gva: Some(super::hw::client::PB_BLOCK_LIST), + size: buffer::BLOCK_LIST_SIZE, + failures: Cell::new(0), + }, + live: Live { + block_ctl: FwObject { gva: None, size: block_ctl::SIZE, failures: Cell::new(0) }, + }, + }, + init: Init { control: Some(std::sync::Arc::new(Ctl { epoch: 0x1_0000_0003 })) }, + engine: Engine { flights: Vec::new() }, + renders: vec![1], + fw: FwModel { + ctx: 5, + epoch: 0x1_0000_0003, + next_counter: 0, + ask_after_reply: false, + reply_fails: false, + replies_this_pass: 0, + }, + } + } + pub fn dev(&self) -> &() { + &() + } + pub fn receipts(&self, i: usize) -> Receipts { + Receipts { renders: self.renders[i] } + } + pub fn devctl(&mut self, reply: &DevCtl, wait: bool) -> Result { + assert!(wait); + crate::ev(crate::Ev::Reply(reply.words)); + self.fw.replies_this_pass += 1; + assert!(self.fw.replies_this_pass <= 64, "unbounded answering in one poll pass"); + // The reply's wait drains the rings: the firmware's next request may queue now. + if self.fw.ask_after_reply { + let q = self.fw.request(); + self.events.grow.0.push_back(q); + } + if self.fw.reply_fails { + Err(EIO) + } else { + Ok(()) + } + } + pub fn queue(&mut self) { + let q = self.fw.request(); + self.events.grow.0.push_back(q); + } + /// One poll pass. + pub fn pass(&mut self) { + self.fw.replies_this_pass = 0; + assert_eq!(self.answer_growth(), Ok(())); + } + /// What Pools::init() does to the ledger. + pub fn reset_ledger(&mut self) { + let p = &mut self.pools; + p.RESET + } + } +''' + +TESTS = r''' +#[cfg(test)] +mod pb_growth_tests { + use crate::g15::firmware::{Firmware, VmState, Flight, Job, HEAP}; + use crate::g15::hw::{client, pte}; + use crate::g15::fw::buffer::{BLOCK_BYTES, MAX_BLOCKS}; + use crate::g15::pools::growth::{max_chunks, CHUNK_BLOCKS}; + use crate::g15::pgtable::Prot; + use crate::{EPROTO, FAIL_ALLOC, FAIL_APPEND, FAIL_MAP}; + use core::sync::atomic::Ordering; + use std::sync::Arc; + use crate::g15::fw::buffer::{self, grow_reply, BLOCK_LIST_RECORDS, GROW_STEP}; + use crate::g15::fw::channels::devctl; + use crate::g15::hw::tuning; + use crate::g15::pools::growth::Ledger; + use crate::g15::{params, OldLedger}; + use crate::{take, Ev, ENOMEM, PARAM, PARAM_DEFAULT}; + + const CAP: u32 = BLOCK_LIST_RECORDS - 1; + /// Refusals answered at the maximum, as the commit states them. + const REFUSALS: u32 = 2; + + fn clamp(v: u32, lo: u32, hi: u32) -> u32 { + v.max(lo).min(hi) + } + + fn soft(param: u32, initial: u32, max: u32) -> u32 { + clamp(clamp(param, 64, tuning::PB_MAX_BLOCKS), initial, max) + } + + /// Event logs compared with a short message at the first difference. + fn same(got: &[Ev], want: &[Ev]) { + if let Some(i) = (0..got.len().max(want.len())).find(|&i| got.get(i) != want.get(i)) { + panic!("event {i} of {}/{}: got {:?}, want {:?}", got.len(), want.len(), got.get(i), want.get(i)); + } + } + + fn level(e: &Ev) -> Option<&'static str> { + match e { + Ev::Log(l, _) => Some(l), + _ => None, + } + } + + /// The accesses, replies and log levels a ledger at `posted` gives for the firmware's + /// requests from counter 0 on, if it keeps asking after every reply: growth to the maximum, + /// MAX_REFUSALS refusals, then one unanswered request. + fn expected(initial: u32, max: u32, soft_max: u32) -> (Vec, u32, u32) { + let mut out = Vec::new(); + let (mut posted, mut refusals, mut logged_refusal) = (initial, 0, false); + let mut counter = 0; + loop { + assert!(counter < 10_000, "the expected trace does not end"); + if refusals >= REFUSALS { + out.push(Ev::Log("warn", String::new())); + break; + } + let n = GROW_STEP.min(max - posted); + for b in posted..posted + n { + out.push(Ev::Record(8 * b as usize, buffer::block_record(HEAP, b) as u64)); + } + if n != 0 { + out.push(Ev::Sync); + out.push(Ev::Producer(posted + n)); + out.push(Ev::Sync); + } + let mut r = [0u32; 14]; + r[grow_reply::OPCODE] = devctl::GROW_REPLY; + r[grow_reply::GREW] = (n != 0) as u32; + r[grow_reply::CONTEXT] = 5; + r[grow_reply::COUNTER] = counter; + r[grow_reply::POSITION] = 0x100 + counter; + r[grow_reply::EPOCH] = 3; + r[grow_reply::EPOCH + 1] = 1; + out.push(Ev::Reply(r)); + posted += n; + let first_refusal = n == 0 && !logged_refusal; + if n == 0 { + refusals += 1; + logged_refusal = true; + } + let info = counter < 2 || (n != 0 && posted > soft_max) || first_refusal; + out.push(Ev::Log(if info { "info" } else { "dbg" }, String::new())); + counter += 1; + } + (out, posted, counter + 1) + } + + fn levels_only(evs: Vec) -> Vec { + evs.into_iter() + .filter(|e| !is_backing(e)) + .map(|e| match e { + Ev::Log(l, _) => Ev::Log(l, String::new()), + e => e, + }) + .collect() + } + + /// Growth to the maximum with the firmware asking after every reply, one pass at a time. + fn run_to_exhaustion(param: u32) { + PARAM.with(|p| p.set(param)); + let mut f = Firmware::new(); + let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); + assert_eq!(f.pools.soft_max(), soft(param, initial, max)); + f.fw.ask_after_reply = true; + f.queue(); + take(); + let mut passes = 0; + while !f.events.grow.is_empty() { + let queued = f.events.grow.len(); + let before = f.pools.stats.requests; + f.pass(); + passes += 1; + // Exactly the requests queued before the pass were taken. + assert_eq!(f.pools.stats.requests - before, queued as u64); + assert!(passes <= 1000, "growth did not end"); + } + let log = take(); + let (want, posted, requests) = expected(initial, max, soft(param, initial, max)); + same(&levels_only(log.clone()), &want); + assert_eq!(passes, requests); + assert_eq!(f.pools.ledger.posted, posted); + assert_eq!(posted, max); + // Every producer store: increasing, in 1..=4095, the last one the maximum. + let producers: Vec = log + .iter() + .filter_map(|e| if let Ev::Producer(p) = e { Some(*p) } else { None }) + .collect(); + assert!(producers.windows(2).all(|w| w[0] < w[1])); + assert!(producers.iter().all(|p| (1..=CAP).contains(p))); + assert_eq!(*producers.last().unwrap(), max); + // Every block above the initial ones was written once, below the list's end. + let records: Vec = log + .iter() + .filter_map(|e| if let Ev::Record(o, _) = e { Some(*o) } else { None }) + .collect(); + assert_eq!(records.len() as u32, max - initial); + assert!(records.iter().all(|o| o + 8 <= buffer::BLOCK_LIST_SIZE)); + let s = &f.pools.stats; + assert_eq!((s.grown, s.refused, s.bad), ((max - initial) as u64, REFUSALS as u64, 1)); + let warn: Vec<&String> = log + .iter() + .filter_map(|e| if let Ev::Log("warn", m) = e { Some(m) } else { None }) + .collect(); + assert_eq!(warn.len(), 1); + assert!(warn[0].contains(&format!("left unanswered: {REFUSALS} refusals at the maximum ({max} blocks)"))); + // A later request: no reply, counted, no second warning. + f.queue(); + f.pass(); + let log = take(); + assert!(log.is_empty(), "later request after exhaustion: {log:?}"); + assert_eq!(f.pools.stats.bad, 2); + // A new ledger, as Pools::init() makes, answers again. + f.reset_ledger(); + f.queue(); + f.pass(); + let log = take(); + assert_eq!(log.iter().filter(|e| matches!(e, Ev::Producer(_))).count(), 1); + assert!(log.contains(&Ev::Producer(initial + GROW_STEP.min(max - initial)))); + } + + #[test] + fn driver_values() { + let l = Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + assert!(tuning::PB_GROW); + assert_eq!((l.initial, l.max, l.posted), (1024, 4095, 1024)); + assert_eq!(PARAM_DEFAULT, 3072); + PARAM.with(|p| p.set(PARAM_DEFAULT)); + assert_eq!(params::pb_max_blocks(), 3072); + assert_eq!(Firmware::new().pools.soft_max(), 3072); + } + + #[test] + fn soft_max_bounds() { + let f = Firmware::new(); + for param in [0, 1, 63, 64, 65, 1023, 1024, 1025, 3071, 3072, 3073, 4094, 4095, 4096, 8192, u32::MAX] { + PARAM.with(|p| p.set(param)); + assert_eq!(params::pb_max_blocks(), clamp(param, 64, tuning::PB_MAX_BLOCKS), "pb_max_blocks, param {param}"); + // Below the initial 1024 the firmware still gets 1024. + assert_eq!(f.pools.soft_max(), clamp(param, 1024, CAP), "soft_max, param {param}"); + } + } + + #[test] + fn default_growth_to_exhaustion() { + run_to_exhaustion(PARAM_DEFAULT); + } + + #[test] + fn soft_max_boundaries() { + // 3074 is reached exactly (1024 + 205 * 10): that answer is not past it. + for param in [0, 1024, 3072, 3074, 3075, 4094, 4095, u32::MAX] { + run_to_exhaustion(param); + } + } + + #[test] + fn queued_snapshot_keeps_order() { + let mut f = Firmware::new(); + f.fw.ask_after_reply = true; + f.queue(); + f.queue(); + take(); + f.pass(); + let log = take(); + let replies: Vec = log + .iter() + .filter_map(|e| if let Ev::Reply(r) = e { Some(r[grow_reply::COUNTER]) } else { None }) + .collect(); + assert_eq!(replies, vec![0, 1], "replies in the first pass"); + // Compare both requests' complete record/barrier/producer/reply/log sequences, + // not just their reply order. Each request finishes before the next one's records. + let (want, _, _) = expected(f.pools.ledger.initial, f.pools.ledger.max, f.pools.soft_max()); + let end = want.iter().position(|e| matches!(e, Ev::Reply(r) if r[grow_reply::COUNTER] == 1)) + .expect("second expected reply") + 2; // Include its log too. + same(&levels_only(log), &want[..end]); + let queued: Vec = f.events.grow.0.iter().map(|q| q[buffer::grow_request::COUNTER]).collect(); + assert_eq!(queued, vec![2, 3]); + assert_eq!(f.pools.stats.requests, 2); + f.pass(); + assert_eq!(f.pools.stats.requests, 4); + assert_eq!(f.events.grow.len(), 2); + } + + #[test] + fn empty_queue_does_nothing() { + let mut f = Firmware::new(); + take(); + f.pass(); + assert!(take().is_empty()); + assert_eq!(f.pools.stats.requests, 0); + } + + #[test] + fn unprocessed_request_does_not_end_the_pass() { + let mut f = Firmware::new(); + f.queue(); + f.queue(); + f.pools.pb_blocks.failures.set(1); + take(); + f.pass(); + let log = take(); + // The second request was answered in the same pass, from the same posted count. + assert_eq!(f.pools.stats.requests, 2, "requests taken in the pass"); + assert_eq!(f.pools.stats.bad, 1); + assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -5")))); + assert!(log.contains(&Ev::Producer(1024 + GROW_STEP))); + assert!(f.events.grow.is_empty()); + } + + #[test] + fn failed_reply_is_logged_and_counted_as_grown() { + let mut f = Firmware::new(); + f.fw.reply_fails = true; + f.queue(); + f.queue(); + f.queue(); + take(); + f.pass(); + let log = take(); + assert_eq!(f.pools.ledger.posted, 1024 + 3 * GROW_STEP); + assert_eq!((f.pools.stats.grown, f.pools.stats.bad), (3 * GROW_STEP as u64, 0)); + // Each failed reply is logged at info level. + let info = log.iter().filter(|e| matches!(e, Ev::Log("info", m) if m.contains("DEVCTL 7: -5"))).count(); + assert_eq!(info, 3); + } + + #[test] + fn invalid_requests_get_no_reply() { + let mut f = Firmware::new(); + let good = f.fw.request(); + let mut cases = Vec::new(); + let mut q = good; + q[buffer::grow_request::TYPE] = 6; + cases.push(q); + let mut q = good; + q[buffer::grow_request::SLOT] = 1; + cases.push(q); + let mut q = good; + q[buffer::grow_request::EPOCH] = 2; + cases.push(q); + let mut q = good; + q[buffer::grow_request::CONTEXT] = 6; + cases.push(q); + for q in cases { + f.events.grow.0.push_back(q); + } + take(); + f.pass(); + let log = take(); + assert!(log.iter().all(|e| matches!(e, Ev::Log("warn", _))), "{log:?}"); + assert_eq!(log.len(), 4); + assert_eq!(f.pools.stats.bad, 4); + assert_eq!(f.pools.ledger, Ledger::new(tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS)); + // No render published: not live. + f.renders[0] = 0; + f.events.grow.0.push_back(good); + f.pass(); + assert!(!take().iter().any(|e| matches!(e, Ev::Reply(_)))); + // A stale counter after an answer. + f.renders[0] = 1; + f.events.grow.0.push_back(good); + f.pass(); + assert!(take().iter().any(|e| matches!(e, Ev::Reply(_)))); + f.events.grow.0.push_back(good); + f.pass(); + assert!(!take().iter().any(|e| matches!(e, Ev::Reply(_)))); + assert_eq!(f.pools.stats.bad, 6); + } + + fn is_backing(e: &Ev) -> bool { + matches!(e, Ev::Alloc(..) | Ev::Free(_) | Ev::Map(..) | Ev::Append(..) | Ev::Unmap(..)) + } + + /// A VM's PB aliases: its count, and the aliases it holds, from block 0 in order. + fn assert_cursor(vm: &VmState, chunks: usize) { + let held = vm.vm.held.borrow(); + assert_eq!(vm.pb_chunks.load(Ordering::Relaxed), held.len(), "count differs from held aliases"); + assert!(held.len() <= chunks); + let mut gva = client::PB_HEAP; + for m in held.iter() { + assert_eq!(m.gva, gva, "aliases not contiguous from the heap's start"); + assert_eq!(m.prot, pte::LEAF_CACHED); + gva += m.size as u64; + } + } + + /// In the order of `log`: no block record and no producer past the blocks the VM with + /// context `slot` holds aliases for at that point. + fn assert_posted_backed(log: &[Ev], slot: u32, start: u64) { + let mut mapped = start; + for e in log { + match e { + Ev::Append(s, gva) if *s == slot => { + assert_eq!(*gva, client::PB_HEAP + mapped * BLOCK_BYTES, "append out of order"); + } + Ev::Map(s, gva, size, _) if *s == slot && *gva == client::PB_HEAP + mapped * BLOCK_BYTES => { + // Counted at its append. + let _ = size; + } + _ => {} + } + if let Ev::Append(s, gva) = e { + if *s == slot { + let size = log + .iter() + .find_map(|m| match m { + Ev::Map(ms, mg, size, _) if ms == s && mg == gva => Some(*size as u64), + _ => None, + }) + .unwrap(); + mapped += size / BLOCK_BYTES; + } + } + if let Ev::Record(off, _) = e { + assert!((*off as u64 / 8) < mapped, "record of block {} with {} mapped", off / 8, mapped); + } + if let Ev::Producer(p) = e { + assert!(*p as u64 <= mapped, "producer {} with {} mapped", p, mapped); + } + } + } + + fn count(log: &[Ev], f: impl Fn(&Ev) -> bool) -> usize { + log.iter().filter(|e| f(e)).count() + } + + /// The firmware asks until the ledger reaches `target` posted blocks, one request a pass. + fn grow_to(f: &mut Firmware, target: u32) { + for _ in 0..1000 { + if f.pools.ledger.posted >= target { + return; + } + f.queue(); + f.pass(); + } + panic!("growth stopped at {} blocks", f.pools.ledger.posted); + } + + #[test] + fn startup_backs_the_initial_blocks() { + let mut f = Firmware::without_flights(); + let log = take(); + let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); + assert_eq!(log, vec![Ev::Alloc(1, initial as usize * BLOCK_BYTES as usize)], "startup allocation"); + assert_eq!((f.pools.backed(), f.pools.chunk_count()), (initial, 1)); + assert_eq!(f.pools.chunk_capacity(), max_chunks(initial, max)); + assert_eq!(max_chunks(initial, max), 97); + // The first prefix of a VM maps the backing, nothing else. + let vm = VmState::new(7); + f.engine.flights.push(Flight { job: Job { vm: vm.clone() } }); + f.map_pb(&vm).unwrap(); + let size = initial as usize * BLOCK_BYTES as usize; + assert_eq!(take(), vec![Ev::Map(7, client::PB_HEAP, size, 1), Ev::Append(7, client::PB_HEAP)]); + assert_cursor(&vm, 1); + // Mapping again maps nothing. + f.map_pb(&vm).unwrap(); + assert!(take().is_empty()); + } + + #[test] + fn pb_leaf_is_gpu_only() { + let prot = Prot::from_pte(pte::LEAF_CACHED); + assert!(prot.ap_is_gpu() && !prot.is_cached_noncoherent(), "PB leaf not GPU-only"); + } + + #[test] + fn growth_backs_before_posting() { + PARAM.with(|p| p.set(PARAM_DEFAULT)); + let mut f = Firmware::new(); + f.fw.ask_after_reply = true; + f.queue(); + take(); + let mut passes = 0; + while !f.events.grow.is_empty() { + f.pass(); + passes += 1; + assert!(passes <= 1000, "growth did not end"); + } + let log = take(); + let (initial, max) = (f.pools.ledger.initial, f.pools.ledger.max); + assert_posted_backed(&log, 5, initial as u64); + // Chunks of at most 32 blocks, allocated only as needed: up to the maximum, no further. + let allocs: Vec = log + .iter() + .filter_map(|e| if let Ev::Alloc(_, b) = e { Some(*b) } else { None }) + .collect(); + assert_eq!(allocs.len() + 1, max_chunks(initial, max)); + assert!(allocs.iter().all(|b| *b <= 4 << 20), "a growth chunk above 4 MiB"); + assert_eq!(CHUNK_BLOCKS as u64 * BLOCK_BYTES, 4 << 20); + assert_eq!(f.pools.backed(), max); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); + assert_cursor(&f.engine.flights[0].job.vm, f.pools.chunk_count()); + } + + /// A failure while backing posts nothing and answers nothing; the request counter and the + /// VM's count stay; the next request is answered. + fn failure_posts_nothing(which: &'static std::thread::LocalKey>) -> Vec { + let mut f = Firmware::new(); + which.with(|c| c.set(1)); + f.queue(); + f.pass(); + let log = take(); + assert_eq!(count(&log, |e| matches!(e, Ev::Record(..) | Ev::Producer(_) | Ev::Reply(_) | Ev::Sync)), 0, + "posted after a backing failure: {log:?}"); + assert_eq!((f.pools.ledger.posted, f.pools.ledger.next_request), (1024, 0), "ledger after a backing failure"); + assert_eq!((f.pools.stats.bad, f.pools.chunk_count()), (1, 1), "chunks after a backing failure"); + assert_cursor(&f.engine.flights[0].job.vm, 1); + assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -12")))); + f.queue(); + f.pass(); + let retry = take(); + assert!(retry.contains(&Ev::Producer(1034))); + assert_eq!(f.pools.chunk_count(), 2); + assert_cursor(&f.engine.flights[0].job.vm, 2); + log + } + + #[test] + fn allocation_failure_posts_nothing() { + let log = failure_posts_nothing(&FAIL_ALLOC); + assert_eq!(count(&log, is_backing), 0); + } + + #[test] + fn map_failure_posts_nothing() { + let log = failure_posts_nothing(&FAIL_MAP); + let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); + assert!(matches!(backing[..], [Ev::Alloc(2, _), Ev::Free(2)]), "{backing:?}"); + } + + #[test] + fn append_failure_posts_nothing() { + let log = failure_posts_nothing(&FAIL_APPEND); + let backing: Vec<&Ev> = log.iter().filter(|e| is_backing(e)).collect(); + assert!( + matches!(backing[..], [Ev::Alloc(2, _), Ev::Map(5, _, _, 2), Ev::Unmap(5, _), Ev::Free(2)]), + "{backing:?}" + ); + } + + #[test] + fn transport_failure_keeps_everything() { + let mut f = Firmware::new(); + f.fw.reply_fails = true; + f.queue(); + f.pass(); + let log = take(); + assert!(log.contains(&Ev::Producer(1034))); + assert_eq!((f.pools.ledger.posted, f.pools.chunk_count()), (1034, 2)); + assert_cursor(&f.engine.flights[0].job.vm, 2); + assert_eq!(count(&log, |e| matches!(e, Ev::Free(_) | Ev::Unmap(..))), 0); + } + + #[test] + fn other_vms_catch_up_before_they_run() { + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + grow_to(&mut f, 1100); + take(); + assert_eq!(f.pools.chunk_count(), 4); + // A's work retires; a new VM runs next: it maps every chunk first. + f.engine.flights.clear(); + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + f.fw.ctx = 6; + f.map_pb(&b).unwrap(); + let log = take(); + assert_eq!(count(&log, |e| matches!(e, Ev::Append(6, _))), 4); + assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..))), 0); + assert_cursor(&b, 4); + // B grows the backing; A, behind, catches up when it runs again. + grow_to(&mut f, 1200); + let log = take(); + assert_posted_backed(&log, 6, 1120); + assert_eq!(count(&log, |e| matches!(e, Ev::Map(5, ..))), 0); + f.engine.flights.clear(); + f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); + f.map_pb(&a).unwrap(); + assert_cursor(&a, f.pools.chunk_count()); + assert_eq!(a.pb_chunks.load(Ordering::Relaxed), f.pools.chunk_count()); + } + + #[test] + fn growth_catches_the_running_vm_up_first() { + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + // B grows the backing while A is idle. + f.engine.flights.clear(); + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + f.map_pb(&b).unwrap(); + f.fw.ctx = 6; + grow_to(&mut f, 1100); + // A runs again; its growth request maps what it lacks before backing more. + f.engine.flights.clear(); + f.engine.flights.push(Flight { job: Job { vm: a.clone() } }); + f.fw.ctx = 5; + take(); + grow_to(&mut f, 1130); + let log = take(); + assert_eq!(a.vm.held.borrow().len(), f.pools.chunk_count(), "running VM not caught up"); + assert_cursor(&a, f.pools.chunk_count()); + assert_posted_backed(&log, 5, 1024); + } + + #[test] + fn partial_catch_up_counts_only_held_aliases() { + let mut f = Firmware::new(); + grow_to(&mut f, 1100); + f.engine.flights.clear(); + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + take(); + // The third alias cannot be made: two held. + FAIL_MAP.with(|c| c.set(0)); + crate::MAP_OK_BEFORE_FAIL.with(|c| c.set(2)); + FAIL_MAP.with(|c| c.set(1)); + assert_eq!(f.map_pb(&b), Err(ENOMEM)); + assert_cursor(&b, 4); + assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2); + // The third alias is made but cannot be held: dropped, still two. + FAIL_APPEND.with(|c| c.set(1)); + assert_eq!(f.map_pb(&b), Err(ENOMEM)); + assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 2, "count after a failed append"); + assert!(take().iter().any(|e| matches!(e, Ev::Unmap(6, _)))); + assert_cursor(&b, 4); + f.map_pb(&b).unwrap(); + assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 4); + assert_cursor(&b, 4); + } + + /// The ledger reset of Pools::init(), not a full recovery. + #[test] + fn ledger_reset_reuses_the_backing() { + let mut f = Firmware::new(); + grow_to(&mut f, 1300); + let chunks = f.pools.chunk_count(); + take(); + f.reset_ledger(); + assert_eq!(f.pools.ledger.posted, 1024); + grow_to(&mut f, 1300); + let log = take(); + assert_eq!(count(&log, |e| matches!(e, Ev::Alloc(..) | Ev::Map(..))), 0); + assert_eq!(f.pools.chunk_count(), chunks); + assert_posted_backed(&log, 5, f.pools.backed() as u64); + } + + #[test] + fn rejected_requests_allocate_nothing() { + let mut f = Firmware::new(); + let mut q = f.fw.request(); + q[buffer::grow_request::EPOCH] = 2; // a halt epoch that has passed + f.events.grow.0.push_back(q); + f.pass(); + let log = take(); + assert_eq!(count(&log, is_backing), 0); + assert_eq!(f.pools.stats.bad, 1); + } + + #[test] + fn catch_up_refuses_flights_of_another_vm() { + let mut f = Firmware::new(); + let a = f.engine.flights[0].job.vm.clone(); + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + assert_eq!(f.map_pb(&b), Err(EPROTO), "catch-up with a flight of another VM"); + assert_eq!(b.pb_chunks.load(Ordering::Relaxed), 0); + assert_eq!(count(&take(), is_backing), 0); + // A count past the chunks. + f.engine.flights.pop(); + a.pb_chunks.store(f.pools.chunk_count() + 1, Ordering::Relaxed); + assert_eq!(f.map_pb(&a), Err(EPROTO), "catch-up with a count past the chunks"); + } + + #[test] + fn grant_refuses_flights_of_two_vms() { + let mut f = Firmware::new(); + let b = VmState::new(6); + f.engine.flights.push(Flight { job: Job { vm: b.clone() } }); + f.renders.push(1); + f.queue(); + f.pass(); + let log = take(); + assert_eq!( + count(&log, |e| is_backing(e) || matches!(e, Ev::Reply(_) | Ev::Producer(_) | Ev::Record(..))), + 0, + "grant with flights of two VMs" + ); + assert!(log.iter().any(|e| matches!(e, Ev::Log("err", m) if m.contains("not answered: -71")))); + } + + #[test] + fn constructor_is_the_old_one_capped() { + let mut values: Vec = (0..=4200).collect(); + values.extend([8191, 8192, u32::MAX - 1, u32::MAX]); + let mut changed = 0u64; + let mut cases = 0u64; + for grow in [true, false] { + for &blocks in &values { + for &max_blocks in &values { + let old = OldLedger::new(grow, blocks, max_blocks); + let new = Ledger::new(grow, blocks, max_blocks); + cases += 1; + assert_eq!( + (new.initial, new.max, new.posted), + (old.initial.min(CAP), old.max.min(CAP), old.posted.min(CAP)), + "new({grow}, {blocks}, {max_blocks})" + ); + assert!((1..=CAP).contains(&(new.posted % BLOCK_LIST_RECORDS))); + if old.max <= CAP { + assert_eq!((new.initial, new.max), (old.initial, old.max)); + } else { + changed += 1; + } + } + } + } + // Only inputs the old constructor took to the whole 4096-block heap changed. + assert!(changed > 0 && changed < cases); + // The driver's values are below the cap, so its ledger is the old one. + let d = (tuning::PB_GROW, tuning::PB_BLOCKS, tuning::PB_MAX_BLOCKS); + let (old, new) = (OldLedger::new(d.0, d.1, d.2), Ledger::new(d.0, d.1, d.2)); + if d.2 <= CAP { + assert_eq!((old.initial, old.max, old.posted), (new.initial, new.max, new.posted)); + } + println!("constructor cases {cases}, changed {changed}"); + } +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument('--output', type=Path, help='Work directory (default: a temporary one)') + p.add_argument('--ref', help='Read the tested bodies at this git revision') + p.add_argument('--mutation', choices=sorted(MUTATIONS) + sorted(CONTROLS)) + p.add_argument('--mutations', action='store_true', + help='Also require every mutation to fail and every control to pass') + a = p.parse_args() + import tempfile + out = (a.output or Path(tempfile.mkdtemp(prefix='g15-pb-growth-'))).resolve() + out.mkdir(parents=True, exist_ok=True) + + def read(path, ref=None): + ref = ref or a.ref + return (subprocess.check_output(['git', 'show', f'{ref}:{path}'], cwd=ROOT, text=True, + stderr=subprocess.DEVNULL) + if ref else (ROOT / path).read_text()) + + # The embedded reference is the commit's own text, when the commit is here. + try: + old_git = item(read(GROWTH.replace("drivers/gpu/drm/asahi/", "drivers/gpu/drm/asahi-m3/"), OLD_REF), ' pub(crate) const fn new(') + except (subprocess.CalledProcessError, ValueError): + old_git = None + if old_git is not None and old_git != OLD_LEDGER_NEW: + raise SystemExit('embedded Ledger::new() differs from ' + OLD_REF) + + def run(mutation): + paths = (POOLS, GROWTH, BUFFER, CHANNELS, INITDATA, PARAMS, HW, CORE, ENGINE, PGTABLE) + src = {path: read(path) for path in paths} + if mutation: + spec = {**MUTATIONS, **CONTROLS}[mutation] + for path, old, new in (spec if isinstance(spec, list) else [spec]): + if src[path].count(old) != 1: + raise SystemExit(f'mutation {mutation}: anchor not found once') + src[path] = src[path].replace(old, new) + pools, hw = src[POOLS], src[HW] + bodies = [item(pools, s) for s in (' pub(crate) fn answer_growth(&mut self) -> Result {', + ' fn answer_request(&mut self, request: &[u32; 14]) -> Result {', + ' pub(crate) fn map_pb(&mut self, vm: &VmState) -> Result {', + ' fn running_vm(&self, ctx: u32) -> Result> {', + ' fn back_pb(&mut self, end: u32, ctx: u32) -> Result {')] + soft_max = item(pools, ' pub(crate) fn soft_max(&self) -> u32 {') + backed = item(pools, ' pub(crate) fn backed(&self) -> u32 {') + chunk = item(pools, 'struct PbChunk {') + '\n' + item(pools, 'impl PbChunk {') + start = pools.index(POOLS_NEW_BACKING[0]) + end = pools.index('\n', pools.index(POOLS_NEW_BACKING[1], start)) + 1 + pools_new_backing = pools[start:end] + if pools.count(PB_LEAF_ASSERT) != 1: + raise SystemExit('pools.rs does not assert the PB leaf GPU-only') + # Every prefix maps the PB into its VM before anything else. + prefix_into = item(src[ENGINE], ' fn publish_prefix_into(&mut self, job: &Arc, f: &mut Flight) -> Result {') + statements = [l.strip() for l in prefix_into.splitlines()[1:] if l.strip() and not l.strip().startswith('//')] + if not statements[0].startswith('self.map_pb(&job.vm)'): + raise SystemExit('publish_prefix_into does not start with the PB catch-up') + pgtable = src[PGTABLE] + pg_consts = '\n'.join(re.search(rf'^const {n}: \w+ = [^;]+;', pgtable, re.M).group(0) + for n in ('UAT_AP_SHIFT', 'UAT_AP_BITS', 'UAT_HIGH_BITS_SHIFT', 'UAT_MEMATTR_SHIFT', + 'UAT_MEMATTR_BITS', 'MEMATTR_CACHED', 'AP_GPU')) + prot = (item(pgtable, 'pub(crate) struct Prot {') + '\nimpl Prot {\n' + + item(pgtable, ' pub(crate) const fn from_pte(pte: u64) -> Self {') + '\n' + + item(pgtable, ' pub(crate) const fn is_cached_noncoherent(&self) -> bool {') + + '\n pub fn ap_is_gpu(&self) -> bool { self.ap == AP_GPU }\n}\n') + init = item(pools, ' fn init(&mut self) -> Result {') + for line in LEDGER_RESET: + if init.count(line) != 1: + raise SystemExit('Pools::init() does not hold: ' + line) + reset = '\n'.join('p.' + line[len('self.'):] for line in LEDGER_RESET) + pb_max_blocks = item(src[PARAMS], 'pub(crate) fn pb_max_blocks() -> u32 {') + m = re.search(r'g15_pb_max_blocks: u32 \{\s*default: (\d+),', src[CORE]) + if not m: + raise SystemExit('g15_pb_max_blocks default not found') + param_default = int(m.group(1)) + sizes = consts(item(hw, 'pub(crate) mod sizes {'), ['PB_BLOCKS']) + tuning = consts(item(hw, 'pub(crate) mod tuning {'), ['PB_GROW', 'PB_BLOCKS', 'PB_MAX_BLOCKS']) + client = consts(item(hw, 'pub(crate) mod client {'), ['PB_HEAP', 'PB_BLOCK_LIST']) + pte = consts(item(hw, 'pub(crate) mod pte {'), ['LEAF_CACHED']) + control = consts(item(src[INITDATA], 'pub(crate) mod control {'), ['HALT_COUNT']) + growth = src[GROWTH] + clamp_fn = item(growth, 'const fn clamp(v: u32, lo: u32, hi: u32) -> u32 {') + tested = (''.join(bodies) + soft_max + backed + chunk + pools_new_backing + pb_max_blocks + growth + + src[BUFFER] + sizes + tuning + pte + prot) + dir = out / (mutation or 'actual') + dir.mkdir(exist_ok=True) + for path in (GROWTH, BUFFER, CHANNELS): + (dir / Path(path).name).write_text(src[path]) + code = (PREFIX + f'pub const PARAM_DEFAULT: u32 = {param_default};\n' + + 'pub mod g15 {\n' + ' pub mod fw {\n' + f' #[path = "{dir / "channels.rs"}"] pub mod channels;\n' + f' #[path = "{dir / "buffer.rs"}"] pub mod buffer;\n' + ' pub mod initdata { pub mod control {\n' + control + '\n} }\n' + ' }\n' + f' pub mod pools {{ #[path = "{dir / "growth.rs"}"] pub mod growth; }}\n' + ' pub mod hw {\n' + ' pub mod sizes {\n' + sizes + '\n}\n' + ' pub mod tuning {\n' + tuning + '\n}\n' + ' pub mod client {\n' + client + '\n}\n' + ' pub mod pte {\n' + pte + '\n}\n' + ' }\n' + ' pub mod pgtable {\n' + pg_consts + '\n' + prot + '\n }\n' + ' pub mod boot {\n pub fn eproto() -> crate::Error { crate::EPROTO }\n }\n' + ' /// The checked backing allocation, failing where a test injects it.\n' + ' pub mod object {\n' + ' use crate::prelude::*;\n' + ' pub struct HeldPages;\n' + ' pub fn gpu_backing(_: &crate::driver::AsahiDevice, bytes: usize)\n' + ' -> Result<(crate::gem::ObjectRef, HeldPages)> {\n' + ' if crate::fail(&crate::FAIL_ALLOC) { return Err(ENOMEM); }\n' + ' let id = crate::NEXT_ID.with(|n| { let id = n.get(); n.set(id + 1); id });\n' + ' crate::ev(crate::Ev::Alloc(id, bytes));\n' + ' Ok((crate::gem::ObjectRef { gem: crate::ARef::new(crate::gem::Object { id, size: bytes }) }, HeldPages))\n' + ' }\n' + ' }\n' + ' pub mod params {\n use crate::module_parameters;\n' + pb_max_blocks + '\n }\n' + ' /// Ledger::new() at ' + OLD_REF[:12] + '.\n' + ' pub struct OldLedger { pub initial: u32, pub max: u32, pub posted: u32, pub next_request: u32,\n' + ' pub refusal_logged: bool, pub refusals: u32 }\n' + ' use crate::g15::fw::buffer::{MAX_BLOCKS, MIN_BLOCKS};\n' + ' impl OldLedger {\n' + OLD_LEDGER_NEW + '\n }\n' + clamp_fn + '\n' + ' pub mod firmware {\n' + ' use crate::prelude::*;\n' + ' use std::cell::Cell;\n' + ' use super::fw::buffer::{self, block_ctl};\n' + ' use super::fw::channels::DevCtl;\n' + ' use super::fw::initdata::control;\n' + ' use super::hw::{client, pte, tuning};\n' + ' use super::pools::growth::{self, Ledger};\n' + ' use super::object::{self, HeldPages};\n' + ' use crate::driver::AsahiDevice;\n' + ' use crate::{gem, mmu, ARef};\n' + ' use core::sync::atomic::{AtomicUsize, Ordering};\n' + ' use std::sync::Arc;\n' + + MODEL.replace('p.RESET', reset).replace('POOLS_NEW_BACKING', '') + + chunk + '\n' + ' /// Pools::new\'s PB backing, verbatim.\n' + ' pub fn pools_new_backing(dev: &AsahiDevice) -> Result<(Ledger, KVec)> {\n' + + pools_new_backing + ' Ok((ledger, chunks))\n }\n' + ' impl Pools {\n' + soft_max + '\n' + backed + '\n }\n' + ' impl Firmware {\n' + '\n'.join(bodies) + '\n }\n' + ' }\n' + '}\n' + TESTS) + path = dir / 'growth-answer.rs' + path.write_text(code) + binary = dir / 'growth-answer' + build = subprocess.run(['rustup', 'run', '1.93.1', 'rustc', '--edition=2021', '--test', '-O', + '-C', 'overflow-checks=on', str(path), '-o', str(binary)], + capture_output=True, text=True) + if build.returncode != 0: + raise SystemExit(f'{mutation or "actual"}: build failed\n{build.stderr[-3000:]}') + + def no_core(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + try: + r = subprocess.run([str(binary), '--show-output', 'pb_growth_tests::'], capture_output=True, + text=True, preexec_fn=no_core, timeout=60) + except subprocess.TimeoutExpired: + # A hang is not a test failure: no test is reported failed, so no mutation counts. + return {'mutation': mutation, 'exit': 'timeout', 'summary': [], 'failed': [], 'output': {}, + 'stderr': 'timed out after 60 s'} + summary = [line for line in r.stdout.splitlines() if line.startswith('test result:')] + failed = sorted(line.split()[1].removeprefix('pb_growth_tests::') for line in r.stdout.splitlines() + if line.startswith('test ') and line.endswith('FAILED')) + # Each test's captured output: the panic message of a failure, the counts of a success. + output = {m.group(1).removeprefix('pb_growth_tests::'): m.group(2).strip() + for m in re.finditer(r'---- (\S+) stdout ----\n(.*?)(?=\n---- |\n(?:failures|successes):\n|\Z)', + r.stdout, re.S)} + source_hashes = {path: hashlib.sha256(src[path].encode()).hexdigest() for path in paths} + source_inputs = json.dumps(source_hashes, sort_keys=True, separators=(',', ':')).encode() + return {'mutation': mutation, 'tested_sha256': hashlib.sha256(tested.encode()).hexdigest(), + 'source_sha256': source_hashes, + 'source_inputs_sha256': hashlib.sha256(source_inputs).hexdigest(), + 'exit': r.returncode, 'summary': summary, 'failed': failed, 'output': output, + 'stderr': r.stderr[-2000:]} + + result = run(a.mutation) + receipt = {'ref': a.ref, 'old_ref': OLD_REF, 'old_reference_checked_against_git': old_git is not None, + **result, + 'scope': 'Actual answer_growth/answer_request/soft_max/pb_max_blocks, pools/growth.rs, ' + 'fw/buffer.rs and fw/channels.rs over a model of the firmware objects with an ' + 'ordered access log, and Ledger::new against its body at ' + f'{OLD_REF[:12]}. Not kernel, firmware or GPU execution, timing, or recovery.'} + if a.mutations and not a.mutation: + receipt['mutations'] = {} + for m in sorted(MUTATIONS): + res = run(m) + test, message = INTENDED[m] + got = res['output'].get(test, '') + receipt['mutations'][m] = {'exit': res['exit'], 'failed': res['failed'], 'intended_test': test, + 'intended_failure': test in res['failed'] and message in got, + 'message': '\n'.join(l for l in got.splitlines() + if not l.startswith('note:'))[-600:]} + receipt['controls'] = {} + for m in sorted(CONTROLS): + res = run(m) + receipt['controls'][m] = {'exit': res['exit'], 'failed': res['failed'], 'summary': res['summary']} + (out / 'result.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps({k: v for k, v in receipt.items() if k not in ('output', 'stderr')}, indent=2)) + # The actual bodies and the controls must pass; a single requested mutation must fail. + expect_fail = a.mutation in MUTATIONS + if (receipt['exit'] != 0) != expect_fail: + raise SystemExit(1) + if expect_fail: + test, message = INTENDED[a.mutation] + if test not in receipt['failed'] or message not in receipt['output'].get(test, ''): + raise SystemExit('the mutation timed out or failed elsewhere than intended') + if a.mutations and any(v['exit'] == 0 or not v['intended_failure'] + for v in receipt.get('mutations', {}).values()): + raise SystemExit('a mutation survived or failed elsewhere than intended') + if a.mutations and any(v['exit'] != 0 for v in receipt.get('controls', {}).values()): + raise SystemExit('a control failed') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g15-probe-init-tests.py b/tools/asahi/g15-probe-init-tests.py new file mode 100644 index 0000000000000..9acd1f012413a --- /dev/null +++ b/tools/asahi/g15-probe-init-tests.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the actual G15 probe with the vendored pin-init crate and proc macro. + +Builds the actual vendored host proc macro without a kernel build or downloaded +crates. An existing kernel-build macro may optionally be supplied for comparison. +The G15 probe, platform callback's dispatch block, Device::set_drvdata and +try_pin_init macro are extracted from production. GPU startup and the allocator +are fault-injection models. KBox's actual source is checked to allocate before +write_pin_init; the host uses the same vendored write_pin_init implementation. +This checks allocation/init/binding order, not a GPU or a kernel probe on hardware. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +OLD_REF = "564f61b8189e" +PROBE_PATH = "drivers/gpu/drm/asahi/g15/mod.rs" +# Verbatim inherited eager probe before the lazy initializer fix. +OLD_PROBE = r''' fn probe<'bound>( + pdev: &'bound platform::Device>, + info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound { + let cfg = *info.ok_or(ENODEV)?; + dev_info!(pdev.as_ref(), "G15: probing ({})\n", cfg.name); + match crate::driver::AsahiPlatformDriver::probe_g15(pdev, cfg) { + Ok(driver) => Ok(Driver { _driver: Some(driver) }), + Err(error) if boot::glue::retained() => { + dev_err!( + pdev.as_ref(), + "G15: probe failed after the firmware start ({:?}); device kept bound, firmware retained until reboot\n", + error + ); + Ok(Driver { _driver: None }) + } + Err(error) => Err(error), + } + }''' + + +def item(text, marker): + start = text.index(marker) + at = text.index("{", start) + depth = 1 + end = at + 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PREFIX = r''' +#![allow(dead_code, unused_variables)] +extern crate self as kernel; +use std::{cell::Cell, marker::PhantomData, pin::Pin, sync::atomic::{AtomicBool, Ordering}}; +use pin_init::{pin_data, PinInit, InPlaceWrite}; +pub mod error { pub type Error = i32; } +use error::Error; +type Result = std::result::Result; +const ENODEV: i32 = -19; +const EPROBE_DEFER: i32 = -517; +const GFP_KERNEL: u32 = 0; +macro_rules! dev_info { ($($x:tt)*) => {{}}; } +macro_rules! dev_err { ($($x:tt)*) => {{}}; } +thread_local! { + static FAIL_ALLOC: Cell = const { Cell::new(false) }; + static ALLOCS: Cell = const { Cell::new(0) }; + static PROBES: Cell = const { Cell::new(0) }; + static TOUCH: Cell = const { Cell::new(false) }; + static RETAINED: Cell = const { Cell::new(false) }; + static FAIL_REGISTRATION: Cell = const { Cell::new(0) }; + static COMPLETED_REGISTRATION: Cell = const { Cell::new(0) }; + static ACTIVATED: Cell = const { Cell::new(false) }; + static ATTACH_ERROR: Cell = const { Cell::new(false) }; + static PROBE_ERROR: Cell = const { Cell::new(0) }; +} +struct Core<'a>(PhantomData<&'a ()>); +struct Device { data: Cell<*mut core::ffi::c_void> } +impl Device { + fn as_raw(&self) -> *mut Device { self as *const Device as *mut Device } + // SET_DRVDATA +} +mod bindings { + pub unsafe fn dev_set_drvdata(dev: *mut crate::Device, data: *mut core::ffi::c_void) { + unsafe { (*dev).data.set(data); } + } +} +struct Owned(Pin>); +impl Owned { + fn into_foreign(self) -> *mut T { + // Test ownership transfer mirrors the kernel's retained pinned KBox. + unsafe { Box::into_raw(Pin::into_inner_unchecked(self.0)) } + } +} +struct KBox; +impl KBox { + fn pin_init(init: impl PinInit, _flags: u32) -> Result> { + ALLOCS.with(|v| v.set(v.get() + 1)); + if FAIL_ALLOC.with(Cell::get) { return Err(-12); } + // Kernel KBox::try_pin_init has this same ordering. Allocation is the only + // mocked operation here; write_pin_init is the actual vendored implementation. + let slot = Box::::new_uninit(); + Ok(Owned(slot.write_pin_init(init)?)) + } +} +mod platform { + use super::*; + pub struct Device { pub base: super::Device, pub ctx: PhantomData } + // The fixture runs serially on one thread; this only permits a static probe target. + unsafe impl Sync for Device {} + pub struct Adapter(pub PhantomData); + impl AsRef for Device { + fn as_ref(&self) -> &super::Device { &self.base } + } + pub trait Driver { + type IdInfo; + type Data<'bound>; + fn probe<'bound>( + pdev: &'bound Device>, + info: Option<&'bound Self::IdInfo>, + ) -> impl PinInit, Error> + 'bound; + } +} +struct Config { name: &'static str } +pub struct ThisModule; +pub trait ModuleMetadata { const NAME: &'static std::ffi::CStr; } +pub trait InPlaceModule: Sized { + fn init(module: &'static ThisModule) -> impl PinInit; +} +mod cleanup { + pub struct QueueOwner; + impl QueueOwner { pub fn new() -> crate::Result { Ok(Self) } } +} +mod driver { + use super::*; + #[pin_data] + pub struct Registration { stage: u8, marker: PhantomData } + impl Registration { + pub fn new(name: &'static std::ffi::CStr, _module: &'static ThisModule) -> impl PinInit { + try_pin_init!(Self { + stage: { + let stage = if name.to_bytes() == b"asahi-g15" { 1 } else { 2 }; + // Model driver_attach and a late driver_add_groups failure. A deferred + // worker can also retry G15 while the other registration is in progress. + let _ = callback::>(&PDEV, Some(&&CFG)); + if FAIL_REGISTRATION.get() == stage { return Err(-5); } + COMPLETED_REGISTRATION.set(stage); + stage + }, + marker: PhantomData, + }) + } + } + pub struct AsahiPlatformDriverData<'bound> { value: u8, bound: PhantomData<&'bound ()> } + pub struct AsahiPlatformDriver; + impl AsahiPlatformDriver { + pub fn probe_g15<'bound>( + _pdev: &'bound platform::Device>, _cfg: &'static Config, + ) -> Result> { + PROBES.with(|v| v.set(v.get() + 1)); + RETAINED.with(|v| v.set(TOUCH.with(Cell::get))); + match PROBE_ERROR.with(Cell::get) { + 0 => Ok(AsahiPlatformDriverData { value: 1, bound: PhantomData }), + e => Err(e), + } + } + } +} +mod boot { pub mod glue { + pub fn retained() -> bool { crate::RETAINED.with(core::cell::Cell::get) } + pub unsafe fn asahi_g15_reprobe() { crate::activation_reprobe(); } +}} +''' + +SUFFIX = r''' +fn callback<'bound, T: platform::Driver>( + pdev: &'bound platform::Device>, + info: Option<&'bound T::IdInfo>, +) -> Result { + // CALLBACK +} +fn reset(fail_alloc: bool, touch: bool, error: i32) { + MODULE_READY.store(true, Ordering::Release); + FAIL_ALLOC.set(fail_alloc); TOUCH.set(touch); PROBE_ERROR.set(error); + RETAINED.set(false); ALLOCS.set(0); PROBES.set(0); +} +static CFG: Config = Config { name: "test" }; +static PDEV: platform::Device> = platform::Device { + base: Device { data: Cell::new(core::ptr::null_mut()) }, ctx: PhantomData, +}; +fn activation_reprobe() { + assert_eq!(COMPLETED_REGISTRATION.get(), 2, "activation before all registrations completed"); + ACTIVATED.set(true); + let _ = callback::>(&PDEV, Some(&&CFG)); + // An attach pass may fail after another device starts. Its error is logged, not returned + // to module initialization. The actual helper's void signature is checked separately. + if ATTACH_ERROR.get() { println!("attach failure after probe was not propagated"); } +} +fn module_cases() { + static MODULE: ThisModule = ThisModule; + for failure in [1, 2, 0] { + reset(false, true, 0); + PDEV.base.data.set(core::ptr::null_mut()); + MODULE_READY.store(false, Ordering::Release); + COMPLETED_REGISTRATION.set(0); FAIL_REGISTRATION.set(failure); + ACTIVATED.set(false); ATTACH_ERROR.set(failure == 0); + let init = ::init(&MODULE); + assert_eq!(PROBES.get(), 0, "module initializer eagerly started firmware"); + let result = KBox::pin_init(init, GFP_KERNEL); + if failure != 0 { + assert!(result.is_err(), "late registration failure must fail module init"); + assert_eq!(PROBES.get(), 0, "late registration failure started firmware"); + assert!(!MODULE_READY.load(Ordering::Acquire)); + assert!(!ACTIVATED.get()); + } else { + assert!(result.is_ok(), "attach failure must not unwind a live session"); + assert_eq!(PROBES.get(), 1); + assert!(MODULE_READY.load(Ordering::Acquire)); + assert!(ACTIVATED.get()); + assert!(!PDEV.base.data.get().is_null()); + unsafe { drop(Box::from_raw(PDEV.base.data.get().cast::>())); } + } + println!("module-registration-{failure}: PASS"); + } +} +fn main() { + let cfg = &CFG; + for (name, allocation_fails, touched, error, expected, bound, calls) in [ + ("allocation-before-start", true, true, 0, Err(-12), false, 0), + ("allocation-before-retained-failure", true, true, -5, Err(-12), false, 0), + ("pre-start-failure", false, false, -19, Err(-19), false, 1), + ("retained-start-failure-binds", false, true, -5, Ok(0), true, 1), + ("retained-registration-failure-binds", false, true, -12, Ok(0), true, 1), + ("successful-registration-binds", false, true, 0, Ok(0), true, 1), + ] { + reset(allocation_fails, touched, error); + let pdev = platform::Device { base: Device { data: Cell::new(core::ptr::null_mut()) }, ctx: PhantomData }; + let result = callback::>(&pdev, Some(&cfg)); + assert_eq!(result, expected, "{name}: callback result"); + assert_eq!(PROBES.get(), calls, "{name}: firmware probe before drvdata allocation"); + assert_eq!(ALLOCS.get(), 1, "{name}: driver-data allocations"); + assert_eq!(!pdev.base.data.get().is_null(), bound, "{name}: bound data"); + if bound { + // Fake kernel teardown after asserting installation. Production suppresses manual unbind. + let data = unsafe { Box::from_raw(pdev.base.data.get().cast::>()) }; + assert_eq!(data._driver.is_some(), error == 0, "{name}: retained failure state"); + } + println!("{name}: PASS"); + } + module_cases(); +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--kernel-build", type=Path, help="Use this existing host proc macro instead of compiling vendored host dependencies") + p.add_argument("--output", type=Path) + p.add_argument("--mutations", action="store_true") + p.add_argument("--rustc", default=os.environ.get("HOSTRUSTC", "rustc")) + a = p.parse_args() + out = (a.output or Path(tempfile.mkdtemp(prefix="g15-probe-init-"))).resolve() + out.mkdir(parents=True, exist_ok=True) + if a.kernel_build is None: + a.kernel_build = out / "host-deps" + dependencies = a.kernel_build / "rust" + dependencies.mkdir(parents=True, exist_ok=True) + crates = [ + ("proc_macro2", "rust/proc-macro2/lib.rs", "2021", [], + ['feature="proc-macro"', "wrap_proc_macro", "proc_macro_span_file", "proc_macro_span_location"]), + ("quote", "rust/quote/lib.rs", "2018", ["proc_macro2"], ['feature="proc-macro"']), + ("syn", "rust/syn/lib.rs", "2021", ["proc_macro2", "quote"], + ['feature="' + feature + '"' for feature in ["clone-impls", "derive", "full", "parsing", "printing", "proc-macro", "visit", "visit-mut"]]), + ("pin_init_internal", "rust/pin-init/internal/src/lib.rs", "2021", ["proc_macro2", "quote", "syn"], + ["kernel", "USE_RUSTC_FEATURES"]), + ] + commands = [] + for name, path, edition, external, cfgs in crates: + macro_crate = name == "pin_init_internal" + output = dependencies / ("lib" + name + (".so" if macro_crate else ".rlib")) + command = [a.rustc, "--edition=" + edition, "--crate-name=" + name, + "--crate-type=" + ("proc-macro" if macro_crate else "rlib"), + "--cap-lints=allow", str(ROOT / path), "-L", "dependency=" + str(dependencies), "-o", str(output)] + if macro_crate: + command += ["--extern", "proc_macro"] + for dependency in external: + command += ["--extern", dependency + "=" + str(dependencies / ("lib" + dependency + ".rlib"))] + for cfg in cfgs: + command += ["--cfg", cfg] + commands.append(command) + built = subprocess.run(command, capture_output=True, text=True) + (out / (name + "-build.log")).write_text(built.stdout + built.stderr) + if built.returncode: + raise RuntimeError("vendored host dependency failed: " + name + "\n" + built.stderr) + (out / "host-dependency-commands.json").write_text(json.dumps(commands, indent=2) + "\n") + sources = {name: (ROOT / name).read_text() for name in [PROBE_PATH, "rust/kernel/platform.rs", "rust/kernel/device.rs", "rust/kernel/init.rs", "rust/kernel/alloc/kbox.rs", "drivers/gpu/drm/asahi/asahi_core.rs", "drivers/gpu/drm/asahi/g15/glue.c"]} + probe = item(sources[PROBE_PATH], " fn probe<'bound>(") + driver = item(sources[PROBE_PATH], "#[pin_data]\npub(crate) struct Driver<'bound>").replace("pub(crate) ", "") + macro = item(sources["rust/kernel/init.rs"], "macro_rules! try_pin_init") + set_data = item(sources["rust/kernel/device.rs"], " pub fn set_drvdata") + callback = sources["rust/kernel/platform.rs"] + callback = callback[callback.index(" let data = T::probe(pdev, info);"):] + callback = callback[:callback.index(" Ok(0)") + len(" Ok(0)")] + ready = "static MODULE_READY: AtomicBool = AtomicBool::new(false);" + assert ready in sources[PROBE_PATH] + activate = item(sources[PROBE_PATH], "pub(crate) fn activate()") + core = sources["drivers/gpu/drm/asahi/asahi_core.rs"] + module_struct = item(core, "#[pin_data]\nstruct DriverModule") + module_init = item(core, " fn init(module:") + module_code = module_struct + "\nimpl InPlaceModule for DriverModule {\n" + module_init + "\n}\nimpl ModuleMetadata for DriverModule { const NAME: &'static std::ffi::CStr = c\"asahi\"; }\n" + kbox = item(sources["rust/kernel/alloc/kbox.rs"], " fn try_pin_init(") + assert "Box::<_, A>::new_uninit(flags)?.write_pin_init(init)" in kbox + reprobe = item(sources["drivers/gpu/drm/asahi/g15/glue.c"], "void asahi_g15_reprobe(void)\n{") + c_prefix = r''' +#include +#include +#include +struct device_driver { int unused; }; +static struct device_driver driver; +static int platform_bus_type; +static bool present; +static int attach_result, finds, attaches, logs, started; +#define pr_err(...) ((void)++logs) +static const struct device_driver *driver_find(const char *name, const int *bus) { + (void)name; assert(bus == &platform_bus_type); finds++; + return present ? &driver : 0; +} +static int driver_attach(const struct device_driver *drv) { + assert(drv == &driver); attaches++; started++; return attach_result; +} +''' + c_suffix = r''' +int main(void) { + for (int test = 0; test < 3; test++) { + present = test != 0; attach_result = test == 2 ? -ENOMEM : 0; + finds = attaches = logs = started = 0; + asahi_g15_reprobe(); + assert(finds == 1); + assert(attaches == (test != 0)); + assert(started == (test != 0)); + assert(logs == (test == 0 || test == 2)); + } +} +''' + c_source = out / "actual-activation.c" + c_source.write_text(c_prefix + reprobe + c_suffix) + c_binary = out / "actual-activation" + subprocess.run([os.environ.get("HOSTCC", "cc"), "-std=c11", "-Wall", "-Wextra", "-Werror", str(c_source), "-o", str(c_binary)], check=True) + subprocess.run([str(c_binary)], check=True) + library = out / "libpin_init.rlib" + subprocess.run([a.rustc, "--edition=2021", "--crate-type=rlib", "--crate-name=pin_init", "--cfg", 'feature="std"', str(ROOT / "rust/pin-init/src/lib.rs"), "--extern", f"pin_init_internal={a.kernel_build / 'rust/libpin_init_internal.so'}", "-L", f"dependency={a.kernel_build / 'rust'}", "-o", str(library)], check=True) + receipt = {"rustc": subprocess.check_output([a.rustc, "--version"], text=True).strip(), "pin_init_source_sha256": {str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() for p in (ROOT / "rust/pin-init").rglob("*.rs")}, "proc_macro_sha256": hashlib.sha256((a.kernel_build / "rust/libpin_init_internal.so").read_bytes()).hexdigest(), "source_sha256": {n: hashlib.sha256(s.encode()).hexdigest() for n, s in sources.items()}, "cases": 9, "actual_C_activation_cases": 3, "mutants": {}, "scope": __doc__} + variants = {"actual": (probe, module_code)} + if a.mutations: + try: + old = subprocess.check_output(["git", "show", f"{OLD_REF}:{PROBE_PATH}"], cwd=ROOT, text=True, stderr=subprocess.DEVNULL) + assert item(old, " fn probe<'bound>(") == OLD_PROBE + receipt["old_reference_checked_against_git"] = True + except subprocess.CalledProcessError: + receipt["old_reference_checked_against_git"] = False + variants["eager-probe-before-allocation"] = (OLD_PROBE, module_code) + needle = " None\n" + assert probe.count(needle) == 1 + variants["retained-error-unbinds"] = (probe.replace(needle, " return Err(error);\n"), module_code) + gate = " if !MODULE_READY.load(Ordering::Acquire) {\n return Err(EPROBE_DEFER);\n }\n" + assert probe.count(gate) == 1 + variants["no-registration-readiness-gate"] = (probe.replace(gate, ""), module_code) + late = " #[cfg(CONFIG_DRM_ASAHI_G15)]\n _g15_ready: g15::activate(),\n" + assert module_code.count(late) == 1 + early = module_code.replace(late, "").replace(" _driver <-", late + " _driver <-", 1) + variants["activation-before-last-registration"] = (probe, early) + for name, (method, module_variant) in variants.items(): + # Macro definitions precede the fake registration, just as kernel prelude exports them. + source = macro + "\n" + PREFIX.replace("#![allow(dead_code, unused_variables)]\n", "").replace(" // SET_DRVDATA", set_data) + "\n" + ready + "\n" + driver + "\nimpl platform::Driver for Driver<'_> {\n type IdInfo = &'static Config;\n type Data<'bound> = Driver<'bound>;\n" + method + "\n}\nmod g15 { use super::*; pub type Driver<'a> = super::Driver<'a>; " + activate + "\n}\ntype G15Registration = driver::Registration>>;\n" + module_variant + "\n" + SUFFIX.replace(" // CALLBACK", callback) + source = "#![allow(dead_code, unused_variables, private_bounds, private_interfaces)]\n" + source + src = out / (name + ".rs"); src.write_text(source) + binary = out / name + subprocess.run([a.rustc, "--edition=2021", "--cfg", "CONFIG_DRM_ASAHI_G15", str(src), "--extern", f"pin_init={library}", "-L", f"dependency={a.kernel_build / 'rust'}", "-o", str(binary)], check=True) + r = subprocess.run([str(binary)], capture_output=True, text=True) + receipt["mutants" if name != "actual" else "control"] = receipt.get("mutants" if name != "actual" else "control", {}) + result = {"exit": r.returncode, "stdout": r.stdout, "stderr": r.stderr} + if name == "actual": + receipt["control"] = result + if r.returncode: raise SystemExit(r.stderr) + if r.stdout.count(": PASS\n") != 9: raise SystemExit("control did not run every case") + else: + receipt["mutants"][name] = result + if r.returncode == 0: raise SystemExit("SURVIVED: " + name) + intended = {"eager-probe-before-allocation": "allocation-before-start: firmware probe before drvdata allocation", "retained-error-unbinds": "retained-start-failure-binds: callback result", "no-registration-readiness-gate": "late registration failure started firmware", "activation-before-last-registration": "activation before all registrations completed"}[name] + result["intended_failure"] = intended in r.stderr + if not result["intended_failure"]: raise SystemExit("INVALID unrelated failure: " + name + "\n" + r.stderr) + (out / "result.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps(receipt, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/g15-queue-credit-tests.py b/tools/asahi/g15-queue-credit-tests.py new file mode 100644 index 0000000000000..2c8d7f64fb3f3 --- /dev/null +++ b/tools/asahi/g15-queue-credit-tests.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run actual native logical-queue credit/new/drop bodies on host services. + +Scheduler/VM/allocation services are models. This tests host capacity and Rust +field-drop ordering, not hardware execution-state reservation or DRM scheduling. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[2] + + +def extract(text, marker): + start = text.index(marker); opening = text.index("{", start) + depth, end = 1, opening + 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +CODE = r''' +#![allow(dead_code)] +use std::{sync::{Arc as StdArc,Mutex,Barrier,OnceLock,atomic::{AtomicBool,AtomicI32,AtomicU32,AtomicU64,Ordering}},ops::Deref,cell::Cell}; +#[derive(Copy,Clone,Debug,PartialEq,Eq)]struct Error(i32); +type Result = std::result::Result; +const EIO:Error=Error(-5);const EINVAL:Error=Error(-22);const ENOMEM:Error=Error(-12);const GFP_KERNEL:()=(); +thread_local!{static FAIL:Cell=const{Cell::new(0)};static RACE_FIRST:Cell=const{Cell::new(false)};} +static RACE_GATE:OnceLock=OnceLock::new(); +trait RacyCounter{fn racy_reserve(&self)->std::result::Result;} +impl RacyCounter for StdArc{fn racy_reserve(&self)->std::result::Result{ + let count=self.load(Ordering::Acquire); + RACE_FIRST.with(|flag|if flag.replace(false){RACE_GATE.get_or_init(||Barrier::new(16)).wait();}); + if countbool{FAIL.with(|f|f.get()==n)} +struct Arc(StdArc);impl Clone for Arc{fn clone(&self)->Self{Self(self.0.clone())}} +impl Deref for Arc{type Target=T;fn deref(&self)->&T{&self.0}} +impl Arc{fn wrap(v:T)->Self{Self(StdArc::new(v))}fn new(v:T,_:())->Result{if failure(4){Err(ENOMEM)}else{Ok(Self::wrap(v))}}} +struct KBox(Box);implKBox{fn new(v:T,_:())->Result{if failure(5){Err(ENOMEM)}else{Ok(Self(Box::new(v)))}}} +struct Trace{counter:StdArc,drains:Mutex>,prunes:AtomicU32} +struct Device{trace:StdArc} +struct Shared{logical_queues:StdArc,crashed:AtomicBool,dev:Device} +struct Status(AtomicI32);impl Status{fn get(&self)->i32{self.0.load(Ordering::SeqCst)}} +struct VmState{status:Status,trace:StdArc}impl VmState{fn prune_registry(&self){self.trace.prunes.fetch_add(1,Ordering::SeqCst);}} +mod mmu{use super::*;pub struct Vm{pub state:Arc}} +mod vm{use super::*;pub fn lookup(_:&Gpu,vm:&mmu::Vm)->Result>{if failure(1){Err(ENOMEM)}else{Ok(vm.state.clone())}}} +mod hw{pub const USER_RANGE:std::ops::Range=0x100000000..0x2000000000;pub mod sizes{__LIMIT__}} +mod uapi{pub const drm_asahi_priority_DRM_ASAHI_PRIORITY_REALTIME:u32=3;} +const USC_WINDOW:u64=0x100000000; +struct Gpu{shared:Arc} +struct Scheduler{trace:StdArc} +impl Scheduler{fn new(dev:&Device)->Result{if failure(3){Err(ENOMEM)}else{Ok(Self{trace:dev.trace.clone()})}}} +impl Drop for Scheduler{fn drop(&mut self){let n=self.trace.counter.load(Ordering::SeqCst);assert!(n>0,"scheduler must drain before credit release");self.trace.drains.lock().unwrap().push(n);}} +struct Timeline;impl Timeline{fn new()->Result>{if failure(2){Err(ENOMEM)}else{Ok(Arc::wrap(Self))}}} +__STATE__ +__CREDIT__ +__CREDIT_IMPL__ +__CREDIT_DROP__ +__QUEUE__ +__QUEUE_DROP__ +impl Queue{__NEW__} +fn setup()->Gpu{ + let counter=StdArc::new(AtomicU32::new(0));let trace=StdArc::new(Trace{counter:counter.clone(),drains:Mutex::new(Vec::new()),prunes:AtomicU32::new(0)}); + Gpu{shared:Arc::wrap(Shared{logical_queues:counter,crashed:AtomicBool::new(false),dev:Device{trace}})} +} +fn vm(gpu:&Gpu,status:i32)->mmu::Vm{mmu::Vm{state:Arc::wrap(VmState{status:Status(AtomicI32::new(status)),trace:gpu.shared.dev.trace.clone()})}} +fn new(gpu:&Gpu,status:i32,priority:u32,base:u64)->Result>{KBox::new(Queue::new(gpu,vm(gpu,status),priority,base)?,GFP_KERNEL)} +fn count(gpu:&Gpu)->u32{gpu.shared.logical_queues.load(Ordering::SeqCst)} +fn main(){ + // One device shared across different VM objects/files: exactly58 live queues. + let gpu=setup();let mut queues=Vec::new();for _ in 0..hw::sizes::MAX_QUEUES{queues.push(new(&gpu,0,2,0x100000000).expect("live logical queue below cap"));} + assert_eq!(count(&gpu),58,"reported logical capacity must be reserved"); + assert!(matches!(new(&gpu,0,2,0x100000000),Err(ENOMEM)),"59th device queue must refuse"); + assert_eq!(count(&gpu),58);drop(queues); + assert_eq!(count(&gpu),0,"release returns all logical credits"); + assert_eq!(*gpu.shared.dev.trace.drains.lock().unwrap(),(1..=58).rev().collect::>(),"every scheduler drains before its capacity returns"); + assert!(new(&gpu,0,2,0x100000000).is_ok(),"capacity reusable after drain"); + assert_eq!(count(&gpu),0); + // Failure at lookup/timeline/scheduler/state/outerKBox must never leak a credit. + for point in 1..=5{ + let gpu=setup();FAIL.with(|f|f.set(point));assert!(new(&gpu,0,2,0x100000000).is_err());FAIL.with(|f|f.set(0)); + assert_eq!(count(&gpu),0,"allocation failure must return logical credit"); + let drains=gpu.shared.dev.trace.drains.lock().unwrap().clone(); + assert_eq!(drains,if point>=4{vec![1]}else{vec![]},"allocation rollback drains before credit release"); + let retry=new(&gpu,0,2,0x100000000).expect("allocation failure must leave reusable capacity");assert_eq!(count(&gpu),1);drop(retry);assert_eq!(count(&gpu),0); + } + // Existing GPU/VM/range refusals do not consume capacity. + for (status,priority,base) in [(-5,2,0x100000000),(0,4,0x100000000),(0,2,1),(0,2,0x2000000000)]{ + let gpu=setup();assert!(new(&gpu,status,priority,base).is_err());assert_eq!(count(&gpu),0); + } + let gpu=setup();gpu.shared.crashed.store(true,Ordering::SeqCst);assert!(new(&gpu,0,2,0x100000000).is_err());assert_eq!(count(&gpu),0); + // CAS contention:128 acquisitions across16threads, held until all have attempted. + let gpu=setup();let ready=StdArc::new(Barrier::new(17));let release=StdArc::new(Barrier::new(17));let successes=StdArc::new(AtomicU32::new(0));let mut threads=Vec::new(); + for _ in 0..16{let shared=gpu.shared.clone();let ready=ready.clone();let release=release.clone();let successes=successes.clone();threads.push(std::thread::spawn(move||{ + RACE_FIRST.with(|flag|flag.set(true));let mut held=Vec::new();for _ in 0..8{if let Ok(credit)=QueueCredit::acquire(&shared){successes.fetch_add(1,Ordering::SeqCst);held.push(credit);}} + ready.wait();release.wait();drop(held); + }));} + ready.wait();assert_eq!(successes.load(Ordering::SeqCst),58,"concurrent reservations respect shared cap");assert_eq!(count(&gpu),58,"concurrent reservations respect shared cap");release.wait();for t in threads{t.join().unwrap();}assert_eq!(count(&gpu),0,"concurrent releases return capacity"); + // An already-corrupt/saturated counter must not wrap into an admitted credit. + gpu.shared.logical_queues.store(u32::MAX,Ordering::SeqCst);assert!(matches!(QueueCredit::acquire(&gpu.shared),Err(ENOMEM)));assert_eq!(count(&gpu),u32::MAX); + println!("PASS actual Queue::new/field order/credit:58/59,5 allocation failures,admission negatives,16-thread CAS and saturation"); +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--mutations", action="store_true") + args = parser.parse_args(); args.output.mkdir(parents=True, exist_ok=True) + queue = (ROOT / "drivers/gpu/drm/asahi/g15/queue.rs").read_text() + hardware = (ROOT / "drivers/gpu/drm/asahi/g15/hw.rs").read_text() + limit = next(line.strip() for line in hardware.splitlines() if "const MAX_QUEUES:" in line) + replacements = {"__LIMIT__": limit, "__STATE__": extract(queue, "pub(crate) struct QueueState {"), + "__CREDIT__": extract(queue, "struct QueueCredit {"), "__CREDIT_IMPL__": extract(queue, "impl QueueCredit {"), + "__CREDIT_DROP__": extract(queue, "impl Drop for QueueCredit {"), "__QUEUE__": extract(queue, "pub(crate) struct Queue {"), + "__QUEUE_DROP__": extract(queue, "impl Drop for Queue {"), "__NEW__": extract(queue, " pub(crate) fn new(gpu: &Gpu,")} + code = CODE + for old, new in replacements.items(): code = code.replace(old, new) + variants = {"control": code}; expected = {} + if args.mutations: + changes = { + "cap-not-enforced": ("count < hw::sizes::MAX_QUEUES", "count < u32::MAX", "59th device queue must refuse"), + "counter-not-reserved": ("then(|| count + 1)", "then(|| count)", "reported logical capacity must be reserved"), + "credit-not-released": ("let previous = self.shared.logical_queues.fetch_sub(1, Ordering::AcqRel);\n debug_assert!(previous != 0);", "let _ = &self.shared;", "release returns all logical credits"), + "release-before-drain": ("pub(crate) struct Queue {\n", "pub(crate) struct Queue {\n _credit: QueueCredit,\n", "scheduler must drain before credit release"), + "nonatomic-reservation": (".fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| {\n (count < hw::sizes::MAX_QUEUES).then(|| count + 1)\n })", ".racy_reserve()", "concurrent reservations respect shared cap"), + "advertised-cap-off-by-one": ("count < hw::sizes::MAX_QUEUES", "count <= hw::sizes::MAX_QUEUES", "59th device queue must refuse"), + } + for name, (old, new, message) in changes.items(): + if code.count(old) != 1: raise RuntimeError(f"{name}: mutation anchor count {code.count(old)}") + variant = code.replace(old, new) + if name == "release-before-drain": + old_tail = " // Last field: all preceding fields (especially scheduler) drop before capacity returns.\n _credit: QueueCredit,\n" + if variant.count(old_tail) != 1: raise RuntimeError("missing tail credit") + variant = variant.replace(old_tail, "") + variants[name] = variant; expected[name] = message + variants["control-after"] = code + receipt = {"scope": __doc__, "sources": {"queue.rs": hashlib.sha256(queue.encode()).hexdigest(), + "hw.rs": hashlib.sha256(hardware.encode()).hexdigest()}, "variants": {}} + for name, source in variants.items(): + path=args.output/(name+".rs");path.write_text(source);binary=args.output/name + built=subprocess.run([os.environ.get("RUSTC","rustc"),"--edition=2021",str(path),"-o",str(binary)],capture_output=True,text=True) + (args.output/(name+".build.log")).write_text(built.stdout+built.stderr) + if built.returncode: raise RuntimeError(f"{name}: NOBUILD, not a runtime kill: {built.stderr}") + ran=subprocess.run([str(binary.resolve())],capture_output=True,text=True,timeout=30) + receipt["variants"][name]={"exit":ran.returncode,"stdout":ran.stdout,"stderr":ran.stderr,"intended_failure":name in expected and expected[name] in ran.stderr} + if name.startswith("control") and ran.returncode:raise RuntimeError(ran.stdout+ran.stderr) + if name in expected and (ran.returncode==0 or expected[name] not in ran.stderr):raise RuntimeError(f"{name}: wrong runtime failure: {ran.stderr}") + (args.output/"result.json").write_text(json.dumps(receipt,indent=2)+"\n");print(json.dumps(receipt,indent=2)) + + +if __name__ == "__main__": main() diff --git a/tools/asahi/g15-retention-tests.py b/tools/asahi/g15-retention-tests.py new file mode 100644 index 0000000000000..4812a32b0b1e3 --- /dev/null +++ b/tools/asahi/g15-retention-tests.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run actual G15 retention/backend/boot-completion bodies with drop-count fault injection. + +Allocators, firmware objects and hardware operations are host models. Production +ownership transfers and error handling are extracted unchanged. No firmware or +device is accessed. Mutants must build and fail an intended runtime assertion. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] +G15 = ROOT / "drivers/gpu/drm/asahi/g15" +CONTROL_REF = "31da169251bd34d5810a0874563d9a019b85bf56" + + +def item(text, marker): + start = text.index(marker) + end = text.index("{", start) + 1 + depth = 1 + while depth: + depth += (text[end] == "{") - (text[end] == "}") + end += 1 + return text[start:end] + + +PREFIX = r''' +#![allow(dead_code, unused_variables)] +use std::{cell::{Cell, RefCell, RefMut}, mem::MaybeUninit, ops::{Deref, DerefMut}, + sync::{Arc, atomic::{AtomicBool, Ordering}}}; +type Error = i32; +type Result = std::result::Result; +const GFP_KERNEL: u32 = 0; +const EINVAL: Error = -22; +macro_rules! dev_info { ($($args:tt)*) => {{}}; } +macro_rules! dev_err { ($($args:tt)*) => {{}}; } +thread_local! { + static FAIL: Cell = const { Cell::new(0) }; + static OWNER_DROPS: Cell = const { Cell::new(0) }; + static BUFFER_DROPS: Cell = const { Cell::new(0) }; +} +fn step(site: u32) -> Result { + if FAIL.get() & (1 << site) != 0 { Err(-100 - site as i32) } else { Ok(()) } +} +struct Owner; +impl Drop for Owner { fn drop(&mut self) { OWNER_DROPS.set(OWNER_DROPS.get() + 1); } } +struct Buffer; +impl Drop for Buffer { fn drop(&mut self) { BUFFER_DROPS.set(BUFFER_DROPS.get() + 1); } } +struct Mutex(RefCell); +impl Mutex { + fn new(value: T) -> Self { Self(RefCell::new(value)) } + fn lock(&self) -> RefMut<'_, T> { self.0.borrow_mut() } +} +struct KBox(Box); +impl KBox { + fn new_uninit(_flags: u32) -> Result>> { + step(if std::any::type_name::().ends_with("Orphan") { 7 } else { 5 })?; + Ok(KBox(Box::new_uninit())) + } + // KBOX_NEW + fn into_inner(value: Self) -> T { *value.0 } +} +impl KBox> { + fn write(storage: Self, value: T) -> KBox { KBox(Box::write(storage.0, value)) } +} +impl Deref for KBox { type Target = T; fn deref(&self) -> &T { &self.0 } } +impl DerefMut for KBox { fn deref_mut(&mut self) -> &mut T { &mut self.0 } } +struct AsahiDevice; +impl AsahiDevice { fn as_ref(&self) -> &Self { self } } +struct Config; +static CONFIG: Config = Config; +mod mmu { pub struct Uat; } +mod regs { + pub struct Identity; + pub struct Regs; + pub const ASC_RUN: u32 = 1; + impl Regs { pub fn set_asc_control(&self, _: u32) -> super::Result { super::step(11) } } +} +mod object { + use super::*; + pub struct Objects { _owner: Owner, retained: Vec> } + impl Objects { + pub fn new() -> Self { Self { _owner: Owner, retained: vec![] } } + pub fn buffer(&mut self) -> Arc { + let buffer = Arc::new(Buffer); self.retained.push(buffer.clone()); buffer + } + } +} +mod initdata { pub struct InitData(pub super::Owner); } +mod channel { pub struct Channels(pub super::Owner); } +mod event { + pub struct Events(pub super::Owner); + impl Events { + pub fn new() -> Self { Self(super::Owner) } + pub fn reserve(&mut self) -> super::Result { super::step(12) } + } +} +mod graph { + use super::*; + pub struct Graphs(Arc); + impl Graphs { + pub fn new(_: &AsahiDevice, objects: &mut object::Objects) -> Result { + let buffer = objects.buffer(); step(1)?; Ok(Self(buffer)) + } + } +} +mod slots { + use super::*; + pub struct Slots(Arc); + impl Slots { + pub fn new(objects: &mut object::Objects) -> Result { + let buffer = objects.buffer(); step(2)?; Ok(Self(buffer)) + } + } +} +mod pools { + use super::*; + pub struct Pools(Arc); + impl Pools { + pub fn new(_: &AsahiDevice, objects: &mut object::Objects) -> Result { + let buffer = objects.buffer(); step(3)?; Ok(Self(buffer)) + } + } +} +mod vm { + use super::*; + pub struct Registry; + impl Registry { pub fn new() -> Result> { step(4)?; Ok(Arc::new(Self)) } } +} +mod engine { pub struct Engine; impl Engine { pub fn new() -> Self { Self } } } +mod recovery { pub struct Recovery; impl Recovery { pub fn new() -> Self { Self } } } +struct Shared { + dev: AsahiDevice, + orphan: Mutex>>, + orphaned: AtomicBool, +} +impl Shared { + fn new() -> Arc { + Arc::new(Self { dev: AsahiDevice, orphan: Mutex::new(None), orphaned: AtomicBool::new(false) }) + } + // ORPHAN +} +// FIRMWARE +impl Firmware { + fn init_graphs(&mut self) -> Result { step(6) } + // BACKEND +} +mod boot { + use super::*; + use super::{object::Objects, initdata::InitData, channel::Channels, event::Events}; + pub struct Session { _owner: Owner, uat: Arc, regs: regs::Regs } + impl Session { + pub fn new() -> Self { Self { _owner: Owner, uat: Arc::new(mmu::Uat), regs: regs::Regs } } + // UAT + } + // BOOTED + // ORPHAN_STRUCT + fn run(_: &mut Session, _: &Config, _: &mut Objects, _: &mut Events, + built: &mut Option<(InitData, Channels, u32)>, _: ()) -> Result { + step(8)?; + *built = Some((InitData(Owner), Channels(Owner), 1234)); + step(9) + } + pub fn start(shared: &Arc) -> Result { + let mut session = Session::new(); + let mut objects = Objects::new(); + let d = AsahiDevice; + let cfg = &CONFIG; + let control = 0; + let id = regs::Identity; + let board_images = (); + // START_TAIL +} +fn booted() -> boot::Booted { + boot::Booted { session: boot::Session::new(), init: initdata::InitData(Owner), + objects: object::Objects::new(), channels: channel::Channels(Owner), + events: event::Events::new(), id: regs::Identity, max_frequency_khz: 1234 } +} +fn orphan() -> boot::Orphan { + let boot::Booted { session, init, objects, channels, events, .. } = booted(); + boot::Orphan { session, _objects: objects, channels: Some(channels), _init: Some(init), events } +} +fn reset(fail: u32) { FAIL.set(fail); OWNER_DROPS.set(0); BUFFER_DROPS.set(0); } +fn retained(label: &str) { + assert_eq!(OWNER_DROPS.get(), 0, "{label}: live session owner dropped"); + assert_eq!(BUFFER_DROPS.get(), 0, "{label}: firmware buffer dropped"); +} +fn main() { + for oom in [false, true] { + reset(if oom { 1 << 7 } else { 0 }); + let shared = Shared::new(); shared.orphan(orphan()); + retained("orphan allocation"); + assert_eq!(shared.orphaned.load(Ordering::Acquire), !oom); + assert_eq!(shared.orphan.lock().is_some(), !oom); + std::mem::forget(shared); + } + for site in 1..=6 { + for orphan_oom in [false, true] { + reset((1 << site) | if orphan_oom { 1 << 7 } else { 0 }); + let shared = Shared::new(); + let result = Firmware::start_backend(&AsahiDevice, &CONFIG, booted(), &shared); + assert!(matches!(result, Err(e) if e == -100 - site as i32), "backend error changed"); + retained("backend failure"); + assert_eq!(shared.orphan.lock().is_some(), !orphan_oom); + std::mem::forget(shared); + } + } + for site in [8, 9, 11, 12] { + for orphan_oom in [false, true] { + reset((1 << site) | if orphan_oom { 1 << 7 } else { 0 }); + let shared = Shared::new(); let result = boot::start(&shared); + let expected = -100 - site as i32; + assert!(matches!(result, Err(e) if e == expected), "boot error changed"); + retained("boot completion"); + assert_eq!(shared.orphan.lock().is_some(), !orphan_oom); + std::mem::forget(shared); + } + } + reset(0); + let shared = Shared::new(); + let booted = boot::start(&shared).expect("normal boot"); + assert_eq!(booted.max_frequency_khz, 1234); + let uat: Arc = booted.session.uat(); + let firmware = Firmware::start_backend(&AsahiDevice, &CONFIG, booted, &shared).expect("normal backend"); + assert!(Arc::ptr_eq(&uat, &firmware.session.uat())); + assert!(!shared.orphaned.load(Ordering::Acquire)); + retained("normal success"); + std::mem::forget((shared, firmware)); + println!("PASS 23 retention cases: orphan OOM, six backend failure sites, four boot-completion sites and normal success"); + println!("normal success owner_drops={} buffer_drops={}", OWNER_DROPS.get(), BUFFER_DROPS.get()); +} +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path) + parser.add_argument("--mutations", action="store_true") + args = parser.parse_args() + out = (args.output or Path(tempfile.mkdtemp(prefix="g15-retention-"))).resolve() + out.mkdir(parents=True, exist_ok=True) + mod = (G15 / "mod.rs").read_text() + boot = (G15 / "boot.rs").read_text() + kbox = (ROOT / "rust/kernel/alloc/kbox.rs").read_text() + orphan = item(mod, " pub(crate) fn orphan(") + backend = item(mod, " fn start_backend(") + start = item(boot, "pub(crate) fn start(") + tail = start[start.index(" let mut events = Events::new();"):] + source = PREFIX.replace(" // ORPHAN\n", orphan + "\n") + allocator_new = item(kbox, " pub fn new(x: T, flags: Flags)") + allocator_new = allocator_new.replace("pub fn new(x: T, flags: Flags) -> Result", + "fn new(x: T, flags: u32) -> Result") + source = source.replace(" // KBOX_NEW", allocator_new.replace("Box::write", "KBox::write")) + source = source.replace("// FIRMWARE", item(mod, "pub(crate) struct Firmware")) + source = source.replace(" // BACKEND", backend) + source = source.replace(" // UAT", item(boot, " pub(crate) fn uat(")) + source = source.replace(" // BOOTED", item(boot, "pub(crate) struct Booted")) + source = source.replace(" // ORPHAN_STRUCT", item(boot, "pub(crate) struct Orphan")) + source = source.replace(" // START_TAIL", tail) + variants = {"actual": source} + if args.mutations: + old = subprocess.check_output( + ["git", "show", CONTROL_REF + ":drivers/gpu/drm/asahi/g15/mod.rs"], + cwd=ROOT, text=True) + variants["consume-orphan-before-allocation"] = source.replace(orphan, item(old, " pub(crate) fn orphan(")) + variants["unguarded-registry-allocation"] = source.replace(backend, item(old, " fn start_backend(")) + receipt = {"scope": __doc__, "sources": { + name: hashlib.sha256(text.encode()).hexdigest() + for name, text in [("mod.rs", mod), ("boot.rs", boot), ("kbox.rs", kbox)]}, + "rustc": subprocess.check_output([os.environ.get("HOSTRUSTC", "rustc"), "--version"], text=True).strip(), + "cases": 23, "public_old_source_control": CONTROL_REF, "variants": {}} + for name, text in variants.items(): + if name != "actual" and text == source: + raise RuntimeError("NOT_APPLIED " + name) + path = out / (name + ".rs") + path.write_text(text) + binary = out / name + subprocess.run([os.environ.get("HOSTRUSTC", "rustc"), "--edition=2021", + str(path), "-o", str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + receipt["variants"][name] = {"exit": result.returncode, "stdout": result.stdout, + "stderr": result.stderr} + if name == "actual": + if result.returncode: + raise RuntimeError(result.stderr) + else: + expected = {"consume-orphan-before-allocation": "orphan allocation: live session owner dropped", + "unguarded-registry-allocation": "backend failure: live session owner dropped"}[name] + if result.returncode == 0 or expected not in result.stderr or "left: 5" not in result.stderr: + raise RuntimeError("INVALID mutant outcome " + name + "\n" + result.stderr) + receipt["variants"][name]["intended_owner_drops"] = 5 + after = subprocess.run([str(out / "actual")], capture_output=True, text=True, check=True) + receipt["actual_after_mutants"] = {"exit": after.returncode, "stdout": after.stdout, "stderr": after.stderr} + receipt["fixed_invariants"] = {"live_session_owner_drops": 0, "firmware_buffer_drops": 0, + "checked_cases": 23} + (out / "result.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps(receipt, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/g15-stats-drain-tests.py b/tools/asahi/g15-stats-drain-tests.py new file mode 100644 index 0000000000000..2bd9cc2cacd0b --- /dev/null +++ b/tools/asahi/g15-stats-drain-tests.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Run the G15 receive-ring drain's actual bodies, before and after the stats ring is noted. + +The harness extracts Channels::drain_one(), note_stats(), peek_stats() and drain_rings() from +drivers/gpu/drm/asahi/g15/channel.rs, and power/pstate.rs whole. It runs them against a +model of the firmware objects that logs every load, store and barrier in order. The reference is +drain_rings() as of 9f09c85f8d3b, before the stats ring was noted. It is embedded below, because +a shallow checkout lacks that commit, and checked against git when the commit is present. + +What it shows, for the stats ring and the event, log and trace rings drained beside it: +- every (consumer, producer) pair, both inside and outside the 256-slot ring, gives the same + stores the firmware sees (offsets, values, order, write barriers) and the same final state; +- the producer is loaded before the consumer, as before. Inside the ring the consumer is loaded + again before the store; outside the ring both are loaded again by the earlier drain_one(); +- with a producer that moves between loads, the consumer store is the earlier drain's for the + producer loaded by the same code; +- the noted records: the pending slots in ring order (wrap, empty, full), the tag at +0 and the + word at +0x20, read after a read barrier, the newest power-state record winning; +- a full barrier after the last record load and before the consumer store that frees the slots; +- peek_stats() stores nothing. + +Not shown: kernel, firmware or GPU execution, nor timing. The noting drain loads up to 510 more +record words, so it takes longer than the earlier drain. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import resource +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +G15 = "drivers/gpu/drm/asahi/g15" +CHANNEL = f"{G15}/channel.rs" +PSTATE = f"{G15}/power/pstate.rs" +HW = f"{G15}/hw.rs" +FW_CHANNELS = f"{G15}/fw/channels.rs" +OLD_REF = "9f09c85f8d3bae8512712686f5a91067fc4eb3de" + +# drain_rings() at OLD_REF (the stats ring consumed unread), verbatim. +OLD_DRAIN_RINGS = ''' pub(crate) fn drain_rings(&self, dev: &kernel::device::Device, events: &mut Events) -> Result { + events.drain(dev, &self.event)?; + for i in 0..rings::LOG_RINGS { + Self::drain_one(&self.log, i * 0x30)?; + } + Self::drain_one(&self.trace, 0)?; + Self::drain_one(&self.stats, 0) + }''' + +MUTATIONS = { + # Store the consumer even when it already equals the producer. + 'unconditional-consume': (CHANNEL, 'if self.stats.read()? != producer {', 'if true {'), + # Leave the stats ring unconsumed when an index is out of the ring. + 'skip-fallback': (CHANNEL, ' Self::drain_one(&self.stats, 0)\n }', ' Ok(())\n }'), + # Store a producer loaded after the records were noted. + 'resample-producer': (CHANNEL, 'self.stats.state.w32(STATE_READ, producer)?', + 'self.stats.state.w32(STATE_READ, self.stats.state.r32(STATE_WRITE)?)?'), + # Store the consumer without the full barrier after the record loads. + 'no-release-barrier': (CHANNEL, ' mb();\n self.stats.state.w32(STATE_READ, producer)?;', + ' self.stats.state.w32(STATE_READ, producer)?;'), + # Read the records without the read barrier. + 'no-read-barrier': (CHANNEL, ' rmb();\n for slot in slots {', ' for slot in slots {'), + # Walk the slots without the wrap. + 'no-wrap': (PSTATE, '(read + i) % STATS_SLOTS', 'read + i'), + # Keep the oldest power-state record instead of the newest. + 'oldest-power-state': (PSTATE, 'if tag == TAG_POWER_STATE {', 'if tag == TAG_POWER_STATE && self.last.is_none() {'), +} + + +def item(text, start): + a = text.index(start) + b = text.index("{", a) + 1 + depth = 1 + while depth: + depth += (text[b] == "{") - (text[b] == "}") + b += 1 + return text[a:b] + + +def const(text, name): + m = re.search(rf'pub\(crate\) const {name}: (\w+) = ([^;]+);', text) + if not m: + raise SystemExit(f'{name} not found') + return f'pub const {name}: {m.group(1)} = {m.group(2)};' + + +PREFIX = r''' +#![allow(dead_code, unused_imports)] +use std::{cell::RefCell, sync::Arc}; +type Result = std::result::Result; +const EINVAL: i32 = -22; +mod kernel { pub mod device { pub struct Device; } } +mod power { + #[path = "PSTATE_PATH"] + pub mod pstate; +} +use power::pstate::{self, StatsSeen}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Ev { Load(u8, usize, u32), Store(u8, usize, u32), Rmb, Mb, Wmb, Events } +thread_local! { static LOG: RefCell> = RefCell::new(Vec::new()); } +fn log(e: Ev) { LOG.with(|l| l.borrow_mut().push(e)); } +fn take_log() -> Vec { LOG.with(|l| std::mem::take(&mut *l.borrow_mut())) } +fn wmb() { log(Ev::Wmb); } +fn rmb() { log(Ev::Rmb); } +fn mb() { log(Ev::Mb); } + +/// A firmware object: little-endian bytes, bounds-checked like FwObject's ptr(). A word the +/// firmware writes (a producer) is given as successive values, one per load; the last repeats. +struct FwObject { id: u8, mem: RefCell>, fw: RefCell)>> } +impl FwObject { + fn new(id: u8, size: usize) -> Arc { + Arc::new(Self { id, mem: RefCell::new(vec![0; size]), fw: RefCell::new(Vec::new()) }) + } + fn set(&self, at: usize, v: u32) { self.mem.borrow_mut()[at..at + 4].copy_from_slice(&v.to_le_bytes()); } + fn firmware(&self, at: usize, values: &[u32]) { self.fw.borrow_mut().push((at, values.to_vec())); } + fn r32(&self, offset: usize) -> Result { + if offset.checked_add(4).map_or(true, |e| e > self.mem.borrow().len()) { return Err(EINVAL); } + let mut fw = self.fw.borrow_mut(); + if let Some((_, vals)) = fw.iter_mut().find(|(at, _)| *at == offset) { + let v = if vals.len() > 1 { vals.remove(0) } else { vals[0] }; + drop(fw); + self.set(offset, v); + } else { drop(fw); } + let m = self.mem.borrow(); + let v = u32::from_le_bytes(m[offset..offset + 4].try_into().unwrap()); + log(Ev::Load(self.id, offset, v)); + Ok(v) + } + fn w32(&self, offset: usize, value: u32) -> Result { + if offset.checked_add(4).map_or(true, |e| e > self.mem.borrow().len()) { return Err(EINVAL); } + self.set(offset, value); + log(Ev::Store(self.id, offset, value)); + Ok(()) + } +} + +pub(crate) struct Ring { pub(crate) state: Arc, pub(crate) ring: Arc } +impl Ring { +RING_READ +} + +struct Events; +impl Events { + fn drain(&mut self, _dev: &kernel::device::Device, _ring: &Ring) -> Result { log(Ev::Events); Ok(()) } +} +''' + +MAIN = r''' +// Object ids in the access log. +const EVENT: u8 = 1; const LOG_ID: u8 = 2; const TRACE: u8 = 3; const STATS: u8 = 4; const STATS_RECORDS: u8 = 5; + +struct World { event: Ring, log: Ring, trace: Ring, stats: Ring } + +/// The rings: log and trace with fixed indices, some behind; the stats ring at `read` with the +/// producer's successive loads `producers`; record s carries tag `tag(s)` and word `index(s)`. +fn world(read: u32, producers: &[u32], records: usize, recs: &dyn Fn(usize) -> (u32, u32)) -> World { + let ring = |id: u8, ring_id: u8, state: usize, size: usize| Ring { state: FwObject::new(id, state), ring: FwObject::new(ring_id, size) }; + let w = World { + event: ring(EVENT, 11, rings::STATE_SIZE, 0x40), + log: ring(LOG_ID, 12, rings::LOG_STATE_SIZE, 0x40), + trace: ring(TRACE, 13, rings::STATE_SIZE, 0x40), + stats: ring(STATS, STATS_RECORDS, rings::STATE_SIZE, records), + }; + for i in 0..rings::LOG_RINGS { + w.log.state.set(i * 0x30 + STATE_READ, (i * 7) as u32); + w.log.state.set(i * 0x30 + STATE_WRITE, if i % 2 == 0 { (i * 7) as u32 } else { (i * 7 + 3) as u32 }); + } + w.trace.state.set(STATE_READ, 40); + w.trace.state.set(STATE_WRITE, 41); + w.stats.state.set(STATE_READ, read); + w.stats.state.firmware(STATE_WRITE, producers); + for s in 0..records / pstate::STATS_RECORD { + let (tag, index) = recs(s); + w.stats.ring.set(s * pstate::STATS_RECORD + pstate::STATS_TAG, tag); + w.stats.ring.set(s * pstate::STATS_RECORD + pstate::POWER_STATE_INDEX, index); + w.stats.ring.set(s * pstate::STATS_RECORD + 4, 0xdead_0000 | s as u32); + } + w +} +fn old(w: &World) -> OldChannels { OldChannels { event: w.event.clone(), log: w.log.clone(), trace: w.trace.clone(), stats: w.stats.clone() } } +fn new(w: &World) -> Channels { Channels { event: w.event.clone(), log: w.log.clone(), trace: w.trace.clone(), stats: w.stats.clone(), stats_seen: StatsSeen::default() } } +impl Clone for Ring { fn clone(&self) -> Self { Ring { state: self.state.clone(), ring: self.ring.clone() } } } + +/// What the firmware sees: stores and write barriers, in order. +fn visible(log: &[Ev]) -> Vec { log.iter().copied().filter(|e| matches!(e, Ev::Store(..) | Ev::Wmb)).collect() } +fn state(w: &World) -> Vec> { [&w.event, &w.log, &w.trace, &w.stats].iter().map(|r| r.state.mem.borrow().clone()).collect() } +fn stats_index_loads(log: &[Ev]) -> Vec<(usize, u32)> { log.iter().filter_map(|e| match e { Ev::Load(STATS, at, v) => Some((*at, *v)), _ => None }).collect() } +fn record_loads(log: &[Ev]) -> Vec { log.iter().filter_map(|e| match e { Ev::Load(STATS_RECORDS, at, _) => Some(*at), _ => None }).collect() } + +fn pattern(s: usize) -> (u32, u32) { ((s % 20) as u32, (s as u32) ^ 0x5a) } + +/// The expected notes of slots `read..write` (with wrap) under `recs`. +fn expect(read: u32, write: u32, recs: &dyn Fn(usize) -> (u32, u32)) -> (StatsSeen, Vec) { + let mut seen = StatsSeen::default(); + let mut loads = Vec::new(); + let mut s = read; + while s != write { + let (tag, index) = recs(s as usize); + seen.records += 1; + seen.tags[(tag as usize).min(pstate::TAGS - 1)] += 1; + if tag == pstate::TAG_POWER_STATE { seen.last = Some(index); } + loads.push(s as usize * pstate::STATS_RECORD + pstate::STATS_TAG); + loads.push(s as usize * pstate::STATS_RECORD + pstate::POWER_STATE_INDEX); + s = (s + 1) % 256; + } + (seen, loads) +} + +fn main() { + let dev = kernel::device::Device; + let size = rings::STATS_RING_SIZE; + // The record reads stay inside the ring object; the kernel holds the same with static_assert. + assert_eq!(pstate::STATS_RECORD * pstate::STATS_SLOTS as usize, rings::STATS_RING_SIZE); + assert!(pstate::STATS_TAG + 4 <= pstate::STATS_RECORD && pstate::POWER_STATE_INDEX + 4 <= pstate::STATS_RECORD); + assert_eq!(pstate::STATS_SLOTS, RING_SLOTS); + + // 1. Every consumer/producer pair, inside and outside the ring, with a steady producer. + let mut idx: Vec = (0..256).collect(); + idx.extend([256, 257, 0x1000, 0x7fff_ffff, u32::MAX]); + let mut pairs = 0u64; + for &r in &idx { for &p in &idx { + let wo = world(r, &[p], size, &pattern); + let mut ev = Events; + take_log(); + old(&wo).drain_rings(&dev, &mut ev).unwrap(); + let lo = take_log(); + let wn = world(r, &[p], size, &pattern); + let mut ch = new(&wn); + ch.drain_rings(&dev, &mut ev).unwrap(); + let ln = take_log(); + assert_eq!(visible(&lo), visible(&ln), "stores for read {r:#x} producer {p:#x}"); + assert_eq!(state(&wo), state(&wn), "state for read {r:#x} producer {p:#x}"); + let (io, inew) = (stats_index_loads(&lo), stats_index_loads(&ln)); + assert_eq!(io, [(STATE_WRITE, p), (STATE_READ, r)]); + assert_eq!(inew[..2], io[..], "producer before consumer"); + if r < 256 && p < 256 { + let (seen, loads) = expect(r, p, &pattern); + assert_eq!(inew, [(STATE_WRITE, p), (STATE_READ, r), (STATE_READ, r)], "consumer loaded again before the store"); + assert_eq!(record_loads(&ln), loads, "records {r}..{p}"); + assert_eq!(ch.stats_seen, seen); + let rmb = ln.iter().position(|e| *e == Ev::Rmb).expect("read barrier"); + let second = ln.iter().position(|e| *e == Ev::Load(STATS, STATE_READ, r)).unwrap(); + assert!(second < rmb, "barrier after the indices"); + if let Some(first) = ln.iter().position(|e| matches!(e, Ev::Load(STATS_RECORDS, ..))) { + assert!(rmb < first, "barrier before the records"); + } + let last_record = ln.iter().rposition(|e| matches!(e, Ev::Load(STATS_RECORDS, ..))).unwrap_or(rmb); + let reread = ln.iter().rposition(|e| *e == Ev::Load(STATS, STATE_READ, r)).unwrap(); + assert!(last_record < reread, "consumer loaded again after the records"); + // The record loads complete before the store hands their slots back. + let mbs: Vec = ln.iter().enumerate().filter(|(_, e)| **e == Ev::Mb).map(|(i, _)| i).collect(); + match ln.iter().position(|e| matches!(e, Ev::Store(STATS, ..))) { + Some(store) => { + assert_eq!(mbs.len(), 1, "one full barrier before the consumer store"); + assert!(last_record < mbs[0] && mbs[0] < store, "full barrier after the records, before the store"); + assert_eq!(ln.get(store + 1), Some(&Ev::Wmb)); + } + None => assert!(mbs.is_empty()), + } + } else { + assert_eq!(inew, [(STATE_WRITE, p), (STATE_READ, r), (STATE_WRITE, p), (STATE_READ, r)], "outside the ring: the earlier drain_one"); + assert!(record_loads(&ln).is_empty() && !ln.contains(&Ev::Rmb) && !ln.contains(&Ev::Mb)); + assert_eq!(ch.stats_seen, StatsSeen::default()); + } + // peek_stats() on the same ring stores nothing and notes what the drain would. + let wp = world(r, &[p], size, &pattern); + let cp = new(&wp); + take_log(); + let peeked = cp.peek_stats().unwrap(); + let lp = take_log(); + assert!(visible(&lp).is_empty(), "peek stores"); + assert_eq!(peeked, ch.stats_seen); + pairs += 1; + }} + + // 2. A producer that moves between loads. + // Inside the ring: one producer load per drain, the same as the earlier drain's. + for (r, ps) in [(10u32, vec![20u32, 30]), (250, vec![3, 9]), (5, vec![5, 6])] { + let wo = world(r, &ps, size, &pattern); + let wn = world(r, &ps, size, &pattern); + let (o, mut n) = (old(&wo), new(&wn)); + let mut ev = Events; + take_log(); o.drain_rings(&dev, &mut ev).unwrap(); o.drain_rings(&dev, &mut ev).unwrap(); let lo = take_log(); + n.drain_rings(&dev, &mut ev).unwrap(); let first = take_log(); + n.drain_rings(&dev, &mut ev).unwrap(); let second = take_log(); + let both: Vec = first.iter().chain(second.iter()).copied().collect(); + assert_eq!(visible(&lo), visible(&both), "moving producer {ps:?}"); + let (s1, l1) = expect(r, ps[0], &pattern); + assert_eq!(record_loads(&first), l1); + let (_, l2) = expect(ps[0], ps[1], &pattern); + assert_eq!(record_loads(&second), l2, "each record noted once"); + assert_eq!(n.stats_seen.records, s1.records + l2.len() as u64 / 2); + } + // Outside the ring: the earlier drain_one() loads the producer again and stores that load. + for (r, ps) in [(10u32, vec![300u32, 40]), (300, vec![40, 41]), (10, vec![u32::MAX, 10])] { + let wn = world(r, &ps, size, &pattern); + let mut n = new(&wn); + let mut ev = Events; + take_log(); n.drain_rings(&dev, &mut ev).unwrap(); let ln = take_log(); + // The earlier drain with the producer the fallback loads. + let wo = world(r, &ps[1..], size, &pattern); + old(&wo).drain_rings(&dev, &mut ev).unwrap(); let lo = take_log(); + assert_eq!(visible(&ln), visible(&lo), "fallback store for {ps:?}"); + assert_eq!(stats_index_loads(&ln), [(STATE_WRITE, ps[0]), (STATE_READ, r), (STATE_WRITE, ps[1]), (STATE_READ, r)]); + assert_eq!(n.stats_seen, StatsSeen::default()); + } + + // 3. Wrap, empty and full, with the parsing. + let recs = |s: usize| match s { + 254 => (pstate::TAG_POWER_STATE, 3), + 2 => (pstate::TAG_POWER_STATE, 7), + 0 => (4, 99), + 1 => (200, 98), + _ => (9, 97), + }; + let wn = world(250, &[5], size, &recs); + let mut n = new(&wn); let mut ev = Events; + n.drain_rings(&dev, &mut ev).unwrap(); take_log(); + assert_eq!(n.stats_seen.records, 11); + assert_eq!(n.stats_seen.last, Some(7), "the newest power-state record, across the wrap"); + assert_eq!(n.stats_seen.tags[pstate::TAG_POWER_STATE as usize], 2); + assert_eq!(n.stats_seen.tags[4], 1); + assert_eq!(n.stats_seen.tags[pstate::TAGS - 1], 1); + assert_eq!(n.stats_seen.tags[9], 7); + assert_eq!(wn.stats.state.mem.borrow()[0..4], 5u32.to_le_bytes()); + for (r, p, count) in [(0u32, 0u32, 0u64), (128, 128, 0), (255, 255, 0), (10, 9, 255), (0, 255, 255), (255, 254, 255), (255, 0, 1)] { + let wn = world(r, &[p], size, &pattern); + let mut n = new(&wn); + n.drain_rings(&dev, &mut ev).unwrap(); + let l = take_log(); + assert_eq!(n.stats_seen.records, count, "{r}..{p}"); + assert_eq!(l.iter().filter(|e| matches!(e, Ev::Store(STATS, ..))).count(), (count != 0) as usize); + } + // Notes accumulate across drains. + let wn = world(0, &[100, 200, 200], size, &pattern); + let mut n = new(&wn); + for _ in 0..3 { n.drain_rings(&dev, &mut ev).unwrap(); } + take_log(); + assert_eq!(n.stats_seen, expect(0, 200, &pattern).0); + + // 4. A record that cannot be loaded (not reachable in the kernel: see the size checks above). + // The drain then fails before its store and keeps its notes. + let wn = world(250, &[5], 0x3000, &pattern); + let mut n = new(&wn); + take_log(); + assert_eq!(n.drain_rings(&dev, &mut ev), Err(EINVAL)); + let l = take_log(); + assert!(visible(&l).iter().all(|e| !matches!(e, Ev::Store(STATS, ..)))); + assert_eq!(n.stats_seen, StatsSeen::default()); + + // 5. The HardwareDataA frequency single against the host's arithmetic. + let mut singles = 0u32; + for sign in [0u32, 1] { for exp in 0u32..=255 { + for frac in [0u32, 1, 2, 0x3ff, 0x40_0000, 0x40_0001, 0x55_5555, 0x7f_fffe, 0x7f_ffff] { + let bits = sign << 31 | exp << 23 | frac; + let f = f32::from_bits(bits); + let want = if f == 0.0 { Some(0) } else if !f.is_finite() || f < 0.0 { None } else { + let r = (f as f64 * 1000.0).round_ties_even(); + if r > u32::MAX as f64 { None } else { Some(r as u32) } + }; + assert_eq!(pstate::f32_mhz_to_khz(bits), want, "{bits:#010x}"); + singles += 1; + } + }} + + println!("PASS actual G15 drain bodies: {pairs} consumer/producer pairs, moving producers, wrap/empty/full parsing, peek, record fault, {singles} singles"); +} +''' + + +def main(): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument('--output', type=Path, help='Work directory (default: a temporary one)') + p.add_argument('--ref', help='Read the tested bodies at this git revision') + p.add_argument('--mutation', choices=sorted(MUTATIONS)) + p.add_argument('--mutations', action='store_true', help='Also require every mutation to fail') + a = p.parse_args() + import tempfile + out = (a.output or Path(tempfile.mkdtemp(prefix='g15-stats-drain-'))).resolve() + out.mkdir(parents=True, exist_ok=True) + + def read(path, ref=None): + ref = ref or a.ref + return (subprocess.check_output(['git', 'show', f'{ref}:{path}'], cwd=ROOT, text=True, + stderr=subprocess.DEVNULL) + if ref else (ROOT / path).read_text()) + + # The embedded reference is the commit's own text, when the commit is here. + try: + old_git = item(read(CHANNEL.replace("drivers/gpu/drm/asahi/", "drivers/gpu/drm/asahi-m3/"), OLD_REF), ' pub(crate) fn drain_rings(') + except (subprocess.CalledProcessError, ValueError): + old_git = None + if old_git is not None and old_git != OLD_DRAIN_RINGS: + raise SystemExit('embedded drain_rings() differs from ' + OLD_REF) + + def run(mutation): + src = {CHANNEL: read(CHANNEL), PSTATE: read(PSTATE)} + if mutation: + path, old, new = MUTATIONS[mutation] + if src[path].count(old) != 1: + raise SystemExit(f'mutation {mutation}: anchor not found once') + src[path] = src[path].replace(old, new) + channel, hw, fw = src[CHANNEL], read(HW), read(FW_CHANNELS) + drain_one = item(channel, ' fn drain_one(ring: &Ring, offset: usize) -> Result {') + bodies = [item(channel, s) for s in ( + ' fn note_stats(ring: &Ring, seen: &mut StatsSeen) -> Result> {', + ' pub(crate) fn peek_stats(&self) -> Result {', + ' pub(crate) fn drain_rings(')] + ring_read = item(channel, ' pub(crate) fn read(&self) -> Result {') + consts = '\n'.join(const(fw, n) for n in ('STATE_READ', 'STATE_WRITE', 'RING_SLOTS')) + consts += '\nmod rings {\n' + '\n'.join( + const(hw, n) for n in ('STATE_SIZE', 'LOG_RINGS', 'LOG_STATE_SIZE', 'STATS_RING_SIZE')) + '\n}\n' + tested = drain_one + ''.join(bodies) + ring_read + src[PSTATE] + dir = out / (mutation or 'actual') + dir.mkdir(exist_ok=True) + (dir / 'pstate.rs').write_text(src[PSTATE]) + code = (PREFIX.replace('PSTATE_PATH', str(dir / 'pstate.rs')).replace('RING_READ', ring_read) + consts + + 'struct OldChannels { event: Ring, log: Ring, trace: Ring, stats: Ring }\n' + 'impl OldChannels {\n' + drain_one + '\n' + OLD_DRAIN_RINGS + '\n}\n' + 'struct Channels { event: Ring, log: Ring, trace: Ring, stats: Ring, stats_seen: StatsSeen }\n' + 'impl Channels {\n' + drain_one + '\n' + '\n'.join(bodies) + '\n}\n' + MAIN) + path = dir / 'drain.rs' + path.write_text(code) + binary = dir / 'drain' + subprocess.run(['rustup', 'run', '1.93.1', 'rustc', '--edition=2021', '-O', '-C', 'overflow-checks=on', + '-C', 'panic=abort', str(path), '-o', str(binary)], check=True) + + def no_core(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + r = subprocess.run([str(binary)], capture_output=True, text=True, preexec_fn=no_core, timeout=300) + return {'mutation': mutation, 'tested_sha256': hashlib.sha256(tested.encode()).hexdigest(), + 'exit': r.returncode, 'stdout': r.stdout, 'stderr': r.stderr[-2000:]} + + result = run(a.mutation) + receipt = {'ref': a.ref, 'old_ref': OLD_REF, 'old_reference_checked_against_git': old_git is not None, + **result, + 'scope': 'Actual drain_one/note_stats/peek_stats/drain_rings and power/pstate.rs against ' + f'drain_rings at {OLD_REF[:12]}, over a model of the firmware objects with an ' + 'ordered access log. Not kernel, firmware or GPU execution, and not timing: the ' + 'noting drain loads more words and takes longer.'} + if a.mutations and not a.mutation: + receipt['mutations'] = {} + for m in sorted(MUTATIONS): + res = run(m) + receipt['mutations'][m] = {'exit': res['exit'], 'stderr': res['stderr'][-400:]} + (out / 'result.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps({k: v for k, v in receipt.items() if k != 'stderr'}, indent=2)) + # The actual bodies must pass; a single requested mutation must fail. + if (receipt['exit'] != 0) != bool(a.mutation): + raise SystemExit(1) + if a.mutations and any(v['exit'] == 0 for v in receipt.get('mutations', {}).values()): + raise SystemExit('a mutation survived') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/g15-zero-words-tests.py b/tools/asahi/g15-zero-words-tests.py new file mode 100644 index 0000000000000..5b728e95fc03e --- /dev/null +++ b/tools/asahi/g15-zero-words-tests.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Test the actual J516C record builder's narrow zero-word exemption. + +Uses synthetic public data and temporary host copies of production Rust modules. +No GPU, firmware file, private fixture, kernel build or hardware access. The +golden T6030 inputs below are from SiliconForest 9f09c85f8d3b, except the full +hw.rs file refreshed at 709f0f08316a for the reviewed shared PB allocator changes. +Its T6030 configuration module is independently pinned to the original bytes. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile + + +G15 = "drivers/gpu/drm/asahi/g15" +GOLDEN_FILES = { + "board/data.rs": "7c5ff1077ca3296cbfc9e7144294162b6dea78a3774c38eeed4b0f59a14d7a5c", + "board/template.rs": "286872f5474a67534f214e83e659253299f85143f3dc59035fa80894022e2549", + "hw.rs": "e4619590e670ed88d2a9bb76b03853081c615d3d0ad1c6a68321ce8e615ef8a0", + "power/idle.rs": "4928ecf7699b3ca85ff5143eb6e996efd40d47321e29a238a02c8873bc9f102c", +} +GOLDEN_FUNCTIONS = { + ("hw.rs", "pub(crate) mod t6030 {"): "0079ae08fbe0e51cbf14e48711981a81048d1a1d3fc5495ab8d5c376283a760c", + ("initdata.rs", "fn templates("): "c3115c9926d3d7d93317cb8510e93b04f30b0c256811d7137568e822dd93b8ee", + ("initdata.rs", "pub(crate) fn prepare_board("): "c59f968f125397efc63f75a49d3a8543c15c1fd4d50362f15683563398e64bc6", + ("board.rs", "pub(crate) fn from_adt("): "cecd0b28db5bd94d3b75c1bbb5f35204e8bf0699f74e5b3d08228c59025f6ac2", +} + + +def function(source, marker): + start = source.index(marker) + depth = 0 + for end in range(source.index("{", start), len(source)): + depth += (source[end] == "{") - (source[end] == "}") + if depth == 0: + return source[start:end + 1] + raise ValueError("unterminated function: " + marker) + + +CASES = r''' +#[cfg(test)] +mod zero_words_regressions { + use super::*; + + // Frozen pre-5131caff FROM_FILE offsets from SiliconForest 9f09c85f8d3b. + // This list is deliberately independent of both new lists. + const OLD: [usize; 74] = [ + 224,232,244,248,252,256,260,264,268,728,732,736,740,744,748, + 992,1000,1012,1016,1024,1036,1496,1500,1504,1508,1512,1516, + 4192,4196,4200,4204,4208,4212,4216,12504,12528,19492,20012, + 20016,23056,23060,27176,27180,45528,45532,45536,45540,45544, + 45548,45552,45556,45560,45564,45568,45572,45576,45580,45584, + 45588,45592,45596,45748,45836,45840,45844,45848,45900,45904, + 45908,45912,45932,45940,45948,45956, + ]; + + fn synthetic() -> (Vec, Vec, [Opp; 1], [u32; PROPS.len()]) { + let mut file = vec![0u8; SIZE]; + for (n, &at) in OLD.iter().enumerate() { + put32(&mut file, at, 0x13570000 | (n as u32 + 1)).unwrap(); + } + put32(&mut file, MAX_POWER, 54321).unwrap(); + for (n, key) in [b"RT0g", b"RT1g", b"RT2g", b"RT3g", b"RT0r", b"RT0f"].iter().enumerate() { + put32(&mut file, SENSOR_ROWS + 12*n, u32::from_le_bytes(**key)).unwrap(); + } + // Public, synthetic OPPs: state0 off; eight voltage groups, five paired. + let mut main = vec![Opp {mhz:0, mv:[125;4], cells:4, mw:0}]; + for (mv, freqs) in [ + (600, &[300u32][..]), (650, &[500][..]), (700, &[700][..]), + (750, &[800,900][..]), (800, &[950,1000][..]), + (850, &[1050,1100][..]), (900, &[1150,1200][..]), + (950, &[1250,1300][..]), + ] { + for &mhz in freqs { + main.push(Opp {mhz, mv:[mv;4], cells:4, mw:mhz*40}); + } + } + let cs = [Opp {mhz:24, mv:[635,0,0,0], cells:1, mw:0}]; + let mut props = [1u32; PROPS.len()]; + props[PERIOD_MS] = 8; + props[AVG_TC_MS] = 32; + props[PPM_TC_MS] = 32; + props[MIN_SRAM_UV] = 815000; + props[CSAFR_MIN_SRAM_UV] = 815000; + for at in [AVG_KI,AVG_KP,FD_KI,FD_KP,PERF_KI,PERF_KI2,PERF_KP, + PERF_KP2,PPM_KI,PPM_KP,PWR_KI,PWR_KP,SE_KI,SE_KI1,SE_KP,SE_KP1] { + props[at] = 0x3f800000; // finite normal 1.0, not a calibration claim + } + (file, main, cs, props) + } + + fn inputs<'a>(main: &'a [Opp], cs: &'a [Opp], props: [u32; PROPS.len()]) -> Inputs<'a> { + Inputs {props, zones:&[], main, cs, afr:cs, slots:&[], empty_writable:&[], + ttbat:0x100000, usc_base:0x1000000000, user_end:0x2000000000, + timestamps:0xfffffc2ffc000000, max_power_mw:54321} + } + + #[test] + fn old74_is_exact_disjoint_partition() { + assert_eq!(FROM_FILE.len(), 32); + assert_eq!(UNNEEDED.len(), 42); + let mut union = FROM_FILE.to_vec(); + union.extend_from_slice(&UNNEEDED); + union.sort_unstable(); + assert!(union.windows(2).all(|w| w[0] < w[1]), "duplicate or overlapping offsets"); + assert!(union.iter().all(|&at| at % 4 == 0 && at + 4 <= SIZE)); + let mut old = OLD.to_vec(); old.sort_unstable(); + assert_eq!(union, old, "the exemption must not grow beyond old74"); + } + + #[test] + fn actual_from_file_copies_only_retained32() { + let (file, _, _, _) = synthetic(); + let mut out = vec![0u8; SIZE]; + from_file(&mut out, &file).unwrap(); + for at in (0..SIZE).step_by(4) { + let expected = if FROM_FILE.contains(&at) {get32(&file, at).unwrap()} else {0}; + assert_eq!(get32(&out, at).unwrap(), expected, "copy offset +{at:#x}"); + } + } + + #[test] + fn actual_builder_leaves42_zero_and_preserves32() { + let (file, main, cs, props) = synthetic(); + let mut out = vec![0u8; SIZE]; + from_file(&mut out, &file).unwrap(); + build(&mut out, &inputs(&main, &cs, props)).unwrap(); + for &at in UNNEEDED.iter() { + assert_ne!(get32(&file, at).unwrap(), 0, "fixture must exercise a real difference"); + assert_eq!(get32(&out, at).unwrap(), 0, "built offset +{at:#x}"); + } + for &at in FROM_FILE.iter() { + assert_eq!(get32(&out, at).unwrap(), get32(&file, at).unwrap(), "retained +{at:#x}"); + } + // Model a matching board file with nonzero legacy values only at the42. + let mut matching = out.clone(); + for &at in UNNEEDED.iter() { + put32(&mut matching, at, get32(&file, at).unwrap()).unwrap(); + } + assert_eq!(cross_check(&out, &matching), None); + } + + #[test] + fn actual_cross_check_exempts_only_zero_at_exact42() { + let mut built = vec![0u8; SIZE]; + let mut file = vec![0u8; SIZE]; + for at in (0..SIZE).step_by(4) { + put32(&mut file, at, 0x12345678).unwrap(); + let expected = if UNNEEDED.contains(&at) {None} else {Some((at,0,0x12345678))}; + assert_eq!(cross_check(&built, &file), expected, "file mismatch +{at:#x}"); + // No exemption for a nonzero built mismatch, even at the42. + put32(&mut built, at, 1).unwrap(); + assert_eq!(cross_check(&built, &file), Some((at,1,0x12345678)), "built mismatch +{at:#x}"); + put32(&mut built, at, 0).unwrap(); + put32(&mut file, at, 0).unwrap(); + } + } +} +''' + + +MUTANTS = { + "broad-cross-check-exemption": ("cross_check", "UNNEEDED.contains(&at)", "true"), + "nonzero-built-exempted": ("cross_check", "b == 0 && UNNEEDED.contains(&at)", "UNNEEDED.contains(&at)"), + "legacy-copy-reintroduced": ("from_file", "FROM_FILE.iter()", "FROM_FILE.iter().chain(UNNEEDED.iter())"), +} + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--rustc", default=os.environ.get("HOSTRUSTC", "rustc")) + parser.add_argument("--mutations", action="store_true") + parser.add_argument("--evidence", type=Path) + args = parser.parse_args() + root = args.source_root.resolve() + sources = {name: (root / G15 / "board" / name).read_bytes() + for name in ("record.rs", "words.rs", "template.rs", "data.rs")} + golden = {} + for name, expected in GOLDEN_FILES.items(): + actual = sha((root / G15 / name).read_bytes()) + if actual != expected: + raise SystemExit("T6030 golden file changed: " + name) + golden[name] = actual + for (name, marker), expected in GOLDEN_FUNCTIONS.items(): + actual = sha(function((root / G15 / name).read_text(), marker).encode()) + if actual != expected: + raise SystemExit("T6030 golden path changed: " + marker) + golden[name + ":" + marker] = actual + + if args.evidence: + args.evidence.mkdir(parents=False, exist_ok=False) + receipt = {"sources": {name: sha(data) for name, data in sources.items()}, + "t6030_golden_inputs": golden, "mutants": {}, + "harness_sha256": sha(Path(__file__).read_bytes()), + "compiler": subprocess.check_output([args.rustc, "--version"], text=True).strip()} + with tempfile.TemporaryDirectory(prefix="g15-zero-words-") as temp: + temp = Path(temp) + for name, data in sources.items(): + (temp / name).write_bytes(data) + original = sources["record.rs"].decode() + wrapper = "mod board {\n" + for name in ("template", "data", "words", "record"): + wrapper += f'#[path = "{temp / name}.rs"] pub(crate) mod {name};\n' + if name == "template": + wrapper += "pub(crate) use template::{Line, Template};\n" + wrapper += "}\n" + (temp / "host.rs").write_text(wrapper) + + def run(name, record): + (temp / "record.rs").write_text(record + CASES) + binary = temp / name + command = [args.rustc, "--edition=2021", "--test", "-O", "-C", "overflow-checks=on", + str(temp / "host.rs"), "-o", str(binary)] + compiled = subprocess.run(command, capture_output=True, text=True) + if args.evidence: + (args.evidence / (name + "-compile.log")).write_text(compiled.stdout + compiled.stderr) + if compiled.returncode: + raise RuntimeError(name + " did not compile:\n" + compiled.stderr) + tested = subprocess.run([str(binary)], capture_output=True, text=True) + if args.evidence: + (args.evidence / (name + "-test.log")).write_text(tested.stdout + tested.stderr) + return tested + + actual = run("actual", original) + if actual.returncode: + raise SystemExit(actual.stdout + actual.stderr) + receipt["test_summary"] = next(line for line in actual.stdout.splitlines() + if line.startswith("test result:")) + if args.mutations: + for name, (func, before, after) in MUTANTS.items(): + scope = function(original, "pub(crate) fn " + func + "(") + if scope.count(before) != 1: + raise RuntimeError("mutation site changed: " + name) + changed = original.replace(scope, scope.replace(before, after, 1), 1) + mutant = run(name, changed) + if not mutant.returncode: + raise RuntimeError("mutant survived: " + name) + regression = ("actual_from_file_copies_only_retained32" if func == "from_file" + else "actual_cross_check_exempts_only_zero_at_exact42") + if regression + " ... FAILED" not in mutant.stdout: + raise RuntimeError("new regression did not catch mutant: " + name) + receipt["mutants"][name] = "caught at runtime" + receipt["source_files_unchanged"] = all( + (root / G15 / "board" / name).read_bytes() == data for name, data in sources.items()) + if not receipt["source_files_unchanged"]: + raise SystemExit("source changed during validation; rerun on a frozen candidate") + receipt["limits"] = "Host synthetic record data only; two private hardware-fixture tests remain ignored." + if args.evidence: + for name, data in sources.items(): + (args.evidence / name).write_bytes(data) + (args.evidence / "synthetic-tests.rs").write_text(CASES) + (args.evidence / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps(receipt, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/asahi/mt7932-association-events-tests.py b/tools/asahi/mt7932-association-events-tests.py index 720671701782e..bce20bdc099d2 100644 --- a/tools/asahi/mt7932-association-events-tests.py +++ b/tools/asahi/mt7932-association-events-tests.py @@ -38,6 +38,7 @@ def function(source, name): #include typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; #define READ_ONCE(x) (x) +#define BIT(n) (1U << (n)) #define ARRAY_SIZE(x) (sizeof(x)/sizeof(*(x))) #define GFP_ATOMIC 0 #define IEEE80211_CHAN_DISABLED 1 @@ -48,6 +49,7 @@ def function(source, name): #define WLAN_REASON_UNSPECIFIED 1 #define WLAN_EID_EXTENSION 255 #define WLAN_EID_EXT_HE_CAPABILITY 35 +#define WLAN_EID_HT_CAPABILITY 45 #define dev_info(...) ((void)0) #define dev_dbg_ratelimited(...) ((void)0) #define dev_dbg(...) ((void)0) @@ -88,6 +90,9 @@ def function(source, name): }; static bool usable=true; static unsigned cal_calls,queue_calls,scheduled; +static unsigned long jiffies; +static bool retried; +static unsigned first_elapsed,cal_elapsed,queue_elapsed,final_budget; static int queue_error,cal_error; static struct ieee80211_channel channel; static struct ieee80211_channel *ieee80211_get_channel(void*w,int f){channel.center_freq=f;return &channel;} @@ -95,6 +100,7 @@ def function(source, name): static bool cfg80211_chandef_usable(void*w,const struct cfg80211_chan_def*d,int flags){return usable;} static bool mt7932_assoc_phy_valid(const void*b,const void*p,size_t n){return true;} static const struct element *cfg80211_find_ext_elem(int id,const void*p,size_t n){return NULL;} +static const struct element *cfg80211_find_elem(int id,const void*p,size_t n){return NULL;} static void netif_carrier_off(void*n){} static void netif_stop_queue(void*n){} static void schedule_work(void*w){scheduled++;} @@ -118,14 +124,14 @@ def function(source, name): static void done(struct mt7932*m){u8 b[2]={};receive(m,0xee,0,b,sizeof(b),0x42);} static void cal_null(struct mt7932*m,u8 seq){u8 b[20]={};b[3]=1;receive(m,0xd6,seq,b,sizeof(b),0);} static int mt_association_calibration(struct mt7932*m){ - assert(command_locked);cal_calls++;if(cal_error)return cal_error; + assert(command_locked);cal_calls++;jiffies+=cal_elapsed;if(cal_error)return cal_error; int e=mt7932_cal_begin(&m->cal_state,1);if(e)return e; cal_null(m,181);assert(m->cal_state.done&&m->cal_state.received==1); return m->cal_state.error; } static int mt_request(struct mt7932*m,u8 cid,bool runtime,bool set,bool wait,const void*body,size_t n){ assert(command_locked&&cid==0x6b&&runtime&&set&&!wait&&n==32); - queue_calls++;return queue_error; + queue_calls++;jiffies+=queue_elapsed;return queue_error; } enum scenario {SUCCESS,DUP_SUCCESS,DUP_BEFORE_SUCCESS,DUP_TIMEOUT,CANCEL,LATE_ERROR,UNOWNED_REPORT,DUP_BAD_CHANNEL,NO_START,DONE_WITHOUT_REPORT,DISCONNECT,DUP_FLOOD}; static enum scenario which; @@ -133,8 +139,8 @@ def function(source, name): static unsigned first_waits,final_waits,total_wait_budget; static unsigned long wait_for_completion_timeout(struct completion*c,unsigned long timeout){ assert(!command_locked);total_wait_budget+=timeout; - if(c==&active->assoc_start){assert(timeout==5000);first_waits++;if(which!=NO_START)start(active,active->connect_channel);if(which==DUP_BEFORE_SUCCESS)start(active,active->connect_channel);} - else {assert(c==&active->assoc_done&&timeout==10000);final_waits++; + if(c==&active->assoc_start){assert(timeout==(retried?10000:5000));first_waits++;jiffies+=first_elapsed;if(which!=NO_START)start(active,active->connect_channel);if(which==DUP_BEFORE_SUCCESS)start(active,active->connect_channel);} + else {assert(c==&active->assoc_done&&timeout==(retried?final_budget:10000));final_waits++; if(which==DUP_SUCCESS||which==DUP_TIMEOUT||which==DUP_BAD_CHANNEL)start(active,which==DUP_BAD_CHANNEL?6:active->connect_channel); if(which==DUP_FLOOD)for(unsigned i=0;i<1000;i++)start(active,active->connect_channel); if(which==CANCEL||which==LATE_ERROR){active->disconnecting=which==CANCEL;active->connect_error=which==CANCEL?-ECANCELED:-EIO;complete(c);report(active,true);done(active);} @@ -145,13 +151,15 @@ def function(source, name): } if(!c->done)return 0;if(c->done!=UINT_MAX)c->done--;return timeout; } -static int worker_wait(struct mt7932*m){int ret=0;u8 queue[32];command_locked=1; +#define time_after_eq(a,b) ((long)((a)-(b))>=0) +#define max_t(t,a,b) ((t)(a)>(t)(b)?(t)(a):(t)(b)) +static int worker_wait(struct mt7932*m){int ret=0;u8 queue[32];bool retry=retried;unsigned long deadline=jiffies+10000,timeout;command_locked=1; WAIT -report: +out: assert(command_locked);command_locked=0;return ret?:m->connect_error; } static struct mt7932 fresh(void){return (struct mt7932){.netdev=(void*)1,.connecting=true,.rf_ready=true,.connect_channel=11,.smart_version=12,.peer_ids={{1,2},{3,4,5,6}}};} -static void reset(struct mt7932*m,enum scenario s){*m=fresh();active=m;which=s;cal_calls=queue_calls=scheduled=first_waits=final_waits=total_wait_budget=0;queue_error=cal_error=0;usable=true;command_locked=0;} +static void reset(struct mt7932*m,enum scenario s){*m=fresh();active=m;which=s;cal_calls=queue_calls=scheduled=first_waits=final_waits=total_wait_budget=0;queue_error=cal_error=0;usable=true;command_locked=0;jiffies=first_elapsed=cal_elapsed=queue_elapsed=0;retried=false;final_budget=10000;} int main(void){struct mt7932 m;unsigned cases=0; for(enum scenario s=SUCCESS;s<=DUP_FLOOD;s++){ reset(&m,s);int e=worker_wait(&m); @@ -181,6 +189,19 @@ def function(source, name): reset(&m,SUCCESS);m.cal_request_count=32;receive(&m,0xd7,92,request,16,0);assert(m.cal_request_count==32&&m.link_failed&&!scheduled);cases++; reset(&m,SUCCESS);receive(&m,0xd7,93,request,15,0);assert(!m.cal_request_count&&m.link_failed);cases++; reset(&m,SUCCESS);usable=false;start(&m,11);assert(m.connect_error==-EPROTO&&m.assoc_start.done&&m.assoc_done.done&&!m.connect_center);cases++; + // The retry shares one deadline across start, calibration, queue and done. + reset(&m,SUCCESS);retried=true;first_elapsed=7000;cal_elapsed=1000;queue_elapsed=500;final_budget=1500; + assert(!worker_wait(&m)&&first_waits==1&&final_waits==1&&jiffies==8500);cases++; + reset(&m,SUCCESS);retried=true;first_elapsed=7000;cal_elapsed=3000; + assert(worker_wait(&m)==-ETIMEDOUT&&cal_calls==1&&!queue_calls&&!final_waits);cases++; + reset(&m,SUCCESS);retried=true;first_elapsed=7000;cal_elapsed=2000;queue_elapsed=1000; + assert(worker_wait(&m)==-ETIMEDOUT&&queue_calls==1&&!final_waits);cases++; + reset(&m,NO_START);retried=true; + assert(worker_wait(&m)==-ETIMEDOUT&&total_wait_budget==10000&&!final_waits);cases++; + // A well-formed firmware rejection is retryable; malformed ownership is not. + reset(&m,SUCCESS);u8 rejected[52]={};put_unaligned_le16(1,rejected); + receive(&m,0xee,0,rejected,sizeof(rejected),0x40); + assert(m.connect_error==-ECONNREFUSED&&m.assoc_start.done&&m.assoc_done.done&&!m.peer_valid);cases++; printf("PASS %u association event/wait/calibration ownership controls\n",cases); } ''' @@ -195,12 +216,18 @@ def main(): root=args.source_root;out=args.out;out.mkdir(parents=True,exist_ok=True) link=(root/DRIVER/"link.c").read_text();rx=(root/DRIVER/"mt7932.c").read_text() parts={"CHANNEL":function(link,"mt_channel_event"),"LINK":function(link,"mt_link_event"),"RECEIVE":function(rx,"mt_receive")} - worker=function(link,"mt_connect_work") + worker=function(link,"mt_connect_attempt") start=worker.index("\t/* Genuine D7 requests") - parts["WAIT"]=worker[start:worker.index("\nreport:",start)] + parts["WAIT"]=worker[start:worker.index("\nout:",start)] headers=[] for n in ["protocol.h","channels.h","bandwidth.h","association.h"]: - headers.append(re.sub(r"^#include .*\n","",(root/DRIVER/n).read_text(),flags=re.M)) + header = re.sub(r"^#include .*\n", "", (root/DRIVER/n).read_text(), flags=re.M) + if n == "association.h": + # These event fixtures do not use the sleeping crypto/admission helpers. + start = header.index("/* The first AKM") + end = header.index("struct mt7932_peer_ids") + header = header[:start] + header[end:] + headers.append(header) cal=(root/DRIVER/"calibration.h").read_text() headers.append(cal[cal.index("struct mt7932_cal_completion {"):cal.index("struct mt7932_cal_segment {")]) parts["HEADERS"]="\n".join(headers) diff --git a/tools/asahi/mt7932-connect-result-tests.py b/tools/asahi/mt7932-connect-result-tests.py index ac7241b210bd7..2f8a4bf9d4c62 100755 --- a/tools/asahi/mt7932-connect-result-tests.py +++ b/tools/asahi/mt7932-connect-result-tests.py @@ -75,8 +75,8 @@ static int wiphy_net(int*w){return 0;} static void genlmsg_multicast_netns(int*f,int n,struct sk_buff*m,int p,int g,gfp_t t){} ENCODER -struct mt7932 {bool stopping,connect_open,assoc_request_seen,connecting,connected,disconnecting,connect_cancelled,connect_publishing,connect_lost,rf_ready; - int connect_error,response_lock,command_mutex,data_lock,wiphy,power_work;u16 disconnect_reason;struct net_device *netdev;struct cfg80211_bss *connect_bss;u8 connect_bssid[6],connect_pmk[32],assoc_request_ies[512];unsigned assoc_request_ie_len; +struct mt7932 {bool stopping,connect_open,assoc_request_seen,connecting,connected,disconnecting,connect_cancelled,connect_publishing,connect_lost,rf_ready,connect_pmk_valid; + int connect_error,response_lock,command_mutex,data_lock,wiphy,power_work;u16 disconnect_reason;struct net_device *netdev;struct cfg80211_bss *connect_bss;u8 connect_bssid[6],connect_pmk[32],connect_sae_pwd[63],connect_sae_pwd_length,assoc_request_ies[512];unsigned assoc_request_ie_len; struct {struct {bool ht_supported;}ht_cap;}band2;bool power_tim,data_ready,station_signal_valid,station_rx_rate_valid,station_tx_rate_valid;unsigned long station_tx_bytes,station_rx_bytes,station_tx_packets,station_rx_packets,station_connected;}; static unsigned long jiffies=1; static unsigned notifications,disconnections,retirements,releases,authorizations,carrier,work_count; @@ -96,7 +96,7 @@ static void netif_carrier_on(struct net_device*n){carrier++;} static void netif_wake_queue(struct net_device*n){} static void cfg80211_port_authorized(struct net_device*n,const u8*b,const u8*p,size_t l,gfp_t f){authorizations++;} -static void report_worker(struct mt7932 *m,int ret){ +static void report_worker(struct mt7932 *m,int ret,bool retirement_attempted){ struct cfg80211_connect_resp_params response={};unsigned long flags;bool submitted=true,peer_owned=true,cancelled;u16 reason; REPORT int main(void){struct net_device n={1};struct cfg80211_bss b={}; @@ -104,7 +104,11 @@ for(unsigned i=0;i +#include +#include +#include +#include +#include +#define READ_ONCE(x) (x) +#define spin_lock_irqsave(l,f) ((void)(l),(f)=0) +#define spin_unlock_irqrestore(l,f) ((void)(l),(void)(f)) +#define dev_dbg(...) ((void)0) +struct completion { bool done; }; +struct mt7932 { + int command_mutex,response_lock,connect_error; + bool owe_cold_prepared,connect_owe,connect_cancelled,disconnecting,link_failed,rf_ready; + bool disconnect_seen,disconnect_event_received,assoc_request_seen,owner; + unsigned assoc_request_ie_len,connect_center,connection_generation; + struct completion assoc_start,assoc_done,disconnect_done; + unsigned char connect_chandef[32]; +}; +static unsigned locked, attempts, retirements, snapshots, preparations; +static int first_error, second_error, retire_error, prepare_error; +static bool initial_auth, initial_submitted, cancel_before, cancel_during, rf_before, rf_during; +static bool first_disconnecting, late_error; +static void mutex_lock(int *m) { assert(!locked); locked=1; } +static void mutex_unlock(int *m) { assert(locked); locked=0; } +static bool mt_rf_allowed(struct mt7932 *m) { return m->rf_ready; } +static void reinit_completion(struct completion *c) { c->done=false; } +static void mt_transport_snapshot(struct mt7932 *m) { snapshots++; } +static int mt_connect_prepare(struct mt7932 *m, bool *owned) +{ + assert(locked && !m->owner);preparations++; + if (prepare_error) return prepare_error; + *owned=m->owner=true; + return 0; +} +static int mt_connect_attempt(struct mt7932 *m, bool *submitted, bool *owned, bool *auth_submitted, bool retry) +{ + assert(locked && attempts<2 && retry==(attempts==1)); + if (retry || preparations) { + assert(!m->owner && !m->connect_error && !m->disconnecting); + assert(!m->disconnect_seen && !m->disconnect_event_received); + assert(!m->assoc_start.done && !m->assoc_done.done && !m->disconnect_done.done); + assert(!m->assoc_request_seen && !m->assoc_request_ie_len && !m->connect_center); + for (unsigned i=0;iconnect_chandef);i++) assert(!m->connect_chandef[i]); + assert(m->connection_generation==9+preparations+retry); + } else { + m->connect_cancelled=cancel_before; + m->rf_ready=!rf_before; + m->disconnecting=first_disconnecting; + if (late_error) m->connect_error=first_error; + } + *submitted=retry || initial_submitted; + *auth_submitted=retry || initial_auth; + *owned=true; + m->owner=true; + attempts++; + return retry ? second_error : first_error; +} +static int mt_retire_connection(struct mt7932 *m) +{ + assert(!locked && m->disconnecting && m->owner); + retirements++; + if (retire_error) { m->link_failed=true; return retire_error; } + m->owner=false; + m->connect_error=-ECONNRESET; /* Expected typed disconnect event. */ + m->connect_cancelled=cancel_during; + m->rf_ready=!rf_during; + return 0; +} +RETIRE_HELPER +static int worker(struct mt7932 *m) +{ + unsigned long flags; + bool submitted=false,peer_owned=false,retry=false,retirement_attempted=false,auth_submitted=false; + int ret; +WORKER +report: + assert(locked); + // Mirror the final report's cleanup decision, so a failed retirement cannot + // be repeated or release an uncertain peer through the unsubmitted path. + if (ret && submitted && !retirement_attempted) retirements++; + else if (ret && peer_owned && !submitted && !retirement_attempted) assert(!retirement_attempted); + mutex_unlock(&m->command_mutex); + return ret; +} +static struct mt7932 fresh(void) +{ + struct mt7932 m={.owe_cold_prepared=true,.connect_owe=true,.rf_ready=true,.disconnect_seen=true, + .disconnect_event_received=true,.assoc_request_seen=true,.assoc_request_ie_len=8, + .connect_center=11,.connection_generation=9, + .assoc_start={true},.assoc_done={true},.disconnect_done={true}}; + memset(m.connect_chandef,0xa5,sizeof(m.connect_chandef)); + attempts=retirements=snapshots=locked=preparations=0; + first_error=-ETIMEDOUT;second_error=retire_error=prepare_error=0;initial_auth=initial_submitted=true; + cancel_before=cancel_during=rf_before=rf_during=first_disconnecting=late_error=false; + return m; +} +int main(void) +{ + struct mt7932 m;unsigned cases=0; + int retry_errors[]={-ETIMEDOUT,-ECONNREFUSED,-ECONNRESET}; + for (unsigned i=0;i<3;i++) { + m=fresh();first_error=retry_errors[i];late_error=true;first_disconnecting=(i==2); + assert(worker(&m)==0 && attempts==2 && retirements==1 && snapshots==1);cases++; + } + m=fresh();second_error=-ETIMEDOUT; + assert(worker(&m)==-ETIMEDOUT && attempts==2 && retirements==2);cases++; + int no_retry_errors[]={0,-EPROTO,-ECANCELED,-EIO,-ENOMEM}; + for (unsigned i=0;i<5;i++) { + m=fresh();first_error=no_retry_errors[i]; + assert(worker(&m)==first_error && attempts==1 && snapshots==0);cases++; + } + for (unsigned i=0;i<7;i++) { + m=fresh(); + switch(i) { + case 0:m.connect_owe=false;break; + case 1:initial_auth=initial_submitted=false;break; + case 2:cancel_before=true;break; + case 3:rf_before=true;break; + case 4:m.link_failed=true;break; + case 5:initial_auth=false;break; + default:m.rf_ready=false;rf_before=true;break; + } + assert(worker(&m)==-ETIMEDOUT && attempts==1 && snapshots==0);cases++; + } + m=fresh();retire_error=-EIO; + assert(worker(&m)==-EIO && attempts==1 && retirements==1 && m.owner);cases++; + m=fresh();cancel_during=true; + assert(worker(&m)==-ECANCELED && attempts==1 && retirements==1 && !m.owner);cases++; + m=fresh();rf_during=true; + assert(worker(&m)==-EIO && attempts==1 && retirements==1 && !m.owner);cases++; + // Cold cleanup precedes the first association and is not repeated once armed. + m=fresh();m.owe_cold_prepared=false;first_error=0; + assert(!worker(&m)&&attempts==1&&preparations==1&&retirements==1&&m.owe_cold_prepared);cases++; + m=fresh();m.owe_cold_prepared=false; + assert(!worker(&m)&&attempts==2&&preparations==1&&retirements==2);cases++; + m=fresh();m.owe_cold_prepared=false;prepare_error=-EINVAL; + assert(worker(&m)==-EINVAL&&!attempts&&!retirements);cases++; + m=fresh();m.owe_cold_prepared=false;retire_error=-EIO; + assert(worker(&m)==-EIO&&!attempts&&retirements==1&&m.owner);cases++; + m=fresh();m.owe_cold_prepared=false;cancel_during=true; + assert(worker(&m)==-ECANCELED&&!attempts&&retirements==1&&!m.owner);cases++; + m=fresh();m.owe_cold_prepared=false;rf_during=true; + assert(worker(&m)==-EIO&&!attempts&&retirements==1&&!m.owner);cases++; + printf("PASS %u OWE cold cleanup, retry, cancellation and ownership controls\n",cases); +} +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--source-root', type=Path, default=ROOT) + parser.add_argument('--out', type=Path, required=True) + args = parser.parse_args() + text = (args.source_root / LINK).read_text() + start = text.index('void mt_connect_work(') + start = text.index('\tmutex_lock(&m->command_mutex);', start) + end = text.index('report:\n', start) + worker = text[start:end] + helper_start = text.index('static int mt_owe_retire_attempt(') + helper_end = text.index('\n/* command_mutex held on entry and return. D7', helper_start) + helper = text[helper_start:helper_end] + args.out.mkdir(parents=True, exist_ok=True) + candidates = {'production': (helper, worker), + 'mutant-repeat-retirement': (helper.replace( + '*retirement_attempted = true;', '*retirement_attempted = false;'), worker), + 'mutant-cancel-reset': (helper.replace( + 'if (m->connect_cancelled)', 'if (false)'), worker), + 'mutant-skip-retirement': (helper.replace( + 'ret = mt_retire_connection(m);', 'ret = 0;'), worker)} + results = {} + for name, (retire_helper, code) in candidates.items(): + source = args.out / (name + '.c') + binary = args.out / name + source.write_text(HARNESS.replace('RETIRE_HELPER', retire_helper).replace('WORKER', code)) + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Werror', + '-Wno-unused-function', '-Wno-unused-parameter', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + str(source), '-o', str(binary)], check=True) + result = subprocess.run([str(binary)], capture_output=True, text=True) + (args.out / (name + '.log')).write_text(result.stdout + result.stderr) + results[name] = { + 'exit': result.returncode, + 'worker_sha256': hashlib.sha256(code.encode()).hexdigest(), + 'retire_helper_sha256': hashlib.sha256(retire_helper.encode()).hexdigest(), + 'binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), + } + (args.out / 'results.json').write_text(json.dumps(results, indent=2) + '\n') + print(name, result.returncode, result.stdout.strip()) + assert (result.returncode == 0) == (name == 'production') + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/neo-stability-tests.py b/tools/asahi/neo-stability-tests.py index 03fd0352089d7..f65cab986c626 100644 --- a/tools/asahi/neo-stability-tests.py +++ b/tools/asahi/neo-stability-tests.py @@ -32,6 +32,8 @@ def function(source, name): #include #include #include +#include +#define SHA256_DIGEST_SIZE 32 typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef int32_t s32; #define READ_ONCE(x) (x) #define WRITE_ONCE(x,v) ((x)=(v)) diff --git a/tools/asahi/t6030-clock-tests.py b/tools/asahi/t6030-clock-tests.py new file mode 100644 index 0000000000000..0b75c3ade6f5d --- /dev/null +++ b/tools/asahi/t6030-clock-tests.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Execute T6030 DCP callbacks and PHY clock functions on modeled registers.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import resource +import subprocess +ROOT = Path(__file__).resolve().parents[2] +FUNCTIONS = { + 'include/linux/soc/apple/dp-tunnel.h': ['apple_dp_tunnel_t6030_pclk'], + 'drivers/gpu/drm/apple/dcp-fabric-core.h': ['dcp_fabric_t6030_pclk'], + 'drivers/gpu/drm/apple/dcp-fabric-session.h': ['dcp_fabric_callback_valid'], + 'drivers/phy/apple/atc.c': [ + 'apple_atc_tunnel_is_t6030', 'atc_t8122_tunnel_apb', 'atc_t8122_tunnel_stop', + 'atc_t8122_tunnel_pclk', 'atc_t8122_tunnel_start', + 'apple_atc_dp_tunnel_rate_source'], + 'drivers/gpu/drm/apple/dcp-fabric.c': [ + 'dcp_route_clock_rate', 'dcp_tunnel_set_rate'], +} +CASES = ['dual-start', 'reverse-start', 'stop-second', 'same-rate', + 'stale-owner', 'same-source', 'retune-one', 'busy-mode', 'busy-pll', + 'invalid-source', 'invalid-rate', 'wrong-generation', 'failed-cleanup', 'failed-final-stop'] +CASES += ['fail-start-' + str(i) for i in range(4)] + +def function(source, name): + match = re.search(r'^(?:static\s+)?(?:inline\s+)?(?:int|void|bool)\s+' + name + + r'\([^;]*?\)\n\{', source, re.M | re.S) + if not match: + raise ValueError('Missing production function: ' + name) + pos = source.index('{', match.start()) + 1 + level = 1 + while level: + level += (source[pos] == '{') - (source[pos] == '}') + pos += 1 + return source[match.start():pos] + '\n' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--callback-ref') + parser.add_argument('--pclk-function', type=Path) + args = parser.parse_args() + bodies = {name: function((ROOT / path).read_text(), name) + for path, names in FUNCTIONS.items() for name in names} + if args.callback_ref: + original = subprocess.check_output( + ['git', 'show', args.callback_ref + ':drivers/gpu/drm/apple/dcp-fabric.c'], + cwd=ROOT, text=True) + bodies['dcp_tunnel_set_rate'] = function(original, 'dcp_tunnel_set_rate') + if args.pclk_function: + bodies['atc_t8122_tunnel_pclk'] = args.pclk_function.read_text() + lines = iter((ROOT / 'drivers/phy/apple/atc.c').read_text().splitlines(keepends=True)) + defines = [] + for line in lines: + if line.startswith('#define'): + definition = line + while line.rstrip().endswith('\\'): + line = next(lines) + definition += line + defines.append(definition) + code = (ROOT / 'tools/asahi/t6030-clock/fixture.c').read_text().replace( + '/* REGISTER_DEFINES */', ''.join(defines) + re.search( + r'enum atcphy_generation \{.*?\n\};', + (ROOT / 'drivers/phy/apple/atc.c').read_text(), re.S).group()).replace( + '/* PRODUCTION_FUNCTIONS */', '\n'.join(bodies.values())) + args.out.mkdir(parents=True, exist_ok=True) + source = args.out / 'production.c' + source.write_text(code) + binary = args.out / 'production' + subprocess.run(['cc', '-std=gnu11', '-g', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + '-Wno-unused-parameter', str(source), '-o', str(binary)], check=True) + def no_core(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + results = [] + for case in CASES: + run = subprocess.run([str(binary), case], capture_output=True, + text=True, preexec_fn=no_core) + results.append({'case': case, 'exit': run.returncode, + 'stdout': run.stdout, 'stderr': run.stderr}) + receipt = {'production_bodies': {name: hashlib.sha256(body.encode()).hexdigest() + for name, body in bodies.items()}, + 'translation_sha256': hashlib.sha256(code.encode()).hexdigest(), + 'hardware_access': False, 'original_callback_ref': args.callback_ref, + 'pclk_function_override': str(args.pclk_function) if args.pclk_function else None, + 'results': results} + (args.out / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps(results, indent=2)) + if any(row['exit'] for row in results): + raise SystemExit(1) +if __name__ == '__main__': + main() diff --git a/tools/asahi/t6030-clock/fixture.c b/tools/asahi/t6030-clock/fixture.c new file mode 100644 index 0000000000000..0a465d503ce03 --- /dev/null +++ b/tools/asahi/t6030-clock/fixture.c @@ -0,0 +1,318 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#include +#include +#include +#include +#include +typedef uint8_t u8; +typedef uint32_t u32; +typedef uint64_t u64; +#define BIT(n) (1U << (n)) +#define GENMASK(h, l) ((~0U >> (31 - (h))) & (~0U << (l))) +#define FIELD_PREP(mask, v) (((u32)(v) << __builtin_ctz(mask)) & (mask)) +#define JOIN_(a, b) a##b +#define JOIN(a, b) JOIN_(a, b) +#define guard(x) __attribute__((unused)) void *JOIN(guard_, __LINE__) = +#define lockdep_assert_held(x) ((void)0) +#define dev_dbg(...) ((void)0) +#define dev_warn(...) ((void)0) +#define dev_err(...) ((void)0) +#define EXPORT_SYMBOL_GPL(x) +#define APPLE_ATCPHY_MODE_USB4 1 +#define APPLE_ATCPHY_MODE_TBT 2 +#define symbol_get(fn) (&fn) +#define symbol_put(fn) ((void)0) +/* REGISTER_DEFINES */ +struct atcphy_hw { + enum atcphy_generation gen; +}; +struct apple_atcphy { + const struct atcphy_hw *hw; + int lock, mode, dp_link_rate; + void *dev; + struct { + u8 *core; + } regs; + bool tunnel_clock_on; + u8 tunnel_rate, tunnel_users, tunnel_dpin_rate[2]; + unsigned tunnel_source[2]; +}; +struct phy { + const void *ops; + struct apple_atcphy *data; +}; +struct apple_dcp; +struct apple_dcp_typec_route { + struct apple_dcp *dcp; + unsigned mux_index, tunnel_dpin; + u64 tunnel_generation; + bool xbar_up; + void *active_xbar; +}; +struct apple_dcp { + int tb_lock; + void *dev; + bool dptx_tunnel, tb_clock_ok; + unsigned dptx_dfp_port; + u64 tb_generation; + struct apple_dcp_typec_route *active_typec_route; + struct { + bool active, xbar_up, clock_ok; + u64 generation; + void *xbar; + } split; +}; +static int apple_atc_dp_phy_ops; +static unsigned writes, polls, apb_requests, stop_requests, crossbar_stops; +static uint64_t faults; +static u8 *registers; +static struct apple_atcphy *running_phy; +static struct apple_atcphy *phy_get_drvdata(struct phy *p) +{ + return p->data; +} +static bool of_machine_is_compatible(const char *s) +{ + return !strcmp(s, "apple,t6030"); +} +static void atcphy_restore_after_pd_off(struct apple_atcphy *p) +{ +} +static void udelay(unsigned us) +{ +} +static u32 readl(const void *p) +{ + u32 v; + memcpy(&v, p, 4); + return v; +} +static void writel(u32 v, void *p) +{ + if ((u8 *)p == registers + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0) { + assert(!(v & DPTX_PCLK2_ENABLE) || (v & DPTX_PCLK1_ENABLE)); + if (running_phy && (running_phy->tunnel_users & BIT(1)) && + running_phy->tunnel_rate) + assert(v & DPTX_PCLK1_ENABLE); + } + writes++; + memcpy(p, &v, 4); +} +static void core_clear32(struct apple_atcphy *p, unsigned off, u32 mask) +{ + writel(readl(p->regs.core + off) & ~mask, p->regs.core + off); +} +static void core_set32(struct apple_atcphy *p, unsigned off, u32 mask) +{ + writel(readl(p->regs.core + off) | mask, p->regs.core + off); +} +static void core_mask32(struct apple_atcphy *p, unsigned off, u32 mask, u32 v) +{ + writel((readl(p->regs.core + off) & ~mask) | (v & mask), + p->regs.core + off); +} +static int apple_atc_dp_tunnel_rate(struct phy *p, unsigned dpin, u8 rate) +{ + abort(); +} +static int dcp_dpxbar_link(void *xbar, bool up) +{ + assert(!up); + crossbar_stops++; + return 0; +} +static int model_poll(void *address, u32 *value) +{ + unsigned i = polls++, off = (u8 *)address - registers; + if (faults & (UINT64_C(1) << i)) + return -ETIMEDOUT; + if (off == AUS_COMMON_DIG_RCAL1) + *value = AUS_COMMON_DIG_RCAL1_ALL_CODES_DONE; + else if (off == AUSPLL_APB_CMD_OVERRIDE) { + u32 cmd = readl(address); + if (cmd & BIT(0)) { + *value = cmd | BIT(1); + apb_requests++; + } else { + *value = cmd & ~BIT(1); + if ((cmd & 0x10000018) == 0x10000018) { + u32 zero = 0; + memcpy(registers + T8122_DP_PCLK_STATUS, &zero, + 4); + stop_requests++; + } + } + } else { + assert(off == T8122_DP_PCLK_STATUS); + *value = readl(address); + } + if (off == AUSPLL_APB_CMD_OVERRIDE && (*value & BIT(0))) { + u32 lock = ACIOPHY_AUSPLL_LOCK; + memcpy(registers + T8122_DP_PCLK_STATUS, &lock, 4); + } + memcpy(address, value, 4); + return 0; +} +#define readl_poll_timeout(address, value, condition, interval, timeout) \ + ({ \ + int err = model_poll((address), &(value)); \ + assert(err || (condition)); \ + err; \ + }) +/* PRODUCTION_FUNCTIONS */ +static int clock_call(struct phy *phy, unsigned source, unsigned dpin, u8 rate) +{ + struct apple_dcp d = { .dptx_tunnel = true, + .dptx_dfp_port = dpin + 1, + .tb_generation = 7 }; + struct apple_dcp_typec_route r = { .dcp = &d, + .mux_index = source, + .tunnel_dpin = dpin, + .tunnel_generation = 7 }; + d.active_typec_route = &r; + int ret = dcp_tunnel_set_rate(&d, phy, 0, rate); + assert(d.tb_clock_ok == (!ret && rate)); + return ret; +} +static u32 tx(void) +{ + return readl(registers + ACIOPHY_LANE_DP_CFG_BLK_TX_DP_CTRL0); +} +int main(int argc, char **argv) +{ + assert(argc == 2); + const char *name = argv[1]; + for (unsigned source = 0; source < 5; source++) + for (unsigned dpin = 0; dpin < 4; dpin++) + assert(dcp_fabric_t6030_pclk(source, dpin) == + apple_dp_tunnel_t6030_pclk(source, dpin)); + u8 regs[0x10000] = { 0 }; + registers = regs; + struct atcphy_hw hw = { .gen = ATCPHY_GENERATION_T8122 }; + struct apple_atcphy d = { .hw = &hw, + .regs.core = regs, + .mode = APPLE_ATCPHY_MODE_USB4 }; + running_phy = &d; + struct phy phy = { .ops = &apple_atc_dp_phy_ops, .data = &d }; + if (!strcmp(name, "wrong-generation")) { + hw.gen = ATCPHY_GENERATION_T8103; + assert(clock_call(&phy, 0, 0, 0x14) == -EOPNOTSUPP && !writes); + return 0; + } + if (!strncmp(name, "fail-start-", 11)) { + unsigned n = strtoul(name + 11, NULL, 10); + faults = UINT64_C(1) << n; + assert(clock_call(&phy, 0, 0, 0x14) == -ETIMEDOUT); + assert(!d.tunnel_clock_on && !d.tunnel_users && !d.tunnel_rate); + assert(!(tx() & T8122_PCLK_ENABLES)); + return 0; + } + if (!strcmp(name, "busy-mode")) { + d.mode = 0; + assert(clock_call(&phy, 0, 0, 0x14) == -EBUSY && !writes); + return 0; + } + if (!strcmp(name, "busy-pll")) { + writel(T8122_AUSPLL_PCLK_DRIVER, regs + AUSPLL_CLKOUT_MASTER); + unsigned saved = writes; + assert(clock_call(&phy, 0, 0, 0x14) == -EBUSY && + writes == saved); + return 0; + } + if (!strcmp(name, "invalid-source")) { + assert(clock_call(&phy, 1, 0, 0x14) == -EOPNOTSUPP && !writes); + return 0; + } + if (!strcmp(name, "invalid-rate")) { + assert(clock_call(&phy, 0, 0, 0x12) == -EINVAL && !writes); + return 0; + } + if (!strcmp(name, "failed-cleanup")) { + faults = BIT(1) | BIT(2); + assert(clock_call(&phy, 0, 0, 0x14) == -ETIMEDOUT); + assert(d.tunnel_clock_on && !d.tunnel_rate && + d.tunnel_users == 1); + unsigned saved = writes; + assert(clock_call(&phy, 2, 1, 0x14) == -EBUSY && + writes == saved); + faults = 0; + assert(!clock_call(&phy, 0, 0, 0)); + assert(!d.tunnel_clock_on && !d.tunnel_users); + return 0; + } + unsigned first = !strcmp(name, "reverse-start") ? 1 : 0, + second = 1 - first; + assert(!clock_call(&phy, 2 * first, first, 0x14)); + assert(d.tunnel_users == BIT(first) && d.tunnel_clock_on); + assert(tx() & DPTX_PCLK1_ENABLE); + if (first) + assert(tx() & DPTX_PCLK2_ENABLE); + unsigned saved_poll = polls, saved_apb = apb_requests; + assert(!clock_call(&phy, 2 * second, second, 0x1e)); + assert(d.tunnel_users == 3 && polls == saved_poll && + apb_requests == saved_apb); + assert((tx() & (DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE)) == + (DPTX_PCLK1_ENABLE | DPTX_PCLK2_ENABLE)); + assert((tx() & DPTX_PCLK1_SELECT) == + FIELD_PREP(DPTX_PCLK1_SELECT, first ? 0 : 1)); + assert((tx() & DPTX_PCLK2_SELECT) == + FIELD_PREP(DPTX_PCLK2_SELECT, first ? 1 : 0)); + if (!strcmp(name, "same-rate")) { + unsigned saved = writes; + assert(!clock_call(&phy, 0, 0, d.tunnel_dpin_rate[0]) && + writes == saved); + } + if (!strcmp(name, "stale-owner")) { + unsigned saved = writes; + assert(clock_call(&phy, 2, 0, 0x06) == -ESTALE && + writes == saved); + assert(clock_call(&phy, 2, 0, 0) == -ESTALE && writes == saved); + } + if (!strcmp(name, "same-source")) { + assert(!clock_call(&phy, 2, 1, 0)); + unsigned saved = writes; + assert(clock_call(&phy, 0, 1, 0x14) == -EBUSY && + writes == saved); + assert(!clock_call(&phy, 2, 1, 0x1e)); + } + if (!strcmp(name, "retune-one")) { + u32 other = tx() & DPTX_PCLK2_SELECT; + assert(!clock_call(&phy, 0, 0, 0x06)); + assert((tx() & DPTX_PCLK1_SELECT) == + FIELD_PREP(DPTX_PCLK1_SELECT, 4)); + assert((tx() & DPTX_PCLK2_SELECT) == other && + polls == saved_poll); + } + unsigned released = !strcmp(name, "stop-second") ? 1 : 0, + remaining = 1 - released; + assert(!clock_call(&phy, 2 * released, released, 0)); + assert(d.tunnel_clock_on && d.tunnel_users == BIT(remaining) && + !stop_requests); + assert(tx() & DPTX_PCLK1_ENABLE); + assert(!!(tx() & DPTX_PCLK2_ENABLE) == !!remaining); + unsigned saved = writes; + assert(!clock_call(&phy, 2 * released, released, 0) && writes == saved); + if (!strcmp(name, "failed-final-stop")) { + faults = UINT64_C(1) << polls; + assert(clock_call(&phy, 2 * remaining, remaining, 0) == + -ETIMEDOUT); + assert(d.tunnel_clock_on && !d.tunnel_rate && + d.tunnel_users == BIT(remaining)); + saved = writes; + assert(clock_call(&phy, 2 * released, released, 0x14) == + -EBUSY && + writes == saved); + faults = 0; + } + assert(!clock_call(&phy, 2 * remaining, remaining, 0)); + assert(!d.tunnel_clock_on && !d.tunnel_users && !d.tunnel_rate && + !(tx() & T8122_PCLK_ENABLES)); + saved = writes; + assert(!clock_call(&phy, 2 * remaining, remaining, 0) && + writes == saved); + printf("PASS %s writes=%u polls=%u: two callbacks, independent selectors, shared PLL, stop ownership\n", + name, writes, polls); + return 0; +} diff --git a/tools/asahi/t6030-display-admission-tests.c b/tools/asahi/t6030-display-admission-tests.c new file mode 100644 index 0000000000000..e2c75eddb5ea5 --- /dev/null +++ b/tools/asahi/t6030-display-admission-tests.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define EPROBE_DEFER 517 +#define DL_FLAG_AUTOREMOVE_CONSUMER 1 +#define EXPORT_SYMBOL_GPL(x) +struct device_node { + const char *compatible; +}; +struct device { + struct device_node *of_node; + atomic_int refs; +}; +struct device_link { + int unused; +}; +static pthread_mutex_t apple_t6030_display_lock = PTHREAD_MUTEX_INITIALIZER; +static struct device *apple_t6030_display; +static struct device_node display_node = { "apple,t6030-display-subsystem" }, + gpu_node = { "apple,agx-t6030" }, + other_node = { "apple,t6031-display-subsystem" }; +static struct device display, duplicate, gpu; +static const char *machine = "apple,t6030"; +static int pmp_calls, link_calls, pmp_result; +static bool link_ok = true; +static atomic_bool pause_link, link_entered, release_link, withdrew; +struct lock_guard { + pthread_mutex_t *lock; + bool once; +}; +static _Thread_local bool locked; +static struct lock_guard lock_take(pthread_mutex_t *lock) +{ + assert(!locked); + assert(!pthread_mutex_lock(lock)); + locked = true; + return (struct lock_guard){ lock, true }; +} +static void lock_drop(struct lock_guard *g) +{ + assert(locked); + locked = false; + assert(!pthread_mutex_unlock(g->lock)); +} +#define guard(kind) \ + struct lock_guard local_guard \ + __attribute__((cleanup(lock_drop))) = lock_take +#define scoped_guard(kind, lock) \ + for (struct lock_guard scoped \ + __attribute__((cleanup(lock_drop))) = lock_take(lock); \ + scoped.once; scoped.once = false) +static bool of_device_is_compatible(const struct device_node *n, + const char *name) +{ + return n && !strcmp(n->compatible, name); +} +static bool of_machine_is_compatible(const char *name) +{ + return !strcmp(machine, name); +} +static bool asahi_g15_board(void) +{ + return of_machine_is_compatible("apple,t6030"); +} +static struct device *get_device(struct device *dev) +{ + assert(locked); + atomic_fetch_add(&dev->refs, 1); + return dev; +} +static void put_device(struct device *dev) +{ + assert(!locked); + if (dev) + assert(atomic_fetch_sub(&dev->refs, 1) > 1); +} +static struct device_link host_link; +static struct device_link *device_link_add(struct device *consumer, + struct device *supplier, int flags) +{ + assert(locked && consumer == &gpu && supplier == &display && + atomic_load(&display.refs) == 2); + assert(flags == DL_FLAG_AUTOREMOVE_CONSUMER); + link_calls++; + if (atomic_load(&pause_link)) { + atomic_store(&link_entered, true); + while (!atomic_load(&release_link)) + sched_yield(); + } + return link_ok ? &host_link : NULL; +} +static int apple_pmp_link_device(struct device *dev) +{ + assert(!locked && dev == &gpu && link_calls > 0); + pmp_calls++; + return pmp_result; +} +#include "production.h" +static void reset(void) +{ + assert(!apple_t6030_display && !locked); + display = (struct device){ .of_node = &display_node, .refs = 1 }; + duplicate = (struct device){ .of_node = &display_node, .refs = 1 }; + gpu = (struct device){ .of_node = &gpu_node, .refs = 1 }; + pmp_calls = link_calls = pmp_result = 0; + link_ok = true; + machine = "apple,t6030"; + atomic_store(&pause_link, false); + atomic_store(&link_entered, false); + atomic_store(&release_link, false); + atomic_store(&withdrew, false); +} +static void *admit_thread(void *unused) +{ + (void)unused; + assert(asahi_g15_pmp_prepare(&gpu) == 0); + return NULL; +} +static void *withdraw_thread(void *unused) +{ + (void)unused; + apple_t6030_display_withdraw(&display); + atomic_store(&withdrew, true); + return NULL; +} +int main(void) +{ + unsigned int cases = 0; + (void)get_device; + (void)put_device; + reset(); + assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && !pmp_calls); + cases++; + /* Component platform bind alone never calls this publication. Retry succeeds only afterward. */ + assert(apple_t6030_display_publish(&display) == 0 && + atomic_load(&display.refs) == 2); + cases++; + assert(asahi_g15_pmp_prepare(&gpu) == 0 && pmp_calls == 1); + cases++; + assert(apple_t6030_display_publish(&display) == 0 && + atomic_load(&display.refs) == 2); + cases++; + assert(apple_t6030_display_publish(&duplicate) == -EBUSY && + apple_t6030_display == &display); + cases++; + apple_t6030_display_withdraw(&duplicate); + assert(apple_t6030_display == &display && + atomic_load(&display.refs) == 2); + cases++; + link_ok = false; + assert(asahi_g15_pmp_prepare(&gpu) == -EINVAL && pmp_calls == 1); + cases++; + link_ok = true; + pmp_result = -EPROBE_DEFER; + assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && pmp_calls == 2); + cases++; + machine = "apple,t6031"; + assert(apple_drm_t6030_link_device(&gpu) == -ENODEV); + cases++; + machine = "apple,t6030"; + gpu.of_node = &other_node; + assert(apple_drm_t6030_link_device(&gpu) == -ENODEV); + cases++; + gpu.of_node = &gpu_node; + apple_t6030_display_withdraw(&display); + assert(atomic_load(&display.refs) == 1 && !apple_t6030_display); + cases++; + assert(asahi_g15_pmp_prepare(&gpu) == -EPROBE_DEFER && pmp_calls == 2); + cases++; + display.of_node = &other_node; + assert(apple_t6030_display_publish(&display) == 0 && + !apple_t6030_display && atomic_load(&display.refs) == 1); + cases++; + reset(); + assert(!apple_t6030_display_publish(&display)); + atomic_store(&pause_link, true); + pthread_t admit, withdraw; + assert(!pthread_create(&admit, NULL, admit_thread, NULL)); + while (!atomic_load(&link_entered)) + sched_yield(); + assert(!pthread_create(&withdraw, NULL, withdraw_thread, NULL)); + usleep(10000); + assert(!atomic_load(&withdrew) && atomic_load(&display.refs) == 2); + atomic_store(&release_link, true); + assert(!pthread_join(admit, NULL) && !pthread_join(withdraw, NULL)); + assert(atomic_load(&withdrew) && atomic_load(&display.refs) == 1 && + !apple_t6030_display); + cases++; + printf("PASS: %u actual display publisher/admission controls, including concurrent link/withdraw\n", + cases); + return 0; +} diff --git a/tools/asahi/t6030-display-admission-tests.py b/tools/asahi/t6030-display-admission-tests.py new file mode 100644 index 0000000000000..a1da987377f18 --- /dev/null +++ b/tools/asahi/t6030-display-admission-tests.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Actual T6030 display publisher and G15 admission, with pthread boundaries. + +Tests initial admission and pointer lifetime, not safe shutdown of retained G15 +firmware after forced display unbind. Full DRM publication placement is checked +as source ordering separately from executable host boundary controls. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shlex +import subprocess +from importlib.machinery import SourceFileLoader +ROOT = Path(__file__).resolve().parents[2] +extract = SourceFileLoader('supplier_test', str(ROOT / 'tools/asahi/t6030-pmp-supplier-tests.py')).load_module() + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--output', required=True, type=Path) + p.add_argument('--mutations', action='store_true') + args = p.parse_args(); args.output.mkdir(parents=True, exist_ok=False) + drv = (ROOT / 'drivers/gpu/drm/apple/apple_drv.c').read_text() + glue = (ROOT / 'drivers/gpu/drm/asahi/g15/glue.c').read_text() + init = extract.function(drv, 'static int apple_drm_init(') + uninit = extract.function(drv, 'static void apple_drm_uninit(') + assert init.index('ret = drm_dev_register(') < init.index('drm_client_setup_with_fourcc(') < init.index('ret = apple_t6030_display_publish(') + assert uninit.index('apple_t6030_display_withdraw(') < uninit.index('apple_drm_quiesce_connectors(') < uninit.index('drm_dev_unplug(') + assert 'drm_dev_unregister(&apple->drm);\n\t\tgoto err_unbind;' in init + code = '\n'.join([extract.function(drv, 'static int apple_t6030_display_publish('), + extract.function(drv, 'static void apple_t6030_display_withdraw('), + extract.function(drv, 'int apple_drm_t6030_link_device('), + extract.function(glue, 'int asahi_g15_pmp_prepare(struct device *dev)\n')]) + variants = [('control', code)] + if args.mutations: + for name, old, new in [ + ('no-ready-gate', 'if (!apple_t6030_display)', 'if (false)'), + ('no-display-get', 'apple_t6030_display = get_device(dev);', 'apple_t6030_display = dev;'), + ('no-display-put', 'put_device(old);', '(void)old;'), + ('replace-live-provider', 'return apple_t6030_display == dev ? 0 : -EBUSY;', 'apple_t6030_display = dev;'), + ('ignore-display-error', 'if (ret)\n\t\treturn ret;', 'if (false)\n\t\treturn ret;'), + ('wrong-gpu', '!of_device_is_compatible(consumer->of_node, "apple,agx-t6030")', 'false'), + ('withdraw-wrong-provider', 'if (apple_t6030_display == dev)', 'if (((void)dev, true))'), + ]: + assert code.count(old) == 1, (name, code.count(old)) + variants.append((name, code.replace(old, new))) + compiler = shlex.split(os.environ.get('CC', 'clang')) + command = compiler + ['-std=gnu11', '-g', '-O1', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', '-pthread', + '-I', str(args.output), str(ROOT / 'tools/asahi/t6030-display-admission-tests.c'), '-o', str(args.output / 'admission')] + results = [] + for name, source in variants: + (args.output / 'production.h').write_text(source) + built = subprocess.run(command, capture_output=True, text=True) + ran = subprocess.run([str(args.output / 'admission')], capture_output=True, text=True) if not built.returncode else None + (args.output / (name + '.log')).write_text(built.stdout + built.stderr + (ran.stdout + ran.stderr if ran else '')) + results.append({'name': name, 'build_rc': built.returncode, 'run_rc': ran.returncode if ran else None}) + (args.output / 'production.h').write_text(code) + good = all(x['build_rc'] == 0 and (x['run_rc'] == 0 if x['name'] == 'control' else x['run_rc'] != 0) for x in results) + (args.output / 'receipt.json').write_text(json.dumps({'passed': good, 'source_sha256': hashlib.sha256(code.encode()).hexdigest(), + 'publication_order_verified': True, 'forced_supplier_unbind_safety_claimed': False, 'command': command, 'results': results}, indent=2) + '\n') + print(json.dumps({'passed': good, 'results': results})) + return not good +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/asahi/t6030-display-linkage-tests.py b/tools/asahi/t6030-display-linkage-tests.py new file mode 100644 index 0000000000000..7ff606c41817d --- /dev/null +++ b/tools/asahi/t6030-display-linkage-tests.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Run the actual G15 Kconfig stanza through the kernel's conf in 18 combinations. + +The test uses a minimal Kconfig containing the actual production G15 stanza and +its dependency/select names, so unrelated architecture/compiler options do not +hide invalid provider/consumer linkage. It writes only the supplied output dir. +""" +import argparse +import hashlib +import itertools +import json +import os +from pathlib import Path +import re +import subprocess +ROOT = Path(__file__).resolve().parents[2] + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--output', required=True, type=Path) + p.add_argument('--conf', required=True, type=Path, help='built scripts/kconfig/conf') + args = p.parse_args(); args.output.mkdir(parents=True, exist_ok=False) + source = (ROOT / 'drivers/gpu/drm/asahi/Kconfig').read_text() + stanza = source.split('config DRM_ASAHI_G15\n', 1)[1].split('\nconfig ', 1)[0] + stanza = 'config DRM_ASAHI_G15\n' + stanza + assert 'depends on DRM_APPLE=y || (DRM_APPLE=m && DRM_ASAHI=m)' in stanza + kconfig = '''mainmenu "T6030 display linkage controls" +config MODULES + bool "Modules" + modules + default y +config DRM_APPLE + tristate "Display" +config DRM_ASAHI + tristate "GPU" +config APPLE_T6030_GPU_PMP_LINK + bool +config CRYPTO_LIB_SHA256 + bool +''' + stanza + source_path = args.output.resolve() / 'Kconfig' + source_path.write_text(kconfig) + results = [] + for gpu, display, request in itertools.product(('n', 'm', 'y'), ('n', 'm', 'y'), ('n', 'y')): + case = args.output / f'asahi-{gpu}-display-{display}-g15-{request}' + case.mkdir() + config = case.resolve() / 'config' + config.write_text(f'CONFIG_MODULES=y\nCONFIG_DRM_ASAHI={gpu}\nCONFIG_DRM_APPLE={display}\nCONFIG_DRM_ASAHI_G15={request}\n') + env = os.environ | {'KCONFIG_CONFIG': str(config)} + result = subprocess.run([str(args.conf.resolve()), '--olddefconfig', str(source_path)], + env=env, cwd=case, capture_output=True, text=True) + (case / 'conf.log').write_text(result.stdout + result.stderr) + actual = dict(re.findall(r'^CONFIG_(\w+)=(\w+)$', config.read_text(), re.M)) + expected = 'y' if request == 'y' and gpu != 'n' and (display == 'y' or display == 'm' and gpu == 'm') else 'n' + got = actual.get('DRM_ASAHI_G15', 'n') + good = result.returncode == 0 and got == expected and actual.get('DRM_ASAHI', 'n') == gpu and actual.get('DRM_APPLE', 'n') == display + results.append({'gpu': gpu, 'display': display, 'g15_requested': request, + 'g15_expected': expected, 'g15_actual': got, 'conf_rc': result.returncode, 'pass': good}) + glue = (ROOT / 'drivers/gpu/drm/asahi/g15/glue.c').read_text() + start = glue.index('int asahi_g15_pmp_prepare(struct device *dev)\n') + body = glue[start:glue.index('\nint asahi_g15_pmp_get(', start)] + assert 'ret = apple_drm_t6030_link_device(dev);' in body + assert 'symbol_get(' not in body + # The provider does not import any of Asahi's glue symbols. Full modpost + # and the actual .modinfo dependency are separately required build gates. + provider = '\n'.join(f.read_text() for f in (ROOT / 'drivers/gpu/drm/apple').glob('*.c')) + assert not re.search(r'\basahi_g15_\w+\s*\(', provider) + good = all(row['pass'] for row in results) + (args.output / 'receipt.json').write_text(json.dumps({'passed': good, + 'stanza_sha256': hashlib.sha256(stanza.encode()).hexdigest(), + 'direct_import_checked': True, 'modpost_or_hardware_claimed': False, 'results': results}, indent=2) + '\n') + print(json.dumps({'passed': good, 'configurations': len(results), 'accepted_g15_linkages': [[r['gpu'], r['display']] for r in results if r['g15_actual'] == 'y']})) + return not good +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/asahi/t6030-pmp-graph-tests.c b/tools/asahi/t6030-pmp-graph-tests.c new file mode 100644 index 0000000000000..37fdc54a057d9 --- /dev/null +++ b/tools/asahi/t6030-pmp-graph-tests.c @@ -0,0 +1,803 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Typed host boundaries for the actual T6030 PMP probe/start/consumer graph. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +typedef uint64_t u64; +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint8_t u8; +#define BIT_ULL(n) (1ULL << (n)) +#define __ffs64(v) __builtin_ctzll(v) +#define BIT(n) (1U << (n)) +#define IS_ENABLED(x) (x) +#define CONFIG_APPLE_T6030_GPU_PMP_LINK 1 +#define PMP_REPORT_READY 1 +#define GFP_KERNEL 0 +#define GENPD_FLAG_ACTIVE_WAKEUP 1 +#define MAX_PHANDLE_ARGS 4 +#define EPROBE_DEFER 517 +#define SZ_4K 4096 +#define __iomem +#define __force +#define container_of(p, t, m) ((t *)((char *)(p) - offsetof(t, m))) +#define READ_ONCE(x) (x) +#define WRITE_ONCE(x, v) ((x) = (v)) +#define EXPORT_SYMBOL_GPL(x) +#define IS_ERR(p) false +#define PTR_ERR(p) (-ENOMEM) +static void host_log(const char *format, ...) +{ + (void)format; +} +#define dev_err(dev, ...) ((void)(dev), host_log(__VA_ARGS__)) +#define dev_info dev_err +#define dev_warn dev_err +#define dev_err_probe(dev, err, ...) ((void)(dev), (err)) +#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0])) +struct device_node { + const char *name, *compatible; + bool available, always_on, no_ack, deferred, uuid, soc_record; + u32 reg; + int refs; + struct device_node *parent, *child, *sibling, *pmp; +}; +struct device { + struct device_node *of_node; + struct device *parent; + void *data; +}; +struct platform_device { + struct device dev; +}; +struct generic_pm_domain { + char *name; + int flags; + int (*power_on)(struct generic_pm_domain *), + (*power_off)(struct generic_pm_domain *); +}; +struct apple_pmp_report_offsets { + u32 tgt_read, tgt_write, actual, status, fast_die, fast_die_lanes; + bool starts_pmp; + const char *name, *fallback, *pwrstate; + bool devices_ordered; +}; +struct list_head { + struct list_head *next, *prev; +}; +#define INIT_LIST_HEAD(h) ((h)->next = (h), (h)->prev = (h)) +static void list_add_tail(struct list_head *n, struct list_head *h) +{ + n->prev = h->prev; + n->next = h; + h->prev->next = n; + h->prev = n; +} +static void list_del(struct list_head *n) +{ + n->prev->next = n->next; + n->next->prev = n->prev; + INIT_LIST_HEAD(n); +} +#define list_add_tail_rcu list_add_tail +#define list_for_each_entry(p, h, m) \ + for (struct list_head *cursor = (h)->next; \ + cursor != (h) && \ + ((p) = container_of(cursor, __typeof__(*(p)), m), 1); \ + cursor = cursor->next) +#define list_for_each_entry_rcu list_for_each_entry +struct apple_pmp_start; +struct apple_pmp_report { + struct device *dev; + const struct apple_pmp_report_offsets *offsets; + void *base; + int lock; + struct list_head list; + void *debugfs; + struct apple_pmp_start *start; +}; +struct completion { + bool done; +}; +struct work_struct { + void (*func)(struct work_struct *); + bool pending; +}; +typedef struct { + u8 b[16]; +} uuid_t; +struct apple_pmp_start_ops { + int (*check)(struct apple_pmp_start *, uuid_t *); + int (*enable)(struct apple_pmp_start *); + bool (*bound)(struct apple_pmp_start *); +}; +struct apple_pmp_start_timing { + unsigned int bind, quiet, ready, ack; +}; +struct apple_pmp_start { + struct apple_pmp_report *rep; + struct list_head list; + struct device_node *pmp; + u64 seed, ack, startup_ack, record_seed; + const struct apple_pmp_start_ops *ops; + const struct apple_pmp_start_timing *timing; + struct work_struct work; + bool queued, enabled; + struct completion started; + int result; + bool stopping; + unsigned int waiters; + struct completion waiters_done; +}; +struct apple_pmp_report_entry { + struct generic_pm_domain genpd; + struct device_node *node, *report_node; + u32 id; + bool no_ack, removed; +}; +#define genpd_to_apple_pmp_report_entry(p) \ + container_of(p, struct apple_pmp_report_entry, genpd) +struct of_phandle_iterator { + struct device_node *node; +}; +struct of_phandle_args { + struct device_node *np; + int args_count, args[MAX_PHANDLE_ARGS]; +}; +#define of_for_each_phandle(it, ret, node, ...) for ((ret) = 0; false;) +static int of_phandle_iterator_args(struct of_phandle_iterator *it, int *a, + int max) +{ + (void)it; + (void)a; + (void)max; + return 0; +} +static int of_genpd_add_subdomain(struct of_phandle_args *p, + struct of_phandle_args *c) +{ + (void)p; + (void)c; + return 0; +} +static int pm_genpd_init(struct generic_pm_domain *pd, void *gov, bool off) +{ + (void)pd; + (void)gov; + (void)off; + return 0; +} +static int pm_genpd_remove(struct generic_pm_domain *pd) +{ + (void)pd; + return 0; +} +static void pm_genpd_remove_device(struct device *d) +{ + (void)d; +} +static int of_genpd_add_provider_simple(struct device_node *n, + struct generic_pm_domain *pd) +{ + (void)n; + (void)pd; + return 0; +} +static void of_genpd_del_provider(struct device_node *n) +{ + (void)n; +} +static int apple_pmp_start_mutex, apple_pmp_reports_lock; +struct lock_guard { + int *lock; + bool once; +}; +static struct lock_guard lock_take(int *l) +{ + assert(!*l); + *l = 1; + return (struct lock_guard){ l, true }; +} +static void lock_drop(struct lock_guard *l) +{ + assert(*l->lock); + *l->lock = 0; +} +#define guard(kind) \ + struct lock_guard local_guard \ + __attribute__((cleanup(lock_drop))) = lock_take +#define scoped_guard(kind, lock) \ + for (struct lock_guard scoped \ + __attribute__((cleanup(lock_drop))) = lock_take(lock); \ + scoped.once; scoped.once = false) +static void mutex_lock(int *l) +{ + assert(!*l); + *l = 1; +} +static void mutex_unlock(int *l) +{ + assert(*l); + *l = 0; +} +#define spin_lock_init(l) (*(l) = 0) +#define spin_lock_irqsave(l, f) ((f) = 0, mutex_lock(l)) +#define spin_unlock_irqrestore(l, f) ((void)(f), mutex_unlock(l)) +static void rcu_read_lock(void) +{ +} +static void rcu_read_unlock(void) +{ +} +static struct list_head apple_pmp_starts, apple_pmp_reports; +static struct device_node report_node, pmp_node, entries[4]; +static struct platform_device report_dev, entry_devs[4]; +static u64 regs[0x14000 / 8]; +static u8 ranges[64], devices[17 * 0x7c]; +static struct work_struct *pending; +static unsigned long jiffies; +static unsigned int ack_polls; +#define msecs_to_jiffies(v) (v) +#define time_before(a, b) ((long)((a) - (b)) < 0) +#define time_after(a, b) time_before(b, a) +static void msleep(unsigned int n) +{ + jiffies += n ? n : 1; +} +static void init_completion(struct completion *c) +{ + c->done = false; +} +static void complete_all(struct completion *c) +{ + c->done = true; +} +#define complete complete_all +#define INIT_WORK(w, f) ((w)->func = (f), (w)->pending = false) +#define system_unbound_wq NULL +static bool queue_work(void *q, struct work_struct *w) +{ + assert(!q && !pending); + pending = w; + w->pending = true; + return true; +} +static unsigned long wait_for_completion_timeout(struct completion *c, + unsigned long n) +{ + assert(!apple_pmp_start_mutex); + if (pending) { + struct work_struct *w = pending; + pending = NULL; + w->pending = false; + w->func(w); + } + jiffies++; + return c->done ? n : 0; +} +static void wait_for_completion(struct completion *c) +{ + assert(c->done); +} +static void cancel_work_sync(struct work_struct *w) +{ + if (pending == w) + pending = NULL; + w->pending = false; +} +static u64 readq(const void *a) +{ + return *(const u64 *)a; +} +static void writeq(u64 v, void *a) +{ + assert(a == (char *)regs + 0x108c0); + *(u64 *)a = v; + regs[0x1180 / 8] = v; +} +#define readq_poll_timeout(address, value, condition, delay, timeout) \ + ({ \ + int r = -ETIMEDOUT; \ + ack_polls++; \ + for (int i = 0; i < 4; i++) { \ + (value) = readq(address); \ + if (condition) { \ + r = 0; \ + break; \ + } \ + } \ + (void)(delay); \ + (void)(timeout); \ + r; \ + }) +#define readq_poll_timeout_atomic readq_poll_timeout +static u32 get_unaligned_le32(const void *p) +{ + u32 v; + memcpy(&v, p, 4); + return v; +} +static void put32(u8 *p, u32 v) +{ + memcpy(p, &v, 4); +} +static struct device_node *of_node_get(struct device_node *n) +{ + if (n) + n->refs++; + return n; +} +static void of_node_put(struct device_node *n) +{ + if (n) { + assert(n->refs > 0); + n->refs--; + } +} +static bool of_device_is_available(const struct device_node *n) +{ + return n && n->available; +} +static bool of_device_is_compatible(const struct device_node *n, const char *s) +{ + return n && n->compatible && !strcmp(n->compatible, s); +} +static bool of_machine_is_compatible(const char *s) +{ + return !strcmp(s, "apple,t6030"); +} +static bool of_property_read_bool(const struct device_node *n, const char *p) +{ + if (!strcmp(p, "apple,always-on")) + return n->always_on; + if (!strcmp(p, "apple,no-ack")) + return n->no_ack; + if (!strcmp(p, "apple,defer-startup-ack")) + return n->deferred; + if (!strcmp(p, "apple,soc-device-record")) + return n->soc_record; + return false; +} +static bool of_property_present(const struct device_node *n, const char *p) +{ + if (!strcmp(p, "apple,tunable-uuid")) + return n->uuid; + if (!strcmp(p, "apple,pmp")) + return !!n->pmp; + return false; +} +static int of_property_read_u32(const struct device_node *n, const char *p, + u32 *v) +{ + assert(!strcmp(p, "reg")); + *v = n->reg; + return 0; +} +static int of_property_count_u32_elems(const struct device_node *n, + const char *p) +{ + if (!strcmp(p, "reg")) + return 1; + assert(!strcmp(p, "apple,pmp")); + return n->pmp ? 1 : -EINVAL; +} +static int of_property_read_string(const struct device_node *n, const char *p, + const char **v) +{ + assert(!strcmp(p, "label")); + *v = n->name; + return 0; +} +static const void *of_get_property(const struct device_node *n, const char *p, + int *len) +{ + assert(n == &pmp_node); + if (!strcmp(p, "apple,tunable-ptd-range")) { + *len = sizeof(ranges); + return ranges; + } + assert(!strcmp(p, "apple,tunable-soc-device")); + *len = sizeof(devices); + return devices; +} +static struct device_node *of_parse_phandle(const struct device_node *n, + const char *p, int i) +{ + assert(!i); + if (!strcmp(p, "apple,pmp")) + return of_node_get(n->pmp); + assert(!strcmp(p, "apple,pmp-report")); + return NULL; +} +static struct device_node *of_get_parent(struct device_node *n) +{ + return of_node_get(n->parent); +} +static struct device_node *next_child(struct device_node *p, + struct device_node *old, bool available) +{ + struct device_node *n = old ? old->sibling : p->child; + of_node_put(old); + while (n && available && !n->available) + n = n->sibling; + return of_node_get(n); +} +#define for_each_child_of_node(p, n) \ + for ((n) = next_child(p, NULL, false); (n); \ + (n) = next_child(p, n, false)) +#define for_each_available_child_of_node(p, n) \ + for ((n) = next_child(p, NULL, true); (n); (n) = next_child(p, n, true)) +static struct device_node *find_report(struct device_node *old, const char *c) +{ + of_node_put(old); + return !old && of_device_is_compatible(&report_node, c) ? + of_node_get(&report_node) : + NULL; +} +#define for_each_compatible_node(n, t, c) \ + for ((n) = find_report(NULL, c); (n); (n) = find_report(n, c)) +static void *dev_get_drvdata(struct device *d) +{ + return d->data; +} +static void platform_set_drvdata(struct platform_device *p, void *v) +{ + p->dev.data = v; +} +static void *platform_get_drvdata(struct platform_device *p) +{ + return p->dev.data; +} +static bool device_is_bound(struct device *d) +{ + (void)d; + return false; +} +static bool device_trylock(struct device *d) +{ + (void)d; + return false; +} +static void device_unlock(struct device *d) +{ + (void)d; +} +static void put_device(struct device *d) +{ + (void)d; +} +static struct platform_device *of_find_device_by_node(struct device_node *n) +{ + (void)n; + return NULL; +} +struct action { + void (*f)(void *); + void *p; +}; +static struct action actions[8]; +static unsigned int action_count; +static void *allocations[8]; +static unsigned int allocation_count; +static void *devm_kzalloc(struct device *d, size_t n, int g) +{ + (void)d; + (void)g; + void *v = calloc(1, n); + assert(v); + allocations[allocation_count++] = v; + return v; +} +#define kzalloc_obj(v) calloc(1, sizeof(v)) +#define kfree free +static char *kstrdup(const char *s, int g) +{ + (void)g; + return strdup(s); +} +static int devm_add_action_or_reset(struct device *d, void (*f)(void *), + void *p) +{ + (void)d; + assert(action_count < 8); + actions[action_count++] = (struct action){ f, p }; + return 0; +} +static void *devm_platform_ioremap_resource(struct platform_device *p, int i) +{ + assert(p == &report_dev && !i); + return regs; +} +static const struct apple_pmp_report_offsets * +of_device_get_match_data(struct device *d); +static void apple_pmp_report_release(void *p) +{ + struct apple_pmp_report *r = p; + list_del(&r->list); +} +static void *apple_pmp_report_debugfs; +static int apple_pmp_report_status_fops; +static const char *dev_name(struct device *d) +{ + (void)d; + return "host-report"; +} +static void *debugfs_create_file(const char *n, int mode, void *dir, void *data, + void *ops) +{ + (void)n; + (void)mode; + (void)dir; + (void)data; + (void)ops; + return NULL; +} +static int of_platform_populate(struct device_node *n, void *a, void *b, + struct device *d); +static int check_error, enable_error; +static bool hw_binds, hw_ready; +static u64 hw_ack; +static unsigned int enables, commands; +static int hw_check(struct apple_pmp_start *s, uuid_t *u) +{ + (void)s; + memset(u, 0, sizeof(*u)); + return check_error; +} +static int hw_enable(struct apple_pmp_start *s) +{ + assert(!pmp_node.available); + enables++; + if (!enable_error) + pmp_node.available = true; + (void)s; + return enable_error; +} +static bool hw_bound(struct apple_pmp_start *s) +{ + if (!hw_binds) + return false; + regs[s->rep->offsets->status / 8] = hw_ready ? 1 : 0; + regs[s->rep->offsets->actual / 8] = hw_ack; + return true; +} +static const struct apple_pmp_start_ops apple_pmp_start_hw_ops = { hw_check, + hw_enable, + hw_bound }; +static const struct apple_pmp_start_timing apple_pmp_start_timing = { 2, 0, 2, + 2 }; +#include "production.h" +static const struct apple_pmp_report_offsets * +of_device_get_match_data(struct device *d) +{ + assert(d == &report_dev.dev); + return &apple_pmp_offsets_t6030; +} +static int of_platform_populate(struct device_node *n, void *a, void *b, + struct device *d) +{ + assert(n == &report_node && !a && !b && d == &report_dev.dev); + /* Actual child probe seeds each request while PMP status is still halted. */ + for (unsigned int i = 0; i < 4; i++) { + int r = apple_pmp_report_entry_probe(&entry_devs[i]); + if (r) + return r; + } + return 0; +} +struct overlay_entry { + u32 reg; + bool always_on, no_ack, deferred; +}; +#include "overlay.h" +static void cleanup(void) +{ + for (unsigned int i = 0; i < 4; i++) { + struct apple_pmp_report_entry *e = entry_devs[i].dev.data; + if (e) + apple_pmp_report_entry_free(e); + entry_devs[i].dev.data = NULL; + } + while (action_count) { + struct action a = actions[--action_count]; + a.f(a.p); + } + for (unsigned int i = 0; i < allocation_count; i++) + free(allocations[i]); + allocation_count = 0; + assert(!report_node.refs && !pmp_node.refs); + for (unsigned int i = 0; i < 4; i++) + assert(!entries[i].refs); + assert(!apple_pmp_start_mutex && !apple_pmp_reports_lock && !pending); +} +static struct apple_pmp_report *setup(void) +{ + INIT_LIST_HEAD(&apple_pmp_starts); + INIT_LIST_HEAD(&apple_pmp_reports); + memset(regs, 0, sizeof(regs)); + memset(ranges, 0, sizeof(ranges)); + memset(devices, 0, sizeof(devices)); + report_node = + (struct device_node){ .name = "report", + .compatible = "apple,t6030-pmp-v2-report", + .available = true, + .child = &entries[0], + .pmp = &pmp_node }; + pmp_node = (struct device_node){ .name = "PMP", + .compatible = "apple,t6000-pmp-v2", + .uuid = true }; + for (unsigned int i = 0; i < 4; i++) { + const struct overlay_entry *o = &overlay_entries[i]; + entries[i] = (struct device_node){ + .name = "entry", + .compatible = "apple,t6000-pmp-v2-report-entry", + .available = true, + .reg = o->reg, + .always_on = o->always_on, + .no_ack = o->no_ack, + .deferred = o->deferred, + .parent = &report_node, + .sibling = i < 3 ? &entries[i + 1] : NULL + }; + entry_devs[i] = (struct platform_device){ + .dev = { .of_node = &entries[i], + .parent = &report_dev.dev } + }; + } + report_dev = + (struct platform_device){ .dev = { .of_node = &report_node } }; + put32(ranges, 10); + put32(ranges + 4, 0x118); + put32(ranges + 8, 4); + put32(ranges + 32, 11); + put32(ranges + 36, 0x11c); + put32(ranges + 40, 4); + for (unsigned int i = 0; i < 17; i++) { + put32(devices + i * 0x7c, i + 1); + if (i >= 7 && i <= 9) + put32(devices + i * 0x7c + 8, 2); + } + ack_polls = 0; + check_error = enable_error = 0; + hw_binds = hw_ready = true; + hw_ack = BIT_ULL(7); + enables = commands = 0; + return NULL; +} +static int g15_vote(void) +{ + struct device supplier = { .of_node = &pmp_node }; + /* Scoped supplier has its first, more specific compatible as on the real DT. */ + pmp_node.compatible = "apple,t6030-pmp-v2"; + int ret = apple_pmp_report_wait_supplier_ready(&supplier, 5000); + if (!ret) + commands++; + return ret; +} +int main(void) +{ + unsigned int cases = 0; + (void)apple_pmp_start_init; + (void)apple_pmp_report_wait_t6030_14; + setup(); + assert(apple_pmp_report_probe(&report_dev) == 0); + struct apple_pmp_report *r = report_dev.dev.data; + /* The old passive descriptor fails the real DCP entry waiter here. */ + if (!r->start) + assert(apple_pmp_report_wait_ready(&entries[0], 30000) == 0); + assert(r->start); + assert(r->start->seed == + (BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9) | BIT_ULL(16))); + assert(r->start->ack == (BIT_ULL(7) | BIT_ULL(8) | BIT_ULL(9)) && + r->start->startup_ack == BIT_ULL(7)); + assert(!pmp_node.available && regs[0x10 / 8] == 0 && enables == 0); + assert(regs[0x1180 / 8] == r->start->seed); + assert(((struct apple_pmp_report_entry *)entry_devs[3].dev.data) + ->no_ack); + assert(apple_pmp_report_wait_ready(&entries[0], 30000) == 0 && + enables == 1 && pmp_node.available); + assert(g15_vote() == 0 && commands == 1 && enables == 1); + cases++; + assert(apple_pmp_report_wait_ready(&entries[1], 3000) == -EIO); + cases++; + regs[0x11c0 / 8] |= BIT_ULL(8); + assert(apple_pmp_report_wait_ready(&entries[1], 3000) == 0); + cases++; + assert(apple_pmp_report_wait_ready(&entries[3], 3000) == -EINVAL); + cases++; + unsigned int polls = ack_polls; + struct apple_pmp_report_entry *storage = entry_devs[3].dev.data; + assert(apple_pmp_report_entry_power_on(&storage->genpd) == 0 && + ack_polls == polls); + cases++; + regs[0x11c0 / 8] &= ~BIT_ULL(7); + assert(g15_vote() == -EIO && commands == 1); + cases++; + cleanup(); + /* Missing display acknowledgement cannot release G15, even if bound+READY. */ + setup(); + assert(apple_pmp_report_probe(&report_dev) == 0); + hw_ack = BIT_ULL(8); + assert(apple_pmp_report_wait_ready(&entries[1], 3000) == -EIO); + cases++; + assert(g15_vote() == -EIO && commands == 0 && enables == 1); + cases++; + cleanup(); + setup(); + assert(apple_pmp_report_probe(&report_dev) == 0); + hw_ready = false; + assert(g15_vote() == -ETIMEDOUT && commands == 0); + cases++; + cleanup(); + setup(); + assert(apple_pmp_report_probe(&report_dev) == 0); + hw_binds = false; + assert(g15_vote() == -ETIMEDOUT && commands == 0); + cases++; + cleanup(); + setup(); + assert(apple_pmp_report_probe(&report_dev) == 0); + check_error = -EIO; + assert(g15_vote() == -EIO && !enables && !pmp_node.available); + cases++; + cleanup(); + setup(); + assert(apple_pmp_report_probe(&report_dev) == 0); + enable_error = -EINVAL; + assert(g15_vote() == -EINVAL && enables == 1 && !pmp_node.available); + cases++; + cleanup(); + setup(); + assert(apple_pmp_report_probe(&report_dev) == 0); + regs[0x1180 / 8] = 0; + assert(g15_vote() == -EBUSY && !enables); + cases++; + cleanup(); + setup(); + assert(apple_pmp_report_probe(&report_dev) == 0); + regs[0x10 / 8] = 1; + assert(g15_vote() == -EBUSY && !enables); + cases++; + cleanup(); + /* No valid handoff means no startup owner; a bound report isn't enough. */ + setup(); + pmp_node.uuid = false; + assert(apple_pmp_report_probe(&report_dev) == 0); + assert(!((struct apple_pmp_report *)report_dev.dev.data)->start); + assert(g15_vote() == -ENODEV && !enables); + cases++; + cleanup(); + /* A drained managed owner is unavailable even while the report stays bound. */ + setup(); + assert(apple_pmp_report_probe(&report_dev) == 0); + r = report_dev.dev.data; + assert(action_count == 2 && actions[1].p == r->start); + action_count--; + apple_pmp_start_drain(r->start); + assert(g15_vote() == -ENODEV && commands == 0 && !enables); + cases++; + cleanup(); + /* Actual describe rejects malformed handoff layouts before any enable. */ + setup(); + put32(ranges + 4, 0x100); + assert(apple_pmp_report_probe(&report_dev) == 0); + assert(g15_vote() == -EINVAL && commands == 0 && !enables); + cases++; + cleanup(); + setup(); + put32(devices + 16 * 0x7c + 8, 2); + assert(apple_pmp_report_probe(&report_dev) == 0); + assert(g15_vote() == -EINVAL && commands == 0 && !enables); + cases++; + cleanup(); + printf("PASS: %u actual overlay/probe/entry/DCP7/managed-start/supplier graph cases\n", + cases); + return 0; +} diff --git a/tools/asahi/t6030-pmp-graph-tests.py b/tools/asahi/t6030-pmp-graph-tests.py new file mode 100644 index 0000000000000..dcef7dcc4c250 --- /dev/null +++ b/tools/asahi/t6030-pmp-graph-tests.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Execute the actual report probe -> entry seed -> DCP -> PMP -> G15 graph. + +The actual overlay supplies the report entries. OF storage, allocation, MMIO, +workqueue execution and the PMP hardware operations are typed host boundaries. +The production probe, descriptor validation, startup, waiter and supplier/command +bodies are extracted verbatim. This is not a firmware or hardware test. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +from importlib.machinery import SourceFileLoader +ROOT = Path(__file__).resolve().parents[2] +extract = SourceFileLoader('supplier_test', str(ROOT / 'tools/asahi/t6030-pmp-supplier-tests.py')).load_module() + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--ref', help='read report source at a Git ref (negative control)') + p.add_argument('--mutations', action='store_true') + p.add_argument('--output', required=True, type=Path) + args = p.parse_args(); args.output.mkdir(parents=True, exist_ok=False) + source = subprocess.check_output(['git', '-C', str(ROOT), 'show', args.ref + ':' + extract.REPORT], text=True) if args.ref else (ROOT / extract.REPORT).read_text() + overlay = (ROOT / 'drivers/soc/apple/t6030-pmp.dtso').read_text() + entries = [] + for node in re.findall(r'report@[0-9a-f]+\s*\{(.*?)\n\s*\};', overlay, re.S): + reg = int(re.search(r'reg = <(0x[0-9a-f]+|[0-9]+)>', node)[1], 0) + entries.append((reg, 'apple,always-on;' in node, 'apple,no-ack;' in node, 'apple,defer-startup-ack;' in node)) + assert len(entries) == 4 + entry_data = ',\n'.join('{%d, %d, %d, %d}' % e for e in entries) + names = ['static int apple_pmp_report_bit(', 'static int apple_pmp_start_check_layout(', + 'static int apple_pmp_start_describe(', 'static void apple_pmp_start_finish(', + 'static bool apple_pmp_start_stopping(', 'static void apple_pmp_start_work(', + 'static void apple_pmp_start_drain(', 'static int apple_pmp_start_init(', + 'static int apple_pmp_start_wait(', 'int apple_pmp_report_wait_ready(', + 'static int apple_pmp_report_set_state(', 'static int apple_pmp_report_entry_set_state(', + 'static int apple_pmp_report_entry_power_on(', 'static int apple_pmp_report_entry_power_off(', + 'static void apple_pmp_report_entry_free(', 'static int apple_pmp_report_entry_probe(', + 'static int apple_pmp_report_probe(', 'static struct device_node *apple_pmp_report_t6030_entry(', + 'static int apple_pmp_report_wait_t6030_14(', 'int apple_pmp_report_wait_supplier_ready('] + if 'static int apple_pmp_report_wait_t6030_bound(' in source: + names.insert(names.index('static struct device_node *apple_pmp_report_t6030_entry('), 'static int apple_pmp_report_wait_t6030_bound(') + offsets = re.search(r'static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030 = \{.*?\n\};', source, re.S)[0] + functions = '\n'.join(extract.function(source, n) for n in names) + validation = re.sub(r'^#include .*\n', '', (ROOT / 'drivers/pmdomain/apple/pmp-report-validation.h').read_text(), flags=re.M) + code = offsets + '\n' + validation + '\n' + functions + # The actual overlay must still describe a halted supplier and disabled DMA/mailbox. + assert overlay.count('status = \"disabled\";') >= 3 + (args.output / 'production.h').write_text(code) + (args.output / 'overlay.h').write_text('static const struct overlay_entry overlay_entries[] = {\n' + entry_data + '\n};\n') + compiler = shlex.split(os.environ.get('CC', 'clang')) + command = compiler + ['-std=gnu11', '-g', '-O1', '-Wall', '-Wextra', '-Werror', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + '-I', str(args.output), str(ROOT / 'tools/asahi/t6030-pmp-graph-tests.c'), + '-o', str(args.output / 'graph')] + result = subprocess.run(command, capture_output=True, text=True) + (args.output / 'build.log').write_text(result.stdout + result.stderr) + if result.returncode: + print(result.stderr); return 1 + result = subprocess.run([str(args.output / 'graph')], capture_output=True, text=True) + (args.output / 'run.log').write_text(result.stdout + result.stderr) + print(result.stdout + result.stderr) + mutations = [] + if args.mutations: + controls = [ + ('no-managed-owner', '.starts_pmp = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK)', '.starts_pmp = false'), + ('no-managed-probe', 'ret = apple_pmp_start_init(rep);', 'ret = 0; (void)apple_pmp_start_init;'), + ('no-child-seeding', 'apple_pmp_report_entry_set_state(&ent->genpd, true);', '(void)0;'), + ('no-storage-policy', 'ent->no_ack = of_property_read_bool(node, "apple,no-ack");', 'ent->no_ack = false;'), + ('external-startup-ack', '!of_property_read_bool(child, "apple,defer-startup-ack")', 'true'), + ('ignore-seeded-state', 'status || request != st->seed', 'status'), + ('ignore-halted-state', 'status || request != st->seed', 'request != st->seed'), + ('ignore-safety-check', 'ret = st->ops->check(st, &image);', 'ret = 0; (void)image;'), + ('ignore-startup-ack', '(ack & st->startup_ack) != st->startup_ack', 'false'), + ('ignore-entry-ack', 'status == PMP_REPORT_READY && (ack & BIT_ULL(id))', '((void)ack, status == PMP_REPORT_READY)'), + ('storage-ack-layout', 'st->ack & ~st->record_seed,', 'st->seed & ~st->record_seed,'), + ('skip-supplier-wait', 'apple_pmp_report_wait_ready(entry, msecs_to_jiffies(timeout_ms))', '((void)timeout_ms, 0)'), + ] + for name, old, new in controls: + assert code.count(old) == 1, (name, code.count(old)) + (args.output / 'production.h').write_text(code.replace(old, new)) + built = subprocess.run(command, capture_output=True, text=True) + # A mutant must compile; compilation failure is not regression evidence. + ran = subprocess.run([str(args.output / 'graph')], capture_output=True, text=True) if not built.returncode else None + (args.output / (name + '.log')).write_text(built.stdout + built.stderr + (ran.stdout + ran.stderr if ran else '')) + mutations.append({'name': name, 'build_rc': built.returncode, 'run_rc': ran.returncode if ran else None}) + (args.output / 'production.h').write_text(code) + assert all(m['build_rc'] == 0 and m['run_rc'] != 0 for m in mutations), mutations + (args.output / 'receipt.json').write_text(json.dumps({'source_ref': args.ref, 'source_sha256': hashlib.sha256(code.encode()).hexdigest(), + 'overlay_sha256': hashlib.sha256(overlay.encode()).hexdigest(), 'overlay_entries': entries, + 'command': command, 'returncode': result.returncode, 'mutations': mutations}, indent=2) + '\n') + return bool(result.returncode) +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/asahi/t6030-pmp-supplier-tests.c b/tools/asahi/t6030-pmp-supplier-tests.c new file mode 100644 index 0000000000000..b581f3863a232 --- /dev/null +++ b/tools/asahi/t6030-pmp-supplier-tests.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Host boundaries for the extracted T6030 PMP selection and power bridge. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#define __init +#define IS_ENABLED(x) (x) +#define EPROBE_DEFER 517 +#define PMP_REPORT_READY 1 +#define DL_FLAG_AUTOREMOVE_CONSUMER 1 +#define BIT_ULL(n) (1ULL << (n)) +typedef uint64_t u64; +typedef uint32_t u32; +typedef uint16_t u16; +typedef uint8_t u8; +struct device_node { + const char *compatible; + bool available, has_reg, has_pmp, has_report, no_ack; + int reg_cells, pmp_cells, report_cells, refs; + u32 reg, phandle; + struct device_node *pmp, *report, *child, *sibling, *parent; +}; +struct device { + struct device_node *of_node; + int refs; + bool bound; +}; +struct platform_device { + struct device dev; +}; +struct device_link { + int unused; +}; +struct of_changeset { + int unused; +}; +struct gate_soc { + int id; +}; +struct apple_pmp_report_offsets { + unsigned int status, actual, tgt_read, tgt_write; + bool starts_pmp, devices_ordered; + const char *name, *fallback, *pwrstate; +}; +struct apple_pmp_report { + void *base; + const struct apple_pmp_report_offsets *offsets; + struct device *dev; + void *start; + struct apple_pmp_report *next; +}; +struct completion { + bool done; +}; +struct apple_pmp_start { + struct apple_pmp_report *rep; + struct apple_pmp_start *next; + bool stopping; + unsigned int waiters; + bool queued; + int result, work; + u64 ack; + struct completion started, waiters_done; +}; +struct host_list { + void *first; +}; +static struct host_list apple_pmp_starts, apple_pmp_reports; +static int apple_pmp_start_mutex; +#define list_for_each_entry(it, head, member) \ + for ((it) = (head)->first; (it); (it) = (it)->next) +#define list_for_each_entry_rcu(it, head, member) \ + list_for_each_entry(it, head, member) +static int rcu_depth; +static void rcu_read_lock(void) +{ + assert(!rcu_depth++); +} +static void rcu_read_unlock(void) +{ + assert(rcu_depth-- == 1); +} +static int apple_pmp_report_bit(struct apple_pmp_start *st, + struct device_node *entry, u32 *id, + bool *record) +{ + (void)st; + *id = entry->reg; + *record = false; + return 0; +} +static struct apple_pmp_start managed_start; +static int ready_result, start_calls; +#define msecs_to_jiffies(n) (n) +#define system_unbound_wq NULL +static bool wait_for_completion_timeout(struct completion *c, + unsigned long timeout) +{ + assert(timeout > 0); + return c->done; +} +static void complete(struct completion *c) +{ + c->done = true; +} +static void queue_work(void *wq, int *work); +static const struct gate_soc gate_t6030 = { 6030 }, gate_t8122 = { 8122 }, + other_soc = { 0 }; +static const struct gate_soc *gate_soc; +static struct device_node supplier_node, other_node, gpu, extra_gpu; +static struct device_node report14, extra_report, report25, entry7, entry17; +static struct device_node *nodes[9]; +static struct device supplier, consumer; +static struct platform_device native_report, passive_report; +static struct device_link link; +static struct apple_pmp_report_offsets native_offsets; +static u64 status_word; +static struct apple_pmp_report native_rep, passive_rep; +static u64 passive_words[0x14000 / sizeof(u64)]; +static bool passive_present, passive_lockable, passive_data; + +static const char *machine; +static const void *pmp_data; +static struct device *pmp_device; +static int pmp_lock, ready_calls, add_calls, add_result; +static int poll_calls, device_locked, power_calls, power_result, link_calls; +static int find_calls, lockable, present_device, link_ok; +static int case_count; + +static struct device_node *of_node_get(struct device_node *node) +{ + if (node) + node->refs++; + return node; +} +static void of_node_put(struct device_node *node) +{ + if (node) { + assert(node->refs > 0); + node->refs--; + } +} +static bool of_device_is_compatible(const struct device_node *node, + const char *name) +{ + return node && node->compatible && !strcmp(node->compatible, name); +} +static bool of_device_is_available(const struct device_node *node) +{ + return node && node->available; +} +static bool of_machine_is_compatible(const char *name) +{ + return !strcmp(machine, name); +} +static struct device_node *of_find_compatible_node(struct device_node *from, + const char *type, + const char *compat) +{ + size_t i = 0; + + assert(!type); + find_calls++; + if (from) { + for (; i < sizeof(nodes) / sizeof(nodes[0]); i++) + if (nodes[i] == from) { + i++; + break; + } + of_node_put(from); + } + for (; i < sizeof(nodes) / sizeof(nodes[0]); i++) + if (of_device_is_compatible(nodes[i], compat)) + return of_node_get(nodes[i]); + return NULL; +} +#define for_each_compatible_node(np, type, compat) \ + for ((np) = of_find_compatible_node(NULL, type, compat); (np); \ + (np) = of_find_compatible_node(np, type, compat)) +static struct device_node *of_get_next_child(struct device_node *parent, + struct device_node *old) +{ + struct device_node *next = old ? old->sibling : parent->child; + + of_node_put(old); + return of_node_get(next); +} +#define for_each_child_of_node(parent, child) \ + for ((child) = of_get_next_child(parent, NULL); (child); \ + (child) = of_get_next_child(parent, child)) +static bool of_property_present(const struct device_node *node, + const char *prop) +{ + if (!strcmp(prop, "apple,pmp")) + return node->has_pmp; + assert(!strcmp(prop, "apple,pmp-report")); + return node->has_report; +} +static bool of_property_read_bool(const struct device_node *node, + const char *prop) +{ + assert(!strcmp(prop, "apple,no-ack")); + return node->no_ack; +} +static int of_property_count_u32_elems(const struct device_node *node, + const char *prop) +{ + if (!strcmp(prop, "reg")) + return node->has_reg ? node->reg_cells : -EINVAL; + assert(!strcmp(prop, "apple,pmp")); + return node->has_pmp ? node->pmp_cells : -EINVAL; +} +static int of_property_read_u32(const struct device_node *node, + const char *prop, u32 *value) +{ + assert(!strcmp(prop, "reg")); + if (!node->has_reg || node->reg_cells < 1) + return -EINVAL; + *value = node->reg; + return 0; +} +static struct device_node *of_parse_phandle(struct device_node *node, + const char *prop, int index) +{ + assert(!index); + if (!strcmp(prop, "apple,pmp")) + return node->has_pmp && node->pmp_cells >= 1 ? + of_node_get(node->pmp) : + NULL; + assert(!strcmp(prop, "apple,pmp-report")); + return node->has_report && node->report_cells >= 1 ? + of_node_get(node->report) : + NULL; +} +static int of_changeset_add_prop_u32(struct of_changeset *cs, + struct device_node *node, const char *prop, + u32 value) +{ + assert(cs && node == &gpu && !strcmp(prop, "apple,pmp")); + assert(value == supplier_node.phandle); + add_calls++; + return add_result; +} +static struct device_node *of_get_parent(struct device_node *node) +{ + return of_node_get(node->parent); +} +static struct platform_device *of_find_device_by_node(struct device_node *node) +{ + struct platform_device *pdev; + + assert(node == &report25 || node == &report14); + if (node == &report25) { + if (!present_device) + return NULL; + pdev = &native_report; + } else { + if (!passive_present) + return NULL; + pdev = &passive_report; + } + pdev->dev.refs++; + return pdev; +} +static bool device_trylock(struct device *dev) +{ + assert(!device_locked); + assert(dev == &native_report.dev || dev == &passive_report.dev); + if (dev == &native_report.dev ? !lockable : !passive_lockable) + return false; + device_locked = dev == &native_report.dev ? 1 : 2; + return true; +} +static void device_unlock(struct device *dev) +{ + assert(device_locked == (dev == &native_report.dev ? 1 : 2)); + device_locked = 0; +} +static bool device_is_bound(const struct device *dev) +{ + return dev->bound; +} +static struct apple_pmp_report * +platform_get_drvdata(struct platform_device *pdev) +{ + assert(device_locked == (pdev == &native_report ? 1 : 2)); + return pdev == &native_report ? &native_rep : + passive_data ? &passive_rep : + NULL; +} +static u64 readq(const void *address) +{ + if (!device_locked) { + assert(apple_pmp_start_mutex && managed_start.waiters); + } else { + assert(device_locked == 1 && native_report.dev.refs > 0); + } + return *(const u64 *)address; +} +#define readq_poll_timeout(address, value, condition, delay, timeout) \ + ({ \ + int poll_ret = -ETIMEDOUT; \ + assert(device_locked && (delay) == 1000 && \ + (timeout) == 5000000); \ + if (device_locked == 2) \ + ready_calls++; \ + else \ + poll_calls++; \ + for (int poll_i = 0; poll_i < 4; poll_i++) { \ + (value) = readq(address); \ + if (condition) { \ + poll_ret = 0; \ + break; \ + } \ + } \ + poll_ret; \ + }) +static struct device *get_device(struct device *dev) +{ + dev->refs++; + return dev; +} +static void put_device(struct device *dev) +{ + assert(dev->refs > 0); + dev->refs--; +} +static struct device_link * +device_link_add(struct device *con, struct device *sup, unsigned int flags) +{ + assert(con == &consumer && sup == &supplier && sup->refs > 0); + assert(flags == DL_FLAG_AUTOREMOVE_CONSUMER && !pmp_lock); + link_calls++; + return link_ok ? &link : NULL; +} +struct lock_guard { + int *lock; + bool once; +}; +static struct lock_guard lock_take(int *lock) +{ + assert(!*lock); + *lock = 1; + return (struct lock_guard){ lock, true }; +} +static void lock_drop(struct lock_guard *guard) +{ + assert(*guard->lock == 1); + *guard->lock = 0; +} +#define guard(kind) \ + struct lock_guard local_guard \ + __attribute__((cleanup(lock_drop))) = lock_take +#define scoped_guard(kind, lock) \ + for (struct lock_guard scoped \ + __attribute__((cleanup(lock_drop))) = lock_take(lock); \ + scoped.once; scoped.once = false) +static int apple_pmp_send_power_command(const void *data, u8 command, u16 id, + u32 enabled) +{ + assert(data == pmp_data && command == 0x0f && id == 5 && enabled == 1); + assert(pmp_lock == 1 && (start_calls == 1 || poll_calls == 1)); + power_calls++; + return power_result; +} +static void queue_work(void *wq, int *work) +{ + assert(!wq && work == &managed_start.work && apple_pmp_start_mutex); + /* Firmware boundary: no READY is fabricated by the readiness reader. */ + start_calls++; + managed_start.result = ready_result; + managed_start.started.done = true; +} +#include "t6030-pmp-production.h" + +static void balanced(void) +{ + for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) + assert(!nodes[i]->refs); + assert(!managed_start.waiters); + assert(!supplier.refs && !consumer.refs && !native_report.dev.refs && + !passive_report.dev.refs); + assert(!apple_pmp_start_mutex && !rcu_depth); + assert(!pmp_lock && !device_locked); +} +static void reset(void) +{ + balanced(); + supplier_node = + (struct device_node){ .compatible = "apple,t6030-pmp-v2", + .phandle = 10 }; + other_node = (struct device_node){ .compatible = "apple,t6030-pmp-v2", + .phandle = 11 }; + gpu = (struct device_node){ .compatible = "apple,agx-t6030", + .available = true }; + extra_gpu = (struct device_node){ 0 }; + report14 = + (struct device_node){ .compatible = "apple,t6030-pmp-v2-report", + .available = true, + .has_pmp = true, + .pmp_cells = 1, + .pmp = &supplier_node, + .child = &entry17 }; + extra_report = (struct device_node){ 0 }; + report25 = (struct device_node){ + .compatible = "apple,j613-25g83-pmp-report", .available = true + }; + entry7 = + (struct device_node){ .compatible = + "apple,t6000-pmp-v2-report-entry", + .available = true, + .has_reg = true, + .reg_cells = 1, + .reg = 7, + .parent = &report14 }; + entry17 = + (struct device_node){ .compatible = + "apple,t6000-pmp-v2-report-entry", + .available = true, + .has_reg = true, + .reg_cells = 1, + .reg = 17, + .sibling = &entry7, + .parent = &report14 }; + supplier = (struct device){ .of_node = &supplier_node, .bound = true }; + consumer = (struct device){ .of_node = &gpu }; + native_report = (struct platform_device){ .dev = { .of_node = &report25, + .bound = true } }; + native_rep = (struct apple_pmp_report){ .base = &status_word, + .offsets = &native_offsets, + .dev = &native_report.dev }; + passive_report = (struct platform_device){ + .dev = { .of_node = &report14, .bound = true } + }; + passive_rep = + (struct apple_pmp_report){ .base = passive_words, + .offsets = &apple_pmp_offsets_t6030, + .dev = &passive_report.dev }; + memset(passive_words, 0, sizeof(passive_words)); + passive_words[apple_pmp_offsets_t6030.status / 8] = PMP_REPORT_READY; + passive_words[apple_pmp_offsets_t6030.actual / 8] = BIT_ULL(7); + passive_present = passive_lockable = passive_data = true; + managed_start = (struct apple_pmp_start){ .rep = &passive_rep, + .ack = BIT_ULL(7) }; + passive_rep.start = &managed_start; + apple_pmp_starts.first = + CONFIG_APPLE_T6030_GPU_PMP_LINK ? &managed_start : NULL; + ready_result = start_calls = 0; + apple_pmp_reports.first = &passive_rep; + machine = "apple,t6030"; + gate_soc = &gate_t6030; + pmp_device = &supplier; + pmp_data = (void *)1; + ready_calls = add_calls = add_result = poll_calls = 0; + power_calls = power_result = link_calls = find_calls = 0; + lockable = present_device = link_ok = 1; + status_word = PMP_REPORT_READY; +} +static int power(void) +{ + return apple_pmp_set_device_power(0x0f, 5, 1); +} +#define RUN(change, result, ready, sent) \ + do { \ + reset(); \ + change; \ + assert(power() == (result)); \ + assert(ready_calls == (ready) && power_calls == (sent)); \ + balanced(); \ + case_count++; \ + } while (0) +int main(void) +{ + struct device_node *initial[] = { + &supplier_node, &other_node, &gpu, &extra_gpu, &report14, + &extra_report, &report25, &entry7, &entry17 + }; + memcpy(nodes, initial, sizeof(nodes)); + /* Keep host boundary functions available when a mutant removes a call. */ + (void)apple_pmp_report_wait_ready; + (void)complete; + (void)of_machine_is_compatible; + (void)of_property_read_bool; + (void)of_property_present; + (void)gate_find_one; + (void)apple_pmp_report_t6030_entry; + (void)apple_pmp_report_wait_t6030_14; + reset(); + assert(apple_pmp_offsets_t6030.starts_pmp == + !!CONFIG_APPLE_T6030_GPU_PMP_LINK); +#if CONFIG_APPLE_T6030_GPU_PMP_LINK + assert(apple_pmp_report_wait_ready(&entry7, 10000) == 0 && + start_calls == 1); + balanced(); + case_count++; + RUN((void)0, 0, 0, 1); + RUN(report14.child = &entry7; entry7.sibling = &entry17; + entry17.sibling = NULL, 0, 0, 1); + RUN(apple_pmp_starts.first = NULL, -ENODEV, 0, 0); + RUN(apple_pmp_starts.first = NULL; + apple_pmp_reports.first = NULL, -EPROBE_DEFER, 0, 0); + RUN(managed_start.stopping = true, -ENODEV, 0, 0); + RUN(ready_result = -ETIMEDOUT, -ETIMEDOUT, 0, 0); + RUN(ready_result = -EINVAL, -EINVAL, 0, 0); + RUN(passive_words[apple_pmp_offsets_t6030.status / 8] = 0, -EIO, 0, 0); + RUN(passive_words[apple_pmp_offsets_t6030.status / 8] = 3, -EIO, 0, 0); + RUN(passive_words[apple_pmp_offsets_t6030.actual / 8] = 0, -EIO, 0, 0); + RUN(passive_words[apple_pmp_offsets_t6030.actual / 8] = BIT_ULL(8), + -EIO, 0, 0); + RUN(passive_words[apple_pmp_offsets_t6030.actual / 8] |= BIT_ULL(9), 0, + 0, 1); + RUN(power_result = -EIO, -EIO, 0, 1); + RUN(machine = "apple,t8122", -ENODEV, 0, 0); + RUN(machine = "apple,t6031", -ENODEV, 0, 0); + RUN(machine = "apple,t8140", -ENODEV, 0, 0); + RUN(supplier_node.compatible = "apple,t8122-pmp-v2", -ENODEV, 0, 0); + RUN(report14.compatible = NULL, -ENODEV, 0, 0); + RUN(extra_report = report14, -ENODEV, 0, 0); + RUN(extra_report = report14; + extra_report.available = false, -ENODEV, 0, 0); + RUN(report14.available = false, -ENODEV, 0, 0); + RUN(report14.has_pmp = false, -ENODEV, 0, 0); + RUN(report14.pmp_cells = 0, -ENODEV, 0, 0); + RUN(report14.pmp_cells = 2, -ENODEV, 0, 0); + RUN(report14.pmp = NULL, -ENODEV, 0, 0); + RUN(report14.pmp = &other_node, -ENODEV, 0, 0); + RUN(entry17.sibling = NULL, -ENODEV, 0, 0); + RUN(entry7.has_reg = false, -ENODEV, 0, 0); + RUN(entry7.reg_cells = 0, -ENODEV, 0, 0); + RUN(entry7.reg_cells = 2, -ENODEV, 0, 0); + RUN(entry7.reg = 8, -ENODEV, 0, 0); + RUN(entry7.available = false, -ENODEV, 0, 0); + RUN(entry7.compatible = "apple,other", -ENODEV, 0, 0); + RUN(entry7.no_ack = true, -ENODEV, 0, 0); + RUN(entry17.reg = 7, -ENODEV, 0, 0); + RUN(entry17.reg = 7; entry17.available = false, -ENODEV, 0, 0); +#else + RUN((void)0, -ENODEV, 0, 0); + assert(!find_calls); +#endif + /* T8122 current14 remains unsupported by this supplier API. */ + RUN(machine = "apple,t8122"; + supplier_node.compatible = "apple,t8122-pmp-v2", -ENODEV, 0, 0); + /* A declared report, including malformed/unresolved data, never falls back. */ + RUN(supplier_node.has_report = true; + supplier_node.report_cells = 0, -ENODEV, 0, 0); + RUN(supplier_node.has_report = true; + supplier_node.report_cells = 1, -ENODEV, 0, 0); + RUN(supplier_node.has_report = true; supplier_node.report_cells = 1; + supplier_node.report = &report14, -ENODEV, 0, 0); + /* Native25 dispatch is untouched, even on another machine/config. */ + RUN(supplier_node.has_report = true; supplier_node.report_cells = 1; + supplier_node.report = &report25; machine = "apple,t8122", 0, 0, 1); + assert(poll_calls == 1); + /* Preserve the existing native25 first-phandle behaviour as well. */ + RUN(supplier_node.has_report = true; supplier_node.report_cells = 2; + supplier_node.report = &report25, 0, 0, 1); + RUN(supplier_node.has_report = true; supplier_node.report_cells = 1; + supplier_node.report = &report25; + present_device = 0, -EPROBE_DEFER, 0, 0); + RUN(supplier_node.has_report = true; supplier_node.report_cells = 1; + supplier_node.report = &report25; + lockable = 0, -EPROBE_DEFER, 0, 0); + RUN(supplier_node.has_report = true; supplier_node.report_cells = 1; + supplier_node.report = &report25; + native_report.dev.bound = false, -EPROBE_DEFER, 0, 0); + RUN(supplier_node.has_report = true; supplier_node.report_cells = 1; + supplier_node.report = &report25; + status_word = 0, -ETIMEDOUT, 0, 0); + RUN(pmp_data = NULL, -ENODEV, 0, 0); + /* GPU phandle changes are scoped, preserve existing good links, and fail closed. */ + for (int soc = 0; soc < 3; soc++) { + struct of_changeset cs; + reset(); + gate_soc = soc == 0 ? &gate_t8122 : + soc == 1 ? &gate_t6030 : + &other_soc; + gpu.compatible = soc == 0 ? "apple,agx-t8122" : + "apple,agx-t6030"; + int active = soc == 0 || + (soc == 1 && CONFIG_APPLE_T6030_GPU_PMP_LINK); + assert(gate_gpu_pmp_link(&cs, &supplier_node) == 0 && + add_calls == active); + balanced(); + case_count++; + add_calls = 0; + gpu.has_pmp = true; + gpu.pmp_cells = 1; + gpu.pmp = &supplier_node; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == 0 && + !add_calls); + balanced(); + case_count++; + gpu.pmp = &other_node; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == + (active ? -EINVAL : 0)); + balanced(); + case_count++; + } + reset(); + struct of_changeset cs; +#if CONFIG_APPLE_T6030_GPU_PMP_LINK + gpu.has_pmp = true; + gpu.pmp_cells = 2; + gpu.pmp = &supplier_node; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == -EINVAL && !add_calls); + balanced(); + case_count++; + reset(); + gpu.has_pmp = true; + gpu.pmp_cells = 1; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == -EINVAL && !add_calls); + balanced(); + case_count++; + reset(); + extra_gpu = gpu; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == 0 && !add_calls); + balanced(); + case_count++; + reset(); + supplier_node.phandle = 0; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == -EINVAL && !add_calls); + balanced(); + case_count++; + reset(); + assert(gate_gpu_pmp_link(&cs, NULL) == -EINVAL && !add_calls); + balanced(); + case_count++; + reset(); + add_result = -ENOMEM; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == -ENOMEM && + add_calls == 1); + balanced(); + case_count++; + reset(); + gpu.available = false; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == 0 && !add_calls); + balanced(); + case_count++; + reset(); + gpu.compatible = NULL; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == 0 && !add_calls); + balanced(); + case_count++; +#else + assert(gate_gpu_pmp_link(&cs, &supplier_node) == 0 && !find_calls); + balanced(); + case_count++; +#endif + /* Existing T8122 linking keeps its original first-phandle semantics. */ + reset(); + gate_soc = &gate_t8122; + gpu.compatible = "apple,agx-t8122"; + gpu.has_pmp = true; + gpu.pmp_cells = 2; + gpu.pmp = &supplier_node; + assert(gate_gpu_pmp_link(&cs, &supplier_node) == 0 && !add_calls); + balanced(); + case_count++; + /* Actual existing consumer link uses the exact supplier and balances refs. */ + reset(); + gpu.has_pmp = true; + gpu.pmp_cells = 1; + gpu.pmp = &supplier_node; + assert(apple_pmp_link_device(&consumer) == 0 && link_calls == 1); + balanced(); + case_count++; + gpu.pmp = &other_node; + assert(apple_pmp_link_device(&consumer) == -EPROBE_DEFER && + link_calls == 1); + balanced(); + case_count++; + gpu.pmp = &supplier_node; + supplier.bound = false; + assert(apple_pmp_link_device(&consumer) == -EPROBE_DEFER); + balanced(); + case_count++; + supplier.bound = true; + link_ok = 0; + assert(apple_pmp_link_device(&consumer) == -EINVAL); + balanced(); + case_count++; + printf("PASS: %d T6030 PMP cases, config=%d; reference, lock and command boundaries checked\n", + case_count, CONFIG_APPLE_T6030_GPU_PMP_LINK); + return 0; +} diff --git a/tools/asahi/t6030-pmp-supplier-tests.py b/tools/asahi/t6030-pmp-supplier-tests.py new file mode 100755 index 0000000000000..3379b2911f480 --- /dev/null +++ b/tools/asahi/t6030-pmp-supplier-tests.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Host controls for default-off T6030 GPU/PMP selection and error propagation. + +Actual selection, dispatch, device-link and serialized power functions are +extracted. OF enumeration/property storage, device binding, MMIO polling, +and locks are controlled host boundaries. The T6030 managed entry wait +and start wait are extracted too; the firmware/workqueue boundary is controlled. +This models ownership/error handling, not real teardown concurrency or DMA. +""" +import argparse +import hashlib +import json +import os +import re +from pathlib import Path +import resource +import shlex +import signal +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[2] +GATE = "drivers/soc/apple/t6030-display-gate.c" +REPORT = "drivers/pmdomain/apple/pmp-report.c" +EXPORT = "drivers/soc/apple/pmp-export.c" + + +def function(source, signature): + start = source.index(signature) + end = source.index("{", start) + 1 + depth = 1 + while depth: + depth += (source[end] == "{") - (source[end] == "}") + end += 1 + return source[start:end] + "\n" + + +def digest(text): + return hashlib.sha256(text.encode()).hexdigest() + + +def no_core(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + + +def run(out, name, code, compiler, configs=(0, 1)): + dest = out / name + dest.mkdir() + (dest / "t6030-pmp-production.h").write_text(code) + results = [] + for config in configs: + binary = dest / f"config-{config}" + command = compiler + ["-std=gnu11", "-g", "-O1", "-Wall", "-Wextra", "-Werror", + "-fsanitize=address,undefined", "-fno-omit-frame-pointer", + f"-DCONFIG_APPLE_T6030_GPU_PMP_LINK={config}", + "-I", str(dest), + str(ROOT / "tools/asahi/t6030-pmp-supplier-tests.c"), + "-o", str(binary)] + built = subprocess.run(command, capture_output=True, text=True) + (dest / f"config-{config}-build.log").write_text(built.stdout + built.stderr) + ran = None + if built.returncode == 0: + ran = subprocess.run([str(binary)], capture_output=True, text=True, + preexec_fn=no_core) + (dest / f"config-{config}-run.log").write_text(ran.stdout + ran.stderr) + results.append({"config": config, "command": command, + "build_rc": built.returncode, "run_rc": ran.returncode if ran else None, + "sanitizer_diagnostic": bool(ran and ( + "AddressSanitizer" in ran.stderr or "runtime error:" in ran.stderr)), + "stdout": ran.stdout if ran else ""}) + return {"name": name, "results": results} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path, + help="new directory for generated sources, logs and receipt") + parser.add_argument("--mutations", action="store_true") + parser.add_argument("--baseline-ref", help="also verify unchanged entry-ready/start consumers") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=False) + gate, report, export = [(ROOT / path).read_text() for path in (GATE, REPORT, EXPORT)] + kconfig = (ROOT / "drivers/soc/apple/Kconfig").read_text() + section = kconfig.split("config APPLE_T6030_GPU_PMP_LINK\n", 1)[1].split("\nconfig ", 1)[0] + assert '\n\tbool\n' in "\n" + section and "\n\tdefault n\n" in section + assert "depends on APPLE_T6030_DISPLAY_GATE" in section + parts = [function(gate, "static struct device_node *__init gate_find_one("), + function(gate, "static int __init gate_gpu_pmp_link("), + function(report, "static struct device_node *apple_pmp_report_t6030_entry("), + function(report, "static int apple_pmp_report_wait_t6030_14("), + function(report, "int apple_pmp_report_wait_supplier_ready("), + function(export, "int apple_pmp_link_device("), + function(export, "int apple_pmp_set_device_power(")] + offsets = re.search(r'static const struct apple_pmp_report_offsets apple_pmp_offsets_t6030 = \{.*?\n\};', report, re.S) + assert offsets + code = offsets.group() + "\n" + function(report, "static int apple_pmp_start_wait(") + "\n" + function(report, "int apple_pmp_report_wait_ready(") + "\n" + "\n".join(parts) + # Existing managed-start consumers must remain byte-identical. + consumers = ["int apple_pmp_report_wait_ready(", "static int apple_pmp_start_wait("] + consumer_hashes = {name: digest(function(report, name)) for name in consumers} + baseline_checked = False + baseline_code = None + if args.baseline_ref: + baseline = subprocess.check_output(["git", "-C", str(ROOT), "show", + f"{args.baseline_ref}:{REPORT}"], text=True) + for name in consumers: + assert function(report, name) == function(baseline, name), name + baseline_gate = subprocess.check_output(["git", "-C", str(ROOT), "show", + f"{args.baseline_ref}:{GATE}"], text=True) + old_gate = function(baseline_gate, "static int __init gate_t8122_gpu_pmp_link(") + old_gate = old_gate.replace("gate_t8122_gpu_pmp_link", "gate_gpu_pmp_link") + old_dispatch = function(baseline, "int apple_pmp_report_wait_supplier_ready(") + baseline_code = code.replace(parts[1], old_gate).replace(function(report, "int apple_pmp_report_wait_supplier_ready("), old_dispatch) + baseline_checked = True + compiler = shlex.split(os.environ.get("CC", "cc")) + version = subprocess.check_output(compiler + ["--version"], text=True).splitlines()[0] + variants = [("control-before", code)] + if args.mutations: + mutations = [ + ("config-scope", "if (!IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) ||", "if (false ||"), + ("machine-scope", '!of_machine_is_compatible("apple,t6030") ||', "false ||"), + ("supplier-compat", '!of_device_is_compatible(supplier->of_node, "apple,t6030-pmp-v2"))', "false)"), + ("duplicate-report", "\t\tif (report) {", "\t\tif (false) {"), + ("disabled-report", "!of_device_is_available(report) ||", "false ||"), + ("report-phandle-shape", 'of_property_count_u32_elems(report, "apple,pmp") != 1', "false"), + ("supplier-identity", "pmp == supplier->of_node && entry ?", "entry ?"), + ("entry-shape", 'of_property_count_u32_elems(child, "reg") != 1', "false"), + ("duplicate-entry", "if (entry ||", "if (false ||"), + ("disabled-entry", "!of_device_is_available(child) ||", "false ||"), + ("entry-compat", '!of_device_is_compatible(child, "apple,t6000-pmp-v2-report-entry") ||', "false ||"), + ("entry-no-ack", 'of_property_read_bool(child, "apple,no-ack"))', "false)"), + ("declared-report-fallback", '!np && !of_property_present(supplier->of_node, "apple,pmp-report")', "!np"), + ("readiness-error", "apple_pmp_report_wait_ready(entry, msecs_to_jiffies(timeout_ms)) : -ENODEV;", + "(apple_pmp_report_wait_ready(entry, msecs_to_jiffies(timeout_ms)), 0) : -ENODEV;"), + ("managed-owner", '.starts_pmp = IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK)', '.starts_pmp = false'), + ("managed-ack", 'status == PMP_REPORT_READY && (ack & BIT_ULL(id))', '((void)ack, status == PMP_REPORT_READY)'), + ("managed-ready", 'status == PMP_REPORT_READY && (ack & BIT_ULL(id))', '((void)status, (ack & BIT_ULL(id)))'), + ("entry-reference", "\tof_node_put(entry);\n\tof_node_put(pmp);", "\tof_node_put(pmp);"), + ("gate-config", "IS_ENABLED(CONFIG_APPLE_T6030_GPU_PMP_LINK) && gate_soc == &gate_t6030", "gate_soc == &gate_t6030"), + ("gate-phandle-shape", 'of_property_count_u32_elems(gpu, "apple,pmp") != 1', "false"), + ("serialized-command", "\tguard(mutex)(&pmp_lock);", "\t(void)&pmp_lock;"), + ] + for name, old, new in mutations: + assert code.count(old) == 1, (name, code.count(old)) + variants.append((name, code.replace(old, new))) + variants.append(("control-after", code)) + results = [run(args.output, name, source, compiler) for name, source in variants] + if baseline_code: + results.append(run(args.output, "control-prior-config-off", baseline_code, compiler, (0,))) + valid = True + for item in results: + control = item["name"].startswith("control-") + built = all(x["build_rc"] == 0 for x in item["results"]) + good = all(x["run_rc"] == 0 and not x["sanitizer_diagnostic"] for x in item["results"]) + killed = any(x["run_rc"] == -signal.SIGABRT or x["sanitizer_diagnostic"] for x in item["results"]) + item["verdict"] = ("PASS" if good else "INVALID") if control else ( + "NOBUILD" if not built else "KILLED" if killed else "SURVIVED") + valid &= item["verdict"] == ("PASS" if control else "KILLED") + receipt = {"passed": valid, "compiler": version, "production_sha256": digest(code), + "fixture_sha256": hashlib.sha256((ROOT / "tools/asahi/t6030-pmp-supplier-tests.c").read_bytes()).hexdigest(), + "entry_ready_consumers_unchanged_checked": baseline_checked, + "consumer_hashes": consumer_hashes, "results": results} + (args.output / "receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps({"passed": valid, "variants": [{"name": x["name"], "verdict": x["verdict"]} + for x in results]})) + return 0 if valid else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/asahi/t6030-wiring-tests.py b/tools/asahi/t6030-wiring-tests.py new file mode 100644 index 0000000000000..3a158d20699c5 --- /dev/null +++ b/tools/asahi/t6030-wiring-tests.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the shared DP wiring admission and DCP mux lookup on modeled DT nodes.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import resource +import subprocess +ROOT = Path(__file__).resolve().parents[2] +FUNCTIONS = { + 'include/linux/soc/apple/dp-tunnel.h': [ + 'apple_dp_tunnel_t6030_pclk', 'apple_dp_tunnel_wiring_dual', + 'apple_dp_tunnel_mux_matches', 'apple_dp_tunnel_t6030_legacy_mux', + 'apple_dp_tunnel_route_wired', 'apple_dp_tunnel_dual_stream'], + 'drivers/gpu/drm/apple/dcp-fabric-core.c': ['dcp_fabric_wiring'], + 'drivers/gpu/drm/apple/dcp-fabric.c': ['dcp_typec_route_endpoints', 'dcp_typec_route_wiring'], +} +CASES = ['tb-source0', 'tb-source2', 'dcp-source0', 'dcp-source2', + 'other-soc', 'odd-source', 'unknown-source', 'missing-source', + 'single-endpoint', 'wrong-provider', 'missing-primary', 'parse-failure', + 'primary-cell1', 'primary-two-args', 'named', 'named-other-soc', + 'named-wrong-cell', 'named-other-provider', 'partial-named', + 'usb4', 'usb4-other-soc', 'bad-usb4'] + +def function(source, name): + match = re.search(r'^[^\n;{}]*\b' + name + r'\([^;]*?\)\n\{', source, re.M | re.S) + if not match: + raise ValueError(name) + pos = source.index('{', match.start()) + 1 + depth = 1 + while depth: + depth += (source[pos] == '{') - (source[pos] == '}') + pos += 1 + return source[match.start():pos] + '\n' + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--old-ref') + parser.add_argument('--old-classifier', choices=['tb', 'dcp']) + args = parser.parse_args() + bodies = {} + for path, names in FUNCTIONS.items(): + source = (ROOT / path).read_text() + for name in names: + body = function(source, name) + old_name = {'tb': 'apple_dp_tunnel_route_wired', 'dcp': 'dcp_typec_route_wiring'}.get(args.old_classifier) + if args.old_ref and name == old_name: + body = function(subprocess.check_output(['git', 'show', args.old_ref + ':' + path], cwd=ROOT, text=True), name) + bodies[name] = body + enum = re.search(r'enum dcp_fabric_wiring \{.*?\n\};', + (ROOT / 'drivers/gpu/drm/apple/dcp-fabric-core.h').read_text(), re.S).group() + fixture = (ROOT / 'tools/asahi/t6030-wiring/fixture.c').read_text() + code = fixture.replace('/* PRODUCTION_FUNCTIONS */', enum + '\n' + '\n'.join(bodies.values())) + args.out.mkdir(parents=True, exist_ok=True) + source, binary = args.out / 'production.c', args.out / 'production' + source.write_text(code) + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Wextra', '-Werror', '-fsanitize=address,undefined', '-fno-omit-frame-pointer', '-Wno-unused-function', str(source), '-o', str(binary)], check=True) + def no_core(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + results = [] + for case in CASES: + run = subprocess.run([str(binary), case], capture_output=True, text=True, preexec_fn=no_core) + results.append({'case': case, 'exit': run.returncode, 'stderr': run.stderr}) + receipt = {'production_bodies': {n: hashlib.sha256(b.encode()).hexdigest() for n, b in bodies.items()}, + 'old_ref': args.old_ref, 'old_classifier': args.old_classifier, + 'hardware_access': False, 'results': results} + (args.out / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps(results, indent=2)) + if any(r['exit'] for r in results): + raise SystemExit(1) +if __name__ == '__main__': + main() diff --git a/tools/asahi/t6030-wiring/fixture.c b/tools/asahi/t6030-wiring/fixture.c new file mode 100644 index 0000000000000..01629ab2c2eb3 --- /dev/null +++ b/tools/asahi/t6030-wiring/fixture.c @@ -0,0 +1,299 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR MIT */ +#include +#include +#include +#include +#include +#include +typedef uint32_t u32; +#define BIT(n) (1u << (n)) +struct mux_chip; +struct mux_control { + struct mux_chip *chip; + unsigned int index; +}; +struct mux_chip { + unsigned int controllers; + struct mux_control mux[3]; +}; +struct device_node { + const char *name, *compatible; + struct device_node *parent, *child, *next, *remote; + unsigned int refs, reg, source, names_count; + bool missing_source, bad_parse; + const char *names[4]; + struct device_node *providers[4]; + unsigned int cells[4], nargs[4]; + struct mux_chip *chip; +}; +struct of_phandle_args { + struct device_node *np; + unsigned int args_count, args[2]; +}; +static bool t6030 = true; +static struct device_node *get(struct device_node *n) +{ + if (n) + n->refs++; + return n; +} +static void of_node_put(struct device_node *n) +{ + if (n) { + assert(n->refs); + n->refs--; + } +} +static void device_node_free(struct device_node **n) +{ + of_node_put(*n); +} +#define __free(name) __attribute__((cleanup(name##_free))) +static struct device_node *of_get_parent(struct device_node *n) +{ + return get(n ? n->parent : NULL); +} +static bool of_node_name_eq(struct device_node *n, const char *s) +{ + return n && !strcmp(n->name, s); +} +static bool of_machine_is_compatible(const char *s) +{ + return t6030 && !strcmp(s, "apple,t6030"); +} +static bool of_device_is_compatible(struct device_node *n, const char *s) +{ + return n && n->compatible && !strcmp(n->compatible, s); +} +static int of_property_match_string(struct device_node *n, const char *prop, + const char *name) +{ + assert(!strcmp(prop, "mux-control-names")); + for (unsigned int i = 0; n && i < n->names_count; i++) + if (!strcmp(n->names[i], name)) + return i; + return -ENODATA; +} +static int of_property_read_u32_index(struct device_node *n, const char *prop, + unsigned int i, u32 *value) +{ + assert(!strcmp(prop, "apple,typec-mux-indices")); + if (!n || n->missing_source || i >= 3) + return -EINVAL; + *value = n->source; + return 0; +} +static int of_property_read_u32(struct device_node *n, const char *prop, + u32 *value) +{ + assert(!strcmp(prop, "reg")); + if (!n) + return -EINVAL; + *value = n->reg; + return 0; +} +static int of_parse_phandle_with_args(struct device_node *n, const char *prop, + const char *cells, int index, + struct of_phandle_args *args) +{ + assert(!strcmp(prop, "mux-controls") && + !strcmp(cells, "#mux-control-cells")); + if (!n || n->bad_parse || index < 0 || + (unsigned)index >= n->names_count) + return -EINVAL; + args->np = get(n->providers[index]); + args->args_count = n->nargs[index]; + args->args[0] = n->cells[index]; + return 0; +} +static struct device_node *of_graph_get_port_by_id(struct device_node *n, + unsigned int id) +{ + for (struct device_node *p = n ? n->child : NULL; p; p = p->next) + if (p->reg == id) + return get(p); + return NULL; +} +static struct device_node *next_child(struct device_node *parent, + struct device_node *prev) +{ + struct device_node *next = prev ? prev->next : + parent ? parent->child : + NULL; + of_node_put(prev); + return get(next); +} +#define for_each_child_of_node_scoped(parent, node) \ + for (struct device_node * node __free(device_node) = \ + next_child(parent, NULL); \ + node; node = next_child(parent, node)) +#define for_each_of_graph_port_endpoint(parent, node) \ + for_each_child_of_node_scoped(parent, node) +static struct device_node * +of_graph_get_remote_port_parent(struct device_node *n) +{ + return get(n ? n->remote : NULL); +} +struct device { + struct device_node *of_node; +}; +struct apple_dcp { + struct device *dev; +}; +struct apple_dcp_typec_route { + struct apple_dcp *dcp; + struct mux_control *xbar, *dpin[2]; + bool dual_stream; +}; +#define IS_ERR(p) ((uintptr_t)(p) >= (uintptr_t)-4095) +#define PTR_ERR(p) ((int)(intptr_t)(p)) +#define dev_warn_once(dev, ...) ((void)(dev)) +static struct mux_control *devm_mux_control_get(struct device *dev, + const char *name) +{ + int i = of_property_match_string(dev->of_node, "mux-control-names", + name); + if (i < 0 || dev->of_node->nargs[i] != 1 || + !dev->of_node->providers[i]->chip || + dev->of_node->cells[i] >= + dev->of_node->providers[i]->chip->controllers) + return (void *)(intptr_t)-EINVAL; + return &dev->of_node->providers[i]->chip->mux[dev->of_node->cells[i]]; +} +/* PRODUCTION_FUNCTIONS */ +int main(int argc, char **argv) +{ + assert(argc == 2); + struct mux_chip chip = { .controllers = 3 }, + other_chip = { .controllers = 3 }; + for (unsigned int i = 0; i < 3; i++) { + chip.mux[i].chip = &chip; + chip.mux[i].index = i; + other_chip.mux[i].chip = &other_chip; + other_chip.mux[i].index = i; + } + struct device_node provider = { + .compatible = "apple,t6020-display-crossbar", .chip = &chip + }; + struct device_node other = { .compatible = + "apple,t6020-display-crossbar", + .chip = &other_chip }; + struct device_node dcp = { .name = "dcpext", + .names_count = 1, + .names = { "typec0" }, + .providers = { &provider, &provider, + &provider }, + .cells = { 0, 1, 2 }, + .nargs = { 1, 1, 1 } }; + struct device_node routes = { .name = "typec-routes", .parent = &dcp }; + struct device_node route_np = { .parent = &routes }; + struct device_node endpoint[2] = { { .remote = &route_np }, + { .remote = &route_np } }; + endpoint[0].next = &endpoint[1]; + struct device_node port = { .reg = 3, .child = &endpoint[0] }; + struct device_node connector = { .child = &port }; + struct device dev = { .of_node = &dcp }; + struct apple_dcp engine = { .dev = &dev }; + struct apple_dcp_typec_route route = { .dcp = &engine, + .xbar = &chip.mux[0] }; + bool expected = true; + int expected_dcp = 0; + const char *test = argv[1]; + if (strstr(test, "source2")) + dcp.source = 2; + if (!strcmp(test, "other-soc")) { + t6030 = false; + expected = false; + } + if (!strcmp(test, "odd-source")) { + dcp.source = 1; + expected = false; + } + if (!strcmp(test, "unknown-source")) { + dcp.source = 4; + expected = false; + } + if (!strcmp(test, "missing-source")) { + dcp.missing_source = true; + expected = false; + } + if (!strcmp(test, "single-endpoint")) { + endpoint[0].next = NULL; + expected = false; + } + if (!strcmp(test, "wrong-provider")) { + provider.compatible = "apple,t8103-display-crossbar"; + expected = false; + } + if (!strcmp(test, "missing-primary")) { + dcp.names_count = 0; + expected = false; + } + if (!strcmp(test, "parse-failure")) { + dcp.bad_parse = true; + expected = false; + } + if (!strcmp(test, "primary-cell1")) { + dcp.cells[0] = 1; + expected = false; + } + if (!strcmp(test, "primary-two-args")) { + dcp.nargs[0] = 2; + expected = false; + } + if (!strcmp(test, "named") || !strcmp(test, "named-other-soc") || + !strcmp(test, "named-wrong-cell") || + !strcmp(test, "named-other-provider") || + !strcmp(test, "partial-named")) { + dcp.names_count = 3; + dcp.names[1] = "typec0-dpin0"; + dcp.names[2] = "typec0-dpin1"; + if (!strcmp(test, "named-other-soc")) + t6030 = false; + if (!strcmp(test, "named-wrong-cell")) { + dcp.cells[2] = 1; + expected = false; + expected_dcp = -EINVAL; + } + if (!strcmp(test, "named-other-provider")) { + dcp.providers[2] = &other; + expected = false; + expected_dcp = -EINVAL; + } + if (!strcmp(test, "partial-named")) { + dcp.names_count = 2; + expected = false; + expected_dcp = -EINVAL; + } + } + if (!strcmp(test, "usb4") || !strcmp(test, "usb4-other-soc") || + !strcmp(test, "bad-usb4")) { + dcp.names_count = 2; + dcp.names[1] = "typec0-usb4"; + if (!strcmp(test, "usb4-other-soc")) + t6030 = false; + if (!strcmp(test, "bad-usb4")) { + dcp.cells[1] = 0; + expected = false; + expected_dcp = -EINVAL; + } + } + if (strncmp(test, "dcp-", 4)) + assert(apple_dp_tunnel_dual_stream(&connector) == expected); + if (strncmp(test, "tb-", 3)) { + int ret = dcp_typec_route_wiring(&route, &connector, 0); + assert(ret == expected_dcp); + if (!ret) { + assert(route.dual_stream == expected); + if (expected) { + assert(route.dpin[0] == &chip.mux[1]); + assert(route.dpin[1] == &chip.mux[2]); + } + } + } + assert(!provider.refs && !other.refs && !dcp.refs && !routes.refs && + !route_np.refs && !port.refs && !endpoint[0].refs && + !endpoint[1].refs); + printf("PASS %s\n", test); + return 0; +} diff --git a/tools/asahi/tb-dp-repair-tests.py b/tools/asahi/tb-dp-repair-tests.py new file mode 100644 index 0000000000000..7e59e8739181f --- /dev/null +++ b/tools/asahi/tb-dp-repair-tests.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Exercise real DP retirement/repair bodies with modeled paths, PM and a queue. + +This checks ordering and reference balance, not kernel structure layout, hardware, +real runtime-PM locking or a complete Thunderbolt workqueue implementation. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess +import tempfile + +ROOT = Path(__file__).resolve().parents[2] + + +def function(text, name): + match = re.search(r'^(?:static\s+)?(?:inline\s+)?(?:int|void|bool)\s+' + + name + r'\([^;]*?\)\n\{', text, re.M | re.S) + if not match: + raise ValueError('Missing actual function: ' + name) + pos = text.index('{', match.start()) + 1 + depth = 1 + while depth: + depth += (text[pos] == '{') - (text[pos] == '}') + pos += 1 + return text[match.start():pos] + '\n' + + +HEADER = r''' +#include +#include +#include +#include +#include +#define CHECK(x) do { if (!(x)) { fprintf(stderr,"CHECK line %d: %s\n",__LINE__,#x); exit(77); } } while (0) +struct list_head {int unused;}; +struct device {int unused;}; +struct tb_switch {struct device dev;}; +struct tb_port {struct tb_switch *sw;bool is_dpin;struct list_head list;}; +struct work_struct {void(*fn)(struct work_struct*);bool pending;}; +struct tb_nhi {unsigned long quirks;}; +#define BIT(n) (1UL<<(n)) +#define QUIRK_NO_SYSTEM_SLEEP BIT(5) +struct tb {int lock;struct device dev;void*wq;struct tb_switch*root_switch;struct tb_nhi*nhi;}; +struct tb_cm {bool hotplug_active;struct work_struct dp_tunnel_work;bool dp_tunnel_pending;}; +struct domain {struct tb tb;struct tb_cm cm;}; +_Static_assert(offsetof(struct domain,cm)==sizeof(struct tb),"mock private storage adjacency"); +#define INIT_WORK(w,handler) ((w)->fn=(handler)) +struct kref {int count;}; +struct tb_path {bool reserved;}; +enum {TB_TUNNEL_DP,TB_TUNNEL_USB3}; +struct tb_tunnel {struct kref kref;struct tb*tb;struct tb_port*src_port,*dst_port;int npaths;void(*destroy)(struct tb_tunnel*);struct list_head list;int type;bool dprx_started,dprx_canceled;void(*callback)(struct tb_tunnel*);struct tb_path*paths[3];}; +static int tb_tunnel_lock,occupied,alloc_calls,alloc_failures,alloc_successes,freed; +static int domain_refs,pm_refs,pm_calls,pm_error,stop_during_pm,requeue_once,lock_depth; +static int restore_calls,scan_calls,unplugged_xdomains; +static struct tb_tunnel *current; +static struct domain domain; +#define container_of(p,t,m) ((t*)((char*)(p)-offsetof(t,m))) +#define READ_ONCE(x) (x) +#define fallthrough __attribute__((fallthrough)) +#define tb_port_dbg(...) ((void)0) +#define tb_tunnel_dbg(...) ((void)0) +#define tb_tunnel_warn(...) ((void)0) +static struct tb_cm*tb_priv(struct tb*t){return (struct tb_cm*)((char*)t+sizeof(*t));} +static struct tb*tcm_to_tb(struct tb_cm*c){return (struct tb*)((char*)c-sizeof(struct tb));} +static void mutex_lock(int*x){if(x==&domain.tb.lock){CHECK(!lock_depth);lock_depth++;}} +static void mutex_unlock(int*x){if(x==&domain.tb.lock){CHECK(lock_depth==1);lock_depth--;}} +#define lockdep_assert_held(x) CHECK((x)==&domain.tb.lock && lock_depth==1) +/* guard()/scoped_guard() as in : the lock is dropped when the scope ends. */ +static int*guard_mutex_lock(int*x){mutex_lock(x);return x;} +static void guard_mutex_unlock(int**x){if(*x)mutex_unlock(*x);} +#define GUARD_CAT2(a,b) a##b +#define GUARD_CAT(a,b) GUARD_CAT2(a,b) +#define guard(type) int*GUARD_CAT(guard_,__COUNTER__) __attribute__((unused,cleanup(guard_##type##_unlock)))=guard_##type##_lock +#define scoped_guard(type,x) \ + for(int*scope __attribute__((cleanup(guard_##type##_unlock)))=guard_##type##_lock(x),*scope_done=NULL; \ + !scope_done;scope_done=scope) +static int tb_domain_unregister_unplugged_xdomains(struct tb*t){return unplugged_xdomains;} +static void tb_scan_switch(struct tb_switch*s){CHECK(lock_depth==1 && s);scan_calls++;} +static void tb_restore_dp_resources(struct tb_switch*s){CHECK(lock_depth==1 && s);restore_calls++;} +static void kref_get(struct kref*k){CHECK(k->count>0);k->count++;} +static void kref_put(struct kref*k,void(*destroy)(struct kref*)){CHECK(k->count>0);if(!--k->count)destroy(k);} +static void tb_path_free(struct tb_path*p){if(p->reserved)occupied--;freed++;free(p);} +#define kfree(x) free(x) +static void list_del(struct list_head*l){current=NULL;} +static void list_del_init(struct list_head*l){} +static bool tb_port_is_dpin(struct tb_port*p){return p->is_dpin;} +static void tb_dp_tunnel_notify(struct tb_tunnel*t,bool active){} +static void tb_tunnel_deactivate(struct tb_tunnel*t){} +static void tb_detach_bandwidth_group(struct tb_port*p){} +static void tb_switch_dealloc_dp_resource(struct tb_switch*s,struct tb_port*p){} +static void tb_configure_sym(struct tb*t,struct tb_port*i,struct tb_port*o,bool active){} +static void pm_runtime_mark_last_busy(struct device*d){} +static void pm_runtime_put_autosuspend(struct device*d){if(d==&domain.tb.dev){CHECK(pm_refs==1);pm_refs--;}} +static int pm_runtime_resume_and_get(struct device*d){ + CHECK(!lock_depth);pm_calls++; + if(pm_error)return pm_error; + pm_refs++; + if(stop_during_pm)domain.cm.hotplug_active=false; + return 0; +} +static void tb_reclaim_usb3_bandwidth(struct tb*t,struct tb_port*i,struct tb_port*o){} +static bool tb_tunnel_is_active(struct tb_tunnel*t){return false;} +static int tb_tunnel_consumed_bandwidth(struct tb_tunnel*t,int*up,int*down){return 0;} +static void tb_configure_asym(struct tb*t,struct tb_port*i,struct tb_port*o,int up,int down){} +static void tb_increase_tmu_accuracy(struct tb_tunnel*t){} +static struct tb_tunnel*tb_find_tunnel(struct tb*t,int type,struct tb_port*in,struct tb_port*out){return current;} +static void tb_enter_redrive(struct tb_port*p){} +static void tb_recalc_estimated_bandwidth(struct tb*t){} +static struct tb*tb_domain_get(struct tb*t){CHECK(domain_refs>=1);domain_refs++;return t;} +static void tb_domain_put(struct tb*t){CHECK(domain_refs>1);domain_refs--;} +static bool queue_work(void*wq,struct work_struct*w){if(w->pending)return false;w->pending=true;return true;} +static void tb_queue_dp_tunnels(struct tb*tb); +static void tb_tunnel_dp(struct tb*t){ + CHECK(lock_depth==1 && domain.cm.hotplug_active && t->root_switch); + alloc_calls++;if(occupied)alloc_failures++;else alloc_successes++; + if(requeue_once){requeue_once=0;tb_queue_dp_tunnels(t);} +} +''' + +FOOTER = r''' +static struct tb_tunnel*new_tunnel(struct tb*tb,struct tb_port*in,struct tb_port*out){ + struct tb_tunnel*t=calloc(1,sizeof(*t));CHECK(t);t->tb=tb;t->src_port=in;t->dst_port=out;t->type=TB_TUNNEL_DP;t->kref.count=1;t->npaths=1; + t->paths[0]=calloc(1,sizeof(struct tb_path));CHECK(t->paths[0]);t->paths[0]->reserved=true;occupied++;t->callback=tb_dp_tunnel_active;current=t;return t; +} +static void flush_queue(void){ + unsigned n=0; + while(domain.cm.dp_tunnel_work.pending){ + CHECK(++n<8);CHECK(domain.cm.dp_tunnel_work.fn);domain.cm.dp_tunnel_work.pending=false; + domain.cm.dp_tunnel_work.fn(&domain.cm.dp_tunnel_work); + } + CHECK(!lock_depth && !pm_refs && domain_refs==1); + if(alloc_successes)CHECK(!domain.cm.dp_tunnel_pending); +} +static void queue_repair(unsigned count){ + mutex_lock(&domain.tb.lock); + for(unsigned i=0;idprx_canceled=true;tb_tunnel_get(t);tb_domain_get(&domain.tb); + worker_tail(&domain.tb,t); + CHECK(!domain.cm.dp_tunnel_work.pending && !alloc_calls && occupied==1 && !freed && domain_refs==1); + mutex_lock(&domain.tb.lock);tb_deactivate_and_free_tunnel(t);mutex_unlock(&domain.tb.lock); + CHECK(!occupied && freed==1 && !lock_depth); + }else if(!strcmp(c,"coalesce")){ + queue_repair(20);CHECK(domain_refs==2);flush_queue();CHECK(alloc_calls==1); + }else if(!strcmp(c,"stop-before-queue")){ + domain.cm.hotplug_active=false;queue_repair(1);CHECK(!domain.cm.dp_tunnel_work.pending && domain_refs==1);flush_queue();CHECK(!alloc_calls && !pm_calls); + }else if(!strcmp(c,"stop-after-queue")){ + queue_repair(1);domain.cm.hotplug_active=false;domain.tb.root_switch=NULL;flush_queue();CHECK(!alloc_calls && !pm_calls); + }else if(!strcmp(c,"stop-during-pm")){ + queue_repair(1);stop_during_pm=1;flush_queue();CHECK(!alloc_calls && pm_calls==1); + }else if(!strcmp(c,"no-root")){ + queue_repair(1);domain.tb.root_switch=NULL;flush_queue();CHECK(!alloc_calls && pm_calls==1); + }else if(!strcmp(c,"pm-failure")){ + queue_repair(1);pm_error=-5;flush_queue();CHECK(!alloc_calls && pm_calls==1); + }else if(!strcmp(c,"requeue-running")){ + queue_repair(1);requeue_once=1;flush_queue();CHECK(alloc_calls==2 && alloc_successes==2); + }else if(!strcmp(c,"freeze-thaw-repair")){ + queue_repair(1);mutex_lock(&domain.tb.lock);CHECK(!tb_freeze_noirq(&domain.tb));mutex_unlock(&domain.tb.lock); + flush_queue();CHECK(!alloc_calls && !pm_calls && domain.cm.dp_tunnel_pending); + mutex_lock(&domain.tb.lock);CHECK(!tb_thaw_noirq(&domain.tb));mutex_unlock(&domain.tb.lock); + CHECK(domain.cm.dp_tunnel_work.pending && domain_refs==2);flush_queue();CHECK(alloc_successes==1); + }else if(!strcmp(c,"pm-failure-retry")){ + queue_repair(1);pm_error=-5;flush_queue();CHECK(!alloc_calls && domain.cm.dp_tunnel_pending); + pm_error=0;mutex_lock(&domain.tb.lock);CHECK(!tb_thaw_noirq(&domain.tb));mutex_unlock(&domain.tb.lock); + flush_queue();CHECK(alloc_successes==1 && pm_calls==2); + }else if(!strcmp(c,"ordinary-retirement")){ + new_tunnel(&domain.tb,&in,&out);mutex_lock(&domain.tb.lock); + tb_dp_resource_unavailable(&domain.tb,&in,"ordinary detach");mutex_unlock(&domain.tb.lock); + CHECK(!occupied && freed==1 && !alloc_calls);flush_queue();CHECK(alloc_successes==1); + }else if(!strcmp(c,"noirq-pm-error-complete-replay")){ + /* Generic host: completion replays the request the noirq pass could not serve. */ + freeze_then_thaw_with_pm_disabled(); + tb_complete(&domain.tb); + CHECK(!lock_depth && !restore_calls && !scan_calls && domain.cm.dp_tunnel_work.pending && domain_refs==2); + flush_queue();CHECK(alloc_calls==1 && alloc_successes==1 && pm_calls==2 && !domain.cm.dp_tunnel_pending); + }else if(!strcmp(c,"apple-complete-acknowledges")){ + /* QUIRK_NO_SYSTEM_SLEEP pairs directly; that serves the request, so nothing is queued again. */ + nhi.quirks=QUIRK_NO_SYSTEM_SLEEP; + freeze_then_thaw_with_pm_disabled(); + tb_complete(&domain.tb); + CHECK(!lock_depth && restore_calls==1 && alloc_calls==1 && !domain.cm.dp_tunnel_pending); + CHECK(!domain.cm.dp_tunnel_work.pending && domain_refs==1); + flush_queue();CHECK(alloc_calls==1 && pm_calls==1); + }else if(!strcmp(c,"complete-no-request")){ + tb_complete(&domain.tb); + CHECK(!lock_depth && !domain.cm.dp_tunnel_work.pending && !alloc_calls && !restore_calls && domain_refs==1); + flush_queue();CHECK(!alloc_calls && !pm_calls); + }else if(!strcmp(c,"complete-xdomain-rescan-replay")){ + /* The rescan's scoped lock is dropped before the pairing lock is taken. */ + unplugged_xdomains=1; + freeze_then_thaw_with_pm_disabled(); + tb_complete(&domain.tb); + CHECK(!lock_depth && scan_calls==1 && domain.cm.dp_tunnel_work.pending); + flush_queue();CHECK(alloc_successes==1 && !domain.cm.dp_tunnel_pending); + }else return 78; + printf("PASS %s\n",c);return 0; +} +''' + +CASES = ['callback-release-before-repair', 'canceled-callback-does-not-repair', 'coalesce', 'stop-before-queue', + 'stop-after-queue', 'stop-during-pm', 'no-root', 'pm-failure', + 'requeue-running', 'ordinary-retirement', 'freeze-thaw-repair', + 'pm-failure-retry', 'noirq-pm-error-complete-replay', + 'apple-complete-acknowledges', 'complete-no-request', + 'complete-xdomain-rescan-replay'] +# The cases a tb_complete() without the replay and the acknowledgement must fail. +COMPLETE_CASES = {'noirq-pm-error-complete-replay', 'apple-complete-acknowledges', + 'complete-xdomain-rescan-replay'} +MUTANTS = { + 'missing-canceled-callback-guard': ('if (tunnel->dprx_canceled)', 'if (false)'), + 'inline-repair': ('\ttb_recalc_estimated_bandwidth(tb);\n\ttb_queue_dp_tunnels(tb);', + '\ttb_recalc_estimated_bandwidth(tb);\n\ttb_tunnel_dp(tb);'), + 'missing-queue-reference': ('\ttb_domain_get(tb);', '\t/* removed get */'), + 'coalescing-leaks-reference': ('\tif (!queue_work(tb->wq, &tcm->dp_tunnel_work))\n\t\ttb_domain_put(tb);', + '\tqueue_work(tb->wq, &tcm->dp_tunnel_work);'), + 'missing-stop-guard': ('\tif (!READ_ONCE(tcm->hotplug_active))', '\tif (false)'), + 'missing-post-resume-guard': ('\tif (tcm->hotplug_active && tb->root_switch)', '\tif (tb->root_switch)'), + 'missing-root-guard': ('\tif (tcm->hotplug_active && tb->root_switch)', '\tif (tcm->hotplug_active)'), + 'missing-pm-error-guard': ('\tif (ret < 0)', '\tif (false)'), + 'missing-pm-put': ('\tpm_runtime_put_autosuspend(&tb->dev);', '\t/* removed PM put */'), + 'missing-probe-init': ('INIT_WORK(&tcm->dp_tunnel_work, tb_dp_tunnel_work);', + '/* removed probe initialization */'), + 'missing-pending-request': ('\ttcm->dp_tunnel_pending = true;', '\t/* lost request */'), + 'pending-never-consumed': ('if (tcm->hotplug_active && tb->root_switch) {\n\t\ttcm->dp_tunnel_pending = false;', + 'if (tcm->hotplug_active && tb->root_switch) {\n\t\t/* pending forever */'), + 'missing-thaw-requeue': ('\ttcm->hotplug_active = true;\n\tif (tcm->dp_tunnel_pending)\n\t\ttb_queue_dp_tunnels(tb);', + '\ttcm->hotplug_active = true;\n\t/* lost thaw request */'), + 'missing-complete-replay': ('was disabled. */\n\tif (tcm->dp_tunnel_pending)\n\t\ttb_queue_dp_tunnels(tb);', + 'was disabled. */\n\t/* lost completion replay */'), + 'missing-apple-ack': ('\t\ttb_restore_dp_resources(tb->root_switch);\n\t\ttcm->dp_tunnel_pending = false;', + '\t\ttb_restore_dp_resources(tb->root_switch);\n\t\t/* request not acknowledged */'), + 'unconditional-complete-replay': ('was disabled. */\n\tif (tcm->dp_tunnel_pending)\n\t\ttb_queue_dp_tunnels(tb);', + 'was disabled. */\n\ttb_queue_dp_tunnels(tb);'), +} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--mutations', action='store_true') + parser.add_argument('--output', type=Path) + parser.add_argument('--complete-ref', help='negative control: take only tb_complete() from ' + 'this commit; exactly the completion cases must fail') + args = parser.parse_args() + tb = (ROOT / 'drivers/thunderbolt/tb.c').read_text() + tunnel = (ROOT / 'drivers/thunderbolt/tunnel.c').read_text() + names = ['tb_tunnel_get', 'tb_tunnel_destroy', 'tb_tunnel_put'] + parts = {n: function(tunnel, n) for n in names} + for name in ['tb_deactivate_and_free_tunnel', 'tb_dp_tunnel_work', + 'tb_queue_dp_tunnels', 'tb_dp_resource_unavailable', 'tb_dp_tunnel_active', + 'tb_freeze_noirq', 'tb_thaw_noirq', 'tb_complete']: + parts[name] = function(tb, name) + if args.complete_ref: + old = subprocess.check_output(['git', 'show', args.complete_ref + ':drivers/thunderbolt/tb.c'], + cwd=ROOT, text=True) + parts['tb_complete'] = function(old, 'tb_complete') + worker = function(tunnel, 'tb_dp_dprx_work') + marker = '\ttunnel->callback(tunnel);' + if worker.count(marker) != 1: + raise ValueError('DPRX completion tail changed; review extraction') + tail = worker[worker.index(marker):].rstrip()[:-1] + if tail.strip() != 'tunnel->callback(tunnel);\n\ttb_tunnel_put(tunnel);\n\ttb_domain_put(tb);': + raise ValueError('DPRX reference-release order changed; review extraction') + production = '\n'.join(parts.values()) + probe = (ROOT / 'drivers/thunderbolt/tb.c').read_text() + probe = probe[probe.index('struct tb *tb_probe(struct tb_nhi *nhi)'):] + initializers = re.findall(r'INIT_WORK\(&tcm->\w+,\s*\w+\);', probe) + production += '\nstatic void init_fixture(struct tb_cm*tcm){\n' + '\n'.join(initializers) + '\n}\n' + results = {} + with tempfile.TemporaryDirectory(prefix='tb-dp-repair-') as tmp: + out = args.output or Path(tmp) + out.mkdir(parents=True, exist_ok=True) + variants = {'real': production} + if args.mutations and not args.complete_ref: + for name, (old, new) in MUTANTS.items(): + if production.count(old) != 1: + raise ValueError('Mutation site changed: ' + name) + variants[name] = production.replace(old, new, 1) + for name, body in variants.items(): + unit = out / (name + '.c') + unit.write_text(HEADER + '#include \n' + body + + '\nstatic void worker_tail(struct tb*tb,struct tb_tunnel*tunnel){\n' + + tail + '\n}\n' + FOOTER) + executable = out / name + subprocess.run(['cc', '-std=gnu11', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-parameter', '-Wno-unused-function', + '-Wno-unused-but-set-variable', '-g', + '-fsanitize=address,undefined', str(unit), '-o', str(executable)], + check=True) + runs = [] + for case in CASES: + run = subprocess.run([str(executable), case], capture_output=True, text=True) + runs.append({'case': case, 'rc': run.returncode, + 'stdout': run.stdout, 'stderr': run.stderr}) + failures = [r for r in runs if r['rc'] != 0] + if args.complete_ref: + failed = {r['case'] for r in failures} + if failed != COMPLETE_CASES or any(r['rc'] != 77 for r in failures): + raise RuntimeError('Control did not fail exactly the completion cases: ' + + json.dumps(sorted(failed))) + elif name == 'real' and failures: + raise RuntimeError(json.dumps(failures)) + if name != 'real' and not any(r['rc'] == 77 for r in failures): + raise RuntimeError('Mutant not caught by a valid runtime assertion: ' + name) + results[name] = runs + receipt = {'scope': __doc__, 'production_bodies': { + n: hashlib.sha256(body.encode()).hexdigest() for n, body in parts.items()}, + 'worker_tail_sha256': hashlib.sha256(tail.encode()).hexdigest(), + 'probe_initializers': initializers, + 'complete_ref': args.complete_ref, + 'results': results} + (out / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps({'cases': len(CASES), 'passed': True, + 'complete_ref': args.complete_ref, + 'control_failed': sorted(COMPLETE_CASES) if args.complete_ref else [], + 'mutants_caught': len(variants) - 1, 'output': str(out)})) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/tiled-split-tests.py b/tools/asahi/tiled-split-tests.py new file mode 100644 index 0000000000000..ecb6cbe311f3c --- /dev/null +++ b/tools/asahi/tiled-split-tests.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only OR MIT +"""Exercise the production DPIN1 dispatch and tiled-display split admission. + +The production bodies are compiled with ASan/UBSan against a host fixture +that stubs only what lies below them. With --mutations, each mutant changes +the extracted bodies in memory (never the source) and must make at least one +case fail. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import resource +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +FUNCTIONS = { + 'drivers/gpu/drm/apple/dcp-fabric-session.h': ['dcp_fabric_binding_request'], + 'drivers/gpu/drm/apple/dcp-fabric-core.c': ['dcp_fabric_tunnel_slot'], + 'drivers/gpu/drm/apple/dcp-fabric.c': [ + 'dcp_typec_port_untileable', 'dcp_tb_split_locked', 'dcp_tb_dispatch_request'], +} +CASES = ['t6030-dpin1-alone', 't6030-dpin1-alone-before-services', + 't6030-dpin1-with-external-owner', 't6030-dpin1-then-dpin0', + 't6030-dpin0-then-dpin1', 't6030-split-off', 't6030-inactive-dpin1', + 'internal-dpin1-alone-waits', 'internal-not-ready-waits', + 'mixed-routes-wait', 'no-routes-wait', 'internal-tile-hint-pending', + 'internal-tiled-join', 'internal-owner-without-port1', + 'split-retired-teardown-first', 'split-existing-release', + 'split-existing-active'] +# (name, old, new): valid C that changes the admission. +MUTATIONS = [ + ('gate-removed', 'if (dcp_typec_port_untileable(port))', + 'if (false && dcp_typec_port_untileable(port))'), + ('gate-before-split-handling', + 'lockdep_assert_held(&dcp_typec_fabric_lock);\n\t/* A retired provider', + 'lockdep_assert_held(&dcp_typec_fabric_lock);\n\tif (dcp_typec_port_untileable(port))\n' + '\t\treturn -EOPNOTSUPP;\n\t/* A retired provider'), + ('readiness-judged', 'if (!route->dcp->external)\n\t\t\treturn false;', + 'if (!route->dcp->external && route->dcp->dptxport[1].enabled)\n\t\t\treturn false;'), + ('empty-untileable', 'return !list_empty(&port->routes);', 'return true;'), + ('any-external-untileable', 'if (!route->dcp->external)\n\t\t\treturn false;', + 'if (route->dcp->external)\n\t\t\treturn true;'), + ('gate-always', 'if (dcp_typec_port_untileable(port))', + 'if (true || dcp_typec_port_untileable(port))'), +] + + +def function(source, name): + # A definition line: not a comment line that mentions the name. + match = re.search(r'^(?![ \t]*(?:\*|/\*|//))[^\n;{}]*\b' + name + r'\([^;]*?\)\n\{', + source, re.M | re.S) + if not match: + raise ValueError(name) + pos = source.index('{', match.start()) + 1 + depth = 1 + while depth: + depth += (source[pos] == '{') - (source[pos] == '}') + pos += 1 + return source[match.start():pos] + '\n' + + +def build(out, bodies, tag): + fixture = (ROOT / 'tools/asahi/tiled-split/fixture.c').read_text() + code = fixture.replace('/* PRODUCTION_FUNCTIONS */', '\n'.join(bodies.values())) + source, binary = out / f'{tag}.c', out / tag + source.write_text(code) + subprocess.run(['cc', '-std=gnu11', '-O1', '-g', '-Wall', '-Wextra', '-Werror', + '-Wno-unused-function', '-Wno-unused-parameter', + '-fsanitize=address,undefined', '-fno-sanitize-recover=all', + str(source), '-o', str(binary)], check=True) + return binary + + +def run(binary): + def no_core(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + results = [] + for case in CASES: + r = subprocess.run([str(binary), case], capture_output=True, text=True, + preexec_fn=no_core) + results.append({'case': case, 'exit': r.returncode, 'stderr': r.stderr.strip()[-300:]}) + return results + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--mutations', action='store_true') + parser.add_argument('--old-ref', help='take the dispatch and split bodies from this ' + 'commit instead (a negative control: expected to fail)') + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + bodies = {} + for path, names in FUNCTIONS.items(): + source = (ROOT / path).read_text() + for name in names: + bodies[name] = function(source, name) + if args.old_ref and name in ('dcp_tb_split_locked', 'dcp_tb_dispatch_request'): + old = subprocess.check_output(['git', 'show', f'{args.old_ref}:{path}'], + cwd=ROOT, text=True) + bodies[name] = function(old, name) + results = run(build(args.out, bodies, 'production')) + failed = [r for r in results if r['exit']] + mutants = [] + if args.mutations: + for name, old, new in MUTATIONS: + text = '\n'.join(bodies.values()) + if text.count(old) != 1: + raise SystemExit(f'mutation {name}: {text.count(old)} matches') + mutated = {k: v.replace(old, new) for k, v in bodies.items()} + mres = run(build(args.out, mutated, 'mutant-' + name)) + caught = [r['case'] for r in mres if r['exit']] + mutants.append({'mutant': name, 'caught_by': caught, 'killed': bool(caught)}) + receipt = {'production_bodies': {n: hashlib.sha256(b.encode()).hexdigest() for n, b in bodies.items()}, + 'old_ref': args.old_ref, 'hardware_access': False, + 'results': results, 'mutants': mutants} + (args.out / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(f'{len(results) - len(failed)}/{len(results)} cases pass') + for r in failed: + print('FAIL', r['case'], r['stderr']) + for m in mutants: + print('KILLED' if m['killed'] else 'SURVIVED', m['mutant'], m['caught_by'][:4]) + if failed or any(not m['killed'] for m in mutants): + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/asahi/tiled-split/fixture.c b/tools/asahi/tiled-split/fixture.c new file mode 100644 index 0000000000000..dd21b5b876ba1 --- /dev/null +++ b/tools/asahi/tiled-split/fixture.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: GPL-2.0-only OR MIT +/* + * Host fixture for the Thunderbolt DPIN1 dispatch and tiled-display split + * admission. The production bodies of dcp_tb_dispatch_request(), + * dcp_tb_split_locked(), dcp_typec_port_untileable(), + * dcp_fabric_tunnel_slot() and dcp_fabric_binding_request() are pasted in; + * only what lies below them (join, connect, teardown, release, attach, + * rebalance) is stubbed, and every stub call is recorded. + */ +#include +#include +#include +#include +#include +#include +#include + +typedef uint64_t u64; +typedef uint32_t u32; +typedef uint8_t u8; + +#define container_of(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member))) +#define READ_ONCE(x) (x) +#define smp_load_acquire(p) (*(p)) +#define lockdep_assert_held(l) ((void)(l)) +#define scoped_guard(type, lock) for (int _sg = 1; _sg; _sg = 0) +#define time_before(a, b) ((long)((a) - (b)) < 0) +#define dev_warn(dev, ...) fprintf(stderr, __VA_ARGS__) +#define EXPORT_SYMBOL_GPL(x) + +struct list_head { struct list_head *next, *prev; }; +#define LIST_HEAD_INIT(n) { &(n), &(n) } +static void INIT_LIST_HEAD(struct list_head *l) { l->next = l->prev = l; } +static bool list_empty(const struct list_head *l) { return l->next == l; } +static void list_add_tail(struct list_head *n, struct list_head *h) +{ + n->prev = h->prev; n->next = h; h->prev->next = n; h->prev = n; +} +#define list_for_each_entry(pos, head, member) \ + for (pos = container_of((head)->next, __typeof__(*pos), member); \ + &pos->member != (head); \ + pos = container_of(pos->member.next, __typeof__(*pos), member)) + +struct mutex { int unused; }; +struct device { int unused; }; +struct mux_chip; +struct mux_control { struct mux_chip *chip; }; +struct mux_chip { struct mux_control mux[3]; unsigned int controllers; }; + +struct dptx_port { bool enabled, connected, tile_hint; }; +struct apple_dcp { + struct device *dev; + bool external; + struct dptx_port dptxport[2]; + struct mutex tb_lock; + bool tb_retiring; + int (*tb_dpin_set_active)(void *, bool); + void *tb_dpin_ctx; + struct { + bool active, ready; + u64 generation; + int (*set_active)(void *, bool); + void *binding; + } split; +}; + +struct dcp_fabric_route { + struct dcp_fabric_route *next; + void *pipeline; + bool tunnel; + unsigned int dpin; + u8 tunnel_clock_blocked; +}; +struct dcp_fabric_port { + unsigned long key; + struct dcp_fabric_port *next; + struct dcp_fabric_route *routes; + struct dcp_fabric_route *owner[2]; + struct dcp_fabric_route *preferred; + u32 candidate_crtcs[2]; + bool wanted, hpd; + void *plan; +}; +struct apple_dcp_typec_route { + struct apple_dcp *dcp; + struct list_head port_link; + struct dcp_fabric_route core; + bool tunnel; + unsigned int tunnel_dpin; + u64 tunnel_generation; + struct mux_control *dpin[2]; + struct mux_control *xbar; +}; +struct apple_dcp_typec_port { + struct dcp_fabric_port core; + struct list_head routes; + struct apple_dcp_typec_route *owner, *secondary_owner; + struct apple_dcp *split_dcp; + unsigned long tile_hint_deadline; +}; +struct dcp_tb_attach_context { + unsigned int dpin; + bool active; + u64 generation; + int (*set_active)(void *, bool); + void *binding; + struct apple_dcp_typec_port *port; + struct apple_dcp_typec_route **slot; + bool planned; +}; +struct dcp_fabric_attach_ops { int unused; }; + +static bool tiled_split = true; +static unsigned long jiffies = 1000; +static int dcp_typec_fabric_lock; +static const struct dcp_fabric_attach_ops dcp_tb_attach_ops; + +/* --- recorded stubs --- */ +static struct { + unsigned int teardown, join, connect, release, attach, rebalance; + unsigned int attach_dpin[4]; + int teardown_ret, connect_ret; + struct apple_dcp *attach_dcp[2]; +} st; + +static int dcp_tb_split_teardown_locked(struct apple_dcp_typec_port *port, bool force) +{ + (void)force; + st.teardown++; + if (port->split_dcp) { + port->split_dcp->split.active = false; + port->split_dcp->split.generation = 0; + port->split_dcp = NULL; + } + return st.teardown_ret; +} + +static int dcp_tb_split_join_locked(struct apple_dcp_typec_port *port, + struct apple_dcp_typec_route *route, + struct mux_control *xbar, u64 generation, + int (*set_active)(void *, bool), void *binding) +{ + (void)xbar; + st.join++; + port->split_dcp = route->dcp; + route->dcp->split.active = true; + route->dcp->split.generation = generation; + route->dcp->split.set_active = set_active; + route->dcp->split.binding = binding; + return 0; +} + +static int dcp_dptx_connect(struct apple_dcp *dcp, unsigned int port) +{ + (void)dcp; (void)port; + st.connect++; + return st.connect_ret; +} + +static int dcp_tb_release_locked(struct apple_dcp_typec_port *port, unsigned int dpin) +{ + st.release++; + if (dpin) + port->secondary_owner = NULL; + else + port->owner = NULL; + return 0; +} + +static struct apple_dcp_typec_route attach_route[2]; + +static int dcp_fabric_run_attach(const struct dcp_fabric_attach_ops *ops, + struct dcp_tb_attach_context *ctx) +{ + struct apple_dcp_typec_route *r = &attach_route[ctx->dpin]; + + (void)ops; + st.attach_dpin[st.attach++] = ctx->dpin; + r->dcp = st.attach_dcp[ctx->dpin]; + r->tunnel = true; + r->tunnel_dpin = ctx->dpin; + r->tunnel_generation = ctx->generation; + *ctx->slot = r; + return 0; +} + +static struct apple_dcp_typec_route * +dcp_typec_rebalance_locked(struct apple_dcp_typec_port *arriving, unsigned int dpin) +{ + (void)arriving; (void)dpin; + st.rebalance++; + return NULL; +} + +/* PRODUCTION_FUNCTIONS */ + +/* --- topology --- */ +static struct apple_dcp dcp_a, dcp_b; +static struct apple_dcp_typec_route route_a, route_b; +static struct apple_dcp_typec_port port; +static struct mux_chip chip; + +static int set_active(void *binding, bool active) { (void)binding; (void)active; return 0; } + +enum kind { NONE, ALL_EXTERNAL, ALL_INTERNAL, MIXED }; + +static void topology(enum kind kind, bool port1_ready) +{ + memset(&st, 0, sizeof(st)); + memset(&dcp_a, 0, sizeof(dcp_a)); + memset(&dcp_b, 0, sizeof(dcp_b)); + memset(&route_a, 0, sizeof(route_a)); + memset(&route_b, 0, sizeof(route_b)); + memset(&port, 0, sizeof(port)); + memset(attach_route, 0, sizeof(attach_route)); + INIT_LIST_HEAD(&port.routes); + chip.controllers = 3; + for (int i = 0; i < 3; i++) + chip.mux[i].chip = &chip; + dcp_a.external = kind == ALL_EXTERNAL || kind == MIXED; + dcp_b.external = kind == ALL_EXTERNAL; + dcp_a.dptxport[1].enabled = dcp_b.dptxport[1].enabled = port1_ready; + route_a.dcp = &dcp_a; + route_b.dcp = &dcp_b; + route_a.xbar = route_b.xbar = &chip.mux[0]; + if (kind != NONE) { + list_add_tail(&route_a.port_link, &port.routes); + list_add_tail(&route_b.port_link, &port.routes); + } + st.attach_dcp[0] = &dcp_a; + st.attach_dcp[1] = &dcp_b; + tiled_split = true; + jiffies = 1000; + port.tile_hint_deadline = 3000; +} + +/* The dpin0 tunnel route already on the port, on @dcp. */ +static void owner_on(struct apple_dcp *dcp) +{ + attach_route[0].dcp = dcp; + attach_route[0].tunnel = true; + attach_route[0].tunnel_dpin = 0; + attach_route[0].tunnel_generation = 5; + attach_route[0].xbar = &chip.mux[0]; + port.owner = &attach_route[0]; +} + +static int request(unsigned int dpin, bool active, u64 generation) +{ + struct dcp_tb_attach_context ctx = { + .dpin = dpin, .active = active, .generation = generation, + .set_active = set_active, .binding = &port, + }; + + return dcp_tb_dispatch_request(&ctx, &port.core); +} + +static int split(bool active, u64 generation) +{ + struct dcp_tb_attach_context ctx = { + .dpin = 1, .active = active, .generation = generation, + .set_active = set_active, .binding = &port, + }; + + return dcp_tb_split_locked(&port, &ctx); +} + +#define CHECK(c) do { if (!(c)) { fprintf(stderr, "%s:%d: %s\n", __func__, __LINE__, #c); exit(1); } } while (0) + +/* T6030: every route an external processor. */ +static void t6030_dpin1_alone(void) +{ + topology(ALL_EXTERNAL, true); + CHECK(split(true, 9) == -EOPNOTSUPP); + CHECK(request(1, true, 9) == 0); + CHECK(st.attach == 1 && st.attach_dpin[0] == 1 && port.secondary_owner); + CHECK(!st.join && !st.connect && !st.teardown); +} + +static void t6030_dpin1_alone_before_services(void) +{ + /* No port 1 service yet: the identity decides, not readiness. */ + topology(ALL_EXTERNAL, false); + CHECK(request(1, true, 9) == 0); + CHECK(st.attach == 1 && !st.join); +} + +static void t6030_dpin1_with_external_owner(void) +{ + topology(ALL_EXTERNAL, true); + owner_on(&dcp_a); + dcp_a.dptxport[0].tile_hint = true; + CHECK(split(true, 9) == -EOPNOTSUPP); + CHECK(request(1, true, 9) == 0); + CHECK(st.attach == 1 && st.attach_dpin[0] == 1 && !st.join); +} + +static void t6030_dpin1_then_dpin0(void) +{ + topology(ALL_EXTERNAL, true); + CHECK(request(1, true, 9) == 0); + CHECK(request(0, true, 10) == 0); + CHECK(st.attach == 2 && st.attach_dpin[0] == 1 && st.attach_dpin[1] == 0); + CHECK(port.owner && port.secondary_owner && !st.join); +} + +static void t6030_dpin0_then_dpin1(void) +{ + topology(ALL_EXTERNAL, true); + CHECK(request(0, true, 10) == 0); + CHECK(request(1, true, 9) == 0); + CHECK(st.attach == 2 && st.attach_dpin[0] == 0 && st.attach_dpin[1] == 1); + CHECK(port.owner && port.secondary_owner && !st.join); +} + +static void t6030_split_off(void) +{ + topology(ALL_EXTERNAL, true); + tiled_split = false; + CHECK(request(1, true, 9) == 0); + CHECK(st.attach == 1 && !st.join); +} + +static void t6030_inactive_dpin1(void) +{ + topology(ALL_EXTERNAL, true); + CHECK(split(false, 9) == -EOPNOTSUPP); + CHECK(request(1, false, 9) == -ESTALE); + CHECK(!st.attach && !st.teardown && !st.release); +} + +/* M1/M2: pipelines that can host a tile keep the dpin0 wait. */ +static void internal_dpin1_alone_waits(void) +{ + topology(ALL_INTERNAL, true); + CHECK(request(1, true, 9) == -ENODEV); + CHECK(!st.attach && !st.join); +} + +static void internal_not_ready_waits(void) +{ + /* Port 1 not announced yet is no reason to give the tile up. */ + topology(ALL_INTERNAL, false); + CHECK(request(1, true, 9) == -ENODEV); + CHECK(!st.attach); +} + +static void mixed_routes_wait(void) +{ + topology(MIXED, true); + CHECK(request(1, true, 9) == -ENODEV); + CHECK(!st.attach); +} + +static void no_routes_wait(void) +{ + topology(NONE, true); + CHECK(request(1, true, 9) == -ENODEV); + CHECK(!st.attach); +} + +static void internal_tile_hint_pending(void) +{ + topology(ALL_INTERNAL, true); + owner_on(&dcp_a); + CHECK(request(1, true, 9) == -ENODEV); + CHECK(!st.attach && !st.join); + jiffies = 4000; + CHECK(request(1, true, 9) == 0); + CHECK(st.attach == 1 && !st.join); +} + +static void internal_tiled_join(void) +{ + topology(ALL_INTERNAL, true); + owner_on(&dcp_a); + dcp_a.dptxport[0].tile_hint = true; + CHECK(request(1, true, 9) == 0); + CHECK(st.join == 1 && st.connect == 1 && !st.attach); + CHECK(port.split_dcp == &dcp_a && dcp_a.split.ready); +} + +static void internal_owner_without_port1(void) +{ + topology(ALL_INTERNAL, true); + owner_on(&dcp_a); + dcp_a.dptxport[1].enabled = false; + dcp_a.dptxport[0].tile_hint = true; + CHECK(request(1, true, 9) == 0); + CHECK(st.attach == 1 && !st.join); +} + +/* An existing split is handled before anything else, on any port. */ +static void split_retired_teardown_first(void) +{ + topology(ALL_EXTERNAL, true); + port.split_dcp = &dcp_a; + dcp_a.split.active = true; + dcp_a.split.generation = 0; + CHECK(request(1, true, 9) == 0); + CHECK(st.teardown == 1 && !port.split_dcp && st.attach == 1); +} + +static void split_existing_release(void) +{ + topology(ALL_EXTERNAL, true); + port.split_dcp = &dcp_a; + dcp_a.split.active = true; + dcp_a.split.generation = 7; + dcp_a.split.set_active = set_active; + dcp_a.split.binding = &port; + CHECK(request(1, false, 7) == 0); + CHECK(st.teardown == 1 && !port.split_dcp && !st.attach); +} + +static void split_existing_active(void) +{ + topology(ALL_EXTERNAL, true); + port.split_dcp = &dcp_a; + dcp_a.split.active = true; + dcp_a.split.generation = 7; + dcp_a.split.set_active = set_active; + dcp_a.split.binding = &port; + CHECK(request(1, true, 7) == -EAGAIN); + dcp_a.split.ready = true; + CHECK(request(1, true, 7) == 0); + CHECK(!st.teardown && !st.attach && port.split_dcp == &dcp_a); +} + +static const struct { const char *name; void (*fn)(void); } cases[] = { + { "t6030-dpin1-alone", t6030_dpin1_alone }, + { "t6030-dpin1-alone-before-services", t6030_dpin1_alone_before_services }, + { "t6030-dpin1-with-external-owner", t6030_dpin1_with_external_owner }, + { "t6030-dpin1-then-dpin0", t6030_dpin1_then_dpin0 }, + { "t6030-dpin0-then-dpin1", t6030_dpin0_then_dpin1 }, + { "t6030-split-off", t6030_split_off }, + { "t6030-inactive-dpin1", t6030_inactive_dpin1 }, + { "internal-dpin1-alone-waits", internal_dpin1_alone_waits }, + { "internal-not-ready-waits", internal_not_ready_waits }, + { "mixed-routes-wait", mixed_routes_wait }, + { "no-routes-wait", no_routes_wait }, + { "internal-tile-hint-pending", internal_tile_hint_pending }, + { "internal-tiled-join", internal_tiled_join }, + { "internal-owner-without-port1", internal_owner_without_port1 }, + { "split-retired-teardown-first", split_retired_teardown_first }, + { "split-existing-release", split_existing_release }, + { "split-existing-active", split_existing_active }, +}; + +int main(int argc, char **argv) +{ + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) + if (argc > 1 && !strcmp(argv[1], cases[i].name)) { + cases[i].fn(); + return 0; + } + fprintf(stderr, "unknown case\n"); + return 2; +} diff --git a/tools/asahi/tunnel-route-tests.py b/tools/asahi/tunnel-route-tests.py index 05d0cbc018802..5be05c91c156f 100644 --- a/tools/asahi/tunnel-route-tests.py +++ b/tools/asahi/tunnel-route-tests.py @@ -11,6 +11,7 @@ ROOT = Path(__file__).resolve().parents[2] FUNCTIONS = { + 'include/linux/soc/apple/dp-tunnel.h': ['apple_dp_tunnel_t6030_pclk'], 'drivers/gpu/drm/apple/dcp-fabric-core.c': ['dcp_fabric_tunnel_clock_blocked'], 'drivers/gpu/drm/apple/dcp-fabric.c': [ 'dcp_fabric_snapshot_port', 'dcp_typec_tunnel_ctl'], diff --git a/tools/asahi/tunnel-route/fixture.c b/tools/asahi/tunnel-route/fixture.c index da8e6c70a212c..dd63853c3bd36 100644 --- a/tools/asahi/tunnel-route/fixture.c +++ b/tools/asahi/tunnel-route/fixture.c @@ -8,87 +8,126 @@ typedef uint8_t u8; typedef uint32_t u32; typedef uint64_t u64; +#define READ_ONCE(x) (x) #define BIT(n) (1U << (n)) #define lockdep_assert_held(x) ((void)0) #define list_for_each_entry(r, head, link) \ - for ((r) = *(head); (r); (r) = (r)->next) + for ((r) = *(head); (r); (r) = (r)->next) /* CORE_HEADER */ struct mux_chip; -struct mux_control { struct mux_chip *chip; }; -struct mux_chip { struct mux_control mux[3]; unsigned int controllers; }; -struct apple_connector { u32 candidate_crtcs; }; -struct apple_crtc { struct { unsigned int index; } base; }; +struct mux_control { + struct mux_chip *chip; +}; +struct mux_chip { + struct mux_control mux[3]; + unsigned int controllers; +}; +struct apple_connector { + u32 candidate_crtcs; +}; +struct apple_crtc { + struct { + unsigned int index; + } base; +}; struct apple_dcp { - struct dcp_fabric_pipeline fabric; - struct apple_crtc *crtc; - bool tb_retiring, external; - void *fixed_phy, *active_typec_route; + struct dcp_fabric_pipeline fabric; + struct apple_crtc *crtc; + bool tb_retiring, external, clamshell, clamshell_free; + void *fixed_phy, *active_typec_route; }; struct apple_dcp_typec_route { - struct apple_dcp_typec_route *next; - struct apple_dcp *dcp; - struct dcp_fabric_route core; - bool tunnel; - unsigned int tunnel_dpin, mux_index; - struct mux_control *xbar, *dpin[2]; + struct apple_dcp_typec_route *next; + struct apple_dcp *dcp; + struct dcp_fabric_route core; + bool tunnel; + unsigned int tunnel_dpin, mux_index; + struct mux_control *xbar, *dpin[2]; }; struct apple_dcp_typec_port { - struct dcp_fabric_port core; - struct dcp_fabric_plan plan; - struct apple_dcp_typec_route *routes, *owner, *secondary_owner, *preferred_route; - struct apple_connector *connector, *secondary_connector; - bool dp_wanted, dp_hpd; + struct dcp_fabric_port core; + struct dcp_fabric_plan plan; + struct apple_dcp_typec_route *routes, *owner, *secondary_owner, + *preferred_route; + struct apple_connector *connector, *secondary_connector; + bool dp_wanted, dp_hpd; }; static bool t6030; static bool of_machine_is_compatible(const char *compatible) { - assert(!strcmp(compatible, "apple,t6030")); - return t6030; + assert(!strcmp(compatible, "apple,t6030")); + return t6030; } #define drm_crtc_index(crtc) ((crtc)->index) -static bool dcp_typec_route_busy(struct apple_dcp_typec_route *r) { return false; } -static bool dcp_typec_tunnel_held(struct apple_dcp *d) { return false; } -static bool dcp_typec_safe_held(struct apple_dcp *d) { return false; } +static bool dcp_typec_route_busy(struct apple_dcp_typec_route *r) +{ + return false; +} +static bool dcp_typec_tunnel_held(struct apple_dcp *d) +{ + return false; +} +static bool dcp_typec_safe_held(struct apple_dcp *d) +{ + return false; +} static enum dcp_fabric_presence_state dcp_hdmi_presence(struct apple_dcp *d) { - return DCP_FABRIC_ABSENT; + return DCP_FABRIC_ABSENT; +} +static bool dcpext_scanout_terminal(struct apple_dcp *d) +{ + return false; +} +static bool dcp_tb_services_ready(struct apple_dcp *d) +{ + return true; } -static bool dcpext_scanout_terminal(struct apple_dcp *d) { return false; } -static bool dcp_tb_services_ready(struct apple_dcp *d) { return true; } /* PRODUCTION_FUNCTIONS */ int main(void) { - struct mux_chip chip = { .controllers = 3 }; - struct apple_crtc crtc = { .base.index = 1 }; - struct apple_dcp dcp = { .crtc = &crtc }; - struct apple_dcp_typec_route source0 = { .dcp = &dcp, .xbar = &chip.mux[0] }; - struct apple_dcp_typec_route source2 = { - .dcp = &dcp, .xbar = &chip.mux[0], .mux_index = 2, - }; - struct apple_dcp_typec_port port = { .routes = &source0 }; - source0.next = &source2; - for (unsigned int i = 0; i < 3; i++) chip.mux[i].chip = &chip; - for (unsigned int soc = 0; soc < 2; soc++) { - t6030 = soc; - for (unsigned int named = 0; named < 2; named++) { - source0.dpin[0] = source2.dpin[0] = named ? &chip.mux[1] : NULL; - source0.dpin[1] = source2.dpin[1] = named ? &chip.mux[2] : NULL; - dcp_fabric_snapshot_port(&port, true); - assert(port.core.routes == &source0.core); - assert(source0.core.next == &source2.core); - assert(!source2.core.next); - assert(source0.core.tunnel_clock_blocked == (soc ? BIT(1) : 0)); - assert(source2.core.tunnel_clock_blocked == (soc ? BIT(1) : 0)); - assert(dcp_typec_tunnel_ctl(&source0, 0) == &chip.mux[1]); - assert(dcp_typec_tunnel_ctl(&source0, 1) == (soc ? NULL : &chip.mux[2])); - assert(dcp_typec_tunnel_ctl(&source2, 0) == &chip.mux[1]); - assert(dcp_typec_tunnel_ctl(&source2, 1) == (soc ? NULL : &chip.mux[2])); - assert(!dcp_typec_tunnel_ctl(&source0, 2)); - /* Late migration checks the real route, even with a stale snapshot. */ - source2.core.tunnel_clock_blocked = 0; - assert(dcp_typec_tunnel_ctl(&source2, 0) == &chip.mux[1]); - } - } - puts("PASS: DCP snapshot, named/legacy controls, invalid DP IN, stale snapshot"); - return 0; + struct mux_chip chip = { .controllers = 3 }; + struct apple_crtc crtc = { .base.index = 1 }; + struct apple_dcp dcp = { .crtc = &crtc }; + struct apple_dcp_typec_route source0 = { .dcp = &dcp, + .xbar = &chip.mux[0] }; + struct apple_dcp_typec_route source2 = { + .dcp = &dcp, + .xbar = &chip.mux[0], + .mux_index = 2, + }; + struct apple_dcp_typec_port port = { .routes = &source0 }; + source0.next = &source2; + for (unsigned int i = 0; i < 3; i++) + chip.mux[i].chip = &chip; + for (unsigned int soc = 0; soc < 2; soc++) { + t6030 = soc; + for (unsigned int named = 0; named < 2; named++) { + source0.dpin[0] = source2.dpin[0] = + named ? &chip.mux[1] : NULL; + source0.dpin[1] = source2.dpin[1] = + named ? &chip.mux[2] : NULL; + dcp_fabric_snapshot_port(&port, true); + assert(port.core.routes == &source0.core); + assert(source0.core.next == &source2.core); + assert(!source2.core.next); + assert(source0.core.tunnel_clock_blocked == 0); + assert(source2.core.tunnel_clock_blocked == 0); + assert(dcp_typec_tunnel_ctl(&source0, 0) == + &chip.mux[1]); + assert(dcp_typec_tunnel_ctl(&source0, 1) == + &chip.mux[2]); + assert(dcp_typec_tunnel_ctl(&source2, 0) == + &chip.mux[1]); + assert(dcp_typec_tunnel_ctl(&source2, 1) == + &chip.mux[2]); + assert(!dcp_typec_tunnel_ctl(&source0, 2)); + /* Late migration checks the real route, even with a stale snapshot. */ + source2.core.tunnel_clock_blocked = 0; + assert(dcp_typec_tunnel_ctl(&source2, 0) == + &chip.mux[1]); + } + } + puts("PASS: DCP snapshot, named/legacy controls, invalid DP IN, stale snapshot"); + return 0; } diff --git a/tools/aurora-ci/README.rst b/tools/aurora-ci/README.rst new file mode 100644 index 0000000000000..687be0ca91e7a --- /dev/null +++ b/tools/aurora-ci/README.rst @@ -0,0 +1,221 @@ +.. SPDX-License-Identifier: GPL-2.0 + +========= +Kernel CI +========= + +The GitHub Actions workflows in ``.github/workflows/kernel-ci*.yml`` check, +build, test and boot this kernel for every change to ``integration/aurora``. +The tools they run live here, so every step can also be run by hand. + +What runs when +============== + +Builds take time and, on some runners, money. Pull requests get the checks +that run on free GitHub-hosted runners on every push; the kernel is built +once, in the merge queue, from exactly the tree that will land. + +==================== ================================================= +Event Jobs +==================== ================================================= +pull request push plan, commits, compile, device trees, host tests, + CI self-test +``ci:build`` label all of the above, then package, KUnit and boot +merge queue everything +push to the branch package (reused from the queue), attestation +nightly package, KUnit and boot, rebuilt from scratch +manual run everything, never reusing an earlier result +==================== ================================================= + +``ci / result`` is the one check branch protection and the merge queue +require. ``result.py`` knows which jobs each kind of run must pass, so a +skipped job never counts as a pass when it was due. + +The jobs +======== + +plan (``plan.py``, ``lookup-build.py``) + Decides between a light and a full run from the event, then names the + build by its inputs: the kernel tree (leaving out ``.github`` and + ``tools/aurora-ci``, which never reach the package) and the parts of + omarchy-pkgs that build it (the linux-aurora recipe, ``build/``, + ``bin/build`` and the helpers it sources). Results are stored as + artifacts under that key, so the same inputs are never built or tested + twice: the push after a merge reuses the queue's packages, and a change to + the CI alone reuses the last build. The nightly run rebuilds from scratch, + because the Arch Linux ARM packages the builder installs change on their + own. + +commits (``check-commits.py``) + Every commit needs a ``Signed-off-by`` trailer (blocking). checkpatch + findings are reported but do not fail the job. A change under + ``include/uapi`` gets a warning: Mesa and other userspace built against + those headers needs a matching update before the kernel ships. Ranges of + more than 100 commits (rebases onto a new upstream) are reported only. + +compile (``compile-check.py``) + Compiles, with ``W=1`` and the shipped configuration, the objects a change + touches (plus a sample of the objects that include a changed header), at + the merge base and at the head, and reports warnings that are new. It also + compiles that set at every commit of the change (up to 20), so a commit + that does not build on its own fails the job and bisection keeps working. + Object selection comes from kbuild, not from guessing at file names. Rust + objects, linking and modpost are left to the package build. + +device trees (``dt-check.py``, ``dt-requirements.txt``) + When a change touches device trees, DT bindings or dtc, builds every Apple + DTB at the base and the head, schema-checks the DTBs whose output changed + and the bindings that changed, and reports only problems the change adds. + DTBs that fail to build and invalid or dropped bindings block; schema and + style findings are reported. A change that touches no DT files finishes in + under a second. + +host tests (``host-tests.py``, ``host-tests.json``) + The hermetic test suites under ``tools/``: Python and Rust tests of driver + logic built against the real sources, and the tools' own tests. Each entry + in ``host-tests.json`` says how to run it; ``known_failure`` marks a suite + that fails today, with the reason, so it is reported without blocking; + ``excluded`` lists the test-like files that cannot run here (hardware, + root, network) and why. A unit test fails when a new test-like file is in + neither list. Suites run without sudo, with ``TMPDIR`` outside the home + directory, and must leave the tree as clean as they found it. + +CI self-test + The unit tests in ``tests/``, actionlint, and zizmor (pedantic persona, + online audits) for the workflows. + +package (``prepare-recipe.py``, ``inspect-package.py``, ``compare-packages.py``) + Builds the kernel with the recipe that ships it: omarchy-pkgs' + ``pkgbuilds/linux-aurora``, run through omarchy-pkgs' own ``bin/build`` in + its builder container. ``prepare-recipe.py`` changes three lines (the + commit, a CI subrelease on ``pkgrel`` so a CI package never has a + published version, and the kernel source, which is read from the checkout + instead of cloned again) and refuses to guess if the recipe changed shape. + + ``inspect-package.py`` checks what installs rely on: a ``-ARCH`` kernel + release, a little-endian arm64 Image with the configured page size, device + trees, ``modules.dep``, ``pkgbase``, every module symbol resolving, and + Rust enabled. ``compare-packages.py`` diffs the result against the target + branch's package: configuration, device trees, modules, image size and + contents. A device tree that stops being built fails the job unless + ``retired-dtbs.txt`` lists it with a reason. + +KUnit (``run-kunit.py``, ``aurora.kunitconfig``, ``kunit-expected.json``) + Every KUnit suite for the Apple, Asahi and Aurora code, built into one + arm64 kernel with the shipped kernel's 16K pages and run in QEMU. Results + are compared with ``kunit-expected.json``, which lists the cases that skip + by design and the kernel warnings known to occur inside passing cases; + anything new, changed or missing fails. + +boot (``boot-test.py``, ``boot-expected.json``) + Boots the packaged kernel, not a test build, in a QEMU ``virt`` guest with + a busybox initramfs, loads every module the package ships, and fails on a + crash or warning in the log, a bad taint, a release mismatch, or a module + that fails to load for a reason other than missing hardware. The guest has + none of the Apple hardware, so this proves the kernel boots and every + module initialises; driver behaviour is tested on Macs. + + Without KVM, QEMU emulates a Neoverse N1 rather than ``-cpu max``: ``max`` + enables whatever the QEMU version emulates, and on QEMU 8 that stalls this + kernel's boot and faults in page clearing. A named model behaves the same + on every QEMU and has the 16K granule. + +Who can start a build +===================== + +Light runs execute a pull request's code on GitHub-hosted runners with a +read-only token, as every public repository's CI does. Full runs start only +from a maintainer action: queueing a pull request, pushing to the branch, a +manual run, or applying ``ci:build`` (which needs triage access). On a branch +of this repository the label applies to every later push; on a fork it +applies to the head it was applied to, and has to be applied again after a +new push. + +On a ``pull_request`` event GitHub runs the workflow files from the pull +request itself, so these rules decide cost, not security: a fork could edit +them. For the same reason no pull request's run is a source of results, and +neither is a manual run of another branch: both run code nobody has reviewed +yet. Earlier packages, verification records and baselines are reused only +from ``kernel-ci.yml`` runs on ``integration/aurora`` and its merge queue +branches (``lookup-build.py``), so no one can plant a package that a later +run would skip testing and attest. The merge queue therefore builds a pull +request again even after a ``ci:build`` run. What keeps untrusted code off +paid runners is GitHub's setting to +require approval before running workflows from outside collaborators, and, +for a runner like Namespace's, a runner group limited to this repository. + +Runners +======= + +Every job runs on GitHub-hosted runners, which cost nothing for a public +repository. ``package`` and ``KUnit`` use the repository variable +``KERNEL_CI_RUNNER`` when it is set: a JSON ``runs-on`` value such as +``["namespace-profile-arm64-linux-apple-12x32"]``, the machine omarchy-pkgs +builds kernels on. Without it they use ``ubuntu-24.04-arm``, where the +package build takes about 45 minutes. + +Repository setup +================ + +These need an administrator: + +- a ``ci:build`` label; +- branch protection for ``integration/aurora`` requiring ``ci / result``; +- the merge queue (merge method: merge commit), so the tree that merges is + the tree that was tested; +- workflow approval for outside collaborators (Settings, Actions, General); +- optionally ``KERNEL_CI_RUNNER`` for faster builds. + +Running the tools locally +========================= + +All tools are Python 3.10+ standard library scripts; run them with +``python3 -I`` from the top of the tree. ``python3 -I -m unittest discover -s +tools/aurora-ci/tests`` runs the unit tests. + +Build a package the way CI does (needs Docker and an omarchy-pkgs checkout):: + + python3 -I tools/aurora-ci/prepare-recipe.py \ + --recipe ../omarchy-pkgs/pkgbuilds/linux-aurora --kernel . \ + --commit "$(git rev-parse HEAD)" --revision 1 + (cd ../omarchy-pkgs && bin/build --mirror edge --arch aarch64 --package linux-aurora) + +Check and compare packages (any directory holding the ``linux-aurora`` and +``linux-aurora-headers`` ``.pkg.tar.zst`` files):: + + python3 -I tools/aurora-ci/inspect-package.py --packages new/ --output new.json --unpacked root/ + python3 -I tools/aurora-ci/compare-packages.py --base old.json --head new.json \ + --retired tools/aurora-ci/retired-dtbs.txt + +Boot it (needs ``qemu-system-aarch64`` and a static aarch64 busybox):: + + python3 -I tools/aurora-ci/boot-test.py --root root/ --busybox /bin/busybox \ + --expected tools/aurora-ci/boot-expected.json --out boot/ + +Run KUnit (``--install-toolchain`` sets up Ubuntu; elsewhere, install LLVM +18, Rust 1.85.0 with rust-src and bindgen-cli 0.71.1):: + + python3 -I tools/aurora-ci/run-kunit.py --output ../kunit-build + +Run the host suites, or one group or suite:: + + python3 -I tools/aurora-ci/host-tests.py --output ../host-tests --jobs 4 + python3 -I tools/aurora-ci/host-tests.py --output ../host-tests --group tools + python3 -I tools/aurora-ci/host-tests.py --list + +Check a range of commits the way a pull request is checked:: + + python3 -I tools/aurora-ci/check-commits.py --base origin/integration/aurora --head HEAD + python3 -I tools/aurora-ci/compile-check.py --tree . --base origin/integration/aurora \ + --head HEAD --config ../omarchy-pkgs/pkgbuilds/linux-aurora/config \ + --fragment arch/arm64/configs/asahi.config --out ../compile --per-commit --llvm=-18 + python3 -I tools/aurora-ci/dt-check.py --tree . --base origin/integration/aurora \ + --head HEAD --out ../dt-check + +Verifying a package +=================== + +Packages built for a commit on ``integration/aurora`` carry a build +provenance attestation:: + + gh attestation verify linux-aurora--aarch64.pkg.tar.zst --repo omacom/linux-aurora diff --git a/tools/aurora-ci/aurora.kunitconfig b/tools/aurora-ci/aurora.kunitconfig new file mode 100644 index 0000000000000..13805653b4b2d --- /dev/null +++ b/tools/aurora-ci/aurora.kunitconfig @@ -0,0 +1,205 @@ +# Combined KUnit configuration for the Apple, Asahi and Aurora suites in +# omacom/linux-aurora (integration/aurora). +# +# Run with tools/testing/kunit/kunit.py run --arch=arm64 (QEMU virt, -cpu max) +# and --make_options LLVM=... . Historical result at 1b4997bbab62: +# 98 suites / 1399 cases (expected: 3 skips, 0 failures). +# Needs a Rust toolchain: the SEP, AOP, T8140 audio and Asahi GPU code is Rust. +# The kernel uses 16K pages and 48-bit VA, as the shipped kernel does: +# PCIE_APPLE (needed by USB4_APPLE_SOC and MT7932) and DRM_ASAHI depend on +# PAGE_SIZE_16KB, and one DART case skips with smaller pages. +# Every option below must survive olddefconfig; kunit.py fails the build if a +# listed option does not end up in .config. + +CONFIG_KUNIT=y + +# Platform +CONFIG_ARCH_APPLE=y +# CONFIG_ARM64_4K_PAGES is not set +CONFIG_ARM64_16K_PAGES=y +CONFIG_ARM64_VA_BITS_48=y +CONFIG_PM=y +CONFIG_OF=y +CONFIG_OF_OVERLAY=y +CONFIG_RUST=y + +# Rust kernel crate: doctests and unit tests (rust/kernel is used by the +# Apple Rust drivers; the fork changes kvec, xarray, allocator and bitfield) +CONFIG_RUST_KERNEL_DOCTESTS=y +CONFIG_RUST_KUNIT_TESTS=y +CONFIG_RUST_ALLOCATOR_KUNIT_TEST=y +CONFIG_RUST_KVEC_KUNIT_TEST=y +CONFIG_RUST_BITMAP_KUNIT_TEST=y +CONFIG_RUST_KUNIT_SELFTEST=y +CONFIG_RUST_STR_KUNIT_TEST=y +CONFIG_RUST_ATOMICS_KUNIT_TEST=y +CONFIG_RUST_BITFIELD_KUNIT_TEST=y + +# Apple SoC core (mailbox, RTKit, PMP; APPLE_PMP selects the Rust RTKit glue) +CONFIG_APPLE_MAILBOX=y +CONFIG_APPLE_RTKIT=y +CONFIG_APPLE_PMP=y +CONFIG_APPLE_AOP=y + +# Apple MCC (T8140 system level cache) +CONFIG_APPLE_MCC=y +CONFIG_APPLE_MCC_KUNIT_TEST=y + +# Apple SEP (Rust) +CONFIG_SPI=y +CONFIG_HW_RANDOM=y +CONFIG_KEYS=y +CONFIG_TRUSTED_KEYS=y +CONFIG_CRYPTO=y +CONFIG_APPLE_SEP=y +CONFIG_APPLE_SEP_KUNIT_TEST=y + +# Apple display gate (T6030/T6031/T8122) +CONFIG_APPLE_T6030_DISPLAY_GATE=y +CONFIG_APPLE_DISPLAY_GATE_KUNIT_TEST=y + +# Apple PMC / PMP report power domains +CONFIG_APPLE_PMC=y +CONFIG_APPLE_PMC_KUNIT_TEST=y +CONFIG_APPLE_PMP_REPORT=y +CONFIG_APPLE_PMP_REPORT_KUNIT_TEST=y +CONFIG_APPLE_PMP_REPORT_START_KUNIT_TEST=y + +# Apple DART IOMMU +CONFIG_IOMMU_SUPPORT=y +CONFIG_APPLE_DART=y +CONFIG_APPLE_DART_KUNIT_TEST=y + +# Apple display controller (DCP) and display fabric +CONFIG_DRM=y +CONFIG_BACKLIGHT_CLASS_DEVICE=y +CONFIG_DRM_APPLE=y +CONFIG_DRM_APPLE_KUNIT_TEST=y +CONFIG_DRM_APPLE_PARSER_KUNIT_TEST=y +CONFIG_DRM_APPLE_DISPLAY_KUNIT_TEST=y +CONFIG_DRM_APPLE_FABRIC_KUNIT_TEST=y + +# Asahi GPU (Rust); enables RUST_DRM_GEM_SHMEM_HELPER for the +# rust_drm_gem_shmem suite in rust/kernel/drm/gem/shmem.rs +CONFIG_DRM_ASAHI=y + +# Apple Neural Engine +CONFIG_DRM_ACCEL=y +CONFIG_DRM_ACCEL_ANE=y +CONFIG_DRM_ACCEL_ANE_KUNIT_TEST=y + +# Apple video decoder and ISP +CONFIG_MEDIA_SUPPORT=y +CONFIG_MEDIA_PLATFORM_SUPPORT=y +CONFIG_MEDIA_CONTROLLER=y +CONFIG_VIDEO_DEV=y +CONFIG_V4L_PLATFORM_DRIVERS=y +CONFIG_VIDEO_APPLE_AVD=y +CONFIG_VIDEO_APPLE_AVD_KUNIT_TEST=y +CONFIG_VIDEO_APPLE_ISP=y +CONFIG_VIDEO_APPLE_ISP_KUNIT_TEST=y +CONFIG_VIDEO_APPLE_ISP_IPC_LAYOUT_KUNIT_TEST=y + +# PCIe, USB4/Thunderbolt +CONFIG_PCI=y +CONFIG_PCI_MSI=y +CONFIG_PCIEASPM=y +CONFIG_PCIE_APPLE=y +CONFIG_PCI_SYSFS_KUNIT_TEST=y +CONFIG_USB4=y +CONFIG_USB4_KUNIT_TEST=y +CONFIG_USB4_APPLE_SOC=y +CONFIG_USB4_APPLE_DPIN_KUNIT_TEST=y + +# Type-C: mux lookup (DT overlay), CD321x (tipd), SN201202x, ATC PHY +CONFIG_USB_SUPPORT=y +CONFIG_TYPEC=y +CONFIG_TYPEC_MUX_KUNIT_TEST=y +CONFIG_I2C=y +CONFIG_TYPEC_TPS6598X=y +CONFIG_TYPEC_TIPD_KUNIT_TEST=y +CONFIG_SPMI=y +CONFIG_TYPEC_SN201202X=y +CONFIG_TYPEC_SN201202X_KUNIT_TEST=y +CONFIG_PHY_APPLE_ATC=y +CONFIG_PHY_APPLE_ATC_KUNIT_TEST=y + +# USB core, xHCI platform, DWC3 and the Apple DWC3 glue +CONFIG_USB=y +CONFIG_USB_XHCI_HCD=y +CONFIG_USB_XHCI_PLATFORM=y +CONFIG_USB_DWC3=y +CONFIG_USB_DWC3_HOST=y +CONFIG_USB_DWC3_APPLE=y +CONFIG_USB_PORT_POWER_KUNIT_TEST=y +CONFIG_USB_QUIRKS_KUNIT_TEST=y +CONFIG_USB_WAKEUP_KUNIT_TEST=y +CONFIG_USB_XHCI_PLATFORM_KUNIT_TEST=y +CONFIG_USB_DWC3_HOST_KUNIT_TEST=y +CONFIG_USB_DWC3_APPLE_KUNIT_TEST=y + +# Apple SMC: core, battery, hwmon +CONFIG_MFD_MACSMC=y +CONFIG_MFD_MACSMC_KUNIT_TEST=y +CONFIG_POWER_SUPPLY=y +CONFIG_MACSMC_POWER=y +CONFIG_MACSMC_POWER_KUNIT_TEST=y +CONFIG_HWMON=y +CONFIG_SENSORS_MACSMC_HWMON=y +CONFIG_SENSORS_MACSMC_HWMON_KUNIT_TEST=y + +# DockChannel HID (keyboard/trackpad transport) +CONFIG_INPUT=y +CONFIG_HID=y +CONFIG_APPLE_DOCKCHANNEL=y +CONFIG_HID_DOCKCHANNEL=y +CONFIG_HID_DOCKCHANNEL_KUNIT_TEST=y +CONFIG_HID_DOCKCHANNEL_PARSER_KUNIT_TEST=y + +# Apple QSPI boot NOR write window +CONFIG_SPI_MEM=y +CONFIG_SPI_APPLE_QSPI=y +CONFIG_SPI_APPLE_QSPI_KUNIT_TEST=y + +# T8140 AOP audio (Rust) +CONFIG_SOUND=y +CONFIG_SND=y +CONFIG_SND_SOC=y +CONFIG_DMADEVICES=y +CONFIG_DMA_ENGINE=y +CONFIG_SND_SOC_APPLE_T8140_AOP_AUDIO=y + +# Cirrus jack codecs: clock switching and playback resume/prepare recovery +# I2C is enabled above; build the codecs in with their real KUnit suites. +CONFIG_SND_SOC_CS42L42=y +CONFIG_SND_SOC_CS42L42_CORE=y +CONFIG_SND_SOC_CS42L42_TEST=y +CONFIG_SND_SOC_CS42L84=y +CONFIG_SND_SOC_CS42L84_TEST=y + +# Wireless and Bluetooth: brcmfmac scan timeout, MT7932 (J700) +CONFIG_NET=y +CONFIG_NETDEVICES=y +CONFIG_WIRELESS=y +CONFIG_CFG80211=y +CONFIG_WLAN=y +CONFIG_WLAN_VENDOR_BROADCOM=y +CONFIG_BRCMFMAC=y +CONFIG_BRCMFMAC_PCIE=y +CONFIG_BRCMFMAC_KUNIT_TEST=y +CONFIG_WLAN_VENDOR_MEDIATEK=y +CONFIG_PCIE_APPLE_PIODMA_DIAG=y +CONFIG_MT7932_FULLMAC=y +CONFIG_MT7932_KUNIT_TEST=y +CONFIG_MT7932_RF_KUNIT_TEST=y +CONFIG_BT=y +CONFIG_BT_MTK7932_PCIE=y +CONFIG_BT_MTK7932_PCIE_KUNIT_TEST=y + +# kexec blockers, trusted-key source registration, long symbol names +CONFIG_KEXEC=y +CONFIG_KEXEC_FILE=y +CONFIG_KEXEC_BLOCK_KUNIT_TEST=y +CONFIG_TRUSTED_SOURCE_KUNIT_TEST=y +CONFIG_KPROBES=y +CONFIG_LONGEST_SYM_KUNIT_TEST=y diff --git a/tools/aurora-ci/boot-expected.json b/tools/aurora-ci/boot-expected.json new file mode 100644 index 0000000000000..a4036b57e482d --- /dev/null +++ b/tools/aurora-ci/boot-expected.json @@ -0,0 +1,38 @@ +{ + "comment": "Known results of boot-test.py in a QEMU virt guest with no Apple hardware. Every entry needs a reason; remove an entry when the module loads cleanly (the test lists those).", + "skip": { + "kernel/kernel/rcu/rcutorture.ko": "starts an RCU torture run that keeps going until the module is unloaded", + "kernel/sound/drivers/snd-mtpav.ko": "claims a fixed legacy MIDI port and interrupt; request_threaded_irq() warns on arm64 machines that have no such interrupt" + }, + "insmod_errors": { + "kernel/drivers/usb/atm/xusbatm.ko": { + "error": "Invalid argument", + "reason": "refuses to load without its vendor and product module parameters" + }, + "kernel/drivers/ptp/ptp_kvm.ko": { + "error": "Operation not supported", + "reason": "needs the KVM PTP clock service, which the test guest does not offer" + }, + "kernel/net/6lowpan/nhc_ghc_ext_dest.ko": { + "error": "Device or resource busy", + "reason": "RFC 7400 compression registers the same next-header ID as the RFC 6282 module loaded before it" + }, + "kernel/net/6lowpan/nhc_ghc_ext_frag.ko": { + "error": "Device or resource busy", + "reason": "RFC 7400 compression registers the same next-header ID as the RFC 6282 module loaded before it" + }, + "kernel/net/6lowpan/nhc_ghc_ext_hop.ko": { + "error": "Device or resource busy", + "reason": "RFC 7400 compression registers the same next-header ID as the RFC 6282 module loaded before it" + }, + "kernel/net/6lowpan/nhc_ghc_ext_route.ko": { + "error": "Device or resource busy", + "reason": "RFC 7400 compression registers the same next-header ID as the RFC 6282 module loaded before it" + }, + "kernel/net/6lowpan/nhc_ghc_udp.ko": { + "error": "Device or resource busy", + "reason": "RFC 7400 compression registers the same next-header ID as the RFC 6282 module loaded before it" + } + }, + "dmesg_allow": [] +} diff --git a/tools/aurora-ci/boot-test.py b/tools/aurora-ci/boot-test.py new file mode 100755 index 0000000000000..bc0de8550d4ff --- /dev/null +++ b/tools/aurora-ci/boot-test.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Boot the packaged kernel in a QEMU virt machine and load its modules. + +This is the shipped kernel, not a test build: the Image and modules come from +the unpacked linux-aurora package. The guest runs a busybox initramfs that +loads every module the package ships (dependencies first, as modules.dep +orders them) and then prints the kernel log. The test fails when: + + - the kernel does not reach userspace, or reports a different release; + - the log shows a crash or warning splat (WARNING:, BUG:, Oops, a panic, + UBSAN/KASAN/KFENCE reports, list or refcount corruption, stalls); + - the kernel taints itself with W, D, B or L; + - a module fails to load for any reason other than missing hardware + (ENODEV/ENXIO), unless boot-expected.json lists that exact error. + +The VM has none of the Apple hardware, so drivers register and wait without +probing. That exercises symbol resolution, module init and exit paths, and +built-in initcalls on every boot; driver behaviour needs real Macs. +""" + +import argparse +import json +import lzma +import gzip +import os +import platform +import re +import shutil +import stat +import subprocess +import sys +import time +from pathlib import Path + +NO_DEVICE = {"No such device", "No such device or address"} +SPLAT = re.compile( + r"(WARNING:|BUG:|Oops|Unable to handle kernel|Internal error:|Kernel panic|" + r"kernel BUG at|UBSAN:|KASAN:|KFENCE:|refcount_t:|list_add corruption|list_del corruption|" + r"detected stall|soft lockup|hard LOCKUP|blocked for more than|Unknown symbol|" + r"disagrees about version of symbol|possible circular locking|possible recursive locking)") +# Taint flags that mean the kernel hit a problem (Documentation/admin-guide/tainted-kernels.rst). +BAD_TAINT = {5: "B (bad page)", 7: "D (died: oops or BUG)", 9: "W (warning)", 14: "L (soft lockup)"} +MARK = "AURORA-CI" + +INIT = """#!/bin/busybox sh +/bin/busybox --install -s /bin +mount -t proc proc /proc +mount -t sysfs sysfs /sys +mount -t devtmpfs devtmpfs /dev 2>/dev/null +echo "{mark}-BOOT kernel=$(uname -r)" +while read -r module; do + if out=$(insmod "/lib/modules/{kver}/$module" 2>&1); then + echo "{mark}-INSMOD ok $module" + else + echo "{mark}-INSMOD fail $module ${{out##*: }}" + fi +done < /load-order +echo "{mark}-TAINT $(cat /proc/sys/kernel/tainted)" +echo "{mark}-LOADED $(wc -l < /proc/modules)" +echo "{mark}-DMESG-BEGIN" +dmesg +echo "{mark}-DMESG-END" +echo "{mark}-DONE" +poweroff -f +""" + + +class CpioWriter: + """Minimal newc (SVR4, no CRC) archive writer for an initramfs.""" + + def __init__(self, stream): + self.stream, self.ino = stream, 0 + + def _entry(self, name, mode, data=b"", rdev=(0, 0)): + self.ino += 1 + encoded = name.encode() + b"\0" + fields = (self.ino, mode, 0, 0, 1, 0, len(data), 0, 0, rdev[0], rdev[1], len(encoded), 0) + self.stream.write(b"070701" + b"".join(b"%08x" % f for f in fields) + encoded) + self._pad(110 + len(encoded)) + self.stream.write(data) + self._pad(len(data)) + + def _pad(self, length): + self.stream.write(b"\0" * (-length % 4)) + + def directory(self, name): + self._entry(name, stat.S_IFDIR | 0o755) + + def file(self, name, data, mode=0o644): + self._entry(name, stat.S_IFREG | mode, data) + + def char_device(self, name, major, minor): + self._entry(name, stat.S_IFCHR | 0o600, rdev=(major, minor)) + + def close(self): + self._entry("TRAILER!!!", 0) + + +def read_module(path): + data = path.read_bytes() + if path.suffix == ".zst": + return subprocess.run(["zstd", "-dc", str(path)], check=True, stdout=subprocess.PIPE).stdout + if path.suffix == ".xz": + return lzma.decompress(data) + if path.suffix == ".gz": + return gzip.decompress(data) + return data + + +def load_order(moddir, skip): + """Modules in modules.order, each after its dependencies; drops skipped + modules and every module that needs one of them.""" + deps = {} + for line in (moddir / "modules.dep").read_text().splitlines(): + module, _, rest = line.partition(":") + deps[module] = rest.split() + order = [m for m in (moddir / "modules.order").read_text().split() if m in deps] + order += sorted(set(deps) - set(order)) + + skipped = {m: f"listed in boot-expected.json: {skip[m]}" for m in skip if m in deps} + for module, needs in deps.items(): + blocked = [d for d in needs if d in skip] + if blocked and module not in skipped: + skipped[module] = f"needs skipped module {blocked[0]}" + + result, seen = [], set() + + def visit(module): + if module in seen or module in skipped: + return + seen.add(module) + for dependency in reversed(deps.get(module, [])): + visit(dependency) + result.append(module) + + for module in order: + visit(module) + return result, skipped + + +def build_initramfs(root, busybox, output, skip): + moddir_list = [p for p in (root / "usr/lib/modules").iterdir() if p.is_dir()] + if len(moddir_list) != 1: + raise RuntimeError("expected one kernel release in the unpacked package") + moddir = moddir_list[0] + kver = moddir.name + order, skipped = load_order(moddir, skip) + + with open(output, "wb") as stream: + cpio = CpioWriter(stream) + for directory in ("bin", "dev", "proc", "sys", "tmp", "lib", "lib/modules", f"lib/modules/{kver}"): + cpio.directory(directory) + cpio.char_device("dev/console", 5, 1) + cpio.file("bin/busybox", Path(busybox).read_bytes(), 0o755) + cpio.file("init", INIT.format(mark=MARK, kver=kver).encode(), 0o755) + cpio.file("load-order", ("\n".join(order) + "\n").encode()) + for name in sorted(p.name for p in moddir.glob("modules.*")): + cpio.file(f"lib/modules/{kver}/{name}", (moddir / name).read_bytes()) + made = set() + for module in order: + parent = str(Path(module).parent) + for depth in range(1, len(Path(parent).parts) + 1): + directory = str(Path(*Path(parent).parts[:depth])) + if directory not in made: + cpio.directory(f"lib/modules/{kver}/{directory}") + made.add(directory) + cpio.file(f"lib/modules/{kver}/{module}", read_module(moddir / module)) + cpio.close() + return kver, moddir / "vmlinuz", order, skipped + + +def kvm_usable(): + """KVM runs an arm64 guest only on an arm64 host.""" + return platform.machine() in ("aarch64", "arm64") and os.access("/dev/kvm", os.R_OK | os.W_OK) + + +def cpu_model(accel, requested): + """The -cpu value: the host CPU under KVM, and under TCG a fixed model + rather than "max". "max" turns on every feature the QEMU version + emulates, which on QEMU 8 means pointer authentication slow enough to + stall boot and page clearing that faults; a named model is the same on + every QEMU. Neoverse N1 has the 16K granule this kernel needs.""" + if requested: + return requested + return "neoverse-n1" if accel == "tcg" else "host" + + +def run_qemu(image, initramfs, log, accel, memory, cpus, timeout, cpu=None): + command = [ + "qemu-system-aarch64", "-M", "virt,gic-version=max", "-cpu", cpu_model(accel, cpu), "-accel", accel, + "-m", str(memory), "-smp", str(cpus), "-nodefaults", "-display", "none", + "-serial", f"file:{log}", "-no-reboot", + "-kernel", str(image), "-initrd", str(initramfs), + "-append", "console=ttyAMA0 earlycon rdinit=/init panic=-1 loglevel=4 log_buf_len=16M", + ] + started = time.monotonic() + try: + result = subprocess.run(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, timeout=timeout) + return result.returncode, result.stdout, time.monotonic() - started, False + except subprocess.TimeoutExpired as expired: + output = expired.stdout or "" + return None, output if isinstance(output, str) else output.decode(), timeout, True + + +def parse(log_text, kver, expected): + report = {"booted": False, "release": None, "done": False, "taint": None, "loaded": None, + "log_complete": False, + "modules": {"ok": [], "no_device": [], "expected": [], "unexpected": []}, + "splats": [], "failures": [], "stale_expectations": []} + errors = expected.get("insmod_errors", {}) + allowed = [(re.compile(a["pattern"]), a["reason"]) for a in expected.get("dmesg_allow", [])] + in_dmesg = first_dmesg_line = False + for line in log_text.splitlines(): + if f"{MARK}-BOOT kernel=" in line: + report["booted"] = True + report["release"] = line.split("kernel=", 1)[1].strip() + elif f"{MARK}-INSMOD " in line: + fields = line.split(f"{MARK}-INSMOD ", 1)[1].split(" ", 2) + status, module = fields[0], fields[1] + error = fields[2].strip() if len(fields) > 2 else "" + if status == "ok": + report["modules"]["ok"].append(module) + elif error in NO_DEVICE: + report["modules"]["no_device"].append(module) + elif errors.get(module, {}).get("error") == error: + report["modules"]["expected"].append(module) + else: + report["modules"]["unexpected"].append({"module": module, "error": error}) + elif f"{MARK}-TAINT " in line: + report["taint"] = int(line.split(f"{MARK}-TAINT ", 1)[1].split()[0]) + elif f"{MARK}-LOADED " in line: + report["loaded"] = int(line.split(f"{MARK}-LOADED ", 1)[1].split()[0]) + elif f"{MARK}-DMESG-BEGIN" in line: + in_dmesg = True + first_dmesg_line = True + continue + elif f"{MARK}-DMESG-END" in line: + in_dmesg = False + elif f"{MARK}-DONE" in line: + report["done"] = True + if in_dmesg and first_dmesg_line: + # A log that still starts at boot has not wrapped, so no early + # splat can have been pushed out of it. + report["log_complete"] = "Booting Linux" in line + first_dmesg_line = False + if (in_dmesg or not report["done"]) and SPLAT.search(line) and MARK not in line: + if not any(pattern.search(line) for pattern, _ in allowed): + report["splats"].append(line.strip()) + report["splats"] = list(dict.fromkeys(report["splats"])) + + if not report["booted"]: + report["failures"].append("the kernel did not reach userspace") + elif report["release"] != kver: + report["failures"].append(f"guest reports kernel {report['release']}, package ships {kver}") + if report["booted"] and not report["done"]: + report["failures"].append("the guest stopped before finishing the module run") + elif report["done"] and not report["log_complete"]: + report["failures"].append("the kernel log wrapped before it was read; raise log_buf_len") + for item in report["modules"]["unexpected"]: + report["failures"].append(f"{item['module']} failed to load: {item['error']}") + for splat in report["splats"]: + report["failures"].append(f"kernel log: {splat}") + if report["taint"]: + bad = [name for bit, name in BAD_TAINT.items() if report["taint"] & (1 << bit)] + if bad: + report["failures"].append(f"kernel tainted {report['taint']}: {', '.join(bad)}") + loaded = set(report["modules"]["ok"]) + report["stale_expectations"] = sorted(m for m in errors if m in loaded) + return report + + +def summary(report, meta): + mods = report["modules"] + lines = ["### Boot test (QEMU virt, shipped kernel)", "", + f"Accelerator `{meta['accel']}` (`-cpu {meta['cpu']}`), {meta['seconds']:.0f} s, " + f"kernel `{meta['kver']}`.", "", + f"Modules: {len(mods['ok'])} loaded, {len(mods['no_device'])} found no device, " + f"{len(mods['expected'])} failed as expected, {len(mods['unexpected'])} failed, " + f"{len(meta['skipped'])} skipped. Taint {report['taint']}.", ""] + if report["failures"]: + lines += ["**Failures**", ""] + [f"- {f}" for f in report["failures"][:60]] + [""] + if report["stale_expectations"]: + lines += ["Now load cleanly (remove from boot-expected.json): " + + ", ".join(f"`{m}`" for m in report["stale_expectations"]), ""] + if meta["skipped"]: + lines += ["
Skipped modules", ""] + lines += [f"- `{m}`: {why}" for m, why in sorted(meta["skipped"].items())] + ["", "
", ""] + if mods["no_device"]: + lines += ["
No device in the VM", ""] + lines += [f"- `{m}`" for m in mods["no_device"]] + ["", "
", ""] + return "\n".join(lines) + "\n" + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--root", type=Path, required=True, help="unpacked linux-aurora package") + parser.add_argument("--busybox", type=Path, required=True, help="static aarch64 busybox") + parser.add_argument("--expected", type=Path, required=True, help="boot-expected.json") + parser.add_argument("--out", type=Path, required=True, help="directory for the log and report") + parser.add_argument("--accel", choices=("auto", "kvm", "tcg"), default="auto") + parser.add_argument("--memory", type=int, default=4096, help="guest memory in MiB") + parser.add_argument("--cpus", type=int, default=min(4, os.cpu_count() or 1)) + parser.add_argument("--timeout", type=int, default=1200, help="seconds before the guest is killed") + parser.add_argument("--cpu", help="QEMU -cpu value (default: host under KVM, neoverse-n1 under TCG)") + args = parser.parse_args(argv) + + args.out.mkdir(parents=True, exist_ok=True) + expected = json.loads(args.expected.read_text()) + initramfs = args.out / "initramfs.cpio" + kver, image, order, skipped = build_initramfs(args.root, args.busybox, initramfs, expected.get("skip", {})) + + accels = {"auto": ["kvm", "tcg"] if kvm_usable() else ["tcg"], "kvm": ["kvm"], "tcg": ["tcg"]}[args.accel] + log = args.out / "console.log" + for accel in accels: + log.unlink(missing_ok=True) + status, qemu_output, seconds, timed_out = run_qemu(image, initramfs, log, accel, args.memory, + args.cpus, args.timeout, args.cpu) + log_text = log.read_text(errors="replace") if log.exists() else "" + # A QEMU that cannot start with KVM exits before the guest prints anything. + if accel == "kvm" and not log_text.strip() and status not in (0, None): + print(f"KVM start failed ({qemu_output.strip()[:300]}); retrying with TCG", file=sys.stderr) + continue + break + initramfs.unlink() + + report = parse(log_text, kver, expected) + if timed_out: + report["failures"].insert(0, f"the guest did not power off within {args.timeout} s") + elif status != 0: + report["failures"].insert(0, f"QEMU exited with status {status}: {qemu_output.strip()[:300]}") + meta = {"kver": kver, "accel": accel, "cpu": cpu_model(accel, args.cpu), "seconds": seconds, + "skipped": skipped, "planned": len(order)} + (args.out / "report.json").write_text(json.dumps(dict(report, **meta), indent=1) + "\n") + + text = summary(report, meta) + print(text, end="") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(text) + for failure in report["failures"][:50]: + print(f"::error title=Boot test::{failure}", file=sys.stderr) + if report["failures"] and not report["booted"]: + print("Last console lines:\n" + "\n".join(log_text.splitlines()[-40:]), file=sys.stderr) + return 1 if report["failures"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/check-commits.py b/tools/aurora-ci/check-commits.py new file mode 100755 index 0000000000000..83c8864ffce83 --- /dev/null +++ b/tools/aurora-ci/check-commits.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Check the commits a pull request or merge group adds. + +For every non-merge commit in base..head: + + Signed-off-by at least one trailer is required (blocking). Carried + commits keep their author and gain the carrier's sign-off, + so the author's own sign-off is not required here. + checkpatch scripts/checkpatch.pl findings, reported for review + (advisory: the job does not fail on them). + +and for the range as a whole: + + UAPI a change under include/uapi/ is flagged: userspace that + builds against those headers (Mesa for the GPU, for one) + needs a matching update before the kernel ships. + +Ranges of more than --max-commits commits (rebases onto a new upstream, +merges of other branches) are reported but never fail: their commits come +from upstream trees with their own rules. + +The checkout may be shallow; history is fetched until base and head meet. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +SIGNED_OFF = re.compile(r"^Signed-off-by:\s*\S.*$", re.M) +CHECKPATCH_LINE = re.compile(r"^(?P[^:\s]+):(?P\d+): (?PERROR|WARNING|CHECK):(?P\w+): (?P.*)$") +DEPTHS = (64, 256, 1024, 4096) + + +def git(repo, *args, check=True, text=True): + return subprocess.run(["git", "-C", str(repo), *args], check=check, text=text, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + +def range_history_complete(repo, base, head): + if git(repo, "merge-base", base, head, check=False).returncode != 0: + return False + shallow = Path(git(repo, "rev-parse", "--path-format=absolute", + "--git-path", "shallow").stdout.strip()) + if not shallow.exists(): + return True + # A recent merge can expose base while an older topic parent is still + # shallow. Such a boundary looks like a root commit to format-patch. + boundaries = set(shallow.read_text().split()) + added = set(git(repo, "rev-list", f"{base}..{head}").stdout.split()) + return boundaries.isdisjoint(added) + + +def ensure_history(repo, remote, base, head): + """Fetch base and head, deepening until they share history. + + Also require every commit added by the range to have its parents. + Returns False if either condition remains unmet at the deepest fetch.""" + for depth in DEPTHS: + if range_history_complete(repo, base, head): + return True + git(repo, "fetch", "--quiet", "--no-tags", f"--depth={depth}", remote, base, head) + return range_history_complete(repo, base, head) + + +def commits_in(repo, base, head): + out = git(repo, "rev-list", "--reverse", "--no-merges", f"{base}..{head}").stdout + return out.split() + + +def describe(repo, sha): + fields = git(repo, "show", "-s", "--format=%an <%ae>%x00%s%x00%B", sha).stdout.split("\x00", 2) + return {"sha": sha, "author": fields[0], "subject": fields[1], "message": fields[2]} + + +def run_checkpatch(repo, sha): + """checkpatch.pl findings for one commit, as dicts.""" + # A patch may contain non-UTF-8 source text. Preserve its bytes for perl; + # only diagnostic output is decoded for the report. + patch = git(repo, "format-patch", "-1", "--stdout", sha, text=False).stdout + result = subprocess.run( + ["perl", "scripts/checkpatch.pl", "--terse", "--showfile", "--show-types", "--no-summary", + "--color=never", "-"], cwd=repo, input=patch, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + findings = [] + for line in result.stdout.decode("utf-8", errors="replace").splitlines(): + match = CHECKPATCH_LINE.match(line.strip()) + if match: + findings.append(match.groupdict()) + return findings + + +def changed_uapi(repo, base, head): + out = git(repo, "diff", "--name-only", f"{base}...{head}", "--", "include/uapi").stdout + return out.split() + + +def escape(text): + return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def check(repo, remote, base, head, max_commits, checkpatch=True): + report = {"base": base, "head": head, "commits": [], "uapi": [], "failures": [], + "mode": "full", "notes": []} + if not ensure_history(repo, remote, base, head): + report["mode"] = "unrelated" + report["notes"].append(f"Complete added history for {base[:12]}..{head[:12]} " + f"was not established within {DEPTHS[-1]} commits; " + "commit checks skipped") + return report + + shas = commits_in(repo, base, head) + report["uapi"] = changed_uapi(repo, base, head) + if len(shas) > max_commits: + report["mode"] = "integration" + report["notes"].append(f"{len(shas)} commits is integration-sized (over {max_commits}); " + "sign-offs are reported, not enforced, and checkpatch is skipped") + for sha in shas: + entry = describe(repo, sha) + entry["signed_off"] = bool(SIGNED_OFF.search(entry["message"])) + del entry["message"] + entry["checkpatch"] = run_checkpatch(repo, sha) if checkpatch and report["mode"] == "full" else [] + report["commits"].append(entry) + if not entry["signed_off"] and report["mode"] == "full": + report["failures"].append(f"{sha[:12]} (\"{entry['subject']}\") has no Signed-off-by trailer") + return report + + +def summary(report): + lines = ["### Commit checks", ""] + commits = report["commits"] + lines.append(f"{len(commits)} commits in `{report['base'][:12]}..{report['head'][:12]}`" + + (f" ({report['mode']} range)" if report["mode"] != "full" else "") + ".") + lines += [""] + [f"- {note}" for note in report["notes"]] + [""] + if report["uapi"]: + lines += ["**UAPI headers changed** — userspace built against them needs a matching update " + "before this kernel ships:", ""] + [f"- `{path}`" for path in report["uapi"]] + [""] + if commits: + lines += ["| Commit | Subject | Signed-off-by | checkpatch |", "|---|---|---|---|"] + for c in commits[:200]: + counts = {level: sum(1 for f in c["checkpatch"] if f["level"] == level) + for level in ("ERROR", "WARNING", "CHECK")} + cp = ", ".join(f"{n} {level.lower()}" for level, n in counts.items() if n) or "clean" + if report["mode"] != "full": + cp = "skipped" + subject = c["subject"].replace("|", "\\|")[:90] + lines.append(f"| `{c['sha'][:12]}` | {subject} | {'yes' if c['signed_off'] else '**no**'} | {cp} |") + if len(commits) > 200: + lines.append(f"| … {len(commits) - 200} more | | | |") + lines.append("") + if report["failures"]: + lines += ["**Failures**", ""] + [f"- {f}" for f in report["failures"]] + [""] + return "\n".join(lines) + "\n" + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--repo", type=Path, default=Path(".")) + parser.add_argument("--remote", default="origin") + parser.add_argument("--base", required=True) + parser.add_argument("--head", required=True) + parser.add_argument("--max-commits", type=int, default=100) + parser.add_argument("--no-checkpatch", action="store_true") + parser.add_argument("--report", type=Path, help="write the findings as JSON") + args = parser.parse_args(argv) + + try: + report = check(args.repo, args.remote, args.base, args.head, args.max_commits, + checkpatch=not args.no_checkpatch) + except subprocess.CalledProcessError as error: + stderr = error.stderr or "" + if isinstance(stderr, bytes): + stderr = stderr.decode("utf-8", errors="replace") + print(f"::error title=Commit checks::{' '.join(error.cmd)} failed: {escape(stderr)}", + file=sys.stderr) + return 1 + if args.report: + args.report.write_text(json.dumps(report, indent=1) + "\n") + + text = summary(report) + print(text, end="") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(text) + if report["uapi"]: + print("::warning title=UAPI headers changed::" + escape( + "This range changes " + ", ".join(report["uapi"]) + + ". Userspace built against these headers needs a matching update before the kernel ships.")) + for c in report["commits"]: + for f in c["checkpatch"]: + if f["level"] == "ERROR": + print(f"::warning file={f['file']},line={f['line']},title=checkpatch {f['type']} " + f"({c['sha'][:12]})::{escape(f['message'])}") + for failure in report["failures"]: + print(f"::error title=Signed-off-by missing::{escape(failure)}", file=sys.stderr) + return 1 if report["failures"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/compare-packages.py b/tools/aurora-ci/compare-packages.py new file mode 100755 index 0000000000000..bc812e4ead877 --- /dev/null +++ b/tools/aurora-ci/compare-packages.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Compare two linux-aurora package descriptions from inspect-package.py. + +Reports how a build differs from a baseline (the target branch, or a package +that already ships): resolved configuration, device trees, modules, image size +and package contents. Kernel releases differ between any two builds, so file +names are compared with the release replaced by a placeholder. + +It fails only for changes that strand installed Macs: + + - a device tree the baseline shipped is gone. The boot bundle picks a DTB by + name, so a Mac whose DTB disappears stops booting the new kernel. A + deliberate retirement is listed, with its reason, in retired-dtbs.txt in + the same change. + +Everything else (modules that became built-in or went away, configuration +changes) is reported for the reviewer. --expect-identical turns every +difference into a failure; it is how CI proves a build matches what ships. +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +LIST_LIMIT = 40 + + +def load(path): + meta = json.loads(Path(path).read_text()) + if meta.get("schema") != 1: + raise ValueError(f"{path}: unsupported metadata schema {meta.get('schema')!r}") + return meta + + +def retired_dtbs(path): + """Names listed in retired-dtbs.txt: '.dtb ' per line.""" + if not path or not Path(path).is_file(): + return {} + retired = {} + for line in Path(path).read_text().splitlines(): + line = line.split("#", 1)[0].strip() + if line: + name, _, reason = line.partition(" ") + retired[name] = reason.strip() + return retired + + +def diff_sets(base, head): + base, head = set(base), set(head) + return sorted(base - head), sorted(head - base) + + +def compare(base, head, retired): + result = {"failures": [], "notes": []} + + removed, added = diff_sets(base["dtbs"], head["dtbs"]) + changed = sorted(n for n in set(base["dtbs"]) & set(head["dtbs"]) if base["dtbs"][n] != head["dtbs"][n]) + stranded = [n for n in removed if n not in retired] + result["dtbs"] = {"removed": removed, "added": added, "changed": changed} + for name in stranded: + result["failures"].append( + f"device tree {name} is no longer built; Macs that boot it would be stranded " + "(list it in tools/aurora-ci/retired-dtbs.txt with a reason if this is deliberate)") + for name in removed: + if name in retired: + result["notes"].append(f"device tree {name} retired: {retired[name]}") + + removed, added = diff_sets(base["modules"], head["modules"]) + builtin = set(head.get("builtin", [])) + # modules.builtin names the .ko a built-in driver would have been; a + # shipped module may carry a compression suffix (.ko.zst). + became_builtin = [m for m in removed if re.sub(r"\.ko\.\w+$", ".ko", m) in builtin] + gone = [m for m in removed if m not in became_builtin] + result["modules"] = {"removed": gone, "became_builtin": became_builtin, "added": added} + + config_changes = [] + for key in sorted(set(base["config"]) | set(head["config"])): + old, new = base["config"].get(key, "absent"), head["config"].get(key, "absent") + if old != new: + config_changes.append({"symbol": key, "base": old, "head": new}) + result["config"] = config_changes + + removed, added = diff_sets(base["package"]["files"], head["package"]["files"]) + result["package_files"] = {"removed": removed, "added": added} + removed, added = diff_sets(base["headers"]["files"], head["headers"]["files"]) + result["headers_files"] = {"removed": removed, "added": added} + result["image"] = {"base": base["image"]["size"], "head": head["image"]["size"], + "page_size": [base["image"].get("page_size"), head["image"].get("page_size")]} + result["toolchain"] = {"removed": diff_sets(base["toolchain"], head["toolchain"])[0], + "added": diff_sets(base["toolchain"], head["toolchain"])[1]} + return result + + +def differences(result): + """Every difference, as text, for --expect-identical.""" + found = [] + for kind in ("removed", "added", "changed"): + found += [f"DTB {kind}: {n}" for n in result["dtbs"][kind]] + for kind in ("removed", "became_builtin", "added"): + found += [f"module {kind}: {n}" for n in result["modules"][kind]] + found += [f"config {c['symbol']}: {c['base']} -> {c['head']}" for c in result["config"]] + for section in ("package_files", "headers_files"): + for kind in ("removed", "added"): + found += [f"{section} {kind}: {n}" for n in result[section][kind]] + if result["image"]["page_size"][0] != result["image"]["page_size"][1]: + found.append(f"image page size: {result['image']['page_size']}") + return found + + +def bullet_list(items, limit=LIST_LIMIT): + lines = [f"- `{item}`" for item in items[:limit]] + if len(items) > limit: + lines.append(f"- … and {len(items) - limit} more (see the JSON report)") + return lines + + +def render(base, head, result, title): + lines = [f"### {title}", "", + f"`{base['package']['file']}` → `{head['package']['file']}`", ""] + delta = result["image"]["head"] - result["image"]["base"] + lines.append(f"Image: {result['image']['base']:,} → {result['image']['head']:,} bytes ({delta:+,}).") + dt = result["dtbs"] + lines.append(f"Device trees: {len(dt['removed'])} removed, {len(dt['added'])} added, " + f"{len(dt['changed'])} changed.") + mods = result["modules"] + lines.append(f"Modules: {len(mods['removed'])} removed, {len(mods['became_builtin'])} now built in, " + f"{len(mods['added'])} added.") + lines.append(f"Configuration: {len(result['config'])} symbols differ.") + lines.append("") + if result["failures"]: + lines += ["**Failures**", ""] + [f"- {f}" for f in result["failures"]] + [""] + for heading, items in (("Device trees removed", dt["removed"]), ("Device trees added", dt["added"]), + ("Device trees changed", dt["changed"]), + ("Modules removed", mods["removed"]), ("Modules now built in", mods["became_builtin"]), + ("Modules added", mods["added"])): + if items: + lines += [f"
{heading} ({len(items)})", ""] + bullet_list(items) + ["", "
", ""] + if result["config"]: + rows = [f"| `{c['symbol']}` | `{c['base']}` | `{c['head']}` |" for c in result["config"][:LIST_LIMIT]] + more = [f"| … {len(result['config']) - LIST_LIMIT} more | | |"] if len(result["config"]) > LIST_LIMIT else [] + lines += [f"
Configuration changes ({len(result['config'])})", "", + "| Symbol | Base | Head |", "|---|---|---|"] + rows + more + ["", "
", ""] + if result["toolchain"]["removed"] or result["toolchain"]["added"]: + lines += ["Toolchain changed: " + ", ".join(f"`-{p}`" for p in result["toolchain"]["removed"]) + + " " + ", ".join(f"`+{p}`" for p in result["toolchain"]["added"]), ""] + for note in result["notes"]: + lines.append(f"- {note}") + return "\n".join(lines) + "\n" + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--base", type=Path, required=True, help="baseline metadata JSON") + parser.add_argument("--head", type=Path, required=True, help="metadata JSON of the build under test") + parser.add_argument("--retired", type=Path, help="retired-dtbs.txt from the tree under test") + parser.add_argument("--report", type=Path, help="write the comparison as JSON") + parser.add_argument("--title", default="Compared with the target branch") + parser.add_argument("--expect-identical", action="store_true", + help="fail on any difference (release names aside)") + args = parser.parse_args(argv) + + try: + base, head = load(args.base), load(args.head) + except (OSError, ValueError) as error: + print(f"::error title=Package comparison::{error}", file=sys.stderr) + return 1 + result = compare(base, head, retired_dtbs(args.retired)) + if args.expect_identical: + result["failures"] += [f"differs from the baseline: {d}" for d in differences(result)] + if args.report: + args.report.write_text(json.dumps(result, indent=1) + "\n") + + text = render(base, head, result, args.title) + print(text, end="") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(text) + for failure in result["failures"][:50]: + print(f"::error title=Package comparison::{failure}", file=sys.stderr) + for name in result["modules"]["removed"][:20]: + print(f"::warning title=Module no longer built::{name}", file=sys.stderr) + return 1 if result["failures"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/compile-check.py b/tools/aurora-ci/compile-check.py new file mode 100755 index 0000000000000..e45d6c4ae55fb --- /dev/null +++ b/tools/aurora-ci/compile-check.py @@ -0,0 +1,1064 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Cheap pull-request compile check for an arm64 kernel tree. + +For the range BASE..HEAD this script: + +* compiles, with W=1, the kbuild objects of the C/assembly files the range + changes (plus a capped set of objects that include changed headers) at the + merge base and at HEAD, and reports W=1 warnings present at HEAD but not at + the merge base ("new" warnings); +* with --per-commit, also compiles the same object set at every non-merge + commit of the range, in order, in one incremental build directory, and + reports commits that do not compile on their own. + +Objects are compiled only when the configuration selects them. Selection is +read from kbuild itself (an injected makefile prints each visited directory's +object lists), because `make path/file.o` compiles an unselected file anyway, +with the wrong flags. + +Output: GitHub workflow annotations on stdout, report.json and summary.md in +--out. Exit status: 0 = no compile errors (warnings allowed), 1 = compile +errors introduced by the range, 2 = the check itself could not run. + +Python 3.10+ standard library only. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import resource +import shutil +import subprocess +import sys +import time +from collections import Counter +from pathlib import Path + +LISTER_MK = r""" +# Injected into kbuild sub-makes through MAKEFILES (see compile-check.py). +ifdef obj +$(obj)/: __aurora_list_objs +PHONY += __aurora_list_objs +__aurora_list_objs: + @$(if $(filter %/scripts/Makefile.build,$(MAKEFILE_LIST)),printf '%s\n' 'AURORA-OBJS|$(obj)|$(strip $(real-obj-y))|$(strip $(real-obj-m))|$(strip $(lib-y))|$(strip $(filter %.o,$(targets)))',:) +endif +""" + +SOURCE_EXTS = (".c", ".S") +HEADER_EXTS = (".h",) +BUILD_FILE_RE = re.compile(r"(^|/)(Makefile[^/]*|Kbuild[^/]*|Kconfig[^/]*)$") + +QUIET_RE = re.compile(r"^ ([A-Z][A-Z0-9_]*(?: \[M\])?) +(\S+)\s*$") +DIAG_RE = re.compile( + r"^(?P[^\s:][^:]*):(?P\d+):(?P\d+): " + r"(?Pwarning|error|fatal error): (?P.*)$") +TOOL_DIAG_RE = re.compile(r"^(?:\S*/)?(?:clang|clang-\d+|ld\.lld|ld\.lld-\d+|ld): " + r"(?Perror|warning): (?P.*)$") +FLAG_RE = re.compile(r"\s\[(-W[^\]]+)\]$") +# W=1 also runs scripts/kernel-doc on every compiled C file. +KDOC_RE = re.compile(r"^(?:Warning|Error): (?P[^\s:][^:]*):(?P\d+) (?P.*)$") +MAKE_ERR_RE = re.compile( + r"^make(?:\[\d+\])?: \*\*\* \[(?P[^\]]*?):(?P\d+): (?P[^\]]+)\] Error \d+") +NO_RULE_RE = re.compile(r"No rule to make target '(?P[^']+)'(?:, needed by '(?P[^']+)')?") + + +def log(msg: str) -> None: + print(msg, file=sys.stderr, flush=True) + + +class SetupError(Exception): + pass + + +def run(cmd, cwd=None, env=None, logfile: Path | None = None, check=False): + t0 = time.monotonic() + p = subprocess.run(cmd, cwd=cwd, env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors="replace") + dt = time.monotonic() - t0 + if logfile is not None: + with logfile.open("a") as f: + f.write(f"$ {' '.join(map(str, cmd))}\n{p.stdout}\n[rc={p.returncode} {dt:.1f}s]\n") + if check and p.returncode: + raise SetupError(f"command failed ({p.returncode}): {' '.join(map(str, cmd))}\n" + + p.stdout[-4000:]) + return p.returncode, p.stdout, dt + + +class Git: + def __init__(self, path: Path): + self.path = path + + def __call__(self, *args, check=True) -> str: + p = subprocess.run(["git", "-C", str(self.path), *args], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, errors="replace") + if check and p.returncode: + raise SetupError(f"git {' '.join(args)} failed: {p.stderr.strip()}") + return p.stdout + + def rev(self, r: str) -> str: + return self("rev-parse", "--verify", "--quiet", r + "^{commit}").strip() + + +# -------------------------------------------------------------------------- +# Change sets + +def name_status(git: Git, a: str, b: str): + """Return (changed paths on the b side, deleted paths, renames old->new).""" + out = git("diff", "--no-ext-diff", "--name-status", "-M", "-z", a, b) + parts = out.split("\0") + changed, deleted, renames = [], [], {} + i = 0 + while i < len(parts) - 1: + st = parts[i] + if not st: + i += 1 + continue + if st[0] in "RC": + old, new = parts[i + 1], parts[i + 2] + changed.append(new) + if st[0] == "R": + renames[old] = new + i += 3 + else: + path = parts[i + 1] + (deleted if st[0] == "D" else changed).append(path) + i += 2 + return changed, deleted, renames + + +def added_lines(git: Git, a: str, b: str) -> dict[str, set[int]]: + out = git("diff", "--no-ext-diff", "--no-color", "-U0", "-M", a, b, "--", + "*.c", "*.h", "*.S") + res: dict[str, set[int]] = {} + cur = None + for line in out.splitlines(): + if line.startswith("+++ "): + cur = line[6:] if line.startswith("+++ b/") else None + if cur is not None: + res.setdefault(cur, set()) + elif line.startswith("@@") and cur is not None: + m = re.match(r"@@ -\S+ \+(\d+)(?:,(\d+))? @@", line) + if m: + start, count = int(m.group(1)), int(m.group(2) or 1) + res[cur].update(range(start, start + count)) + return res + + +# -------------------------------------------------------------------------- +# Header includers: a cheap include graph built from the source text. +# +# Fixed-string `git grep`s for header basenames over the checked-out tree +# (well under 1 s each for the kernel) find candidate #include lines. Each +# spelling is resolved the way the kernel's include paths would resolve it: +# relative to the including file for "..." includes, then include/, +# arch/arm64/include/ and their uapi/ trees, asm/ falling back to +# asm-generic/, and finally a same-subsystem file with a matching path suffix +# (for directories added with ccflags -I). Includes spelled through macros +# and headers forced with -include are not seen. + +INCLUDE_RE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]+)[>"]') +SYS_DIRS = ("include/", "arch/arm64/include/", "include/uapi/", "arch/arm64/include/uapi/") + + +class IncludeIndex: + """Reverse include lookups, filled lazily one batch of basenames at a time. + + Each batch is one fixed-string `git grep` for the header basenames over + the checked-out tree (about 1 s for the kernel), whose hits are parsed as + #include lines. Files the range changes are read at HEAD instead. + """ + + def __init__(self, git: Git, head: str, changed: list[str]): + self.git, self.head = git, head + self.changed = [c for c in changed if c.endswith((".c", ".h", ".S"))] + self.headers = set(git("ls-files", "--", "*.h", "*.c", check=False).split()) + self.headers.update(self.changed) + self.by_base: dict[str, list[tuple[str, str, bool]]] = {} + self.done: set[str] = set() + hb: dict[str, list[str]] = {} + for h in self.headers: + hb.setdefault(os.path.basename(h), []).append(h) + self._hb = hb + + def _load(self, bases): + todo = sorted(set(bases) - self.done) + if not todo: + return + self.done.update(todo) + pats = [] + for b in todo: + pats += ["-e", b] + changed = set(self.changed) + + def add(f, text): + m = INCLUDE_RE.match(text) + if m and os.path.basename(m.group(2)) in todo_set: + self.by_base.setdefault(os.path.basename(m.group(2)), []).append( + (f, m.group(2), m.group(1) == '"')) + + todo_set = set(todo) + out = self.git("grep", "-n", "-F", *pats, "--", "*.c", "*.h", "*.S", check=False) + for line in out.splitlines(): + parts = line.split(":", 2) + if len(parts) == 3 and parts[0] not in changed: + add(parts[0], parts[2]) + if self.changed: + out = self.git("grep", "-n", "-F", *pats, self.head, "--", *self.changed, + check=False) + for line in out.splitlines(): + parts = line.split(":", 3) + if len(parts) == 4: + add(parts[1], parts[3]) + + def resolve(self, includer: str, spell: str, quoted: bool) -> list[str]: + known = self.headers + if quoted: + c = os.path.normpath(os.path.join(os.path.dirname(includer), spell)) + if c in known: + return [c] + for d in SYS_DIRS: + if d + spell in known: + return [d + spell] + if spell.startswith("asm/"): + g = "include/asm-generic/" + spell[4:] + if g in known: + return [g] + tail = "/" + spell.lstrip("./") + return [h for h in self._hb.get(os.path.basename(spell), []) + if h.endswith(tail) and common_prefix(h, includer) >= 2] + + def includers_many(self, hs) -> dict[str, list[str]]: + hs = list(hs) + self._load(os.path.basename(h) for h in hs) + res = {} + for h in hs: + r = set() + for f, spell, quoted in self.by_base.get(os.path.basename(h), []): + if f != h and h in self.resolve(f, spell, quoted): + r.add(f) + res[h] = sorted(r) + return res + + def includers(self, h: str) -> list[str]: + return self.includers_many([h])[h] + + +C_WORDS = set("""auto break case char const continue default define defined do double else +elif endif enum error extern float for goto if ifdef ifndef include inline int long register +restrict return short signed sizeof static struct switch typedef undef union unsigned void +volatile while bool true false NULL u8 u16 u32 u64 s8 s16 s32 s64 size_t ssize_t dma_addr_t +__iomem __user __init __exit __must_check EXPORT_SYMBOL EXPORT_SYMBOL_GPL""".split()) + + +def changed_identifiers(git: Git, ranges: list[tuple[str, str]], path: str) -> set[str]: + """Identifiers on lines a range adds to or removes from `path`.""" + ids: set[str] = set() + for a, b in ranges: + out = git("diff", "--no-ext-diff", "--no-color", "-U0", a, b, "--", path, check=False) + for line in out.splitlines(): + if line[:1] in "+-" and not line.startswith(("+++", "---")): + body = re.sub(r"/\*.*?\*/|//.*$", "", line[1:]) + ids.update(w for w in re.findall(r"\b[A-Za-z_][A-Za-z0-9_]{2,}\b", body) + if w not in C_WORDS) + return ids + + +def owner_dirs(h: str) -> list[str]: + """The header's directory and the directory that likely implements it.""" + d = os.path.dirname(h) + res = [h] + for pat, rep in ((r"^include/(?:uapi/)?linux/(.+)$", r"drivers/\1"), + (r"^include/(?:uapi/)?drm(/.*)?$", r"drivers/gpu/drm\1"), + (r"^include/(?:uapi/)?sound(/.*)?$", r"sound\1"), + (r"^arch/([^/]+)/include/(?:uapi/)?asm(/.*)?$", r"arch/\1\2")): + m = re.match(pat, d) + if m: + res.append(m.expand(rep) + "/" + os.path.splitext(os.path.basename(h))[0]) + return res + + +def common_prefix(a: str, b: str) -> int: + n = 0 + for x, y in zip(a.split("/"), b.split("/")): + if x != y: + break + n += 1 + return n + + +# -------------------------------------------------------------------------- +# Make output parsing + +def norm_path(p: str, srcdir: str, builddir: str) -> str: + if p.startswith("/"): + p = os.path.normpath(p) # e.g. drivers/x/epic/../y.h + if p.startswith(srcdir + "/"): + return p[len(srcdir) + 1:] + if p.startswith(builddir + "/"): + return "/" + p[len(builddir) + 1:] + if not p.startswith("/"): + q = os.path.normpath(p) + return "/" + q + return p + + +def parse_make_output(text: str, srcdir: str, builddir: str): + diags, failed, misc_errors = [], [], [] + obj = None + cur = None + include_chain: list[str] = [] + for line in text.splitlines(): + m = QUIET_RE.match(line) + if m and not line.startswith(" In file"): + obj = m.group(2) + cur = None + include_chain = [] + continue + if line.startswith("In file included from "): + include_chain.append(line[len("In file included from "):].rstrip(":")) + continue + m = DIAG_RE.match(line) + if m: + msg = m.group("msg") + flags = [] + fm = FLAG_RE.search(msg) + if fm: + flags = [f for f in fm.group(1).split(",") if f != "-Werror"] + msg = msg[: fm.start()] + sev = "error" if m.group("sev") in ("error", "fatal error") else "warning" + cur = { + "file": norm_path(m.group("file"), srcdir, builddir), + "line": int(m.group("line")), "col": int(m.group("col")), + "severity": sev, "flag": flags[0] if flags else "", + "message": msg.strip(), "object": obj, "context": [], + } + diags.append(cur) + include_chain = [] + continue + m = KDOC_RE.match(line) + if m: + cur = None + diags.append({"file": norm_path(m.group("file"), srcdir, builddir), + "line": int(m.group("line")), "col": 0, "severity": "warning", + "flag": "kernel-doc", "message": m.group("msg").strip(), + "object": obj, "context": []}) + continue + m = TOOL_DIAG_RE.match(line) + if m: + diags.append({"file": "", "line": 0, "col": 0, "severity": m.group("sev"), + "flag": "", "message": m.group("msg").strip(), "object": obj, + "context": []}) + cur = None + continue + m = MAKE_ERR_RE.match(line) + if m: + t = m.group("target") + if os.path.splitext(t)[1]: # a file target, not a directory + failed.append(t) + cur = None + continue + m = NO_RULE_RE.search(line) + if m and line.startswith("make"): + misc_errors.append(line.strip()) + continue + if cur is not None and len(cur["context"]) < 8 and not line.startswith("make"): + cur["context"].append(line) + return diags, sorted(set(failed)), misc_errors + + +def count_cmds(out: str) -> int: + """Number of kbuild commands (quiet " TAG target" lines) in make output.""" + return sum(1 for l in out.splitlines() if QUIET_RE.match(l)) + + +def norm_msg(msg: str) -> str: + return re.sub(r"\b\d+\b", "N", msg) + + +# -------------------------------------------------------------------------- +# The checker + +class Checker: + def __init__(self, a): + self.a = a + self.tree = Path(a.tree).resolve() + self.out = Path(a.out).resolve() + self.out.mkdir(parents=True, exist_ok=True) + self.logdir = self.out / "logs" + self.logdir.mkdir(exist_ok=True) + self.build = self.out / "build" + self.build.mkdir(exist_ok=True) + self.git_main = Git(self.tree) + self.timings: dict[str, float] = {} + self.cpu: dict[str, float] = {} + self.notes: list[str] = [] + self.unchecked: list[dict] = [] + self.lister = self.out / "kbuild-list-objs.mk" + self.lister.write_text(LISTER_MK) + self.env = dict(os.environ) + self.env["LC_ALL"] = "C" + self.env.pop("MAKEFILES", None) + self.env.pop("MAKEFLAGS", None) + self.selected: dict[str, str] = {} + self.included_c: dict[str, list[str]] = {} + self.obj_diags: dict[str, list[dict]] = {} + self.selected_src: dict[str, str] = {} + self.cur_sha = None + + # -- helpers ---------------------------------------------------------- + @staticmethod + def mark() -> tuple[float, float]: + ru = resource.getrusage(resource.RUSAGE_CHILDREN) + return time.monotonic(), ru.ru_utime + ru.ru_stime + + def tick(self, key: str, start: tuple[float, float]) -> None: + """Add wall and child-CPU seconds since `start` to phase `key`.""" + wall, cpu = self.mark() + self.timings[key] = round(self.timings.get(key, 0.0) + wall - start[0], 2) + self.cpu[key] = round(self.cpu.get(key, 0.0) + cpu - start[1], 2) + + def make_cmd(self, *extra): + a = self.a + cmd = ["make", f"O={self.build}", f"LLVM={a.llvm}", "ARCH=arm64", "W=1", + f"-j{a.jobs}"] + if a.rust == "skip": + # Keep CONFIG_RUST=y (and every C driver that depends on it) in the + # configuration, but do not build rust/ or run its prepare step. + cmd += ["CONFIG_RUST=", "RUSTC=/nonexistent/rustc", "BINDGEN=/nonexistent/bindgen"] + cmd += a.make_arg + cmd += list(extra) + return cmd + + # -- source tree -------------------------------------------------------- + def setup_tree(self): + g = self.git_main + if not self.a.worktree: + # In place (the CI checkout): states are checked out in --tree and + # its original HEAD (branch or detached commit) is restored at exit. + if g("status", "--porcelain", "--untracked-files=no").strip(): + raise SetupError("--tree has local changes; commit or stash them, " + "or use --worktree DIR") + self.src = self.tree + branch = g("symbolic-ref", "-q", "--short", "HEAD", check=False).strip() + self.orig_head = branch or g("rev-parse", "HEAD").strip() + else: + self.src = Path(self.a.worktree).resolve() + if not (self.src / ".git").exists(): + t0 = self.mark() + g("worktree", "add", "--detach", "-f", str(self.src), self.mb) + self.tick("worktree_add", t0) + self.orig_head = None + self.git = Git(self.src) + + def checkout(self, sha: str): + if self.cur_sha == sha: + return + t0 = self.mark() + self.git("checkout", "-q", "--detach", "-f", sha) + self.tick("checkout", t0) + self.cur_sha = sha + + # -- configuration --------------------------------------------------------- + def configure(self, label: str): + t0 = self.mark() + lf = self.logdir / f"config-{label}.log" + cfg = self.build / ".config" + shutil.copyfile(self.a.config, cfg) + if self.a.fragment: + frags = [str(self.src / f) for f in self.a.fragment if (self.src / f).exists()] + if frags: + run(["scripts/kconfig/merge_config.sh", "-m", "-O", str(self.build), str(cfg), + *frags], cwd=self.src, env=self.env, logfile=lf, check=True) + sc = ["scripts/config", "--file", str(cfg), "--disable", "WERROR"] + if not self.a.keep_debug_info: + sc += ["--enable", "DEBUG_INFO_NONE", "--disable", "DEBUG_INFO_DWARF5", + "--disable", "DEBUG_INFO_DWARF4", + "--disable", "DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT"] + run(sc, cwd=self.src, env=self.env, logfile=lf, check=True) + want_rust = "CONFIG_RUST=y" in cfg.read_text().splitlines() + run(self.make_cmd("olddefconfig"), cwd=self.src, env=self.env, logfile=lf, check=True) + have_rust = "CONFIG_RUST=y" in cfg.read_text().splitlines() + if want_rust and not have_rust: + self.notes.append("CONFIG_RUST was dropped by olddefconfig: C objects that depend " + "on Rust drivers are not covered.") + self.frag_ids = self.fragment_ids() + self.tick("config", t0) + + def fragment_ids(self): + ids = [] + for f in self.a.fragment: + ids.append(self.git("rev-parse", f"HEAD:{f}", check=False).strip()) + return ids + + # -- object selection ------------------------------------------------------ + def list_objects(self, label: str) -> tuple[bool, str]: + """Run prepare + the kbuild walk; fill self.selected. Returns (ok, output).""" + t0 = self.mark() + env = dict(self.env) + env["MAKEFILES"] = str(self.lister) + rc, out, _ = run(self.make_cmd("-k", "KBUILD_BUILTIN=", "KBUILD_MODULES=", "."), + cwd=self.src, env=env, logfile=self.logdir / f"list-{label}.log") + self.tick("prepare_and_list", t0) + sel: dict[str, str] = {} + for line in out.splitlines(): + if not line.startswith("AURORA-OBJS|"): + continue + f = line.rstrip("\n").split("|") + if len(f) != 6: + continue + # Objects linked into the kernel or a module first, then other + # objects this directory builds (e.g. X.o behind X.pi.o). + for kind, lst in (("y", f[2]), ("m", f[3]), ("lib", f[4]), ("target", f[5])): + for o in lst.split(): + if o.endswith(".o") and o not in sel: + sel[o] = kind + self.selected = sel + self.selected_src = {} + for o, kind in sel.items(): + s = self.source_of(o) + if s and s not in self.selected_src: + self.selected_src[s] = o + ok = rc == 0 and bool(sel) + return ok, out + + def source_of(self, o: str) -> str | None: + """Source file of object o: X.o <- X.c/X.S, or X..o <- X.c/X.S.""" + b = os.path.normpath(o[:-2]) + cands = [b] + stem, dot, tag = b.rpartition(".") + if dot and "/" not in tag: + cands.append(stem) + for c in cands: + for ext in SOURCE_EXTS: + if (self.src / (c + ext)).is_file(): + return c + ext + return None + + # -- compile ------------------------------------------------------------------- + def compile(self, objs: list[str], label: str): + if not objs: + return {"rc": 0, "loose_diags": [], "failed": [], "misc_errors": [], "seconds": 0.0, + "recompiled": 0} + lf = self.logdir / f"build-{label}.log" + t0 = self.mark() + rc, out, dt = run(self.make_cmd("-k", "-Otarget", *objs), cwd=self.src, + env=self.env, logfile=lf) + self.tick("compile", t0) + diags, failed, misc = parse_make_output(out, str(self.src), str(self.build)) + compiled = sorted({m.group(2) for l in out.splitlines() + if (m := QUIET_RE.match(l)) and m.group(1).split()[0] in ("CC", "AS")} + | set(failed)) + # make only recompiles objects whose inputs changed, so an object's + # diagnostics are those of its most recent compile in this run. + for t in compiled: + self.obj_diags[t] = [d for d in diags if d.get("object") == t] + loose = [d for d in diags if d.get("object") not in self.obj_diags] + return {"rc": rc, "loose_diags": loose, "failed": failed, "misc_errors": misc, + "seconds": round(dt, 2), "recompiled": len(compiled)} + + def effective_diags(self, objs: list[str], loose: list[dict]) -> list[dict]: + """Diagnostics of each object's latest compile (it may predate this state).""" + out = list(loose) + for o in objs: + keys = [o] + s = self.source_of(o) + if s and os.path.splitext(s)[0] + ".o" != o: + keys.append(os.path.splitext(s)[0] + ".o") # X.o behind X.pi.o + for k in keys: + out.extend(self.obj_diags.get(k, [])) + return out + + # -- main flow ---------------------------------------------------------------- + def execute(self): + a = self.a + g = self.git_main + self.base = g.rev(a.base) + self.head = g.rev(a.head) + if not self.base or not self.head: + raise SetupError(f"cannot resolve {a.base} / {a.head} in {self.tree}") + self.mb = g("merge-base", self.base, self.head).strip() + if not self.mb: + raise SetupError("no merge base") + commits = g("rev-list", "--reverse", "--topo-order", "--no-merges", + f"{self.base}..{self.head}").split() + n_merges = len(g("rev-list", "--merges", f"{self.base}..{self.head}").split()) + per_commit = a.per_commit and len(commits) > 0 + if per_commit and len(commits) > a.max_commits: + self.notes.append(f"Per-commit check skipped: {len(commits)} commits exceed " + f"--max-commits {a.max_commits}; only HEAD was compiled.") + per_commit = False + if n_merges: + self.notes.append(f"{n_merges} merge commit(s) in the range are not compiled on " + "their own.") + + # Changed files: whole range, and per commit. + changed, deleted, renames = name_status(g, self.mb, self.head) + per_commit_changes = {} + if per_commit: + for c in commits: + ch, _, _ = name_status(g, c + "^", c) + per_commit_changes[c] = ch + all_changed = set(changed) + for ch in per_commit_changes.values(): + all_changed.update(ch) + + sources = sorted(p for p in all_changed if p.endswith(SOURCE_EXTS)) + headers = sorted(p for p in all_changed if p.endswith(HEADER_EXTS)) + rust_files = sorted(p for p in all_changed if p.endswith(".rs")) + build_files = sorted(p for p in all_changed if BUILD_FILE_RE.search(p)) + + self.setup_tree() + states = [("base", self.mb)] + if per_commit: + states += [(f"commit-{i + 1}", c) for i, c in enumerate(commits)] + if commits[-1] != self.head: + states.append(("head", self.head)) + else: + states.append(("head", self.head)) + + # First state: configure + list, then decide the object set. + self.checkout(self.mb) + self.configure("base") + ok, out = self.list_objects("base") + base_prepare_cmds = count_cmds(out) + if not ok: + raise SetupError("kbuild walk failed at the merge base; see logs/list-base.log\n" + + out[-3000:]) + + ranges = [(self.mb, self.head)] + [(c + "^", c) for c in per_commit_changes] + object_sources, header_cov = self.pick_sources(sources, headers, all_changed, ranges) + if len(object_sources) > a.max_objects: + self.notes.append(f"Object set capped at {a.max_objects} of " + f"{len(object_sources)} candidates.") + object_sources = object_sources[: a.max_objects] + + for p in rust_files: + self.unchecked.append({"path": p, "reason": "Rust source (not compiled by this job)"}) + + # Every object must be compiled at least once in this run so that its + # diagnostics are known (a reused build directory may hold it already). + for src in object_sources: + for o in {self.selected_src.get(src), os.path.splitext(src)[0] + ".o"}: + if o: + (self.build / o).unlink(missing_ok=True) + + results = [] + prev_sha = None + for label, sha in states: + st = {"label": label, "sha": sha} + if label == "base": + st["prepare_cmds"] = base_prepare_cmds + if sha != self.cur_sha: + prev_sha = self.cur_sha + self.checkout(sha) + diff_names = set(g("diff", "--name-only", prev_sha, sha).split()) + if self.a.fragment and self.fragment_ids() != self.frag_ids: + self.configure(label) + relist = True + else: + relist = any(BUILD_FILE_RE.search(p) for p in diff_names) + if relist: + ok, out = self.list_objects(label) + else: + ok, out = self.prepare(label) + st["prepare_cmds"] = count_cmds(out) + st["relisted"] = relist + if not ok: + st["prepare_failed"] = True + st["prepare_log_tail"] = out[-3000:] + pd, _, _ = parse_make_output(out, str(self.src), str(self.build)) + st["diags"] = [d for d in pd if d["severity"] == "error"] + results.append(st) + continue + objs = [] + for s in object_sources: + o = self.selected_src.get(s) + if o: + objs.append(o) + st["objects"] = objs + log(f"[{label}] {sha[:12]}: compiling {len(objs)} object(s)") + r = self.compile(objs, label) + st.update(r) + st["diags"] = self.effective_diags(objs, r.pop("loose_diags")) + results.append(st) + + # Sources never selected at HEAD (or at any state) are reported. + head_sel = set(self.selected_src) + for s, via in self.included_c.items(): + self.notes.append(f"{s} is not an object itself; it is compiled through " + f"{', '.join(via)}, which #include it.") + for s in sources: + if s not in head_sel and s not in self.included_c and (self.src / s).exists(): + if s.startswith(("tools/", "scripts/", "samples/", "Documentation/", "usr/")): + why = "outside kernel objects (host tool, sample or documentation)" + else: + why = "not built by this configuration" + self.unchecked.append({"path": s, "reason": why}) + + return self.analyse(results, commits, renames, header_cov, build_files, per_commit) + + def prepare(self, label: str): + t0 = self.mark() + rc, out, _ = run(self.make_cmd("-k", "prepare"), cwd=self.src, env=self.env, + logfile=self.logdir / f"prepare-{label}.log") + self.tick("prepare_and_list", t0) + return rc == 0, out + + def pick_sources(self, sources, headers, all_changed, ranges): + """Changed sources plus a capped, ranked set of header includers. + + Includers that use identifiers the range adds to or removes from the + header (a renamed function, a changed macro or field) come first, + weighted by how rare each identifier is among the includers; then + files the range changes, then files near the header or its likely + implementation directory. + """ + chosen = [s for s in sources] + header_cov = [] + selected = self.selected_src + changed_set = set(all_changed) + if headers or any(s not in selected for s in sources): + t0 = self.mark() + idx = IncludeIndex(self.git, self.head, sorted(changed_set)) + self.tick("include_index", t0) + # A changed .c that is not an object itself may be #included by one. + for s in sources: + if s in selected or not s.endswith(".c"): + continue + via = [inc for inc in idx.includers(s) if inc in selected] + via.sort(key=lambda c: (0 if c in changed_set else 1, -common_prefix(c, s), c)) + via = via[: self.a.header_objects] + if via: + self.included_c[s] = via + for inc in via: + if inc not in chosen: + chosen.append(inc) + for h in headers: + if h.startswith(("tools/", "scripts/", "Documentation/", "samples/")): + header_cov.append({"header": h, "includers_found": 0, "selected": 0, + "compiled": 0, "note": "outside kernel objects"}) + continue + seen, frontier, cands = {h}, [h], [] + depth = 0 + while frontier and depth < self.a.header_depth: + nxt = [] + level = idx.includers_many(frontier) + for x in frontier: + for inc in level[x]: + if inc in seen: + continue + seen.add(inc) + if inc.endswith(SOURCE_EXTS): + cands.append((depth, inc)) + # Follow headers, and .c files that are #included + # by other files rather than compiled themselves. + if ((inc.endswith(".h") or inc not in selected) + and len(nxt) < self.a.header_fanout): + nxt.append(inc) + frontier = nxt + depth += 1 + sel = [(d, c) for d, c in cands if c in selected] + ref = owner_dirs(h) + sorted(p for p in changed_set if p.endswith(SOURCE_EXTS)) + score = self.identifier_scores(h, [c for _, c in sel], ranges) + + def rank(item): + d, c = item + prox = max(common_prefix(c, r) for r in ref) + return (-score.get(c, 0.0), 0 if c in changed_set else 1, -prox, d, c) + + sel.sort(key=rank) + # Users of changed identifiers first; then the best-ranked file of + # each directory, so the sample covers different users. + pick, dirs = [], set() + for _, c in sel: + if len(pick) >= self.a.header_objects: + break + if score.get(c) or os.path.dirname(c) not in dirs: + pick.append(c) + dirs.add(os.path.dirname(c)) + for _, c in sel: + if len(pick) >= self.a.header_objects: + break + if c not in pick: + pick.append(c) + for c in pick: + if c not in chosen: + chosen.append(c) + header_cov.append({"header": h, "includers_found": len(cands), + "selected": len(sel), "compiled": len(pick), + "users_of_changed_identifiers": sum(1 for c in score if score[c]), + "objects": pick}) + return chosen, header_cov + + def identifier_scores(self, h, cands, ranges) -> dict[str, float]: + """Score each candidate by the changed identifiers of `h` it uses.""" + ids = changed_identifiers(self.git_main, ranges, h) + if not ids or not cands: + return {} + pats = [] + for i in sorted(ids): + pats += ["-e", i] + uses: dict[str, set[str]] = {} + for k in range(0, len(cands), 500): + out = self.git("grep", "-o", "-w", "-F", *pats, "--", *cands[k:k + 500], + check=False) + for line in out.splitlines(): + f, _, w = line.partition(":") + uses.setdefault(f, set()).add(w) + df: Counter = Counter() + for ws in uses.values(): + df.update(ws) + return {f: sum(1.0 / df[w] for w in ws) for f, ws in uses.items()} + + # -- analysis ----------------------------------------------------------------- + def analyse(self, results, commits, renames, header_cov, build_files, per_commit): + base = results[0] + head = results[-1] + added = added_lines(self.git_main, self.mb, self.head) + + def uniq(diags, sev): + seen, out = set(), [] + for d in diags: + if d["severity"] != sev: + continue + k = (d["file"], d["line"], d["col"], d["flag"], d["message"]) + if k in seen: + continue + seen.add(k) + out.append(d) + return out + + def nkey(d, mapping=None): + f = d["file"] + if mapping: + f = mapping.get(f, f) + return (f, d["flag"], norm_msg(d["message"])) + + base_w = Counter(nkey(d, renames) for d in uniq(base.get("diags", []), "warning")) + head_w = uniq(head.get("diags", []), "warning") + remaining = Counter(base_w) + new_w, old_w = [], [] + for d in head_w: + k = nkey(d) + on_added = d["line"] in added.get(d["file"], set()) + d2 = dict(d, on_added_line=on_added) + if remaining[k] > 0: + remaining[k] -= 1 + d2["new"] = False + old_w.append(d2) + else: + d2["new"] = True + new_w.append(d2) + + base_err = Counter(nkey(d, renames) for d in uniq(base.get("diags", []), "error")) + base_failed = set(base.get("failed", [])) + errors = [] + for st in results[1:]: + errs = uniq(st.get("diags", []), "error") + st_errs = [] + for d in errs: + if base_err.get(nkey(d), 0) > 0 and d.get("object") in base_failed: + continue # already broken at the merge base + st_errs.append(d) + failed = [o for o in st.get("failed", []) if o not in base_failed] + if st.get("prepare_failed") or st_errs or failed or st.get("misc_errors"): + errors.append({"label": st["label"], "sha": st["sha"], + "subject": self.git_main("log", "-1", "--format=%s", + st["sha"]).strip(), + "prepare_failed": bool(st.get("prepare_failed")), + "diags": st_errs, "failed_objects": failed, + "misc_errors": st.get("misc_errors", []), + "log_tail": st.get("prepare_log_tail", "")}) + + report = { + "base": self.base, "head": self.head, "merge_base": self.mb, + "commits": commits, "per_commit": per_commit, + "states": [{k: v for k, v in st.items() if k not in ("diags", "prepare_log_tail")} + | {"warnings": len(uniq(st.get("diags", []), "warning")), + "errors": len(uniq(st.get("diags", []), "error"))} + for st in results], + "new_warnings": new_w, + "preexisting_warnings": old_w, + "errors": errors, + "header_coverage": header_cov, + "unchecked": self.unchecked, + "build_files_changed": build_files, + "notes": self.notes, + "timings": self.timings, + "cpu_seconds": self.cpu, + "base_errors": len(uniq(base.get("diags", []), "error")), + } + return report + + +# -------------------------------------------------------------------------- +# Output + +def esc_data(s: str) -> str: + return s.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def esc_prop(s: str) -> str: + return esc_data(s).replace(":", "%3A").replace(",", "%2C") + + +def annotate(kind: str, d: dict, title: str) -> str: + props = [] + if d.get("file") and not d["file"].startswith(""): + props.append(f"file={esc_prop(d['file'])}") + if d.get("line"): + props.append(f"line={d['line']}") + if d.get("col"): + props.append(f"col={d['col']}") + props.append(f"title={esc_prop(title)}") + msg = d["message"] + (f" [{d['flag']}]" if d.get("flag") else "") + return f"::{kind} {','.join(props)}::{esc_data(msg)}" + + +def emit(report: dict, out: Path, total: float) -> int: + lines = [] + for e in report["errors"]: + t = f"Compile error at {e['label']} {e['sha'][:12]}" + for d in e["diags"]: + lines.append(annotate("error", d, t)) + if e["prepare_failed"] and not e["diags"]: + lines.append(f"::error title={esc_prop(t)}::{esc_data('make prepare failed')}") + for o in e["failed_objects"]: + if not any(d.get("object") == o for d in e["diags"]): + lines.append(f"::error title={esc_prop(t)}::{esc_data('failed to build ' + o)}") + for m in e["misc_errors"]: + lines.append(f"::error title={esc_prop(t)}::{esc_data(m)}") + for d in report["new_warnings"]: + lines.append(annotate("warning", d, "New W=1 warning")) + for u in report["unchecked"]: + lines.append(f"::notice file={esc_prop(u['path'])},title=Not compiled::" + f"{esc_data(u['reason'])}") + for n in report["notes"]: + lines.append(f"::notice title=compile-check::{esc_data(n)}") + seen = set() + for l in lines: + if l not in seen: + seen.add(l) + print(l) + + report["timings"]["total"] = round(total, 1) + (out / "report.json").write_text(json.dumps(report, indent=1, default=list) + "\n") + + md = [] + status = "FAIL (compile errors)" if report["errors"] else "pass" + md.append(f"## Kernel compile check: {status}\n") + md.append(f"Range `{report['merge_base'][:12]}..{report['head'][:12]}`, " + f"{len(report['commits'])} commit(s), per-commit " + f"{'on' if report['per_commit'] else 'off'}.\n") + md.append("| state | commit | objects | recompiled | prepare cmds | W=1 warnings | errors " + "| compile s |") + md.append("|---|---|---|---|---|---|---|---|") + for st in report["states"]: + pf = " (prepare failed)" if st.get("prepare_failed") else "" + md.append(f"| {st['label']}{pf} | `{st['sha'][:12]}` | {len(st.get('objects', []))} | " + f"{st.get('recompiled', 0)} | {st.get('prepare_cmds', 0)} | " + f"{st['warnings']} | {st['errors']} | {st.get('seconds', 0)} |") + md.append("") + if report["errors"]: + md.append("### Compile errors\n") + for e in report["errors"]: + md.append(f"- **{e['label']}** `{e['sha'][:12]}` {e['subject']}") + for d in e["diags"][:20]: + md.append(f" - `{d['file']}:{d['line']}:{d['col']}` {d['message']}") + for o in e["failed_objects"]: + md.append(f" - failed: `{o}`") + if e["prepare_failed"]: + md.append(" - `make prepare` failed") + md.append("") + md.append(f"### New W=1 warnings: {len(report['new_warnings'])}\n") + for d in report["new_warnings"][:100]: + tag = "" if d["on_added_line"] else " (line not added by this range)" + md.append(f"- `{d['file']}:{d['line']}:{d['col']}` {d['message']} `{d['flag']}`{tag}") + md.append(f"\nPre-existing W=1 warnings at HEAD (same as merge base): " + f"{len(report['preexisting_warnings'])}\n") + if report["header_coverage"]: + md.append("### Changed headers\n") + md.append("| header | includers found | selected by config | use changed " + "identifiers | compiled |") + md.append("|---|---|---|---|---|") + for h in report["header_coverage"]: + md.append(f"| `{h['header']}` | {h['includers_found']} | {h['selected']} | " + f"{h.get('users_of_changed_identifiers', 0)} | {h['compiled']} |") + md.append("") + if report["unchecked"] or report["build_files_changed"]: + md.append("### Not compiled by this job\n") + for u in report["unchecked"]: + md.append(f"- `{u['path']}`: {u['reason']}") + for b in report["build_files_changed"]: + md.append(f"- `{b}`: build/config file (its effect is only seen by the full build)") + md.append("") + for n in report["notes"]: + md.append(f"> {n}\n") + md.append("Wall time (s): " + ", ".join(f"{k} {v}" for k, v in report["timings"].items())) + md.append("\nCPU time of child processes (s): " + + ", ".join(f"{k} {v}" for k, v in report["cpu_seconds"].items())) + (out / "summary.md").write_text("\n".join(md) + "\n") + return 1 if report["errors"] else 0 + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument("--tree", required=True, help="kernel git checkout") + p.add_argument("--base", required=True, help="target branch tip (or PR base)") + p.add_argument("--head", required=True, help="PR head") + p.add_argument("--config", required=True, help="starting .config") + p.add_argument("--out", required=True, help="output directory (build dir, logs, report)") + p.add_argument("--fragment", action="append", default=[], + help="in-tree config fragment merged at each state (repeatable)") + p.add_argument("--per-commit", action="store_true", help="compile every commit") + p.add_argument("--max-commits", type=int, default=20, + help="skip the per-commit pass for longer ranges") + p.add_argument("--max-objects", type=int, default=150, + help="cap on objects compiled per state") + p.add_argument("--header-objects", type=int, default=8, + help="objects compiled per changed header") + p.add_argument("--header-depth", type=int, default=3, + help="include levels followed from a changed header") + p.add_argument("--header-fanout", type=int, default=100, + help="including headers followed per level") + p.add_argument("--jobs", type=int, default=os.cpu_count() or 4) + p.add_argument("--llvm", default=os.environ.get("LLVM", "1"), + help="kbuild LLVM= value; write --llvm=-18 for a version suffix, " + "or give a prefix such as /usr/lib/llvm-18/bin/") + p.add_argument("--rust", choices=("skip", "toolchain"), default="skip", + help="skip (default): keep CONFIG_RUST=y in the configuration but pass " + "CONFIG_RUST= to make, so rust/ and its prepare step never run and " + "no Rust toolchain is needed; toolchain: build normally (rustc, " + "rust-src and bindgen must be installed)") + p.add_argument("--keep-debug-info", action="store_true", + help="keep the configuration's DEBUG_INFO (default: DEBUG_INFO_NONE, " + "which also drops BTF and makes prepare and compiles faster)") + p.add_argument("--worktree", metavar="DIR", + help="check out states in this git worktree (created if missing) " + "instead of in --tree itself") + p.add_argument("--make-arg", action="append", default=[], + help="extra make argument (repeatable)") + a = p.parse_args(argv) + t0 = time.monotonic() + out = Path(a.out).resolve() + ck = Checker(a) + try: + report = ck.execute() + except SetupError as e: + print(f"::error title=compile-check setup failed::{esc_data(str(e)[:3000])}") + log(str(e)) + return 2 + finally: + if getattr(ck, "orig_head", None): + ck.git_main("checkout", "-q", "-f", ck.orig_head, check=False) + ru = resource.getrusage(resource.RUSAGE_CHILDREN) + report["cpu_seconds"]["total"] = round(ru.ru_utime + ru.ru_stime, 1) + return emit(report, out, time.monotonic() - t0) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/dt-check.py b/tools/aurora-ci/dt-check.py new file mode 100755 index 0000000000000..a86630e3a5ebe --- /dev/null +++ b/tools/aurora-ci/dt-check.py @@ -0,0 +1,1588 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +dt-check.py - report device tree and DT binding problems that a change adds. + +Compares a base and a head commit of a kernel tree and reports only the +problems that exist at head and not at base: + + * every DTB enabled by a minimal config (default CONFIG_ARCH_APPLE=y) is + built with W=1 at both commits; build failures and dtc warnings are + compared for all of them (this costs seconds); + * DTBs whose output bytes changed (or that are new) get a full dt-validate + schema check at both commits; + * DTBs that use a binding the change touched get a dt-validate check + limited (-l) to the touched bindings at both commits; + * changed binding YAML files get `make dt_binding_check DT_SCHEMA_FILES=...` + at both commits; + * changed .dts/.dtsi/.dtso files get scripts/dtc/dt-check-style. + +Warnings are keyed without line numbers or source file names (DTB, node +path, property, schema and message), so moving code around does not create +new warnings. + +Exit status: 0 = no new blocking problems, 1 = new blocking problems, +2 = the check could not run. + +Python 3.10+, standard library only. Needs git, tar, make, a host C +compiler, flex and bison, plus dtschema (dt-validate, dt-mk-schema, +dt-doc-validate, dt-extract-example) and yamllint on PATH. +""" + +import argparse +import collections +import concurrent.futures +import contextlib +import hashlib +import json +import os +import re +import shutil +import struct +import subprocess +import sys +import time + +VERSION = '0.1' +BINDINGS = 'Documentation/devicetree/bindings' + +CLASSES = collections.OrderedDict([ + ('dtb-build', 'DTB fails to build (preprocessor or dtc error)'), + ('binding-invalid', 'binding fails YAML parsing or meta-schema validation'), + ('binding-dropped', 'dt-mk-schema left the binding out of the processed schema'), + ('example-build', 'binding example fails to extract or compile'), + ('schema', 'DTB node fails a binding schema (dt-validate)'), + ('unmatched', 'DTB compatible not documented by any binding (dt-validate -m)'), + ('dtc-warning', 'dtc warning (W=1)'), + ('example-schema', 'binding example fails schema validation, or dtc warns on it'), + ('binding-warning', 'dt-mk-schema warning about a binding'), + ('yamllint', 'yamllint finding in a binding'), + ('style', 'coding style finding (dt-check-style)'), + ('tool-error', 'a checker crashed or could not run'), +]) +DEFAULT_BLOCKING = ('dtb-build', 'binding-invalid', 'binding-dropped', 'example-build') +DTB_SCOPED = ('dtb-build', 'schema', 'unmatched', 'dtc-warning') + +ARCHIVE_PATHS = [ + ':(top,glob)*', # top-level Makefile, Kconfig, ... + ':(top,glob)**/Kconfig*', # Kconfig needs every Kconfig file + 'scripts', + 'include/dt-bindings', + BINDINGS, + 'tools/lib/python', # scripts/jobserver-exec imports from here + ':(top,glob)arch/*/boot/dts/**', # scripts/dtc/include-prefixes targets +] + + +class SetupError(Exception): + """The check could not run (exit status 2).""" + + +def note(msg): + print(f'[dt-check] {msg}', file=sys.stderr, flush=True) + + +# -------------------------------------------------------------------------- +# small helpers + +class Timings: + def __init__(self): + self.t = collections.OrderedDict() + + @contextlib.contextmanager + def __call__(self, name): + t0 = time.monotonic() + try: + yield + finally: + self.t[name] = round(self.t.get(name, 0) + time.monotonic() - t0, 2) + + +def run(cmd, cwd=None, env=None, logfile=None): + t0 = time.monotonic() + p = subprocess.run(cmd, cwd=cwd, env=env, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + out = p.stdout.decode('utf-8', 'replace') + if logfile: + with open(logfile, 'w', encoding='utf-8') as f: + f.write('$ ' + ' '.join(cmd) + '\n' + out + + f'\n[exit {p.returncode}, {time.monotonic() - t0:.1f}s]\n') + return p.returncode, out + + +def git(repo, *args, check=True, binary=False): + p = subprocess.run(['git', '-C', repo, *args], stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if check and p.returncode != 0: + raise SetupError(f"git {' '.join(args)}: " + f"{p.stderr.decode('utf-8', 'replace').strip()}") + return p.stdout if binary else p.stdout.decode('utf-8', 'replace') + + +def git_ok(repo, *args): + return subprocess.run(['git', '-C', repo, *args], stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL).returncode == 0 + + +def sha256(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(1 << 20), b''): + h.update(chunk) + return h.hexdigest() + + +LOC_RE = re.compile(r'(?:[^\s:,()\'"]+/)?[^\s:,()\'"/]+\.(?:dtsi|dtso|dts|h|yaml|tmp)' + r':\d+(?:[.:]\d+)?(?:-\d+(?:[.:]\d+)?)?') +BYTES_RE = re.compile(r"b'(?:[^'\\]|\\.)*'|b\"(?:[^\"\\]|\\.)*\"") + + +def mask(text): + """Remove the parts of a message that move when code moves.""" + text = LOC_RE.sub('', text) + text = BYTES_RE.sub("b'...'", text) + return text + + +def problem(cls, scope, key, message, file=None, line=None, detail=None): + return {'class': cls, 'scope': scope, 'key': [str(k) for k in key], + 'message': message, 'file': file, 'line': line, + 'detail': detail or []} + + +def short_dtb(dtb, arch): + prefix = f'arch/{arch}/boot/dts/' + return dtb[len(prefix):] if dtb.startswith(prefix) else dtb + + +# -------------------------------------------------------------------------- +# git: changed files and tree extraction + +def changed_files(repo, base, head): + raw = git(repo, 'diff', '--name-status', '-z', '-M', '--no-ext-diff', + base, head, binary=True) + parts = raw.split(b'\0') + res, i = [], 0 + while i < len(parts) and parts[i]: + st = parts[i].decode() + i += 1 + if st[0] in 'RC': + old, new = parts[i].decode(), parts[i + 1].decode() + i += 2 + res.append({'status': st[0], 'score': st[1:], 'old': old, 'path': new}) + else: + p = parts[i].decode() + i += 1 + res.append({'status': st[0], 'old': None if st[0] == 'A' else p, + 'path': None if st[0] == 'D' else p}) + return res + + +def dt_symlink_targets(repo, sha): + """Files that include/dt-bindings symlinks point at (e.g. uapi headers).""" + out = git(repo, 'ls-tree', '-r', '-z', sha, '--', 'include/dt-bindings', + binary=True, check=False) + targets = set() + for ent in out.split(b'\0'): + if not ent or b'\t' not in ent: + continue + meta, path = ent.split(b'\t', 1) + mode, _typ, oid = meta.split() + if mode == b'120000': + tgt = git(repo, 'cat-file', 'blob', oid.decode()).strip() + targets.add(os.path.normpath(os.path.join( + os.path.dirname(path.decode()), tgt))) + return targets + + +def classify(path, symlink_targets): + if path.startswith(BINDINGS + '/'): + return 'binding' if path.endswith('.yaml') else 'binding-infra' + if (re.match(r'arch/[^/]+/boot/dts/', path) or + path.startswith('include/dt-bindings/') or path in symlink_targets): + return 'dts' + if path in ('scripts/dtc/dt-check-style',) or \ + path.startswith('scripts/dtc/dt-style-selftest/'): + return 'binding-infra' + if path.startswith('scripts/dtc/') or path == 'scripts/Makefile.dtbs': + return 'tooling' + return None + + +def _archive(repo, sha, specs, dest): + a = subprocess.Popen(['git', '-C', repo, 'archive', '--format=tar', sha, + '--', *specs], stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + t = subprocess.run(['tar', '-x', '-f', '-', '-C', dest], stdin=a.stdout, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + a.stdout.close() + err = a.stderr.read().decode('utf-8', 'replace') + a.wait() + if a.returncode or t.returncode: + raise SetupError(f'git archive {sha[:12]} failed: {err.strip()} ' + f'{t.stderr.decode("utf-8", "replace").strip()}') + + +def _dangling(dest): + res = set() + for root, dirs, files in os.walk(dest): + for n in dirs + files: + p = os.path.join(root, n) + if os.path.islink(p) and not os.path.exists(p): + tgt = os.path.normpath(os.path.join(root, os.readlink(p))) + rel = os.path.relpath(tgt, dest) + if not rel.startswith('..'): + res.add(rel) + return res + + +def extract_tree(repo, sha, dest, arch): + """Write the subset of the tree that DT builds and checks need.""" + if os.path.exists(dest): + shutil.rmtree(dest) + os.makedirs(dest) + specs = [s for s in ARCHIVE_PATHS + [f'arch/{arch}'] + if s.startswith(':(') or git_ok(repo, 'cat-file', '-e', f'{sha}:{s}')] + _archive(repo, sha, specs, dest) + for _ in range(3): + missing = sorted(p for p in _dangling(dest) + if git_ok(repo, 'cat-file', '-e', f'{sha}:{p}')) + if not missing: + break + _archive(repo, sha, missing, dest) + + +# -------------------------------------------------------------------------- +# DTB build (W=1) and its log + +QUIET_RE = re.compile(r'^ ([A-Z][A-Z0-9_-]*)(?: \[C\])?\s+(\S+)\s*$') +MAKE_ERR_RE = re.compile(r'^make(?:\[\d+\])?: \*\*\* \[(?:[^\]]*?): (\S+)\] ' + r'(?:Error \d+|Interrupt)') +DTC_WARN_RE = re.compile(r'^(?:(?P[^\s:]+):(?P\d+)\.\d+(?:-[\d.]+)?: )?' + r'Warning \((?P[^)]+)\): (?P.*)$') +ALSO_RE = re.compile(r'^\s+also defined at (?P[^\s:]+):(?P\d+)') +ERR_RES = [ + re.compile(r'^Error: (?P[^\s:]+):(?P\d+)\.\d+(?:-[\d.]+)? (?P.*)$'), + re.compile(r'^(?:(?P[^\s:]+):(?P\d+)\.\d+(?:-[\d.]+)?: )?' + r'ERROR \((?P[^)]+)\): (?P.*)$'), + re.compile(r'^(?P[^\s:]+):(?P\d+):\d+: (?:fatal )?error: (?P.*)$'), + re.compile(r'^(?PFATAL ERROR: .*)$'), + re.compile(r'^(?PERROR: .*)$'), +] + + +class Side: + def __init__(self, name, sha, out): + self.name = name + self.sha = sha + self.src = os.path.join(out, 'src', name) + self.obj = os.path.join(out, 'build', name) + self.logs = os.path.join(out, 'logs') + self.dtbs = {} # dtb -> info dict + self.failed = {} # dtb -> problem + self.problems = [] # comparable problems + self.schema = None # processed-schema.json path + self.schema_cmd = None + + def rel(self, path): + """Repo-relative path for a path inside this side's src or obj.""" + if not path: + return path + for root in (self.src, self.obj): + if path.startswith(root + '/'): + path = path[len(root) + 1:] + break + else: + if os.path.isabs(path): + rp = os.path.realpath(path) + for root in (os.path.realpath(self.src), os.path.realpath(self.obj)): + if rp.startswith(root + '/'): + return rp[len(root) + 1:] + if path.startswith('scripts/dtc/include-prefixes/'): + rp = os.path.realpath(os.path.join(self.src, path)) + sr = os.path.realpath(self.src) + if rp.startswith(sr + '/'): + return rp[len(sr) + 1:] + return path + + +def parse_dtbs_log(side, out): + """Return {dtb: {'warnings': [...], 'errors': [...]}}, failed set, stray errors.""" + per = collections.defaultdict(lambda: {'warnings': [], 'errors': []}) + failed, stray = set(), [] + cur, last = None, None + for line in out.splitlines(): + m = QUIET_RE.match(line) + if m: + cur = m.group(2) if m.group(1) == 'DTC' and m.group(2).endswith('.dtb') else None + last = None + continue + m = MAKE_ERR_RE.match(line) + if m: + if m.group(1).endswith('.dtb'): + failed.add(m.group(1)) + cur = None + continue + m = DTC_WARN_RE.match(line) + if m and cur: + last = {'file': side.rel(m.group('file')), 'line': m.group('line'), + 'check': m.group('check'), 'msg': m.group('msg'), 'also': []} + per[cur]['warnings'].append(last) + continue + m = ALSO_RE.match(line) + if m and last is not None: + last['also'].append(f"{side.rel(m.group('file'))}:{m.group('line')}") + continue + for rx in ERR_RES: + m = rx.match(line) + if m: + d = m.groupdict() + err = {'file': side.rel(d.get('file')), 'line': d.get('line'), + 'msg': d['msg'], 'text': line.replace(side.src + '/', '')} + (per[cur]['errors'] if cur else stray).append(err) + break + return per, failed, stray + + +def parse_deps(cmdfile, side): + deps, src, in_deps = set(), None, False + with open(cmdfile, encoding='utf-8', errors='replace') as f: + for line in f: + if line.startswith('source_'): + src = line.split(':=', 1)[1].strip() + deps.add(src) + elif line.startswith('deps_'): + in_deps = True + elif in_deps: + s = line.strip() + if not s: + in_deps = False + continue + s = s.rstrip('\\').strip() + if s and not s.startswith('$('): + deps.add(s) + return {side.rel(d) for d in deps}, side.rel(src) + + +def build_side(args, side, env): + """Extract, configure and build all DTBs with W=1.""" + extract_tree(args.tree, side.sha, side.src, args.arch) + if os.path.exists(side.obj): + shutil.rmtree(side.obj) + os.makedirs(side.obj) + frag = os.path.join(side.obj, 'dt-check.config') + with open(frag, 'w') as f: + f.write('\n'.join(args.kconfig) + '\n') + mk = ['make', '-C', side.src, f'O={side.obj}', f'ARCH={args.arch}'] + rc, out = run(mk + [f'KCONFIG_ALLCONFIG={frag}', 'allnoconfig'], env=env, + logfile=os.path.join(side.logs, f'{side.name}-config.log')) + if rc != 0: + raise SetupError(f'{side.name}: allnoconfig failed (see logs/{side.name}-config.log)') + with open(os.path.join(side.obj, '.config')) as f: + cfg = f.read() + for sym in args.kconfig: + if sym.endswith('=y') and f'\n{sym}\n' not in '\n' + cfg: + raise SetupError(f'{side.name}: {sym} did not stick in .config') + rc, out = run(mk + [f'-j{args.jobs}', '-k', '-Otarget', 'W=1', 'dtbs'], env=env, + logfile=os.path.join(side.logs, f'{side.name}-dtbs.log')) + per, failed, stray = parse_dtbs_log(side, out) + side.dtbs_rc = rc + dtsroot = os.path.join(side.obj, 'arch', args.arch, 'boot', 'dts') + for root, _dirs, files in os.walk(dtsroot): + for n in files: + if not n.endswith('.dtb'): + continue + path = os.path.join(root, n) + dtb = os.path.relpath(path, side.obj) + info = {'path': path, 'sha': sha256(path), 'deps': set(), 'source': None, + 'warnings': per[dtb]['warnings'] if dtb in per else []} + cmdfile = os.path.join(root, f'.{n}.cmd') + if os.path.exists(cmdfile): + info['deps'], info['source'] = parse_deps(cmdfile, side) + side.dtbs[dtb] = info + for dtb in failed | {d for d in per if per[d]['errors'] and d not in side.dtbs}: + errs = per[dtb]['errors'] if dtb in per else [] + side.failed[dtb] = errs + if rc != 0 and not side.failed: + side.failed[''] = stray or [{'file': None, 'line': None, + 'msg': out.strip().splitlines()[-1] if out.strip() else 'make failed', + 'text': ''}] + + +# -------------------------------------------------------------------------- +# FDT parsing (compatibles), binding $ref closure + +def fdt_compatibles(path): + with open(path, 'rb') as f: + data = f.read() + if len(data) < 40 or struct.unpack_from('>I', data, 0)[0] != 0xd00dfeed: + return set() + (_magic, _size, off_struct, off_strings, _rsv, _ver, _lc, _cpu, + size_strings, size_struct) = struct.unpack_from('>10I', data, 0) + strings = data[off_strings:off_strings + size_strings] + i, end, out = off_struct, off_struct + size_struct, set() + while i < end: + tok = struct.unpack_from('>I', data, i)[0] + i += 4 + if tok == 1: # FDT_BEGIN_NODE + j = data.index(b'\0', i) + i = (j + 1 + 3) & ~3 + elif tok == 3: # FDT_PROP + ln, nameoff = struct.unpack_from('>II', data, i) + i += 8 + name = strings[nameoff:strings.index(b'\0', nameoff)] + if name == b'compatible': + out.update(x.decode('ascii', 'replace') + for x in data[i:i + ln].split(b'\0') if x) + i = (i + ln + 3) & ~3 + elif tok == 9: # FDT_END + break + return out + + +REF_RE = re.compile(r'''\$ref:\s*['"]?([^\s'"#]+)''') +TOKEN_RE = re.compile(r'[A-Za-z0-9][\w.,+-]*') + + +def binding_closure(src, changed): + """Changed bindings plus every binding that $refs them (transitively).""" + broot = os.path.join(src, BINDINGS) + rev = collections.defaultdict(set) + for root, _dirs, files in os.walk(broot): + for n in files: + if not n.endswith('.yaml'): + continue + path = os.path.join(root, n) + rel = os.path.relpath(path, src) + try: + text = open(path, encoding='utf-8', errors='replace').read() + except OSError: + continue + for ref in REF_RE.findall(text): + if ref.startswith('http://devicetree.org/schemas/'): + tgt = BINDINGS + '/' + ref[len('http://devicetree.org/schemas/'):] + elif ref.startswith('/schemas/'): + tgt = BINDINGS + '/' + ref[len('/schemas/'):] + elif ref.startswith('/'): + continue + else: + tgt = os.path.normpath(os.path.join(os.path.dirname(rel), ref)) + rev[tgt].add(rel) + seen, todo = set(changed), list(changed) + while todo: + f = todo.pop() + for r in rev.get(f, ()): + if r not in seen: + seen.add(r) + todo.append(r) + return seen + + +def binding_selectors(src, files): + """(tokens that may be compatibles, applies-to-everything flag).""" + tokens, everything = set(), False + for rel in files: + path = os.path.join(src, rel) + if not os.path.exists(path): + continue + text = open(path, encoding='utf-8', errors='replace').read() + tokens.update(TOKEN_RE.findall(text)) + m = re.search(r'^select:(.*(?:\n[ \t]+.*)*)', text, re.M) + if m: + body = m.group(1).strip() + if body != 'false' and 'compatible' not in body: + everything = True + if re.search(r'compatible:\s*\n\s+pattern:', text): + everything = True + return tokens, everything + + +# -------------------------------------------------------------------------- +# processed schema + +def schema_cache_key(args, sha, dtschema_version): + tree = git(args.tree, 'rev-parse', f'{sha}:{BINDINGS}').strip() + return f'{dtschema_version}-{tree}' + + +def install_schema(side, json_path, cmd_template): + d = os.path.join(side.obj, BINDINGS) + os.makedirs(d, exist_ok=True) + shutil.copyfile(json_path, os.path.join(d, 'processed-schema.json')) + if cmd_template: + with open(os.path.join(d, '.processed-schema.json.cmd'), 'w') as f: + f.write(cmd_template.replace('@SRCTREE@', side.src)) + side.schema = os.path.join(d, 'processed-schema.json') + + +def parse_schema_log(side, out): + probs = [] + for line in out.splitlines(): + m = re.match(r'^(\S+\.yaml): (.*)$', line) + if not m: + continue + f, msg = side.rel(m.group(1)), m.group(2) + cls = 'binding-dropped' if msg.startswith('ignoring') else 'binding-warning' + probs.append(problem(cls, f, ('mk-schema', mask(msg)), msg, file=f, + line=yaml_key_line(side.src, f, msg))) + return probs + + +def build_schema(args, side, env, cache_dir, key): + """Build (or fetch from cache) processed-schema.json for one side.""" + if cache_dir: + cj = os.path.join(cache_dir, 'schema', key + '.json') + if os.path.exists(cj): + tmpl = open(cj[:-5] + '.cmd').read() if os.path.exists(cj[:-5] + '.cmd') else None + install_schema(side, cj, tmpl) + logtxt = open(cj[:-5] + '.log').read() if os.path.exists(cj[:-5] + '.log') else '' + logtxt = logtxt.replace('@SRCTREE@', side.src) + return parse_schema_log(side, logtxt), True + mk = ['make', '-C', side.src, f'O={side.obj}', f'ARCH={args.arch}'] + rc, out = run(mk + ['dt_binding_schemas'], env=env, + logfile=os.path.join(side.logs, f'{side.name}-schema.log')) + path = os.path.join(side.obj, BINDINGS, 'processed-schema.json') + if rc != 0 or not os.path.exists(path) or os.path.getsize(path) == 0: + raise SetupError(f'{side.name}: building processed-schema.json failed ' + f'(see logs/{side.name}-schema.log)') + side.schema = path + cmdf = os.path.join(side.obj, BINDINGS, '.processed-schema.json.cmd') + tmpl = open(cmdf).read().replace(side.src, '@SRCTREE@') if os.path.exists(cmdf) else None + side.schema_cmd = tmpl + if cache_dir: + os.makedirs(os.path.join(cache_dir, 'schema'), exist_ok=True) + base = os.path.join(cache_dir, 'schema', key) + shutil.copyfile(path, base + '.json.tmp') + os.replace(base + '.json.tmp', base + '.json') + if tmpl: + open(base + '.cmd', 'w').write(tmpl) + open(base + '.log', 'w').write(out.replace(side.src, '@SRCTREE@')) + return parse_schema_log(side, out), False + + +# -------------------------------------------------------------------------- +# dt-validate + +UNEXPECTED_RE = re.compile(r'^(Additional|Unevaluated) properties are not allowed ' + r'\((.*) (?:was|were) unexpected\)$') +NOMATCH_RE = re.compile(r'^(.*) do(?:es)? not match any of the regexes: .*$') +QUOTED_RE = re.compile(r"'((?:[^'\\]|\\.)*)'") +# jsonschema messages start with the instance value ("[[45], [253]] is too +# long", "{...whole node...} is not valid under ..."). Values such as phandle +# numbers change when unrelated nodes are added, so keys drop them. +INSTANCE_RE = re.compile( + r"^(?P.+) (?P(?:is|are) (?:too long|too short|not of type .+|" + r"not valid under any of the given schemas|not one of .+|" + r"greater than(?: or equal to)? .+|less than(?: or equal to)? .+|" + r"not a multiple of .+|expected to be empty|a required property|" + r"a dependency of .+|not allowed for .+)|should not be valid under .+|" + r"does not match .+|does not contain .+|has non-unique elements)$", re.S) + + +def mask_instance(msg): + m = INSTANCE_RE.match(msg) + if m and not m.group('rest').startswith(('is a required property', 'is a dependency of')): + return ' ' + m.group('rest') + if msg.startswith('False schema does not allow '): + return 'False schema does not allow ' + return mask(msg) + + +def schema_short(url): + if not url: + return '?' + url = url.rstrip('#') + for p in ('http://devicetree.org/schemas/', 'http://devicetree.org/meta-schemas/'): + if url.startswith(p): + return url[len(p):] + return url + + +def diag_problems(dtb, diag): + """Problems from one dt-validate JSON diagnostic.""" + t = diag.get('type') + node = diag.get('node') or diag.get('nodename') or '?' + if t == 'unmatched': + compat = diag.get('compatible') or [] + if isinstance(compat, str): + compat = [compat] + return [problem('unmatched', dtb, (node, ','.join(compat)), + f'{node}: compatible {compat} matches no binding', + detail=[diag.get('message', '')])] + pp = [str(x) for x in diag.get('property_path') or []] + schema = schema_short(diag.get('schema')) + msg = diag.get('message', '') + where = node + (': ' + '/'.join(pp) if pp else '') + detail = [c.get('message', '') for c in diag.get('context') or []][:8] + m = UNEXPECTED_RE.match(msg) + if m: + return [problem('schema', dtb, (node, '/'.join(pp), schema, m.group(1) + '-unexpected', n), + f"{where}: property '{n}' is not allowed ({m.group(1).lower()} properties) [{schema}]", + detail=detail, line=None, file=None) | {'prop': n, 'node': node} + for n in QUOTED_RE.findall(m.group(2))] + m = NOMATCH_RE.match(msg) + if m: + return [problem('schema', dtb, (node, '/'.join(pp), schema, 'name-not-matching', n), + f"{where}: '{n}' does not match the allowed names [{schema}]", + detail=detail) | {'prop': n, 'node': node} + for n in QUOTED_RE.findall(m.group(1))] + sp = diag.get('schema_path') or [] + keyword = str(sp[-1]) if sp else '' + p = problem('schema', dtb, (node, '/'.join(pp), schema, keyword, mask_instance(msg)), + f'{where}: {msg} [{schema}]', detail=detail) + p['node'], p['prop'] = node, pp[0] if pp else None + return [p] + + +TEXT_HDR_RE = re.compile(r'^(\S+\.dtb): (.*)$') + + +def text_problems(side, text, path_to_dtb): + """Fallback parser for dt-validate text output (dtschema < 2026.9).""" + probs, cur = [], None + for line in text.splitlines(): + m = TEXT_HDR_RE.match(line) + if m: + dtb = path_to_dtb.get(os.path.abspath(os.path.join(side.obj, m.group(1))), + path_to_dtb.get(m.group(1), m.group(1))) + hdr = m.group(2) + cls = 'unmatched' if 'failed to match any schema' in hdr else 'schema' + cur = problem(cls, dtb, (mask(hdr),), hdr) + probs.append(cur) + elif line.startswith('\t') and cur is not None: + s = line.strip() + if s.startswith('from schema $id:'): + cur['key'].append(schema_short(s.split(':', 1)[1].strip())) + else: + cur['detail'].append(s) + else: + cur = None + return probs + + +def run_validation(args, sides, tasks, env, json_ok, cache_dir): + """tasks: list of (side_name, limit or None, [dtb]). Returns problems per side.""" + jobs = [] + for side_name, limit, dtbs in tasks: + side = sides[side_name] + dtbs = sorted(dtbs, key=lambda d: -os.path.getsize(side.dtbs[d]['path'])) + n = max(1, min(len(dtbs), args.jobs)) + for k in range(n): + chunk = dtbs[k::n] + if chunk: + jobs.append((side_name, limit, chunk)) + results = {name: [] for name in sides} + errors = [] + validated = {name: {} for name in sides} + + def one(idx, side_name, limit, chunk): + side = sides[side_name] + jout = os.path.join(side.logs, f'{side_name}-validate-{idx}.json') + cmd = ['dt-validate', '-m'] + (['-l', limit] if limit else []) + \ + ['-u', os.path.join(side.src, BINDINGS), '-p', side.schema] + if json_ok: + cmd += ['--json-output', jout] + if cache_dir: + cmd += ['--cache-dir', os.path.join(cache_dir, 'validate')] + cmd += [side.dtbs[d]['path'] for d in chunk] + rc, out = run(cmd, cwd=side.obj, env=env, + logfile=os.path.join(side.logs, f'{side_name}-validate-{idx}.log')) + return idx, side_name, limit, chunk, rc, out, jout + + with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as ex: + futs = [ex.submit(one, i, *j) for i, j in enumerate(jobs)] + for fut in concurrent.futures.as_completed(futs): + idx, side_name, limit, chunk, rc, out, jout = fut.result() + side = sides[side_name] + p2d = {side.dtbs[d]['path']: d for d in chunk} + p2d.update({os.path.relpath(side.dtbs[d]['path'], side.obj): d for d in chunk}) + if rc != 0 or 'Traceback (most recent call last)' in out: + tail = out.strip().splitlines()[-1] if out.strip() else f'exit {rc}' + errors.append(f'{side_name}: dt-validate chunk {idx} failed: {tail}') + continue + for d in chunk: + validated[side_name][d] = 'limited' if limit else 'full' + if json_ok: + with open(jout, encoding='utf-8') as f: + diags = json.load(f) + for diag in diags: + fpath = diag.get('file') or '' + dtb = p2d.get(fpath) or p2d.get(os.path.abspath(os.path.join(side.obj, fpath))) + if dtb is None: + continue + for p in diag_problems(dtb, diag): + p['mode'] = 'limited' if limit else 'full' + results[side_name].append(p) + else: + for p in text_problems(side, out, p2d): + p['mode'] = 'limited' if limit else 'full' + results[side_name].append(p) + return results, validated, errors + + +# -------------------------------------------------------------------------- +# source map from dtc -T -T (for annotations of schema problems) + +LOCPIECE_RE = re.compile(r'^(.*):(\d+):\d+-\d+:\d+$') + + +def source_map(side, dtb, arch): + tmp = os.path.join(side.obj, os.path.dirname(dtb), '.' + os.path.basename(dtb) + '.dts.tmp') + dtc = os.path.join(side.obj, 'scripts', 'dtc', 'dtc') + if not (os.path.exists(tmp) and os.path.exists(dtc)): + return {}, {} + rc, out = run([dtc, '-q', '-I', 'dts', '-O', 'dts', '-T', '-T', + '-i', os.path.join(side.src, os.path.dirname(dtb)), + '-i', os.path.join(side.src, 'scripts', 'dtc', 'include-prefixes'), + '-o', '-', tmp]) + if rc != 0: + return {}, {} + nodes, props, stack = {}, {}, [] + + def locs(comment): + res = [] + for piece in comment.split(', '): + m = LOCPIECE_RE.match(piece.strip()) + if m: + res.append((side.rel(m.group(1)), int(m.group(2)))) + return res + + for raw in out.splitlines(): + s = raw.strip() + comment = '' + if s.endswith('*/') and '/*' in s: + comment = s[s.rfind('/*') + 2:-2].strip() + s = s[:s.rfind('/*')].strip() + if s.endswith('{'): + name = s[:-1].strip().split()[-1] + if not stack: + path = '/' + else: + path = (stack[-1].rstrip('/') + '/' + name) + stack.append(path) + nodes[path] = locs(comment) + elif s.startswith('};'): + if stack: + stack.pop() + elif s.endswith(';') and stack and not s.startswith('/'): + name = re.split(r'\s*=\s*|;', s, maxsplit=1)[0].split()[-1] + props[(stack[-1], name)] = locs(comment) + return nodes, props + + +def pick_location(cands, changed): + for f, ln in cands: + if f in changed: + return f, ln + return cands[0] if cands else (None, None) + + +# -------------------------------------------------------------------------- +# dt_binding_check log + +YAMLLINT_RE = re.compile(r'^(?P\S+\.yaml):(?P\d+):\d+: \[(?Perror|warning)\] ' + r'(?P.*?)(?: \((?P[\w-]+)\))?$') +STYLE_RE = re.compile(r'^(?P\S+?):(?P\d+):(?: example (?P\d+))? ' + r'\[(?P[\w-]+)\] (?P.*)$') +YAMLPOS_RE = re.compile(r'^(?P\S+\.yaml):(?P\d+):\d+: (?P.*)$') +FILEMSG_RE = re.compile(r'^(?P\S+\.(?:yaml|dtb|dts)): (?P.*)$') + + +def example_yaml(path): + return re.sub(r'\.example\.dt[bs]$', '.yaml', path) + + +def parse_binding_log(side, out): + probs = [] + tag, target, cur, tb = None, None, None, None + lines = out.splitlines() + for line in lines: + if tb is not None: # inside a Python traceback + if line.startswith(' ') or line.startswith('Traceback'): + continue + exc = line.strip() + tb = None + if tag == 'DTEX' or (tag == 'DTC' and target and 'example' in target): + f = example_yaml(side.rel(target)) + probs.append(problem('example-build', f, ('extract', mask(exc)), + f'example extraction failed: {exc}', file=f)) + elif tag == 'CHKDT': + probs.append(problem('binding-invalid', 'dt-doc-validate', ('crash', mask(exc)), + f'dt-doc-validate crashed: {exc}')) + else: + probs.append(problem('tool-error', tag or 'make', (mask(exc),), + f'{tag or "make"} step crashed: {exc}')) + continue + if line.startswith('Traceback (most recent call last)'): + tb = [] + cur = None + continue + m = QUIET_RE.match(line) + if m: + tag, target, cur = m.group(1), m.group(2), None + continue + m = MAKE_ERR_RE.match(line) + if m: + t = m.group(1) + if '.example.' in t: + f = example_yaml(side.rel(t)) + probs.append(problem('example-build', f, ('make',), + 'example does not build', file=f)) + cur = None + continue + if line.startswith('\t') and cur is not None: + s = line.strip() + if s.startswith('from schema $id:'): + cur['key'].append(schema_short(s.split(':', 1)[1].strip())) + else: + cur['detail'].append(s) + continue + cur = None + if tag == 'LINT': + m = YAMLLINT_RE.match(line) + if m: + f = side.rel(m.group('file')) + rule = m.group('rule') or m.group('lvl') + cls = 'binding-invalid' if rule == 'syntax' else 'yamllint' + probs.append(problem(cls, f, ('yamllint', m.group('lvl'), rule, mask(m.group('msg'))), + f"yamllint {m.group('lvl')}: {m.group('msg')} ({rule})", + file=f, line=int(m.group('line')))) + continue + if tag == 'STYLE': + m = STYLE_RE.match(line) + if m: + f = side.rel(m.group('file')) + probs.append(problem('style', f, (m.group('rule'), mask(m.group('msg'))), + f"[{m.group('rule')}] {m.group('msg')}", + file=f, line=int(m.group('line')))) + continue + m = FILEMSG_RE.match(line) + if m: + f = side.rel(m.group('file')) + probs.append(problem('style', f, ('load', mask(m.group('msg'))), + m.group('msg'), file=f)) + continue + if tag == 'CHKDT': + m = YAMLPOS_RE.match(line) + if m: + f = side.rel(m.group('file')) + probs.append(problem('binding-invalid', f, ('parse', mask(m.group('msg'))), + m.group('msg'), file=f, line=int(m.group('line')))) + continue + m = FILEMSG_RE.match(line) + if m: + f = side.rel(m.group('file')) + cur = problem('binding-invalid', f, ('meta', mask(m.group('msg'))), + m.group('msg'), file=f) + probs.append(cur) + continue + if tag in ('DTC', 'DTEX'): + m = DTC_WARN_RE.match(line) + if m and m.group('file') and '.example.' in m.group('file'): + f = example_yaml(side.rel(m.group('file'))) + probs.append(problem('example-schema', f, + ('dtc', m.group('check'), mask(m.group('msg'))), + f"example dtc warning ({m.group('check')}): {m.group('msg')}", + file=f)) + continue + hit = False + for rx in ERR_RES: + m = rx.match(line) + if m: + f = example_yaml(side.rel(target)) if target else 'example' + probs.append(problem('example-build', f, ('dtc', mask(m.group('msg'))), + f"example does not compile: {m.group('msg')}", file=f)) + hit = True + break + if hit: + continue + m = FILEMSG_RE.match(line) + if m and '.example.' in m.group('file'): + f = example_yaml(side.rel(m.group('file'))) + cur = problem('example-schema', f, ('validate', mask(m.group('msg'))), + m.group('msg'), file=f) + probs.append(cur) + continue + if tag == 'SCHEMA': + m = FILEMSG_RE.match(line) + if m: + probs.extend(parse_schema_log(side, line)) + return probs + + +KEYLINE_RE = re.compile(r'^(\s*)(?:-\s+)?([^\s:#\'"][^:#]*?|\'[^\']*\'|"[^"]*"):(?:\s|$)') + + +def yaml_key_line(src, rel, message): + """Line of the YAML key path a dtschema message starts with (a:b:c: ...).""" + m = re.match(r'^ignoring, error in schema: (.*)$', message) + if m: + path = [x.strip() for x in m.group(1).split(': ')] + else: + head = message.split(': ', 1)[0] + if ' ' in head: + return None + path = [x for x in head.split(':') if x] + if not path: + return None + try: + with open(os.path.join(src, rel), encoding='utf-8', errors='replace') as f: + lines = f.read().splitlines() + except OSError: + return None + stack, best = [], None + for i, line in enumerate(lines, 1): + km = KEYLINE_RE.match(line) + if not km or line.lstrip().startswith('#'): + continue + ind = len(km.group(1)) + key = km.group(2).strip('\'"') + while stack and stack[-1][0] >= ind: + stack.pop() + stack.append((ind, key)) + keys = [k for _, k in stack] + n = 0 + while n < len(keys) and n < len(path) and keys[n] == path[n]: + n += 1 + if n == len(keys) and n > 0 and (best is None or n > best[0]): + best = (n, i) + if n == len(path): + break + return best[1] if best else None + + +def examples_line(src, rel): + try: + with open(os.path.join(src, rel), encoding='utf-8', errors='replace') as f: + for i, line in enumerate(f, 1): + if line.startswith('examples:'): + return i + except OSError: + pass + return None + + +# -------------------------------------------------------------------------- +# style check of DTS sources + +def style_check(side, files, mode, python, env): + checker = os.path.join(side.src, 'scripts', 'dtc', 'dt-check-style') + files = [f for f in files if os.path.exists(os.path.join(side.src, f))] + if not files or not os.path.exists(checker): + return [] + rc, out = run([python, checker, f'--mode={mode}', *files], cwd=side.src, env=env, + logfile=os.path.join(side.logs, f'{side.name}-style.log')) + probs = [] + if 'Traceback (most recent call last)' in out: + exc = [l for l in out.splitlines() if l and not l.startswith(' ')][-1] + return [problem('tool-error', 'dt-check-style', (mask(exc),), + f'dt-check-style crashed: {exc}')] + for line in out.splitlines(): + m = STYLE_RE.match(line) + if m: + f = m.group('file') + probs.append(problem('style', f, (m.group('rule'), mask(m.group('msg'))), + f"[{m.group('rule')}] {m.group('msg')}", file=f, + line=int(m.group('line')))) + return probs + + +# -------------------------------------------------------------------------- +# comparison and output + +def compare(base_probs, head_probs, scope_map): + def key(p, mapped): + s = scope_map.get(p['scope'], p['scope']) if mapped else p['scope'] + return (p['class'], s, tuple(p['key'])) + bc = collections.Counter(key(p, True) for p in base_probs) + hg = collections.defaultdict(list) + for p in head_probs: + hg[key(p, False)].append(p) + new = [] + for k, plist in hg.items(): + extra = len(plist) - bc.get(k, 0) + if extra > 0: + new.extend(plist[len(plist) - extra:]) + hc = collections.Counter({k: len(v) for k, v in hg.items()}) + fixed = [] + bg = collections.defaultdict(list) + for p in base_probs: + bg[key(p, True)].append(p) + for k, plist in bg.items(): + extra = len(plist) - hc.get(k, 0) + if extra > 0: + fixed.extend(plist[:extra]) + return new, fixed + + +def group_new(new, blocking, changed_head, arch): + groups = collections.OrderedDict() + for p in new: + cls = p['class'] + gk = (cls, tuple(p['key'])) if cls in DTB_SCOPED else (cls, p['scope'], tuple(p['key'])) + g = groups.get(gk) + if g is None: + g = groups[gk] = {'class': cls, 'blocking': cls in blocking, + 'message': p['message'], 'scopes': [], 'file': None, + 'line': None, 'detail': p['detail'][:10], 'count': 0} + g['count'] += 1 + sc = short_dtb(p['scope'], arch) if cls in DTB_SCOPED else p['scope'] + if sc not in g['scopes']: + g['scopes'].append(sc) + if p.get('file') and (g['file'] is None or + (p['file'] in changed_head and g['file'] not in changed_head)): + g['file'], g['line'] = p['file'], p.get('line') + order = list(CLASSES) + for g in groups.values(): + g['scopes'].sort() + return sorted(groups.values(), key=lambda g: (not g['blocking'], order.index(g['class']))) + + +def esc_data(s): + return s.replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A') + + +def esc_prop(s): + return esc_data(s).replace(':', '%3A').replace(',', '%2C') + + +def annotate(groups, limit): + shown = 0 + for g in groups: + if shown >= limit: + break + level = 'error' if g['blocking'] else 'warning' + where = '' + if g['file']: + where = f"file={esc_prop(g['file'])}" + if g['line']: + where += f",line={g['line']}" + where += ',' + scopes = g['scopes'] + tail = ', '.join(scopes[:5]) + (f' and {len(scopes) - 5} more' if len(scopes) > 5 else '') + title = f"dt-check {g['class']}" + ( + f" ({len(scopes)} DTB{'' if len(scopes) == 1 else 's'})" if g['class'] in DTB_SCOPED else '') + msg = g['message'] + (f'\nIn: {tail}' if g['class'] in DTB_SCOPED else '') + if g['detail']: + msg += '\n' + '\n'.join(g['detail'][:5]) + print(f"::{level} {where}title={esc_prop(title)}::{esc_data(msg)}", flush=True) + shown += 1 + if len(groups) > shown: + print(f'::notice title=dt-check::{len(groups) - shown} more new problem groups; ' + 'see the job summary or report.json', flush=True) + + +def md_escape(s): + return s.replace('|', '\\|').replace('<', '<').replace('>', '>') + + +def write_summary(path, report): + st = report['status'] + lines = [] + head = {'skipped': 'nothing to check', 'pass': 'no new blocking problems', + 'fail': 'new blocking problems', 'error': 'check could not run'}[st] + lines.append(f"## Device tree check: {st.upper()} ({head})\n") + lines.append(f"base `{report['base'][:12]}` -> head `{report['head'][:12]}`" + + (' (merge base of the given base and head)' if report.get('merge_base_used') else '') + + f"; dtschema {report['versions'].get('dtschema', '?')}\n") + if report.get('error'): + lines.append(f"**Error:** {md_escape(report['error'])}\n") + if st == 'skipped': + lines.append('No device tree, DT header or binding files changed.\n') + if report.get('counts'): + lines.append('| Class | Blocks | Base | Head | New | Fixed |') + lines.append('|---|---|---:|---:|---:|---:|') + for cls, c in report['counts'].items(): + lines.append(f"| {cls} | {'yes' if cls in report['blocking_classes'] else 'no'} | " + f"{c['base']} | {c['head']} | {c['new']} | {c['fixed']} |") + lines.append('') + groups = report.get('new', []) + blk = [g for g in groups if g['blocking']] + adv = [g for g in groups if not g['blocking']] + + def fmt(g): + loc = f" (`{g['file']}{':' + str(g['line']) if g['line'] else ''}`)" if g['file'] else '' + sc = g['scopes'] + where = ', '.join(f'`{s}`' for s in sc[:6]) + (f' and {len(sc) - 6} more' if len(sc) > 6 else '') + return f"- **{g['class']}** {md_escape(g['message'])}{loc} in {where}" + if blk: + lines.append(f'### New blocking problems ({len(blk)})\n') + lines += [fmt(g) for g in blk[:100]] + lines.append('') + if adv: + lines.append(f'### New advisory problems ({len(adv)} groups)\n') + lines.append('
show\n') + lines += [fmt(g) for g in adv[:200]] + if len(adv) > 200: + lines.append(f'- ... {len(adv) - 200} more in report.json') + lines.append('\n
\n') + if st in ('pass', 'fail'): + d = report['dtbs'] + lines.append('### Scope\n') + lines.append(f"- DT-related files changed: {len(report['dt_changed_files'])}") + lines.append(f"- DTBs built: base {d['base_built']}, head {d['head_built']}; " + f"failing at head: {len(d['head_failed'])}") + lines.append(f"- DTBs depending on changed files: {len(d['affected'])}; " + f"output changed or new: {len(d['changed_output'])}") + lines.append(f"- schema check: full {len(d['validated_full'])}, " + f"limited to changed bindings {len(d['validated_limited'])}") + lines.append(f"- bindings checked: {len(report['bindings_checked'])}") + lines.append(f"- fixed problems: {sum(c['fixed'] for c in report['counts'].values())}") + lines.append('') + if report.get('notes'): + lines.append('### Notes\n') + lines += [f'- {md_escape(n)}' for n in report['notes']] + lines.append('') + lines.append('### Timings (s)\n') + lines.append(', '.join(f'{k} {v}' for k, v in report['timings_s'].items()) + '\n') + text = '\n'.join(lines) + '\n' + with open(path, 'w', encoding='utf-8') as f: + f.write(text) + return text + + +# -------------------------------------------------------------------------- +# main flow + +def tool_versions(): + v = {'python': sys.version.split()[0], 'dt-check': VERSION} + for name, cmd in (('dtschema', ['dt-validate', '-V']), ('yamllint', ['yamllint', '--version']), + ('make', ['make', '--version']), ('git', ['git', '--version'])): + try: + rc, out = run(cmd) + v[name] = out.strip().splitlines()[0] if rc == 0 and out.strip() else '?' + except OSError: + v[name] = None + if v.get('yamllint'): + v['yamllint'] = v['yamllint'].replace('yamllint ', '') + return v + + +def check(args, report, timings): + out = args.out + repo = args.tree + env = dict(os.environ) + env['LC_ALL'] = 'C' + env['TMPDIR'] = os.path.join(out, 'tmp') + for k in ('KBUILD_OUTPUT', 'KCONFIG_CONFIG', 'CROSS_COMPILE', 'LLVM'): + env.pop(k, None) + os.makedirs(env['TMPDIR'], exist_ok=True) + os.makedirs(os.path.join(out, 'logs'), exist_ok=True) + + with timings('resolve'): + base = git(repo, 'rev-parse', '--verify', args.base + '^{commit}').strip() + head = git(repo, 'rev-parse', '--verify', args.head + '^{commit}').strip() + report['base_requested'] = base + if base != head and not git_ok(repo, 'merge-base', '--is-ancestor', base, head): + mb = git(repo, 'merge-base', base, head, check=False).strip() + if mb: + base = mb + report['merge_base_used'] = True + note(f'base is not an ancestor of head; using merge base {mb[:12]}') + else: + report['notes'].append('no merge base found (shallow clone?); diffing the given commits') + report['base'], report['head'] = base, head + changed = changed_files(repo, base, head) + targets = dt_symlink_targets(repo, head) | dt_symlink_targets(repo, base) + dt_changed = [] + for c in changed: + kinds = {classify(p, targets) for p in (c['path'], c['old']) if p} + kinds.discard(None) + if kinds: + c['kind'] = sorted(kinds) + dt_changed.append(c) + report['changed_files'] = len(changed) + report['dt_changed_files'] = dt_changed + + if not dt_changed: + report['status'] = 'skipped' + note('no DT-related files changed; nothing to do') + return 0 + + for tool in ('make', 'tar', 'dt-validate', 'dt-mk-schema', 'dt-doc-validate', + 'dt-extract-example', 'flex', 'bison'): + if not shutil.which(tool): + raise SetupError(f'{tool} not found on PATH') + if not shutil.which('yamllint'): + report['notes'].append('yamllint not on PATH; binding lint skipped') + _, help_out = run(['dt-validate', '--help']) + json_ok = '--json-output' in help_out + validate_cache = args.cache_dir if '--cache-dir' in help_out else None + if not json_ok: + report['notes'].append('dt-validate has no --json-output (dtschema < 2026.9); ' + 'using the less precise text parser') + dtschema_version = report['versions'].get('dtschema') or '?' + + # file renames (base path -> head path), changed paths on each side + renames = {c['old']: c['path'] for c in dt_changed if c['status'] == 'R'} + changed_head = {c['path'] for c in dt_changed if c['path']} + changed_base = {c['old'] for c in dt_changed if c['old']} + kinds = set(k for c in dt_changed for k in c['kind']) + bind_head = sorted(c['path'] for c in dt_changed if c['path'] and 'binding' in c['kind'] + and c['path'].endswith('.yaml')) + bind_base = sorted(c['old'] for c in dt_changed if c['old'] and 'binding' in c['kind'] + and c['old'].endswith('.yaml')) + dts_head = sorted(p for p in changed_head if p.endswith(('.dts', '.dtsi', '.dtso')) + and re.match(r'arch/[^/]+/boot/dts/', p)) + dts_base = sorted(p for p in changed_base if p.endswith(('.dts', '.dtsi', '.dtso')) + and re.match(r'arch/[^/]+/boot/dts/', p)) + + sides = {'base': Side('base', base, out), 'head': Side('head', head, out)} + + # 1. build all DTBs with W=1 on both sides + with timings('extract+config+dtbs'): + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex: + futs = [ex.submit(build_side, args, s, env) for s in sides.values()] + for f in futs: + f.result() + B, H = sides['base'], sides['head'] + + # DTB renames follow .dts renames + dtb_map = {} + for old, new in renames.items(): + if old.endswith('.dts') and new.endswith('.dts'): + dtb_map[old[:-4] + '.dtb'] = new[:-4] + '.dtb' + scope_map = dict(renames) + scope_map.update(dtb_map) + + def head_name(dtb): + return dtb_map.get(dtb, dtb) + + base_ok = {head_name(d): d for d in B.dtbs} + head_ok = set(H.dtbs) + head_failed = set(H.failed) + base_failed = {head_name(d) for d in B.failed} + + # DTBs depending on changed files (either side) + affected = {} + for d, info in H.dtbs.items(): + hit = sorted(info['deps'] & changed_head) + if hit: + affected[d] = hit + for d, info in B.dtbs.items(): + hit = sorted(info['deps'] & changed_base) + if hit: + affected.setdefault(head_name(d), []).extend(renames.get(h, h) for h in hit) + affected = {d: sorted(set(v)) for d, v in affected.items()} + changed_output = sorted(d for d in head_ok + if d not in base_ok or H.dtbs[d]['sha'] != B.dtbs[base_ok[d]]['sha']) + new_dtbs = sorted(d for d in head_ok if d not in base_ok and d not in base_failed) + stray = sorted(set(changed_output) - set(affected)) + if stray and 'tooling' not in kinds: + report['notes'].append(f'{len(stray)} DTB(s) changed output without a tracked ' + f'dependency on a changed file: ' + + ', '.join(short_dtb(d, args.arch) for d in stray[:10])) + unused = sorted(p for p in dts_head if not any(p in i['deps'] for i in H.dtbs.values()) + and p not in {e for errs in H.failed.values() for e in [x.get('file') for x in errs]}) + if unused: + report['notes'].append('changed DTS files not used by any DTB built with this config: ' + + ', '.join(unused[:20])) + + # 2. problems that cost nothing extra: build failures and dtc warnings + comparable = {d for d in head_ok if d in base_ok or d in new_dtbs} + for side in (B, H): + for d, errs in side.failed.items(): + e0 = next((e for e in errs if e.get('file')), errs[0] if errs else + {'file': None, 'line': None, 'msg': 'build failed', 'text': ''}) + side.problems.append(problem( + 'dtb-build', d, ('build-failed',), + f"{short_dtb(d, args.arch)} does not build: {e0['msg']}", + file=e0.get('file'), line=int(e0['line']) if e0.get('line') else None, + detail=[e['text'] for e in errs[:10]])) + for d, info in side.dtbs.items(): + if (head_name(d) if side is B else d) not in comparable: + continue + for w in info['warnings']: + side.problems.append(problem( + 'dtc-warning', d, (w['check'], mask(w['msg'])), + f"dtc {w['check']}: {w['msg']}", file=w['file'], + line=int(w['line']) if w['line'] else None, + detail=[f'also defined at {a}' for a in w['also']])) + # a DTB that already failed at base is not new breakage + for d in list(head_failed): + if d in base_failed: + report['notes'].append(f'{short_dtb(d, args.arch)} fails to build at base and head') + + # 3. which DTBs get schema checks + full = set() if args.no_schema else {d for d in changed_output} + if args.full and not args.no_schema: + full = set(head_ok) + limited, limit = set(), None + if (bind_head or bind_base) and not args.no_schema: + with timings('binding-closure'): + clos_h = binding_closure(H.src, bind_head) + clos_b = binding_closure(B.src, bind_base) + tok_h, all_h = binding_selectors(H.src, clos_h) + tok_b, all_b = binding_selectors(B.src, clos_b) + closure = sorted({renames.get(p, p) for p in clos_b} | clos_h) + # dt-core.yaml carries the property type checks; a binding change that + # drops a property definition shows up there, not in the binding itself + limit = ':'.join([p[len(BINDINGS) + 1:] for p in closure] + ['dt-core.yaml']) + tokens = tok_h | tok_b + for d in head_ok: + if d in full: + continue + if all_h or all_b: + limited.add(d) + continue + comp = fdt_compatibles(H.dtbs[d]['path']) + if d in base_ok: + comp |= fdt_compatibles(B.dtbs[base_ok[d]]['path']) + if comp & tokens: + limited.add(d) + report['binding_closure'] = closure + need_validate = bool(full or limited) + need_bcheck = bool(bind_head or bind_base or 'binding-infra' in kinds) + + # 4. processed schema for each side that needs one + if need_validate or need_bcheck: + with timings('schema'): + keys = {n: schema_cache_key(args, s.sha, dtschema_version) for n, s in sides.items()} + need = {'head': True, + 'base': bool(full & set(base_ok)) or bool(limited & set(base_ok)) or bool(bind_base) + or 'binding-infra' in kinds} + shared = keys['base'] == keys['head'] + todo = ['head'] if shared else [n for n in ('base', 'head') if need[n]] + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex: + futs = {n: ex.submit(build_schema, args, sides[n], env, args.cache_dir, keys[n]) + for n in todo} + for n, f in futs.items(): + probs, hit = f.result() + sides[n].problems.extend(probs) + report['schema_cache'][n] = 'hit' if hit else 'built' + if shared and need['base']: + install_schema(B, H.schema, H.schema_cmd or ( + open(os.path.join(H.obj, BINDINGS, '.processed-schema.json.cmd')).read() + .replace(H.src, '@SRCTREE@') + if os.path.exists(os.path.join(H.obj, BINDINGS, '.processed-schema.json.cmd')) + else None)) + B.problems.extend(p for p in H.problems if p['class'] in ('binding-dropped', 'binding-warning')) + report['schema_cache']['base'] = 'same as head' + + # 5. dt-validate + validated = {'base': {}, 'head': {}} + if need_validate: + with timings('dt-validate'): + tasks = [] + if full: + tasks.append(('head', None, sorted(full))) + bfull = sorted(base_ok[d] for d in full if d in base_ok) + if bfull: + tasks.append(('base', None, bfull)) + if limited: + tasks.append(('head', limit, sorted(limited))) + blim = sorted(base_ok[d] for d in limited if d in base_ok) + if blim: + tasks.append(('base', limit, blim)) + res, validated, errors = run_validation(args, sides, tasks, env, json_ok, validate_cache) + for e in errors: + report['notes'].append(e) + if errors: + report['tool_failures'] = errors + ok_pairs = {d for d in validated['head'] + if d not in base_ok or base_ok[d] in validated['base']} + for n, side in sides.items(): + for p in res[n]: + hd = head_name(p['scope']) if n == 'base' else p['scope'] + if hd in ok_pairs: + side.problems.append(p) + + # 6. binding checks + bfiles = {'base': bind_base, 'head': bind_head} + if 'binding-infra' in kinds: + for n, s in sides.items(): + extra = scope_bindings(s.src, args.binding_scope) + bfiles[n] = sorted(set(bfiles[n]) | set(extra)) + if need_bcheck: + with timings('dt_binding_check'): + def bcheck(n): + s = sides[n] + files = [f for f in bfiles[n] if os.path.exists(os.path.join(s.src, f))] + if not files: + return n, [] + rel = ':'.join(f[len(BINDINGS) + 1:] for f in files) + rc, out_ = run(['make', '-C', s.src, f'O={s.obj}', f'ARCH={args.arch}', + f'-j{max(1, args.jobs // 2)}', '-k', '-Otarget', + 'dt_binding_check', f'DT_SCHEMA_FILES={rel}'], env=env, + logfile=os.path.join(s.logs, f'{n}-binding-check.log')) + probs = parse_binding_log(s, out_) + for p in probs: + if p['class'] in ('example-schema', 'example-build') and p['file'] and not p['line']: + p['line'] = examples_line(s.src, p['file']) + elif p['class'] in ('binding-invalid', 'binding-dropped', 'binding-warning') \ + and p['file'] and not p['line']: + p['line'] = yaml_key_line(s.src, p['file'], p['message']) + return n, probs + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex: + for n, probs in ex.map(bcheck, ['base', 'head']): + sides[n].problems.extend(probs) + report['bindings_checked'] = bfiles['head'] + + # 7. style of changed DTS sources + if dts_head and not args.no_style: + with timings('dts-style'): + py = shutil.which('python3') or sys.executable + for n, files in (('base', dts_base), ('head', dts_head)): + sides[n].problems.extend(style_check(sides[n], files, args.style_mode, py, env)) + + # 8. compare + with timings('compare'): + new, fixed = compare(B.problems, H.problems, scope_map) + # style findings in a file that no longer compiles are knock-on noise + broken = {e.get('file') for errs in H.failed.values() for e in errs if e.get('file')} + if broken: + dropped = [p for p in new if p['class'] == 'style' and p['scope'] in broken] + if dropped: + new = [p for p in new if p not in dropped] + report['notes'].append(f'{len(dropped)} new style finding(s) not reported for ' + 'files that fail to compile') + # annotate schema problems with source lines from the head DTB + need_map = sorted({p['scope'] for p in new if p['class'] in ('schema', 'unmatched')}) + maps = {} + for d in need_map[:60]: + maps[d] = source_map(H, d, args.arch) + for p in new: + if p['class'] not in ('schema', 'unmatched') or p['scope'] not in maps: + continue + nodes, props = maps[p['scope']] + node = p.get('node') or p['key'][0] + cands = [] + prop = p.get('prop') + if prop and (node, prop) in props: + cands = props[(node, prop)] + elif prop and (node.rstrip('/') + '/' + prop) in nodes: + cands = nodes[node.rstrip('/') + '/' + prop] + elif node in nodes: + cands = nodes[node] + if cands: + p['file'], p['line'] = pick_location(cands, changed_head) + elif H.dtbs.get(p['scope'], {}).get('source'): + p['file'] = H.dtbs[p['scope']]['source'] + blocking = set(args.block) + groups = group_new(new, blocking, changed_head, args.arch) + counts = collections.OrderedDict() + for cls in CLASSES: + b = sum(1 for p in B.problems if p['class'] == cls) + h = sum(1 for p in H.problems if p['class'] == cls) + nn = sum(1 for p in new if p['class'] == cls) + ff = sum(1 for p in fixed if p['class'] == cls) + if b or h or nn or ff: + counts[cls] = {'base': b, 'head': h, 'new': nn, 'fixed': ff} + + report['counts'] = counts + report['new'] = groups + report['fixed'] = [{'class': p['class'], 'scope': p['scope'], 'message': p['message']} + for p in fixed[:300]] + report['fixed_total'] = len(fixed) + report['dtbs'] = { + 'base_built': len(B.dtbs), 'head_built': len(H.dtbs), + 'head_failed': sorted(short_dtb(d, args.arch) for d in head_failed), + 'new': [short_dtb(d, args.arch) for d in new_dtbs], + 'affected': {short_dtb(d, args.arch): v for d, v in sorted(affected.items())}, + 'changed_output': [short_dtb(d, args.arch) for d in changed_output], + 'validated_full': sorted(short_dtb(d, args.arch) for d, m in validated['head'].items() if m == 'full'), + 'validated_limited': sorted(short_dtb(d, args.arch) for d, m in validated['head'].items() if m == 'limited'), + } + report['bindings_checked'] = report.get('bindings_checked', []) + if any(g['class'] == 'binding-dropped' for g in groups): + report['notes'].append('a binding was dropped from the head schema, so DTB results ' + 'for nodes it covered (including "fixed" counts) are not meaningful') + annotate(groups, args.max_annotations) + if report.get('tool_failures'): + report['status'] = 'error' + report['error'] = '; '.join(report['tool_failures'][:3]) + return 2 + nblock = sum(1 for g in groups if g['blocking']) + report['status'] = 'fail' if nblock else 'pass' + return 1 if nblock else 0 + + +def scope_bindings(src, patterns): + res = [] + broot = os.path.join(src, BINDINGS) + for root, _dirs, files in os.walk(broot): + for n in files: + if not n.endswith('.yaml') or n.startswith('processed-schema'): + continue + rel = os.path.relpath(os.path.join(root, n), src) + if any(p in n for p in patterns): + res.append(rel) + continue + try: + text = open(os.path.join(root, n), encoding='utf-8', errors='replace').read() + except OSError: + continue + if any(p + ',' in text for p in patterns): + res.append(rel) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0], + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--base', required=True, help='base commit (PR target)') + ap.add_argument('--head', required=True, help='head commit (PR result)') + ap.add_argument('--tree', default='.', help='git repository holding both commits') + ap.add_argument('--out', required=True, help='work and report directory') + ap.add_argument('--arch', default='arm64') + ap.add_argument('--kconfig', action='append', default=None, + help='config line for allnoconfig (repeatable; default CONFIG_ARCH_APPLE=y)') + ap.add_argument('--jobs', '-j', type=int, default=os.cpu_count() or 2) + ap.add_argument('--block', default=','.join(DEFAULT_BLOCKING), + help='comma-separated problem classes that fail the check ' + f'(default {",".join(DEFAULT_BLOCKING)}; classes: {",".join(CLASSES)})') + ap.add_argument('--binding-scope', default='apple', + help='comma-separated name patterns for bindings rechecked when binding ' + 'tooling changes (default apple)') + ap.add_argument('--style-mode', default='strict', choices=('relaxed', 'strict'), + help='dt-check-style mode for changed DTS files (default strict)') + ap.add_argument('--cache-dir', help='reuse processed schemas and dt-validate results') + ap.add_argument('--full', action='store_true', help='schema-check every DTB fully') + ap.add_argument('--no-schema', action='store_true', help='skip dt-validate on DTBs') + ap.add_argument('--no-style', action='store_true', help='skip dt-check-style on DTS files') + ap.add_argument('--max-annotations', type=int, default=40) + ap.add_argument('--keep', action='store_true', help='keep extracted sources and build dirs') + args = ap.parse_args(argv) + args.tree = os.path.abspath(args.tree) + args.out = os.path.abspath(args.out) + args.kconfig = args.kconfig or ['CONFIG_ARCH_APPLE=y'] + args.block = [b for b in args.block.split(',') if b] + args.binding_scope = [b for b in args.binding_scope.split(',') if b] + args.cache_dir = os.path.abspath(args.cache_dir) if args.cache_dir else None + for b in args.block: + if b not in CLASSES: + ap.error(f'unknown class {b}') + os.makedirs(args.out, exist_ok=True) + + timings = Timings() + report = {'tool': 'dt-check.py', 'base': args.base, 'head': args.head, + 'merge_base_used': False, 'status': None, 'blocking_classes': args.block, + 'classes': CLASSES, 'versions': tool_versions(), 'notes': [], + 'schema_cache': {}, 'timings_s': timings.t} + t0 = time.monotonic() + try: + rc = check(args, report, timings) + except SetupError as e: + report['status'] = 'error' + report['error'] = str(e) + print(f'::error title=dt-check::{esc_data(str(e))}', flush=True) + rc = 2 + timings.t['total'] = round(time.monotonic() - t0, 2) + report['exit_code'] = rc + with open(os.path.join(args.out, 'report.json'), 'w', encoding='utf-8') as f: + json.dump(report, f, indent=1, default=sorted) + f.write('\n') + text = write_summary(os.path.join(args.out, 'summary.md'), report) + if os.environ.get('GITHUB_STEP_SUMMARY'): + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a', encoding='utf-8') as f: + f.write(text) + if not args.keep: + for d in ('src', 'build', 'tmp'): + shutil.rmtree(os.path.join(args.out, d), ignore_errors=True) + note(f"status {report['status']}, exit {rc}, {timings.t['total']}s") + return rc + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/aurora-ci/dt-requirements.txt b/tools/aurora-ci/dt-requirements.txt new file mode 100644 index 0000000000000..b50f9eaef3099 --- /dev/null +++ b/tools/aurora-ci/dt-requirements.txt @@ -0,0 +1,259 @@ +# Python packages dt-check.py runs (dtschema, yamllint), pinned with hashes. +# Regenerate: uv pip compile --python-version 3.12 --generate-hashes --universal +# with the two top-level pins below as input. +# dtschema==2026.9 +# yamllint==1.38.0 +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via + # jsonschema + # referencing +dtschema==2026.9 \ + --hash=sha256:d50aa00414f09b7aadc776afabe623915b8f94e59d6eddef80920d6505cba671 \ + --hash=sha256:defcbe1be176dec69c65b9700c9cd992cba3b409976a1404b9c0b5080aa5f4c1 + # via -r dt-req.in +jsonschema==4.26.0 \ + --hash=sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326 \ + --hash=sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce + # via dtschema +jsonschema-specifications==2025.9.1 \ + --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ + --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d + # via jsonschema +pathspec==1.1.1 \ + --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 + # via yamllint +pylibfdt==1.7.2.post2 \ + --hash=sha256:b23d03d10ab62ad1f4e7eedde0eb8a7a8872cc08d2a5c2bba7b48bb6fd3c69f4 + # via dtschema +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via yamllint +referencing==0.37.0 \ + --hash=sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231 \ + --hash=sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8 + # via + # jsonschema + # jsonschema-specifications +rfc3987==1.3.8 \ + --hash=sha256:10702b1e51e5658843460b189b185c0366d2cf4cff716f13111b0ea9fd2dce53 \ + --hash=sha256:d3c4d257a560d544e9826b38bc81db676890c79ab9d7ac92b39c7a253d5ca733 + # via dtschema +rpds-py==2026.9.1 \ + --hash=sha256:00ba2d8c7dd4ee537978ddf4b3fbd712bef2d8751603f7f3146b3f4287768e25 \ + --hash=sha256:01445c8d194aa032a08e944f16567672da1c62dbdbefd8b6d0693032e290cf68 \ + --hash=sha256:028ad274ea951dac64491b5d1e65712a4aeabfdbdb9fccf797b57bd899b0c495 \ + --hash=sha256:0483515261947e4e8b8e1375bf7463e7eb6ccfb3d86e7b554d90cd5285f20f32 \ + --hash=sha256:068c37bba854ec2fe42f7365c640af11dd9895890ccbf2df5070d0c059bd7f96 \ + --hash=sha256:074a4d198bc34d9a8ea425114fc3ded6d11ec01f6a314a8db67454a5152d8834 \ + --hash=sha256:07deecbfce94c78473018bc7d10b337cc651d12df87a1eb2cb3e4024bc9c33d0 \ + --hash=sha256:08dae4a4095150a7c4545a1fb40b98e1ab1744fbc2770d92c977b9dadaa49ab6 \ + --hash=sha256:0da298fb372dc192610a4b9ecbc68a0cd8b675bbbd1fc519d01b41cfd658333e \ + --hash=sha256:0f045bb053c9057720d72c56dffe30dffdc05997b2897a827b9325f0ab6623fa \ + --hash=sha256:10e208f2425d973938afcd56e28a7c4be32e27b6a60b5d381f49fb9d8acf9759 \ + --hash=sha256:136a1c3fe4402b7008bc81cb62ee538481795b61a7e83df88dff3b3f02b726ff \ + --hash=sha256:159a7aab5c5e8b112c8830f54717ce56da1252ebdbb526f5be2df2309280b9e7 \ + --hash=sha256:172e47169583f46ce118cbec68e6795d0da0f4606b488b6434f8276bca0a058c \ + --hash=sha256:1c2d1f6da5128eabf34e963d7163a818846075a52568250d006c4c953b40f903 \ + --hash=sha256:1d55198263bb51f557550c6ed2e6d1cb6a6fed6eb5c9120b741c5926bef8a45d \ + --hash=sha256:1d77b649e6f7cdf12ca5c2a98dad0ad37f9ea9b6f960408a92f0cb12bb3d04d9 \ + --hash=sha256:1e8d4d79d828299bf44a55db22a9388ab967b49d17132c88eab0f4360b48da8e \ + --hash=sha256:22ffd29a63d71fb1b81552c21f2c2b734949b7ac751a9be70675a939a900839b \ + --hash=sha256:2693b2728bbcc48d09a981a356954b0c47c53ff25b545856f28a889ea619f69a \ + --hash=sha256:270bdcdaac5d5b6f73c5e22e7e135c7f2a50e789f71d9e241d5be8d90026e19a \ + --hash=sha256:2711d29b653b3bce48a63d18b9c6b53274669e6d6c4094dddeb4d9a0e45128b2 \ + --hash=sha256:2c16ab111bc27c646ba8aa005d0527754edc538ebb636f0b1bf8e244b48d1945 \ + --hash=sha256:306ee1850d8105b5baf977e78d45fcadd12c1a54678d614c9baf217708446e91 \ + --hash=sha256:3231c4c0e521dafa5be0c9f114ee2c2ad46650836f2d72caa86801950c3e7044 \ + --hash=sha256:3890a6aa36e6baa53d5258a2a25d3ef8b37ad165a6ab27a892d7c3e3a432cd69 \ + --hash=sha256:3a72c11530d71abfb66c8d7696a2f86c43e63fca8b948f1a784ac490f4ec688e \ + --hash=sha256:3b5a6f40f0a1486b4b36c888123afc67acdbd9f33235927acf5ff295429a0ba3 \ + --hash=sha256:3c91c210ae7645626c608400e3519b4a642f837cce09ca830db3beb2e9f274d4 \ + --hash=sha256:3cd182d7291d29b92c521a0069d9c01ba6193628a9a105531d11b40a6d731a33 \ + --hash=sha256:3e524c7874ac72884d28e16dd5b8d839fd09e0fe76b020d3fbca23212a7b8c52 \ + --hash=sha256:3e93b2cd69a9830be33e03945cd7cda940a0a8bfcfbff41d6144f0cb0d3d8bd9 \ + --hash=sha256:3edae8c5ddfdb6985d49ae9d150516e5076888879022f91a26c2de9276ce0bdb \ + --hash=sha256:3f0e9ac28fc067d4d34b88ae43c48e9489455c97fee9633d851f7eeed5a05d35 \ + --hash=sha256:42e75466f83cd43f6026c81eab74246efb2bdadafb307b85700632d06c68f299 \ + --hash=sha256:44b32a7c4f0da3d28af31c259e38ddcff096f855e205ed0671d02fcf44f1ea1c \ + --hash=sha256:457866b85daf5034296666168b84a69e0b2e89dc4f1af102b46f6448a60b9063 \ + --hash=sha256:45bc6bccf78b20fd834237d18db64965d7ee68ba7f60440a26c7ab71e7b8d51a \ + --hash=sha256:46d80bc76b51a6c24f9944368c28d38b8bcbcea1da4f2f8d3ebc31a67e8c6ec6 \ + --hash=sha256:4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12 \ + --hash=sha256:492e5e428cbe126221611f47e068f01660352feec4ad18bc0f5ea9b2ae88fb14 \ + --hash=sha256:4b26b03d9d2658ee2fa234f8f4f19f38a09773fe5261028025032e26d4d35af0 \ + --hash=sha256:4c0d2cb595a420b34d5086db0add011e26e2c09d6a024afbac4228bf8f863a30 \ + --hash=sha256:4cfaf02209061880210819934de2f4f6aa83dc04dafe6770276acc240a56da31 \ + --hash=sha256:501909f2e4a1e2dee528ef766fe3c469060ebc17e54a8383d404ba07a81a6f02 \ + --hash=sha256:50906f5aea24b5a865cbd0a589698288631d9f3a54c3a937c83aefa95a0d14af \ + --hash=sha256:54ac2158a6f96cfbabff0b2eedaf94b90c5ec7ca8317fcadc61e1c2b2e0ff6ef \ + --hash=sha256:56c6952a9b15047466d0c2347c446a761d4527f89976156341e68f0ce5cc08b0 \ + --hash=sha256:56cd8b3f77d7b6812f533b662186a1f28316931166ddc00fb893b1b0db7e9888 \ + --hash=sha256:57492a550a1d88d29d003247e5f78dd8cf04a701fac0e4c8db8745a6d2504e0a \ + --hash=sha256:5943980471829f6de242a20b109de3111ba6b77e3af0ffc587028ac854b05e6c \ + --hash=sha256:5c6ee90dee3e85e055ddfd502d611643d9b0fd94c818220bda84ec3dacd9b27b \ + --hash=sha256:5c90e7fa02e8f5de0d10c17595c568ada48c5302e749462c0ea1a4c362111a86 \ + --hash=sha256:5ce8943f79c2210f7abcc28e86367b03b28d95027fd01c46d2472373ae70c86f \ + --hash=sha256:617f59cde379b4f648a09797b7f683d04b90a46344cddab85639da5aff0f5531 \ + --hash=sha256:6307a0da524939decb8ca4a3933b8ab62525794411d6984fca6726e732804af6 \ + --hash=sha256:684fd492fff4fead00587544e059be2bbcb6f93454f21fa2a91b66fc7508be82 \ + --hash=sha256:6b5b393eda5ea42cca1c1a6665f2a4882b4fd5d1777e41ce0545a107fb008c9d \ + --hash=sha256:6b723eb406dec5bc9ec516c73ab9c3239a3284e017f7eb89ee2b3258bd504fb7 \ + --hash=sha256:6b9bf3135b4ad5981df9a73d71a35272d650a2985ae9c2746357b24d59de2448 \ + --hash=sha256:6beb738155fe8ab8091afdfa5a3226b21c2b1593f1e50ebb90eb25b44dbc0391 \ + --hash=sha256:6c0dbbcc19735fe5f8b0a54c07659d154a9e69f47e15d0a6ab7299215daf62cb \ + --hash=sha256:6cdc537c8633d7fd92a82e2e0d2ab74320a3f63d5e59fb9cf08711e08fe151c4 \ + --hash=sha256:6eae33003518fd4cb4f83a218d5371469dd3001aa3b87128c005b07762f7fe5e \ + --hash=sha256:740d0a99cf9de0b17a3943388e9294a59becf75e7c43421f387bd3c7a9901f7c \ + --hash=sha256:75c38c50ab9aca840225d9a9a3810bf11d04bd5c1f186cabbb8aee56db3e9b15 \ + --hash=sha256:761fdae6728ceb99ab182fad2f0cc1e262f610834dc891aea1d1a2a2e634776f \ + --hash=sha256:7664419f27db41d4f1c43a78dccda7dd6e8ef2428df3ee01d0c2a07a6b071297 \ + --hash=sha256:76d3af9732d2dab69f28179b40ba2d87e2f1d5824b4a694780aa787d685e8f36 \ + --hash=sha256:78326f4cb4427a56ba4996c0762b63be45f06b85f086526420d2b3a66e40f84d \ + --hash=sha256:7868b85224291c6cb6759f9b5adb9745f486d226f62b16a614dd5a2a5ab2b35b \ + --hash=sha256:815d26356930846a40c7bc1366e7b1b0320ab8a063e66c11298a208bed0fd237 \ + --hash=sha256:8171b44a054e5c67fd748ada04187f1250bf35b95f85e52ab64bcf3331a923bb \ + --hash=sha256:821b2755db9194409254012f429c56643416fb96ef9be090be82ec8826b7f477 \ + --hash=sha256:837c6b305e26fe0f75b15c92cf3b2ba29e0ae19dc40b1c557b026cb426347d0c \ + --hash=sha256:839dde845559254f34885267c6878f60d61d5205180226d976fe488d45fa128e \ + --hash=sha256:84a6ecc0c940169190d2c23bd969debd48c94dbc855acd60188a68d71d421608 \ + --hash=sha256:8601470267d938bcb7f3ab1a336100af51a4fd5b6ed030ef52461bb3ef5e7e07 \ + --hash=sha256:88b5268892fde430d5531f95bc560b6efbbd67c929662c586afd729a96e7461c \ + --hash=sha256:8aa5dda18d39b6143eb24809d158f9252c88f402749b6f1b62a506cc7d96cc35 \ + --hash=sha256:926bdd3e3b5998ddf70cc64bc8cf57209571f9044542913afb673799fec77dd0 \ + --hash=sha256:96beca19ec79de272e8668585380ff9092c47077c1d7a1e098e00bbd921f4785 \ + --hash=sha256:9a0460d43603d1fd9ef59c30278531e15d78581721ddb538fa560aa7817ea4ad \ + --hash=sha256:a03d57b86d2a51d0a66c92177e2be154ad015f357791d306e714569999cdb4cc \ + --hash=sha256:a36b70596407634ca82d4b989a3729074a008537a0522e4c8046a67c729103e9 \ + --hash=sha256:a3a52a3ba86436ab3aef510fbe21512abc2ddd1993005dfe50514bd2284ef025 \ + --hash=sha256:a3dbc5ed9514908d5046107d7b1346bde71eea61de6e0e4919c19354f97e769f \ + --hash=sha256:a431156bb41865fc14cd5d79bb9d7bbed83110b0159e34e62ae30951f96c0009 \ + --hash=sha256:a575404ebc9cf2e91edd32eaf570ec1430eb900d4f56724ba7dd4bc1fc9c176d \ + --hash=sha256:a5cf77eb04f20b720be95265a3e00eb2a14814074255cc27069c551b2db53118 \ + --hash=sha256:a8763f20692da7df39b0afdd1ba3042b004c50a45994f76c2d9a25641f7673db \ + --hash=sha256:ab4b2fda7c2b542f7f9d886cc6a838c5079d2b76f72e6081411faba11adde2c9 \ + --hash=sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab \ + --hash=sha256:b242c27c8f836305a4a72df9cdd564386ac57b807bd252a063223331c9316b37 \ + --hash=sha256:b4f062343e7ad3fa94f2c66e5ae667dee47ee74dd41a9057c4fbe163236a123d \ + --hash=sha256:b5b8b0753718d258fd454283fbd57e14545d3b40583fa672e27cb4f987626bcc \ + --hash=sha256:be3e47e2d91aa3942ff9bf4077a505226005abfc39b6f7554a91c1b9393986b9 \ + --hash=sha256:befc2d6a953e563f8a7bfd87a42c22ebf8a3e980dcb7b6a4d17b70b0e914e8a3 \ + --hash=sha256:bf35d0568abda97233239ce32896d3ad53fccc537832c104e30c94aa5fb93569 \ + --hash=sha256:c933c6678c6f116ff8af47a4c6db0868b8ace74af0343016c0ef00f00272ea69 \ + --hash=sha256:c9d1aca01f49170fdcf5c92761b1fafe97f554b721ca4570c5949fff778f0d4b \ + --hash=sha256:cdeaa99ce822dca76cfb1b993e9120c5ea212f2eb66d48950ad63c349668a018 \ + --hash=sha256:ce4d4f52e2a4324396caddbd45a97d8d7be5f42edd25d2355282a9c34f9b2f7f \ + --hash=sha256:d1028417bb44037eb3069c1009bd7b7277212876cda22fbe565b0bca9fab6d2c \ + --hash=sha256:d151e148117294133bf8af7eeace085e7e87432db15ab6adf640330298a47f6f \ + --hash=sha256:d7841166b7fa64c9c56404617ae4341448847482d45933b13135d26c130519e5 \ + --hash=sha256:d7fca4eb6df565e2a928f1c7dad92d27db8f9df0f449e76423ed5d7e713ed445 \ + --hash=sha256:d95a354e02393eada6d7351184671aced9d4cce109dabf927cb7aa99624352a1 \ + --hash=sha256:d9edf30457d74eebfd76b045535e36f1cd89062566a128a0db2145ca042d787e \ + --hash=sha256:dbc2673f9223d420c91145599b3ba45a8a50c207d1976908e5fb5ddb0c9b9429 \ + --hash=sha256:e01b3c878c8641913e688edd1b3f08658c6783d29cf6b826bd3c0d1ae7a1ffaa \ + --hash=sha256:e21c1429e205828ea886a2293a4a2c8e01f4c25d9893ca330e97a6cf73f52e7b \ + --hash=sha256:e43d4a1f673e8a1cbd8533e809e02b4bf9d4f2280269bb640436556312121250 \ + --hash=sha256:e6d198bad4e49dd6732fbd636e2fc5c082f45c8cad0b4acb756b00c82c76072e \ + --hash=sha256:e6ea1cda8d8c688278430e4268a42f5e5da3bdd74578dfadc0820c3f1766ce83 \ + --hash=sha256:ea394a937f17a54c51239348bdbe2e3518124c8d4a8951ba04a311d3095bd18f \ + --hash=sha256:eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd \ + --hash=sha256:eb61be926bb81567c1f48bdc8aa22b9855048dc2efd53871f9f7e6e9a5632346 \ + --hash=sha256:eba5d173f7d5708b22a93815017a4611873ed54db9f268077c0dd1ed99cfc858 \ + --hash=sha256:ec450527cbf485e13c8d3602a54f428ab0432fdade0ede75efd74b735421c871 \ + --hash=sha256:eef6a03b0b6d08d0835ccfa8ec8d1bc70525e3801387567137b50c557695e6da \ + --hash=sha256:ef0d8c843e2827d6c120ab4687e9423fb1d893db1df27b7c1506615bcb9734a0 \ + --hash=sha256:ef6b65b03247c54692ad4fd9ee97cb772781927db72e3cb05e70b3db6d1ff14f \ + --hash=sha256:f3d6ed6a98cfd19155996605474982cc470d7601746a6439078f1a5a3fa8b050 \ + --hash=sha256:fce4b85234a0cbad67bf8e6e1201ee815d172c9aebad75f25645bc4d834f8e31 \ + --hash=sha256:fda1d96e542c37b6c804547dbf489c129fe7c97183a76a5ec275909ba1a063df \ + --hash=sha256:fdcd198979b4ecffcc1beba366a7fbcf4eb41243691a82fe52ceb0b902f09c12 \ + --hash=sha256:fe5ad0664ec772b02c45859041aa17655709cced7a31005817fbbbd988c25567 + # via + # jsonschema + # referencing +ruamel-yaml==0.19.1 \ + --hash=sha256:27592957fedf6e0b62f281e96effd28043345e0e66001f97683aa9a40c667c93 \ + --hash=sha256:53eb66cd27849eff968ebf8f0bf61f46cdac2da1d1f3576dd4ccee9b25c31993 + # via dtschema +typing-extensions==4.16.0 ; python_full_version < '3.13' \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via referencing +yamllint==1.38.0 \ + --hash=sha256:09e5f29531daab93366bb061e76019d5e91691ef0a40328f04c927387d1d364d \ + --hash=sha256:fc394a5b3be980a4062607b8fdddc0843f4fa394152b6da21722f5d59013c220 + # via -r dt-req.in diff --git a/tools/aurora-ci/host-tests.json b/tools/aurora-ci/host-tests.json new file mode 100644 index 0000000000000..d286dde870e53 --- /dev/null +++ b/tools/aurora-ci/host-tests.json @@ -0,0 +1,2247 @@ +{ + "comment": [ + "Hermetic host test suites, run by host-tests.py from the top of the tree.", + "{output} is a fresh path that does not exist yet; its parent does.", + "expect: pass (exit 0, the default) or fail (non-zero; mutation checks that report a caught mutant that way).", + "known_failure: runs, is reported, and does not fail the job; remove it once the suite passes.", + "commits: history the suite reads; host-tests.py --fetch-commits fetches it into a shallow checkout.", + "Every test-like file under the discovery roots is listed in suites or excluded (tests/test_host_tests.py checks).", + "helpers: files that look like suites but are run by a listed suite.", + "serial: the suite measures time; it runs alone, after the parallel batch." + ], + "apt": [ + "bison", + "bubblewrap", + "clang", + "dbus", + "device-tree-compiler", + "flex", + "libarchive-tools", + "libnotify-bin", + "lld", + "llvm", + "make", + "makepkg", + "python3-gi", + "zstd" + ], + "discovery": [ + "tools/asahi", + "tools/asahi-m3", + "tools/aurora-*", + "tools/testing/apple", + "tools/testing/selftests/drivers/apple", + "tools/testing/selftests/drivers/gpu/apple-*", + "tools/testing/selftests/drivers/gpu/asahi-g15" + ], + "suites": [ + { + "name": "tools/asahi/apple-dart-fw-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/apple-dart-fw-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/apple-dart-range-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/apple-dart-range-tests.py" + ], + "known_failure": "its mocks predate the current apple-dart.c (no u32, fw_mirror/fw_handoff, apple_dart_fw_invalidate; apple_dart_hw_sync_locked takes three arguments now)" + }, + { + "name": "tools/asahi/avd-codec-tail-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/avd-codec-tail-tests.py", + "--mutations", + "--output", + "{output}" + ] + }, + { + "name": "tools/asahi/avd-av1-tail-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/avd-av1-tail-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/avd-av1-tail-tests.py --mutation round-up", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/avd-av1-tail-tests.py", + "--out", + "{output}", + "--mutation", + "round-up" + ], + "expect": "fail", + "note": "inverted: exit status is the controls program status, non-zero when the mutant is caught" + }, + { + "name": "tools/asahi/avd-av1-tail-tests.py --mutation cpu-unaligned", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/avd-av1-tail-tests.py", + "--out", + "{output}", + "--mutation", + "cpu-unaligned" + ], + "expect": "fail", + "note": "inverted: exit status is the controls program status, non-zero when the mutant is caught" + }, + { + "name": "tools/asahi/bcm4377-id-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/bcm4377-id-tests.py", + "--out", + "{output}", + "--old-ref", + "87ddae1397f6f70817cb57df76477a86bdf96424" + ], + "commits": [ + "87ddae1397f6f70817cb57df76477a86bdf96424" + ] + }, + { + "name": "tools/asahi/bcm4377-resume-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/bcm4377-resume-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/bcm4377-resume-tests.py --mutation epoch", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/bcm4377-resume-tests.py", + "--out", + "{output}", + "--mutation", + "epoch" + ], + "expect": "pass", + "note": "self-checking: exits 0 when the mutant aborts" + }, + { + "name": "tools/asahi/bcm4377-resume-tests.py --mutation close", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/bcm4377-resume-tests.py", + "--out", + "{output}", + "--mutation", + "close" + ], + "expect": "pass", + "note": "self-checking: exits 0 when the mutant aborts" + }, + { + "name": "tools/asahi/bcm4377-resume-tests.py --mutation producer-fence", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/bcm4377-resume-tests.py", + "--out", + "{output}", + "--mutation", + "producer-fence" + ], + "expect": "pass", + "note": "self-checking: exits 0 when the mutant aborts" + }, + { + "name": "tools/asahi/bcm4377-resume-tests.py --mutation active-fence", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/bcm4377-resume-tests.py", + "--out", + "{output}", + "--mutation", + "active-fence" + ], + "expect": "pass", + "note": "self-checking: exits 0 when the mutant aborts" + }, + { + "name": "tools/asahi/bcm4377-resume-tests.py --mutation board", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/bcm4377-resume-tests.py", + "--out", + "{output}", + "--mutation", + "board" + ], + "expect": "pass", + "note": "self-checking: exits 0 when the mutant aborts" + }, + { + "name": "tools/asahi/bcm4377-resume-tests.py --old-ref 36fab84538bed0f2f823c0e6745e4b0f5e6c2a1e", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/bcm4377-resume-tests.py", + "--out", + "{output}", + "--old-ref", + "36fab84538bed0f2f823c0e6745e4b0f5e6c2a1e" + ], + "expect": "pass", + "note": "self-checking: original driver (87ddae1397f6^) must abort cases 0,1,2,28", + "commits": [ + "36fab84538bed0f2f823c0e6745e4b0f5e6c2a1e" + ] + }, + { + "name": "tools/asahi/brcmf-assoc-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-assoc-tests.py", + "--output", + "{output}" + ] + }, + { + "name": "tools/asahi/brcmf-assoc-tests.py --mutation short-ie", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-assoc-tests.py", + "--output", + "{output}", + "--mutation", + "short-ie" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/brcmf-assoc-tests.py --mutation header-length", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-assoc-tests.py", + "--output", + "{output}", + "--mutation", + "header-length" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/brcmf-assoc-tests.py --mutation skb-capacity", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-assoc-tests.py", + "--output", + "{output}", + "--mutation", + "skb-capacity" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/brcmf-assoc-tests.py --mutation cmplt-header", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-assoc-tests.py", + "--output", + "{output}", + "--mutation", + "cmplt-header" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/brcmf-assoc-tests.py --old-association f58ae50192550e6fc67c956f361f525881a1f969", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-assoc-tests.py", + "--output", + "{output}", + "--old-association", + "f58ae50192550e6fc67c956f361f525881a1f969" + ], + "expect": "fail", + "note": "inverted; old getter from ecd1ea4d0f06^", + "commits": [ + "f58ae50192550e6fc67c956f361f525881a1f969" + ] + }, + { + "name": "tools/asahi/brcmf-bss-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-bss-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/brcmfmac-scan-timeout-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmfmac-scan-timeout-tests.py", + "--output", + "{output}" + ] + }, + { + "name": "tools/asahi/brcmfmac-scan-timeout-tests.py --old-source {output}/old.c", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmfmac-scan-timeout-tests.py", + "--output", + "{output}", + "--old-source", + "{output}/old.c" + ], + "expect": "pass", + "note": "self-checking; cfg80211.c from f58ae5019255^", + "setup": [ + [ + "sh", + "-c", + "mkdir -p \"$1\" && git show 661c0a5740e8f36816f7e79a784c36c026097368:drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c > \"$1/old.c\"", + "sh", + "{output}" + ] + ], + "commits": [ + "661c0a5740e8f36816f7e79a784c36c026097368" + ] + }, + { + "name": "tools/asahi/brcmf-pcie-dma-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-pcie-dma-tests.py", + "--output", + "{output}" + ] + }, + { + "name": "tools/asahi/brcmf-pcie-dma-tests.py --mutation narrow-stream", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-pcie-dma-tests.py", + "--output", + "{output}", + "--mutation", + "narrow-stream" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/brcmf-pcie-dma-tests.py --mutation ignore-aperture", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-pcie-dma-tests.py", + "--output", + "{output}", + "--mutation", + "ignore-aperture" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/brcmf-pcie-dma-tests.py --mutation revision-wide", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-pcie-dma-tests.py", + "--output", + "{output}", + "--mutation", + "revision-wide" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/brcmf-pcie-dma-tests.py --old-dma 654386a164a1e93063f879491f10bebfc1439291", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-pcie-dma-tests.py", + "--output", + "{output}", + "--old-dma", + "654386a164a1e93063f879491f10bebfc1439291" + ], + "expect": "fail", + "note": "inverted; probe from dbdeefb63947^", + "commits": [ + "654386a164a1e93063f879491f10bebfc1439291" + ] + }, + { + "name": "tools/asahi/brcmf-pcie-dma-tests.py --firmware-revision b0de24a8c59735f21023c93592dd9a2eb9e155a4", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-pcie-dma-tests.py", + "--output", + "{output}", + "--firmware-revision", + "b0de24a8c59735f21023c93592dd9a2eb9e155a4" + ], + "expect": "fail", + "note": "inverted; firmware table from 2d17f269503c^", + "commits": [ + "b0de24a8c59735f21023c93592dd9a2eb9e155a4" + ] + }, + { + "name": "tools/asahi/brcmf-roam-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/brcmf-roam-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/compute-preempt-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/compute-preempt-tests.py", + "--out", + "{output}" + ], + "rust": "1.85.0" + }, + { + "name": "tools/asahi/compute-preempt-tests.py --old-ref f885ea0de2cd638bda1e0d3d38a901d927a503f5", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/compute-preempt-tests.py", + "--out", + "{output}", + "--old-ref", + "f885ea0de2cd638bda1e0d3d38a901d927a503f5" + ], + "rust": "1.85.0", + "expect": "pass", + "note": "self-checking; policy from c6f4ea502bdd^", + "commits": [ + "f885ea0de2cd638bda1e0d3d38a901d927a503f5" + ] + }, + { + "name": "tools/asahi/dcp14-backlight-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/dcp14-backlight-tests.py" + ] + }, + { + "name": "tools/asahi/dcp14-backlight-tests.py --old-source {output}/old.c", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/dcp14-backlight-tests.py", + "--old-source", + "{output}/old.c" + ], + "expect": "pass", + "note": "self-checking (-DOLD); iomfb_v14_7.c from 2e5a8e777ada^", + "setup": [ + [ + "sh", + "-c", + "mkdir -p \"$1\" && git show e19eb212f57ec32db9026666c89dd610062e9f24:drivers/gpu/drm/apple/iomfb_v14_7.c > \"$1/old.c\"", + "sh", + "{output}" + ] + ], + "commits": [ + "e19eb212f57ec32db9026666c89dd610062e9f24" + ] + }, + { + "name": "tools/asahi/dcp14-panel-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/dcp14-panel-tests.py" + ] + }, + { + "name": "tools/asahi/dcp14-panel-tests.py --old-source {output}/old.c", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/dcp14-panel-tests.py", + "--old-source", + "{output}/old.c" + ], + "expect": "pass", + "note": "self-checking; iomfb_v14_7.c from 7d5a612eceaa^", + "setup": [ + [ + "sh", + "-c", + "mkdir -p \"$1\" && git show a5713d126ea098fd3b6b833fc4022762540aac6c:drivers/gpu/drm/apple/iomfb_v14_7.c > \"$1/old.c\"", + "sh", + "{output}" + ] + ], + "commits": [ + "a5713d126ea098fd3b6b833fc4022762540aac6c" + ] + }, + { + "name": "tools/asahi/devcoredump-init-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/devcoredump-init-tests.py" + ] + }, + { + "name": "tools/asahi/devcoredump-init-tests.py --old-source {output}/old.c", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/devcoredump-init-tests.py", + "--old-source", + "{output}/old.c" + ], + "expect": "pass", + "note": "self-checking (-DOLD); devcoredump.c from ebd3dc459ac1^", + "setup": [ + [ + "sh", + "-c", + "mkdir -p \"$1\" && git show fb7cca304cb381dc589fadbed479c6f3b36e5e23:drivers/base/devcoredump.c > \"$1/old.c\"", + "sh", + "{output}" + ] + ], + "commits": [ + "fb7cca304cb381dc589fadbed479c6f3b36e5e23" + ] + }, + { + "name": "tools/asahi/display26-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/display26-tests.py" + ] + }, + { + "name": "tools/asahi/dma-heap-fd-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/dma-heap-fd-tests.py", + "--out", + "{output}", + "--old-ref", + "94380ecd5ff9b2a2b9f6e8a0b5c465159ccfaf71" + ], + "commits": [ + "94380ecd5ff9b2a2b9f6e8a0b5c465159ccfaf71" + ] + }, + { + "name": "tools/asahi/mt7932-association-events-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/mt7932-association-events-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/mt7932-association-events-tests.py --old-source {output}/old.c", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/mt7932-association-events-tests.py", + "--out", + "{output}", + "--old-source", + "{output}/old.c" + ], + "expect": "pass", + "note": "self-checking; link.c from c980f7435d50^ (the fix). The test commit itself (f885ea0de2cd) is test-only, so its parent is NOT a valid old source.", + "setup": [ + [ + "sh", + "-c", + "mkdir -p \"$1\" && git show 818bde483e1618783005ac76e1c58057a4bb4cfa:drivers/net/wireless/mediatek/mt7932/link.c > \"$1/old.c\"", + "sh", + "{output}" + ] + ], + "commits": [ + "818bde483e1618783005ac76e1c58057a4bb4cfa" + ] + }, + { + "name": "tools/asahi/mt7932-connect-result-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/mt7932-connect-result-tests.py", + "--output", + "{output}" + ] + }, + { + "name": "tools/asahi/mt7932-connect-result-tests.py --old-source {output}/old.c", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/mt7932-connect-result-tests.py", + "--output", + "{output}", + "--old-source", + "{output}/old.c" + ], + "expect": "pass", + "note": "self-checking; link.c from 6c3ee18a0725^", + "setup": [ + [ + "sh", + "-c", + "mkdir -p \"$1\" && git show 7db4eadeeba6512726fc527cb624ae99742de607:drivers/net/wireless/mediatek/mt7932/link.c > \"$1/old.c\"", + "sh", + "{output}" + ] + ], + "commits": [ + "7db4eadeeba6512726fc527cb624ae99742de607" + ] + }, + { + "name": "tools/asahi/mt7932-owe-retry-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/mt7932-owe-retry-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/neo-auxiliary-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/neo-auxiliary-tests.py", + "--output", + "{output}" + ], + "rust": "1.85.0" + }, + { + "name": "tools/asahi/neo-auxiliary-tests.py --mutation", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/neo-auxiliary-tests.py", + "--output", + "{output}", + "--mutation" + ], + "rust": "1.85.0", + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/neo-auxiliary-tests.py --old", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/neo-auxiliary-tests.py", + "--output", + "{output}", + "--old" + ], + "rust": "1.85.0", + "expect": "fail", + "note": "inverted; reads 990a45b609dd", + "commits": [ + "990a45b609ddcaa49f5386e1c11530599226c0e1" + ] + }, + { + "name": "tools/asahi/neo-stability-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/neo-stability-tests.py", + "--output", + "{output}" + ] + }, + { + "name": "tools/asahi/neo-stability-tests.py --mutation sep-syscore", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/neo-stability-tests.py", + "--output", + "{output}", + "--mutation", + "sep-syscore" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/neo-stability-tests.py --mutation wake-irq", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/neo-stability-tests.py", + "--output", + "{output}", + "--mutation", + "wake-irq" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/neo-stability-tests.py --mutation stale-policy", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/neo-stability-tests.py", + "--output", + "{output}", + "--mutation", + "stale-policy" + ], + "expect": "fail", + "note": "inverted" + }, + { + "name": "tools/asahi/neo-stability-tests.py --old-usb", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/neo-stability-tests.py", + "--output", + "{output}", + "--old-usb" + ], + "expect": "fail", + "note": "inverted; reads 990a45b609dd", + "commits": [ + "990a45b609ddcaa49f5386e1c11530599226c0e1" + ] + }, + { + "name": "tools/asahi/neo-stability-tests.py --old-regulatory", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/neo-stability-tests.py", + "--output", + "{output}", + "--old-regulatory" + ], + "expect": "fail", + "note": "inverted; reads 990a45b609dd", + "commits": [ + "990a45b609ddcaa49f5386e1c11530599226c0e1" + ] + }, + { + "name": "tools/asahi/pmp-j613-hwmon-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/pmp-j613-hwmon-tests.py" + ] + }, + { + "name": "tools/asahi/qspi-nvram-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/qspi-nvram-tests.py", + "--output", + "{output}" + ] + }, + { + "name": "tools/asahi/rtkit-oslog-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/rtkit-oslog-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/rtkit-oslog-tests.py --mutate admission", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/rtkit-oslog-tests.py", + "--out", + "{output}", + "--mutate", + "admission" + ], + "expect": "fail", + "note": "inverted (note: flag is --mutate)" + }, + { + "name": "tools/asahi/rtkit-oslog-tests.py --mutate channel", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/rtkit-oslog-tests.py", + "--out", + "{output}", + "--mutate", + "channel" + ], + "expect": "fail", + "note": "inverted (note: flag is --mutate)" + }, + { + "name": "tools/asahi/rtkit-oslog-tests.py --mutate counter", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/rtkit-oslog-tests.py", + "--out", + "{output}", + "--mutate", + "counter" + ], + "expect": "fail", + "note": "inverted (note: flag is --mutate)" + }, + { + "name": "tools/asahi/rtkit-oslog-tests.py --mutate reset", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/rtkit-oslog-tests.py", + "--out", + "{output}", + "--mutate", + "reset" + ], + "expect": "fail", + "note": "inverted (note: flag is --mutate)" + }, + { + "name": "tools/asahi/rtkit-oslog-tests.py --ref 0cee491386e3dff5a09b60d5703f6a7be71b0883", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/rtkit-oslog-tests.py", + "--out", + "{output}", + "--ref", + "0cee491386e3dff5a09b60d5703f6a7be71b0883" + ], + "expect": "fail", + "note": "inverted; rtkit.c from 1369d18b4ec7^", + "commits": [ + "0cee491386e3dff5a09b60d5703f6a7be71b0883" + ] + }, + { + "name": "tools/asahi/tbnet-refill-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/tbnet-refill-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/tbnet-refill-tests.py --mutant refill-count", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/tbnet-refill-tests.py", + "--out", + "{output}", + "--mutant", + "refill-count" + ], + "expect": "fail", + "note": "inverted (note: flag is --mutant)" + }, + { + "name": "tools/asahi/tbnet-refill-tests.py --mutant no-retry", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/tbnet-refill-tests.py", + "--out", + "{output}", + "--mutant", + "no-retry" + ], + "expect": "fail", + "note": "inverted (note: flag is --mutant)" + }, + { + "name": "tools/asahi/tbnet-refill-tests.py --mutant no-cancel", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/tbnet-refill-tests.py", + "--out", + "{output}", + "--mutant", + "no-cancel" + ], + "expect": "fail", + "note": "inverted (note: flag is --mutant)" + }, + { + "name": "tools/asahi/tbnet-refill-tests.py --mutant worker-allocation", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/tbnet-refill-tests.py", + "--out", + "{output}", + "--mutant", + "worker-allocation" + ], + "expect": "fail", + "note": "inverted (note: flag is --mutant)" + }, + { + "name": "tools/asahi/tbnet-refill-tests.py --ref f7f5fc8e97cf5238c8669d9fca0d024b40174209", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/tbnet-refill-tests.py", + "--out", + "{output}", + "--ref", + "f7f5fc8e97cf5238c8669d9fca0d024b40174209" + ], + "expect": "fail", + "note": "inverted; main.c from 71700a159448^", + "commits": [ + "f7f5fc8e97cf5238c8669d9fca0d024b40174209" + ] + }, + { + "name": "tools/asahi/tunnel-route-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/tunnel-route-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/m3-batching-tests.rs", + "group": "tools", + "argv": [ + "{output}", + "--test-threads=1" + ], + "setup": [ + [ + "rustc", + "--edition=2021", + "--test", + "tools/asahi/m3-batching-tests.rs", + "-o", + "{output}" + ] + ], + "rust": "1.85.0" + }, + { + "name": "tools/asahi/m3-runtime-tests.rs", + "group": "tools", + "argv": [ + "{output}", + "--test-threads=1" + ], + "setup": [ + [ + "rustc", + "--edition=2021", + "--test", + "tools/asahi/m3-runtime-tests.rs", + "-o", + "{output}" + ] + ], + "rust": "1.85.0" + }, + { + "name": "tools/asahi/pmp-bootargs-tests.rs", + "group": "tools", + "argv": [ + "{output}", + "--test-threads=1" + ], + "setup": [ + [ + "rustc", + "--edition=2021", + "--test", + "tools/asahi/pmp-bootargs-tests.rs", + "-o", + "{output}" + ] + ], + "rust": "1.85.0" + }, + { + "name": "tools/asahi/pmp-power-tests.rs", + "group": "tools", + "argv": [ + "{output}", + "--test-threads=1" + ], + "setup": [ + [ + "rustc", + "--edition=2021", + "--test", + "tools/asahi/pmp-power-tests.rs", + "-o", + "{output}" + ] + ], + "rust": "1.85.0" + }, + { + "name": "tools/asahi-m3/test_completion_progress.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi-m3/test_completion_progress.py" + ] + }, + { + "name": "tools/asahi-m3/test_fault_snapshot.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi-m3/test_fault_snapshot.py" + ] + }, + { + "name": "tools/asahi-m3/test-retirement-progress.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi-m3/test-retirement-progress.py" + ] + }, + { + "name": "tools/asahi-m3/test-gem-object-size.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi-m3/test-gem-object-size.py" + ], + "rust": "1.85.0" + }, + { + "name": "tools/aurora-sep/tests/test_adt_extract.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_adt_extract.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_air_gpu.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_air_gpu.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_desktop_fixes.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_desktop_fixes.py", + "-v" + ], + "commits": [ + "05d6db3921dec6a6667a7ec8b1b0c75f0f66bafe" + ] + }, + { + "name": "tools/aurora-sep/tests/test_diag_sensor.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_diag_sensor.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_download_resume.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_download_resume.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_esp_history.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_esp_history.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_extract_mesa_calibration.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_extract_mesa_calibration.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_frozen_dependencies.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_frozen_dependencies.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_host_entropy.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_host_entropy.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_host_state_check.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_host_state_check.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m1n1_rollback.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m1n1_rollback.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_air.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_air.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_air_default.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_air_default.py", + "-v" + ], + "commits": [ + "4f2a68f8b88a6b407f072356c248d22f076b401a" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_air_mesa.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_air_mesa.py", + "-v" + ], + "commits": [ + "4f2a68f8b88a6b407f072356c248d22f076b401a" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_boot_profile.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_boot_profile.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_cmdline.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_cmdline.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_flow.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_flow.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_gpu_check.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_gpu_check.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_handoff.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_handoff.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_kernel_only.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_kernel_only.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_max.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_max.py", + "-v" + ], + "commits": [ + "2ca6ff6e3480e11dfd51b720763c0fbd99fa015e", + "26cdc069a3dfa86b901ac3c407bad14cc2fb7efc" + ], + "serial": true + }, + { + "name": "tools/aurora-sep/tests/test_m3_persistent.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_persistent.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_pro_mesa.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_pro_mesa.py", + "-v" + ], + "commits": [ + "26cdc069a3dfa86b901ac3c407bad14cc2fb7efc", + "df0c433027e9eb283b14a56894efcc817c8a3cfb", + "1d41e1b7ff5db2b31de8fcee56959d70ce152c0a" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_pro_perf.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_pro_perf.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_report.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_report.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3_stack_assembly.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3_stack_assembly.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_m3max_kit.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_m3max_kit.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_neo_gpu.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_neo_gpu.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_packaging.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_packaging.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_pm_domains.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_pm_domains.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_ready_pam_hint.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_ready_pam_hint.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_refkey_seal_admission.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_refkey_seal_admission.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_release.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_release.py", + "-v" + ], + "commits": [ + "d14f756c9a65752ca532b93391f775387ba4f191", + "df0c433027e9eb283b14a56894efcc817c8a3cfb", + "1d41e1b7ff5db2b31de8fcee56959d70ce152c0a" + ] + }, + { + "name": "tools/aurora-sep/tests/test_sbio_exchange.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_sbio_exchange.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_sensor_recheck.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_sensor_recheck.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_sensor_wake.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_sensor_wake.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_sep_ready.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_sep_ready.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_store_shipped.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_store_shipped.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_t8112_sep_dma.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_t8112_sep_dma.py", + "-v" + ], + "setup": [ + [ + "make", + "-s", + "O={output}/kbuild", + "ARCH=arm64", + "allnoconfig" + ], + [ + "make", + "-s", + "O={output}/kbuild", + "ARCH=arm64", + "scripts_dtc" + ] + ], + "env": { + "DTC": "{output}/kbuild/scripts/dtc/dtc" + } + }, + { + "name": "tools/aurora-sep/tests/test_t8122_sep.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_t8122_sep.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_unsupported_soc.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_unsupported_soc.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_verify_bringup.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_verify_bringup.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/tests/test_xart_locator.py", + "group": "aurora-sep", + "argv": [ + "python3", + "-m", + "unittest", + "discover", + "-s", + "tools/aurora-sep/tests", + "-p", + "test_xart_locator.py", + "-v" + ] + }, + { + "name": "tools/aurora-sep/recipes/mesa-m3/tests/test-udev-rule.sh", + "group": "tools", + "argv": [ + "sh", + "tools/aurora-sep/recipes/mesa-m3/tests/test-udev-rule.sh" + ] + }, + { + "name": "tools/aurora-sep/recipes/mesa-m3/tests/test-session-env.sh (sh)", + "group": "tools", + "argv": [ + "sh", + "tools/aurora-sep/recipes/mesa-m3/tests/test-session-env.sh" + ] + }, + { + "name": "tools/aurora-sep/recipes/mesa-m3/tests/test-session-env.sh (bash)", + "group": "tools", + "argv": [ + "bash", + "tools/aurora-sep/recipes/mesa-m3/tests/test-session-env.sh" + ] + }, + { + "name": "tools/aurora-sep/recipes/mesa-m3/tests/test-user-setup.sh", + "group": "tools", + "argv": [ + "sh", + "tools/aurora-sep/recipes/mesa-m3/tests/test-user-setup.sh" + ] + }, + { + "name": "tools/aurora-sep/recipes/mesa-m3/tests/test-uwsm.sh", + "group": "tools", + "argv": [ + "sh", + "tools/aurora-sep/recipes/mesa-m3/tests/test-uwsm.sh" + ] + }, + { + "name": "tools/aurora-display-notifier/tests/test_notifier.py", + "group": "tools", + "argv": [ + "make", + "-C", + "tools/aurora-display-notifier/", + "test" + ] + }, + { + "name": "tools/aurora-display-notifier/tests/integration_dbus.py", + "group": "tools", + "argv": [ + "dbus-run-session", + "--", + "env", + "AURORA_TEST_PRIVATE_BUS=1", + "python3", + "tools/aurora-display-notifier/tests/integration_dbus.py" + ] + }, + { + "name": "tools/aurora-tb/pr14-regression.py", + "group": "tools", + "argv": [ + "python3", + "tools/aurora-tb/pr14-regression.py", + "--negative-controls" + ] + }, + { + "name": "tools/aurora-tb/tbnet-lifecycle-test.py", + "group": "tools", + "argv": [ + "python3", + "tools/aurora-tb/tbnet-lifecycle-test.py", + "--source", + ".", + "--output", + "{output}", + "--negative-controls" + ], + "known_failure": "its fixture predates the receive refill retry (1d25feb6bb33): no rx_refill_work member or cancel_delayed_work_sync mock" + }, + { + "name": "tools/aurora-wake/test-s2idle-rtc-regression.sh", + "group": "tools", + "argv": [ + "bash", + "tools/aurora-wake/test-s2idle-rtc-regression.sh" + ] + }, + { + "name": "tools/aurora-userpages/userpages.c", + "group": "tools", + "argv": [ + "cc", + "-O2", + "-g", + "-Wall", + "-Wextra", + "-Werror", + "-Wl,-z,max-page-size=65536", + "tools/aurora-userpages/userpages.c", + "-o", + "{output}" + ] + }, + { + "name": "tools/testing/apple/check-backlight.py", + "group": "tools", + "argv": [ + "python3", + "tools/testing/apple/check-backlight.py", + "--output", + "{output}" + ], + "known_failure": "its include path lacks linux/math64.h, which dcp_backlight.h now includes" + }, + { + "name": "tools/testing/apple/check-backlight-retry.py", + "group": "tools", + "argv": [ + "python3", + "tools/testing/apple/check-backlight-retry.py", + "--output", + "{output}" + ], + "known_failure": "the extracted code now calls min(), div_u64() and mul_u32_u32(), which the fixture does not mock" + }, + { + "name": "tools/testing/apple/check-lifetime.py", + "group": "tools", + "argv": [ + "python3", + "tools/testing/apple/check-lifetime.py", + "--output", + "{output}" + ], + "known_failure": "the apple_dcp_afkep mock lacks lock and stopping, and spin_lock_irqsave is not mocked" + }, + { + "name": "tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh", + "group": "tools", + "argv": [ + "sh", + "tools/testing/selftests/drivers/gpu/apple-afkv2/run-afkv2-tests.sh" + ], + "known_failure": "the send and edid stages predate afk.c changes (afk_send_command_common, reply_len)" + }, + { + "name": "tools/testing/selftests/drivers/gpu/apple-afkv2/test-admission.py", + "group": "tools", + "argv": [ + "python3", + "tools/testing/selftests/drivers/gpu/apple-afkv2/test-admission.py" + ] + }, + { + "name": "tools/testing/selftests/drivers/gpu/apple-dcp/plane-layout.py", + "group": "tools", + "argv": [ + "python3", + "tools/testing/selftests/drivers/gpu/apple-dcp/plane-layout.py", + "--mutations", + "--output", + "{output}" + ] + }, + { + "name": "tools/testing/selftests/drivers/gpu/apple-dcp/run.sh", + "group": "tools", + "argv": [ + "sh", + "tools/testing/selftests/drivers/gpu/apple-dcp/run.sh", + "--output", + "{output}" + ] + }, + { + "name": "tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py", + "group": "tools", + "argv": [ + "python3", + "tools/testing/selftests/drivers/gpu/test_t8140_external_bw.py", + "-v" + ] + }, + { + "name": "tools/testing/selftests/drivers/net/mt7932_e2e_test.py", + "group": "tools", + "argv": [ + "python3", + "tools/testing/selftests/drivers/net/mt7932_e2e_test.py", + "-v" + ] + }, + { + "name": "tools/asahi/tb-dp-repair-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/tb-dp-repair-tests.py", + "--mutations", + "--output", + "{output}" + ] + }, + { + "name": "tools/asahi/t6030-clock-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/t6030-clock-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/t6030-wiring-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/t6030-wiring-tests.py", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/tiled-split-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/tiled-split-tests.py", + "--mutations", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/t6030-pmp-supplier-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/t6030-pmp-supplier-tests.py", + "--mutations", + "--output", + "{output}" + ] + }, + { + "name": "tools/asahi/atc-usb3-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/atc-usb3-tests.py", + "--mutations", + "--out", + "{output}" + ] + }, + { + "name": "tools/asahi/g15-input-error-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/g15-input-error-tests.py", + "--mutations", + "--output", + "{output}" + ], + "rust": "1.93.1", + "note": "Actual production-body host control plus runtime mutants; kernel services modeled, no private hardware fixtures." + }, + { + "name": "tools/asahi/g15-queue-credit-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/g15-queue-credit-tests.py", + "--mutations", + "--output", + "{output}" + ], + "rust": "1.93.1", + "note": "Actual production-body host control plus runtime mutants; kernel services modeled, no private hardware fixtures." + }, + { + "name": "tools/asahi/g15-probe-init-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/g15-probe-init-tests.py", + "--mutations", + "--output", + "{output}" + ], + "rust": "1.93.1", + "note": "Actual production-body host control plus runtime mutants; kernel services modeled, no private hardware fixtures." + }, + { + "name": "tools/asahi/g15-retention-tests.py", + "group": "tools", + "commits": [ + "31da169251bd34d5810a0874563d9a019b85bf56" + ], + "argv": [ + "python3", + "tools/asahi/g15-retention-tests.py", + "--mutations", + "--output", + "{output}" + ], + "rust": "1.93.1", + "note": "Actual G15 and KBox ownership bodies with drop-count allocation failures; old-source P1 controls must fail at runtime. Public synthetic inputs only." + }, + { + "name": "tools/asahi/g15-maintained-handoff-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/g15-maintained-handoff-tests.py", + "--mutations", + "--output", + "{output}" + ], + "rust": "1.93.1", + "note": "Actual production-body host control plus runtime mutants; kernel services modeled, no private hardware fixtures." + }, + { + "name": "tools/asahi/g15-zero-words-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/g15-zero-words-tests.py", + "--mutations", + "--evidence", + "{output}" + ], + "rust": "1.93.1", + "note": "Actual production-body host control plus runtime mutants; kernel services modeled, no private hardware fixtures." + }, + { + "name": "tools/asahi/g15-pb-growth-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/g15-pb-growth-tests.py", + "--mutations", + "--output", + "{output}" + ], + "rust": "1.93.1", + "note": "Actual production-body host control plus runtime mutants; kernel services modeled, no private hardware fixtures." + }, + { + "name": "tools/asahi/g15-stats-drain-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/g15-stats-drain-tests.py", + "--mutations", + "--output", + "{output}" + ], + "rust": "1.93.1", + "note": "Actual production-body host control plus runtime mutants; kernel services modeled, no private hardware fixtures." + }, + { + "name": "tools/testing/selftests/drivers/gpu/asahi-g15/run.py", + "group": "tools", + "argv": [ + "python3", + "tools/testing/selftests/drivers/gpu/asahi-g15/run.py" + ], + "rust": "1.93.1", + "note": "Public synthetic Rust/C host cases; two unavailable private-hardware fixture cases remain explicitly ignored." + }, + { + "name": "tools/asahi/t6030-pmp-graph-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/t6030-pmp-graph-tests.py", + "--mutations", + "--output", + "{output}" + ], + "note": "Actual production-body host boundaries and runtime mutants; no firmware or hardware qualification." + }, + { + "name": "tools/asahi/t6030-display-admission-tests.py", + "group": "tools", + "argv": [ + "python3", + "tools/asahi/t6030-display-admission-tests.py", + "--mutations", + "--output", + "{output}" + ], + "note": "Actual production-body host boundaries and runtime mutants; no firmware or hardware qualification." + }, + { + "name": "tools/asahi/t6030-display-linkage-tests.py", + "group": "tools", + "setup": [ + [ + "make", + "-s", + "O={output}-conf", + "allnoconfig" + ] + ], + "argv": [ + "python3", + "tools/asahi/t6030-display-linkage-tests.py", + "--conf", + "{output}-conf/scripts/kconfig/conf", + "--output", + "{output}" + ], + "note": "Build only the host Kconfig tool in an isolated output directory; check all built-in/module dependency combinations." + } + ], + "helpers": { + "tools/testing/selftests/drivers/gpu/apple-dcp/run.py": "the implementation behind tools/testing/selftests/drivers/gpu/apple-dcp/run.sh, which is listed", + "tools/testing/selftests/drivers/gpu/asahi-g15/uat-owner.c": "compiled and executed by the registered asahi-g15/run.py actual reservation-owner fixture", + "tools/testing/selftests/drivers/gpu/asahi-g15/pmp-bridge.c": "compiled and executed by the registered asahi-g15/run.py maintained PMP fixture", + "tools/asahi/t6030-pmp-graph-tests.c": "compiled and executed by the corresponding registered Python actual-body host suite", + "tools/asahi/t6030-display-admission-tests.c": "compiled and executed by the corresponding registered Python actual-body host suite" + }, + "excluded": { + "tools/aurora-sep/tests/coordinator/runtime-check.sh": "Needs /dev/sep-bio, a running kernel booted with trusted.source=applesep, the hw_random sysfs interface and fprintd. Coordinator payload for an approved device.", + "tools/aurora-sep/tests/coordinator/deletion-check.sh": "Needs /dev/sep-bio, fprintd over the system bus and a previously enrolled expendable test account; spans a reboot.", + "tools/aurora-sep/tests/coordinator/sep-live-count.py": "Opens /dev/sep-bio on a running kernel. Not picked up by `unittest discover` (no test_ prefix) - keep it that way.", + "tools/aurora-wake/s2idle-rtc-regression.sh": "Real suspend-to-idle cycles with rtcwake on an Apple machine, as root. Its argument checks run hermetically via test-s2idle-rtc-regression.sh (in suites.json).", + "tools/aurora-wake/attended-bt-wake.sh": "J700 with a paired Bluetooth keyboard, a person pressing keys, root.", + "tools/aurora-wake/bt-wake-cycle.sh": "One s2idle cycle on a J700 with Bluetooth wake, root.", + "tools/aurora-wake/bt-wake-power.sh": "Alternating RTC-timed sleeps on a J700, root.", + "tools/aurora-wake/bt-wake-emulated.sh": "Drives two other machines over SSH (target and a BlueZ host).", + "tools/aurora-wake/hog-keyboard.py": "BLE HID keyboard emulator for a second BlueZ host (system bus + Bluetooth adapter).", + "tools/aurora-tb/tb-pcie-report": "Reads live Thunderbolt/PCIe state and the kernel log with a dock attached (may need sudo).", + "tools/aurora-m3/m3-serial.py": "macOS helper (sudo, macvdmtool) recording a second Mac's serial console over a USB-C cable.", + "tools/testing/selftests/drivers/apple/j813-peripherals.sh": "Physical J813 check (exits 4 unless /proc/device-tree/compatible has apple,j813 and euid 0).", + "tools/testing/selftests/drivers/net/mt7932_e2e.py": "End-to-end Wi-Fi on a J700 via nmcli and the real network. Its helper unit tests (mt7932_e2e_test.py) are in suites.json.", + "tools/testing/selftests/arm64/user4k/": "kselftest programs for the user4k granule API: need an arm64 Aurora kernel booted with user4k=on (and an aarch64 toolchain to build). check-any-quarter.py / runtime-app.py consume output of those programs.", + "tools/aurora-sep/recipes/mesa-m3/tests/test-probe.sh": "Needs a built mesa-m3-probe (C, links Mesa/EGL/Wayland) and a headless weston for the wayland case.", + "tools/aurora-sep/recipes/mesa-m3/tests/test-confirm.sh": "Needs a built mesa-m3-wlcheck (Wayland client) passed as its argument.", + "tools/aurora-sep/recipes/mesa-m3/tests/build-checks.sh": "Runs inside the PKGBUILD package() step against a freshly built Mesa tree (makepkg, aarch64).", + "tools/aurora-bindgen/build.py": "Downloads the pinned bindgen 0.73.2 source archive (codeload.github.com) and crates.io dependencies, then cargo build --locked -j2 (89 s here incl. its own test.py run; passed). Viable as an optional cached job; GitHub runners have network.", + "tools/aurora-bindgen/test.py": "Hermetic once a patched bindgen exists (--bindgen PATH, 2 s, passed). --baseline-bindgen must be an unpatched 0.73.2; the system 0.73.1 fails the tag-only equality check by design.", + "tools/asahi-m3/completion_progress.py, fault_snapshot.py, retirement_progress.py": "Libraries that read /sys/kernel/debug/asahi-m3/* or /dev/kmsg on an M3 machine. Their offline tests are in suites.json.", + "tools/aurora-sep/air-gpu/*.sh, tools/aurora-sep/pro/*, tools/aurora-sep/neo/*": "On-device helpers; exercised through fixtures by tools/aurora-sep/tests (test_air_gpu, test_m3_pro_perf, test_neo_gpu).", + "drivers/**/*-test.c, *_test.c, tools/testing/kunit/configs/{apple_firmware,arm64_user4k*}.config": "KUnit suites: need a kernel build and tools/testing/kunit/kunit.py (arm64 under QEMU). Not host scripts.", + "tools/aurora-sep/tests: env-gated cases": "Skipped unless set: AURORA_ADT_FIXTURE (real ADT dump), AIR_GPU_TEST_VULKAN=1 (real Vulkan job on a render node), AURORA_PRO_MESA_PKG / AURORA_M1N1_PKG (release packages); test_m3max_kit real-package cases need /nonexistent/... packages. The rest of each module runs." + } +} diff --git a/tools/aurora-ci/host-tests.py b/tools/aurora-ci/host-tests.py new file mode 100755 index 0000000000000..5ccd282a1ee65 --- /dev/null +++ b/tools/aurora-ci/host-tests.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Run the tree's hermetic host test suites, as listed in host-tests.json. + +These are the suites under tools/ that need no Apple hardware, root, network +or running kernel: Python and Rust tests of driver logic built against the +real sources, their mutation checks, and the tools' own tests. Each entry +runs from the top of the tree: + + argv the command; {output} is a fresh path that does not exist + yet (its parent does), for suites that write results + setup commands run first, same substitution (e.g. compile a test) + env extra environment, same substitution + expect "pass" (exit 0, the default) or "fail" (non-zero: some + mutation checks report a caught mutant that way) + rust the toolchain, passed as RUSTUP_TOOLCHAIN + commits history the suite reads (--fetch-commits fetches it) + known_failure why the suite fails today; it runs and is reported, but does + not fail the job, and passing again is reported too + serial the suite measures time, so it runs alone after the others + +Suites run in a controlled environment: TMPDIR outside /home, no +XDG_CONFIG_HOME, no bytecode files, and a sudo that always refuses, so no +suite can change the machine it runs on. The tree must be as clean after the +run as before it. Suites that cannot run here are listed under "excluded" +with the reason; the unit tests check that every test-like file under the +"discovery" roots is listed in one place or the other. +""" + +import argparse +import concurrent.futures +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +MANIFEST = Path(__file__).with_name("host-tests.json") +SUDO_STUB = "#!/bin/sh\necho \"sudo is not available to host test suites: $*\" >&2\nexit 1\n" + + +def load_manifest(path=MANIFEST): + manifest = json.loads(Path(path).read_text()) + names = [suite["name"] for suite in manifest["suites"]] + if len(names) != len(set(names)): + raise ValueError(f"{path}: suite names must be unique") + for suite in manifest["suites"]: + if suite.get("expect", "pass") not in ("pass", "fail"): + raise ValueError(f"{suite['name']}: expect must be pass or fail") + return manifest + + +def select(manifest, groups=None, only=None): + suites = [s for s in manifest["suites"] if not groups or s.get("group", "tools") in groups] + if only: + unknown = set(only) - {s["name"] for s in suites} + if unknown: + raise ValueError(f"unknown suite(s): {', '.join(sorted(unknown))}") + suites = [s for s in suites if s["name"] in only] + return suites + + +def install(manifest, suites): + packages = manifest.get("apt", []) + if packages: + subprocess.run(["sudo", "apt-get", "update", "-q"], check=True) + subprocess.run(["sudo", "apt-get", "install", "-y", "-q", "--no-install-recommends", *packages], check=True) + for version in sorted({s["rust"] for s in suites if s.get("rust")}): + subprocess.run(["rustup", "toolchain", "install", version, "--profile", "minimal"], check=True) + # Ubuntu 24.04 keeps unprivileged user namespaces from bubblewrap, which + # the desktop-session suites run their helpers in. + sysctl = Path("/proc/sys/kernel/apparmor_restrict_unprivileged_userns") + if sysctl.exists() and sysctl.read_text().strip() != "0": + subprocess.run(["sudo", "sysctl", "-w", "kernel.apparmor_restrict_unprivileged_userns=0"], check=True) + + +def fetch_commits(tree, suites, remote="origin"): + """Fetch the history the suites read. Returns (fetched, unavailable).""" + def present(commit): + return subprocess.run(["git", "-C", str(tree), "cat-file", "-e", f"{commit}^{{commit}}"], + stderr=subprocess.DEVNULL).returncode == 0 + + def fetch(*commits): + return subprocess.run(["git", "-C", str(tree), "fetch", "--quiet", "--no-tags", "--depth=1", remote, + *commits], stderr=subprocess.DEVNULL).returncode == 0 + + missing = [c for c in sorted({c for s in suites for c in s.get("commits", [])}) if not present(c)] + if missing and not fetch(*missing): + # One unknown commit fails the whole fetch; find which. + for commit in missing: + fetch(commit) + return [c for c in missing if present(c)], {c for c in missing if not present(c)} + + +def tree_state(tree): + result = subprocess.run(["git", "-C", str(tree), "status", "--porcelain", "--untracked-files=all"], + check=True, text=True, stdout=subprocess.PIPE) + return set(result.stdout.splitlines()) + + +def base_environment(scratch): + env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") + env.pop("XDG_CONFIG_HOME", None) + tmp = scratch / "tmp" + stub = scratch / "bin" + tmp.mkdir() + stub.mkdir() + (stub / "sudo").write_text(SUDO_STUB) + (stub / "sudo").chmod(0o755) + env.update(TMPDIR=str(tmp), PATH=f"{stub}{os.pathsep}{env.get('PATH', '')}") + return env + + +def safe_name(name): + return re.sub(r"[^A-Za-z0-9._-]+", "_", name).strip("_")[:120] + + +def run_suite(suite, tree, output, env, unavailable=frozenset()): + workdir = output / safe_name(suite["name"]) + workdir.mkdir(parents=True) + lacking = sorted(set(suite.get("commits", [])) & unavailable) + if lacking: + note = f"history unavailable: {', '.join(c[:12] for c in lacking)}" + (workdir / "log.txt").write_text(note + "\n") + return {"name": suite["name"], "status": "skipped", "seconds": 0.0, + "known_failure": suite.get("known_failure"), "log": note} + fresh = workdir / "out" + fill = lambda text: text.replace("{output}", str(fresh)) # noqa: E731 + suite_env = dict(env, **{k: fill(v) for k, v in suite.get("env", {}).items()}) + if suite.get("rust"): + suite_env["RUSTUP_TOOLCHAIN"] = suite["rust"] + log, started, status = [], time.monotonic(), None + timeout = suite.get("timeout", 1200) + for step in suite.get("setup", []) + [suite["argv"]]: + command = [fill(part) for part in step] + log.append("$ " + " ".join(command)) + try: + result = subprocess.run(command, cwd=tree, env=suite_env, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=timeout, stdin=subprocess.DEVNULL) + except subprocess.TimeoutExpired as expired: + output_text = expired.stdout or "" + log.append(output_text if isinstance(output_text, str) else output_text.decode(errors="replace")) + status = "timeout" + break + except OSError as error: + log.append(str(error)) + status = "could not start" + break + log.append(result.stdout) + if step is not suite["argv"] and result.returncode: + status = f"setup failed ({result.returncode})" + break + if step is suite["argv"]: + want_pass = suite.get("expect", "pass") == "pass" + ok = (result.returncode == 0) == want_pass + status = "pass" if ok else f"exit {result.returncode}, expected {'0' if want_pass else 'non-zero'}" + text = "\n".join(log) + (workdir / "log.txt").write_text(text) + return {"name": suite["name"], "status": status, "seconds": round(time.monotonic() - started, 1), + "known_failure": suite.get("known_failure"), "log": text} + + +def verdict(result): + if result["status"] == "skipped": + return "skipped" + if result["known_failure"]: + return "known failure, now passes" if result["status"] == "pass" else "known failure" + return "pass" if result["status"] == "pass" else "FAIL" + + +def report(results, dirtied, fetched, unavailable=frozenset()): + failed = [r for r in results if verdict(r) == "FAIL"] + known = [r for r in results if r["known_failure"] and verdict(r) != "skipped"] + skipped = [r for r in results if verdict(r) == "skipped"] + lines = ["### Host test suites", "", + f"{len(results)} suites: {sum(verdict(r) == 'pass' for r in results)} passed, {len(failed)} failed, " + f"{len(known)} known failures, {len(skipped)} skipped; " + f"{sum(r['seconds'] for r in results):.0f} s of suite time.", ""] + if fetched: + lines += [f"Fetched {len(fetched)} commit(s) the suites read.", ""] + if unavailable: + lines += [f"**{len(unavailable)} commit(s) the suites read are not in this repository**, so " + f"{len(skipped)} suite(s) were skipped: " + ", ".join(f"`{c[:12]}`" for c in sorted(unavailable)), ""] + if failed: + lines += ["**Failed**", ""] + [f"- `{r['name']}`: {r['status']}" for r in failed] + [""] + if known: + lines += ["Known failures (not blocking):", ""] + lines += [f"- `{r['name']}`: {r['known_failure']}" + + (" **Now passes: remove `known_failure`.**" if r["status"] == "pass" else "") for r in known] + lines.append("") + if dirtied: + lines += ["**The suites changed the tree:**", ""] + [f"- `{line}`" for line in dirtied[:40]] + [""] + lines += ["
Every suite", "", "| Suite | Result | Seconds |", "|---|---|---|"] + lines += [f"| `{r['name']}` | {verdict(r)} | {r['seconds']} |" for r in results] + return "\n".join(lines + ["", "
", ""]) + "\n", failed + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--manifest", type=Path, default=MANIFEST) + parser.add_argument("--tree", type=Path, default=Path(".")) + parser.add_argument("--group", action="append", help="run only this group (repeatable)") + parser.add_argument("--only", action="append", help="run only this suite (repeatable)") + parser.add_argument("--output", type=Path, help="directory for each suite's output and log") + parser.add_argument("--jobs", type=int, default=1, help="suites to run at once") + parser.add_argument("--list", action="store_true", help="list the selected suites and exit") + parser.add_argument("--install", action="store_true", + help="install what the selected suites need (apt, Rust, sysctl) and exit") + parser.add_argument("--fetch-commits", action="store_true", + help="fetch the history the selected suites read before running them") + args = parser.parse_args(argv) + + try: + manifest = load_manifest(args.manifest) + suites = select(manifest, args.group, args.only) + except ValueError as error: + parser.error(str(error)) + if args.list: + for suite in suites: + print(f"{suite['name']}: {' '.join(suite['argv'])}") + return 0 + if args.install: + install(manifest, suites) + return 0 + if not args.output: + parser.error("--output is required to run suites") + tree = args.tree.resolve() + fetched, unavailable = fetch_commits(tree, suites) if args.fetch_commits else ([], set()) + if args.output.exists(): + shutil.rmtree(args.output) + args.output.mkdir(parents=True) + output = args.output.resolve() + + in_ci = bool(os.environ.get("GITHUB_ACTIONS")) + before = tree_state(tree) + with tempfile.TemporaryDirectory(prefix="aurora-ci-host-", dir="/tmp") as scratch: + env = base_environment(Path(scratch)) + # Serial suites measure time; they run alone, after the parallel batch. + parallel = [s for s in suites if not s.get("serial")] + serial = [s for s in suites if s.get("serial")] + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, args.jobs)) as pool: + futures = [pool.submit(run_suite, suite, tree, output, env, unavailable) for suite in parallel] + results = [] + for future in futures + [None] * len(serial): + result = future.result() if future else run_suite(serial[len(results) - len(parallel)], + tree, output, env, unavailable) + results.append(result) + if in_ci: + print(f"::group::{result['name']}: {verdict(result)} ({result['seconds']} s)") + print(result["log"]) + print("::endgroup::") + else: + print(f"{result['name']}: {verdict(result)} ({result['seconds']} s)") + dirtied = sorted(tree_state(tree) - before) + + text, failed = report(results, dirtied, fetched, unavailable) + if not in_ci: + print(text.split("
")[0], end="") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(text) + (output / "results.json").write_text(json.dumps( + [{k: v for k, v in r.items() if k != "log"} for r in results], indent=1) + "\n") + for r in failed: + print(f"::error title=Host test failed::{r['name']}: {r['status']}", file=sys.stderr) + for r in results: + if r["known_failure"] and r["status"] == "pass": + print(f"::warning title=Known failure now passes::{r['name']}: remove known_failure", file=sys.stderr) + if unavailable: + print(f"::warning title=History unavailable::{len(unavailable)} commit(s) the suites read could not be " + "fetched; the suites that need them were skipped", file=sys.stderr) + if dirtied: + print(f"::error title=Host tests changed the tree::{len(dirtied)} paths, first {dirtied[0]}", file=sys.stderr) + return 1 if failed or dirtied else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/inspect-package.py b/tools/aurora-ci/inspect-package.py new file mode 100755 index 0000000000000..db7cb4e5cbeb5 --- /dev/null +++ b/tools/aurora-ci/inspect-package.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Describe a built linux-aurora package pair and check what installs rely on. + +Reads linux-aurora--aarch64.pkg.tar.zst and its -headers package, unpacks +them, and writes a metadata record (kernel release, image header, device +trees, modules, resolved configuration, toolchain from .BUILDINFO) that +compare-packages.py diffs against another build. It fails when the package +would break an install: + + - the kernel release must end in -ARCH (boot bundle DTB discovery); + - vmlinuz must be a little-endian arm64 Image whose page size matches the + configuration; + - the package must carry device trees, modules.dep and pkgbase; + - every module symbol must resolve (depmod -e against System.map); + - the configuration must keep CONFIG_RUST=y (Rust drivers ship). +""" + +import argparse +import hashlib +import json +import os +import re +import shutil +import struct +import subprocess +import sys +import tempfile +from pathlib import Path + +SCHEMA = 1 +KVER_TOKEN = "" +PAGE_SIZES = {1: "4K", 2: "16K", 3: "64K"} +CONFIG_PAGE_SIZE = {"CONFIG_ARM64_4K_PAGES": "4K", "CONFIG_ARM64_16K_PAGES": "16K", + "CONFIG_ARM64_64K_PAGES": "64K"} + + +class PackageError(Exception): + pass + + +def find_packages(directory): + main, headers = [], [] + for path in sorted(directory.glob("*.pkg.tar.zst")): + if path.name.startswith("linux-aurora-headers-"): + headers.append(path) + elif re.match(r"linux-aurora-[0-9]", path.name): + main.append(path) + if len(main) != 1 or len(headers) != 1: + raise PackageError(f"expected one linux-aurora and one linux-aurora-headers package in " + f"{directory}, found {[p.name for p in main + headers]}") + return main[0], headers[0] + + +def unpack(package, destination): + """Unpack a .pkg.tar.zst with the system zstd and tar; return member names.""" + destination.mkdir(parents=True, exist_ok=True) + names = subprocess.run(["tar", "--zstd", "-tf", str(package)], check=True, text=True, + stdout=subprocess.PIPE).stdout.splitlines() + subprocess.run(["tar", "--zstd", "-xf", str(package), "-C", str(destination)], check=True) + return names + + +def parse_pkginfo(text): + info = {} + for line in text.splitlines(): + if " = " in line and not line.startswith("#"): + key, value = line.split(" = ", 1) + info.setdefault(key, []).append(value) + return info + + +def parse_config(text): + config = {} + for line in text.splitlines(): + if line.startswith("CONFIG_") and "=" in line: + key, value = line.split("=", 1) + config[key] = value + else: + match = re.match(r"# (CONFIG_\w+) is not set$", line) + if match: + config[match.group(1)] = "n" + return config + + +def image_header(path): + """Decode the arm64 Image header (Documentation/arch/arm64/booting.rst).""" + with open(path, "rb") as image: + header = image.read(64) + if len(header) < 64 or header[56:60] != b"ARM\x64": + raise PackageError(f"{path.name} is not an arm64 Image (no ARM\\x64 magic)") + text_offset, image_size, flags = struct.unpack_from("> 1) & 3, "unspecified"), + } + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def normalise(names, kernelrelease): + return sorted(name.replace(kernelrelease, KVER_TOKEN) for name in names + if name and not name.endswith("/")) + + +def run_depmod(root, kernelrelease, system_map): + """Return the module symbol problems depmod reports, as plain lines.""" + result = subprocess.run( + ["depmod", "-b", str(root / "usr"), "-e", "-F", str(system_map), "-n", kernelrelease], + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True) + lines = [line.strip() for line in result.stderr.splitlines() if line.strip()] + if result.returncode: + lines.append(f"depmod exited with status {result.returncode}") + return [line.replace(str(root), "") for line in lines] + + +def describe(main_pkg, headers_pkg, workdir): + root = workdir / "root" + main_names = unpack(main_pkg, root) + headers_root = workdir / "headers" + headers_names = unpack(headers_pkg, headers_root) + + module_dirs = [p for p in (root / "usr/lib/modules").iterdir() if p.is_dir()] + if len(module_dirs) != 1: + raise PackageError(f"expected one kernel release under usr/lib/modules, found " + f"{[p.name for p in module_dirs]}") + moddir = module_dirs[0] + kernelrelease = moddir.name + build = headers_root / "usr/lib/modules" / kernelrelease / "build" + config_path = build / ".config" + if not config_path.is_file(): + raise PackageError(f"headers package has no build/.config for {kernelrelease}") + config = parse_config(config_path.read_text()) + + pkginfo = parse_pkginfo((root / ".PKGINFO").read_text()) + buildinfo = parse_pkginfo((root / ".BUILDINFO").read_text()) if (root / ".BUILDINFO").exists() else {} + toolchain = sorted(p for p in buildinfo.get("installed", []) + if re.match(r"(clang|llvm|lld|rust|rustup|pahole|dtc|gcc|binutils|kmod)[0-9]*-", p)) + + image = moddir / "vmlinuz" + header = image_header(image) if image.is_file() else None + dtbs = {p.name: sha256(p) for p in sorted((moddir / "dtbs").glob("*.dtb"))} + order = (moddir / "modules.order").read_text().split() if (moddir / "modules.order").exists() else [] + modules = sorted(str(p.relative_to(moddir)) for p in moddir.rglob("*.ko*")) + builtin = (moddir / "modules.builtin").read_text().split() if (moddir / "modules.builtin").exists() else [] + depmod = run_depmod(root, kernelrelease, build / "System.map") if (build / "System.map").exists() \ + else ["System.map missing from the headers package"] + + page_config = [size for key, size in CONFIG_PAGE_SIZE.items() if config.get(key) == "y"] + checks = [ + ("kernel release ends in -ARCH", kernelrelease.endswith("-ARCH"), kernelrelease), + ("vmlinuz is a little-endian arm64 Image", bool(header) and header["endian"] == "le", + json.dumps(header) if header else "missing"), + ("Image page size matches the configuration", + bool(header) and page_config == [header["page_size"]], + f"Image {header['page_size'] if header else '?'}, config {page_config}"), + ("package carries device trees", bool(dtbs), f"{len(dtbs)} DTBs"), + ("modules.dep was generated", (moddir / "modules.dep").is_file(), ""), + ("pkgbase is linux-aurora", + (moddir / "pkgbase").is_file() and (moddir / "pkgbase").read_text().strip() == "linux-aurora", ""), + ("every module symbol resolves", not depmod, "; ".join(depmod[:20])), + ("CONFIG_RUST=y", config.get("CONFIG_RUST") == "y", config.get("CONFIG_RUST", "absent")), + ("modules.order lists only shipped modules", + all(entry in modules for entry in order), f"{len(order)} in modules.order"), + ] + + return { + "schema": SCHEMA, + "kernelrelease": kernelrelease, + "package": { + "file": main_pkg.name, + "version": pkginfo.get("pkgver", ["?"])[0], + "size": main_pkg.stat().st_size, + "sha256": sha256(main_pkg), + "files": normalise(main_names, kernelrelease), + }, + "headers": { + "file": headers_pkg.name, + "size": headers_pkg.stat().st_size, + "sha256": sha256(headers_pkg), + "files": normalise(headers_names, kernelrelease), + }, + "toolchain": toolchain, + "image": dict(header or {}, size=image.stat().st_size if image.is_file() else 0, + sha256=sha256(image) if image.is_file() else None), + "dtbs": dtbs, + "modules": modules, + "builtin": builtin, + "config": config, + "depmod": depmod, + "checks": [{"name": n, "ok": bool(ok), "detail": d} for n, ok, d in checks], + } + + +def summary(meta): + lines = [f"### Package `{meta['package']['file']}`", "", + f"Kernel release `{meta['kernelrelease']}`; {len(meta['modules'])} modules, " + f"{len(meta['dtbs'])} device trees, image {meta['image']['size'] // (1 << 20)} MiB " + f"({meta['image'].get('page_size', '?')} pages).", "", + "| Check | Result | Detail |", "|---|---|---|"] + for check in meta["checks"]: + detail = (check["detail"] or "").replace("|", "\\|") + lines.append(f"| {check['name']} | {'pass' if check['ok'] else '**FAIL**'} | {detail} |") + if meta["toolchain"]: + lines += ["", "Toolchain: " + ", ".join(f"`{p}`" for p in meta["toolchain"])] + return "\n".join(lines) + "\n\n" + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--packages", type=Path, required=True, + help="directory holding the two .pkg.tar.zst files") + parser.add_argument("--output", type=Path, required=True, help="metadata JSON to write") + parser.add_argument("--unpacked", type=Path, + help="keep the unpacked kernel package here (for the boot test)") + args = parser.parse_args(argv) + + with tempfile.TemporaryDirectory(prefix="aurora-ci-inspect-") as scratch: + try: + main_pkg, headers_pkg = find_packages(args.packages) + meta = describe(main_pkg, headers_pkg, Path(scratch)) + except (PackageError, OSError, subprocess.CalledProcessError) as error: + print(f"::error title=Package inspection::{error}", file=sys.stderr) + return 1 + if args.unpacked: + if args.unpacked.exists(): + shutil.rmtree(args.unpacked) + shutil.move(str(Path(scratch) / "root"), str(args.unpacked)) + + args.output.write_text(json.dumps(meta, indent=1, sort_keys=True) + "\n") + text = summary(meta) + print(text, end="") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(text) + failed = [c for c in meta["checks"] if not c["ok"]] + for check in failed: + print(f"::error title=Package check failed::{check['name']}: {check['detail']}", file=sys.stderr) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/kunit-expected.json b/tools/aurora-ci/kunit-expected.json new file mode 100644 index 0000000000000..9c7520e8be6c4 --- /dev/null +++ b/tools/aurora-ci/kunit-expected.json @@ -0,0 +1,23 @@ +{ + "comment": [ + "Known results of run-kunit.py with aurora.kunitconfig. Keys are suite.test[.param].", + "results: cases that do not pass, with the status they are expected to have and why.", + "warnings: kernel splats (WARNING, BUG, ...) inside a case, by call site and count. KTAP still", + "reports these cases as passed, so they are tracked here; a new or missing one is a mismatch." + ], + "results": { + "apple-dcp-fabric.fabric_scenario_test.S35_dual_retry_off": { + "status": "SKIP", + "reason": "Unconditional: scenario 35 of the fabric scenario table only calls kunit_skip(\"TB match-data retry policy is tested by W3\") (drivers/gpu/drm/apple/dcp-fabric-test.c). No other code in the tree is named W3; the retry policy is not exercised by this case." + }, + "rust_kernel_kunit.rust_test_kunit_always_disabled_test": { + "status": "SKIP", + "reason": "By design (upstream Rust KUnit selftest): the case carries #[cfg(not(all()))], so #[kunit_tests] reports it as skipped to show that cfg'd-out tests are not run (rust/kernel/kunit.rs)." + }, + "trusted_source.trusted_source_after_mark_readonly": { + "status": "SKIP", + "reason": "By design when built in: the case skips while system_state < SYSTEM_RUNNING, because built-in KUnit runs before mark_readonly(). It only runs when trusted_source_kunit is loaded as a module after boot, which kunit.py run does not do (security/keys/trusted-keys/tests/trusted_source_kunit.c)." + } + }, + "warnings": {} +} diff --git a/tools/aurora-ci/lookup-build.py b/tools/aurora-ci/lookup-build.py new file mode 100755 index 0000000000000..37bf6275fc13a --- /dev/null +++ b/tools/aurora-ci/lookup-build.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Name a kernel build by its inputs and find earlier results for it. + +A build is identified by what goes into it: the kernel tree without the CI +itself (.github and tools/aurora-ci never reach the package), and the parts +of omarchy-pkgs that build it (the linux-aurora recipe, the builder image +definition, bin/build and the helpers it sources). The same key means the +same package, so CI never builds or tests one twice; a change to the CI +alone, or to omarchy-pkgs outside those parts, reuses the last build. What +the key cannot see (package updates in the Arch Linux ARM repositories the +builder installs from) is what the nightly rebuild is for. + + kernel-package- the packages and their metadata (reused instead of + building again) + kernel-verified- written when every full-run job passed; a later + run with the same key skips KUnit and the boot test + kernel-metadata- package metadata for a kernel (its tree without + the CI), the baseline a later change is compared with + +This script computes the key, pins the omarchy-pkgs commit it was computed +from, and looks those artifacts up through the Actions API. It accepts only +artifacts that kernel-ci.yml uploaded after review: from the branch, the +merge queue, the nightly run or a manual run of the branch, never from a +pull request (see TRUSTED_EVENTS). +""" + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import urllib.parse +import urllib.request +from pathlib import Path + +# What omarchy-pkgs' bin/build uses to build linux-aurora: the recipe, the +# builder image and its build script, bin/build itself and the helpers that +# bin/build and build/build.sh source. +RECIPE_PARTS = ("pkgbuilds/linux-aurora", "build", "bin/build", "helpers/docker-helpers.sh", + "helpers/message-helpers.sh", "helpers/package-metadata.sh", "helpers/paths.sh") +# Paths in the kernel tree that never reach the package. +CI_PATHS = {"": {".github"}, "tools": {"aurora-ci"}} + + +def git(repo, *args): + return subprocess.run(["git", "-C", str(repo), *args], check=True, text=True, + stdout=subprocess.PIPE).stdout.strip() + + +def kernel_id(repo, rev): + """Hash of the tree at rev, leaving out CI_PATHS.""" + digest = hashlib.sha256() + for directory, skip in sorted(CI_PATHS.items()): + spec = f"{rev}:{directory}" if directory else f"{rev}^{{tree}}" + listing = subprocess.run(["git", "-C", str(repo), "ls-tree", "--full-tree", spec], text=True, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + if listing.returncode and directory: + continue # no such directory at this commit + listing.check_returncode() + for line in listing.stdout.splitlines(): + name = line.split("\t", 1)[1] + if name in skip or (not directory and name in CI_PATHS): + continue + digest.update(f"{directory}/{line}\n".encode()) + return digest.hexdigest()[:40] + + +def ensure_commit(repo, sha, remote="origin"): + """Fetch a commit's trees (no file contents) if the checkout lacks it.""" + if subprocess.run(["git", "-C", str(repo), "cat-file", "-e", f"{sha}^{{commit}}"], + stderr=subprocess.DEVNULL).returncode: + git(repo, "fetch", "--quiet", "--no-tags", "--depth=1", "--filter=blob:none", remote, sha) + + +def build_key(kernel_tree, recipe_trees): + text = f"kernel {kernel_tree}\n" + "".join(f"{part} {recipe_trees[part]}\n" for part in RECIPE_PARTS) + return hashlib.sha256(text.encode()).hexdigest()[:32] + + +# Runs whose artifacts may be reused. Every artifact name is chosen by the +# workflow that uploads it, and a pull request, or a manual run of a branch, +# runs that branch's own copy of the workflow files, before anyone reviewed +# them. So a name alone proves nothing: reuse only artifacts from runs of +# reviewed code, the branch itself and the merge queue's branches, which +# only a merged or queued pull request reaches. +TRUSTED_WORKFLOWS = {".github/workflows/kernel-ci.yml"} +TRUSTED_EVENTS = {"push", "merge_group", "schedule", "workflow_dispatch"} +TRUSTED_BRANCH = "integration/aurora" +QUEUE_PREFIX = f"gh-readonly-queue/{TRUSTED_BRANCH}/" + + +def trusted_branch(event, branch): + if event == "merge_group": + return branch.startswith(QUEUE_PREFIX) + return branch == TRUSTED_BRANCH + + +class Api: + def __init__(self, url, repository, token, repository_id): + self.url, self.repository, self.token = url.rstrip("/"), repository, token + self.repository_id = int(repository_id) + self._runs = {} + + def get(self, path, **query): + address = f"{self.url}/repos/{self.repository}/{path}" + if query: + address += "?" + urllib.parse.urlencode(query) + request = urllib.request.Request(address, headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "X-GitHub-Api-Version": "2022-11-28", + }) + with urllib.request.urlopen(request, timeout=60) as response: + return json.load(response) + + def trusted_run(self, run_id): + """Whether a run is this repository's kernel CI on reviewed code: + the branch or the merge queue, never a pull request or another + branch.""" + if run_id not in self._runs: + run = self.get(f"actions/runs/{run_id}") + self._runs[run_id] = ( + run.get("event") in TRUSTED_EVENTS + and trusted_branch(run.get("event"), run.get("head_branch") or "") + and run.get("path") in TRUSTED_WORKFLOWS + and (run.get("repository") or {}).get("id") == self.repository_id + and (run.get("head_repository") or {}).get("id") == self.repository_id) + return self._runs[run_id] + + def newest_artifact(self, name): + """Run ID of the newest unexpired artifact with this exact name that a + trusted run uploaded, or ''.""" + data = self.get("actions/artifacts", name=name, per_page=100) + live = [a for a in data.get("artifacts", []) + if not a.get("expired") and a.get("name") == name + and a.get("workflow_run", {}).get("repository_id") == self.repository_id + and a.get("workflow_run", {}).get("head_repository_id") == self.repository_id] + live.sort(key=lambda a: a.get("created_at", ""), reverse=True) + for artifact in live: + if self.trusted_run(artifact["workflow_run"]["id"]): + return str(artifact["workflow_run"]["id"]) + return "" + + +def lookup(api, kernel, recipe_repo, base_sha, force): + kernel_tree = kernel_id(kernel, "HEAD") + recipe_commit = git(recipe_repo, "rev-parse", "HEAD") + recipe_trees = {part: git(recipe_repo, "rev-parse", f"HEAD:{part}") for part in RECIPE_PARTS} + key = build_key(kernel_tree, recipe_trees) + out = {"key": key, "kernel_tree": kernel_tree, "recipe_commit": recipe_commit, + "package_run": "", "verified_run": "", "base_tree": "", "baseline_run": ""} + if not force: + out["package_run"] = api.newest_artifact(f"kernel-package-{key}") + out["verified_run"] = api.newest_artifact(f"kernel-verified-{key}") + if base_sha: + ensure_commit(kernel, base_sha) + out["base_tree"] = kernel_id(kernel, base_sha) + out["baseline_run"] = api.newest_artifact(f"kernel-metadata-{out['base_tree']}") + return out + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--kernel", type=Path, required=True, help="checkout of the commit to build") + parser.add_argument("--recipe-repo", type=Path, required=True, help="omarchy-pkgs checkout") + parser.add_argument("--base-sha", default="", help="commit whose build is the comparison baseline") + parser.add_argument("--force", default="false", help="'true' ignores earlier builds and results") + args = parser.parse_args(argv) + + api = Api(os.environ.get("GITHUB_API_URL", "https://api.github.com"), + os.environ["GITHUB_REPOSITORY"], os.environ["GITHUB_TOKEN"], + os.environ["GITHUB_REPOSITORY_ID"]) + try: + result = lookup(api, args.kernel, args.recipe_repo, args.base_sha, args.force == "true") + except (subprocess.CalledProcessError, OSError, KeyError, ValueError) as error: + print(f"::error title=Build lookup::{error}", file=sys.stderr) + return 1 + + print(json.dumps(result, indent=1)) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as out: + for name, value in result.items(): + out.write(f"{name}={value}\n") + if os.environ.get("GITHUB_STEP_SUMMARY"): + reuse = [] + if result["package_run"]: + reuse.append(f"packages from run {result['package_run']}") + if result["verified_run"]: + reuse.append(f"verified by run {result['verified_run']}") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(f"Build key `{result['key']}` (kernel `{result['kernel_tree'][:12]}`, " + f"omarchy-pkgs `{result['recipe_commit'][:12]}`)" + + (f"; reusing {', '.join(reuse)}" if reuse else "") + ".\n\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/plan.py b/tools/aurora-ci/plan.py new file mode 100755 index 0000000000000..be5b4c6de625e --- /dev/null +++ b/tools/aurora-ci/plan.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Decide what a kernel CI run does, from the GitHub event that started it. + +A light run is the checks that cost nothing on GitHub-hosted runners: commit +hygiene, host test suites and the CI's own tests. A full run adds the +package build, KUnit and the boot test. Builds are paid for, so they run only +when someone with write access asks for one: + + merge_group full: the queue tests exactly what will land + push to the branch full: provenance for what landed (reuses the + queue's build when the tree is unchanged) + schedule full, never reusing an earlier build: catches + changes in omarchy-pkgs and in the packages its + builder installs + workflow_dispatch full, never reusing an earlier build + pull_request, labeled full when the label is the build label. Applying + a label needs triage access, so this is a + maintainer asking for this exact head commit. + pull_request, other full when the build label is already on a branch + of this repository (only writers can push there); + light for forks, where a new push needs the label + re-applied, and light otherwise. +""" + +import argparse +import json +import os +import sys +from pathlib import Path + +DEFAULT_LABEL = "ci:build" +ZERO_SHA = "0" * 40 +PR_REFRESH_ACTIONS = {"opened", "synchronize", "reopened", "ready_for_review"} + + +def plan(event_name, event, label=DEFAULT_LABEL): + """Return the plan as a dict of strings, ready for $GITHUB_OUTPUT.""" + out = {"tier": "light", "reason": "", "base_sha": "", "head_sha": "", + "pr_number": "", "fork": "false", "force": "false"} + + if event_name == "pull_request": + pr = event["pull_request"] + action = event.get("action", "") + labels = {item["name"] for item in pr.get("labels", [])} + fork = pr["head"]["repo"] is None or pr["head"]["repo"]["full_name"] != pr["base"]["repo"]["full_name"] + out.update(base_sha=pr["base"]["sha"], head_sha=pr["head"]["sha"], + pr_number=str(pr["number"]), fork=str(fork).lower()) + if action == "labeled" and event.get("label", {}).get("name") == label: + out.update(tier="full", reason=f"`{label}` applied to {pr['head']['sha'][:12]}") + elif action in PR_REFRESH_ACTIONS and label in labels and not fork: + out.update(tier="full", reason=f"`{label}` is on this pull request") + elif action in PR_REFRESH_ACTIONS and label in labels: + out["reason"] = (f"fork pull request: re-apply `{label}` to build " + f"{pr['head']['sha'][:12]}") + else: + out["reason"] = (f"checks only; the merge queue builds the kernel, or add `{label}` " + "for a build now") + elif event_name == "merge_group": + group = event["merge_group"] + out.update(tier="full", reason="merge queue", base_sha=group["base_sha"], + head_sha=group["head_sha"]) + elif event_name == "push": + before = event.get("before", "") + out.update(tier="full", reason=f"push to {event.get('ref', '?')}", head_sha=event["after"]) + if before and before != ZERO_SHA and not event.get("forced"): + out["base_sha"] = before + elif event_name == "schedule": + # Rebuild from scratch: the Arch Linux ARM packages the builder + # installs change without any change to this tree or omarchy-pkgs. + out.update(tier="full", reason="nightly rebuild", force="true") + elif event_name == "workflow_dispatch": + out.update(tier="full", reason="manual run", force="true") + else: + raise ValueError(f"unsupported event {event_name!r}") + return out + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--event-name", default=os.environ.get("GITHUB_EVENT_NAME")) + parser.add_argument("--event-path", type=Path, default=os.environ.get("GITHUB_EVENT_PATH")) + parser.add_argument("--label", default=DEFAULT_LABEL) + args = parser.parse_args(argv) + if not args.event_name or not args.event_path: + parser.error("--event-name and --event-path (or the GitHub environment) are required") + + try: + result = plan(args.event_name, json.loads(args.event_path.read_text()), args.label) + except (KeyError, TypeError, ValueError) as error: + print(f"::error title=CI plan::cannot plan this {args.event_name} event: {error!r}", file=sys.stderr) + return 1 + + print(json.dumps(result, indent=1)) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as out: + for key, value in result.items(): + out.write(f"{key}={value}\n") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(f"### Plan: {result['tier']} run\n\n{result['reason']}.\n\n") + if result["tier"] == "light": + print(f"::notice title=Checks only::{result['reason']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/prepare-recipe.py b/tools/aurora-ci/prepare-recipe.py new file mode 100755 index 0000000000000..95f8584260697 --- /dev/null +++ b/tools/aurora-ci/prepare-recipe.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Point the packaging recipe for linux-aurora at the commit under test. + +CI builds the kernel with the recipe that ships it: omacom/omarchy-pkgs +pkgbuilds/linux-aurora, run through that repository's own bin/build. This +script makes the smallest change that lets the recipe build an unpublished +commit, and refuses to guess when the recipe no longer looks the way it +expects: + + _commit the commit under test + pkgrel the recipe's pkgrel plus a CI subrelease (2 -> 2.), so a + CI package never has the version of a published one and bin/build + does not skip it as already published + source the kernel entry reads a bare repository placed next to the + PKGBUILD (aurora-ci-source) instead of cloning GitHub again + +Every other line of the recipe, the builder image and the build tooling stay +exactly as they are on omarchy-pkgs. +""" + +import argparse +import difflib +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +SOURCE_DIR = "aurora-ci-source" +SOURCE_REF = "refs/heads/aurora-ci" +UPSTREAM_SOURCE = "linux::git+https://github.com/omacom/linux-aurora.git#commit=${_commit}" +LOCAL_SOURCE = "linux::git+file://${startdir}/" + SOURCE_DIR + "#commit=${_commit}" + +COMMIT_LINE = re.compile(r"^_commit=([0-9a-f]{40})$", re.M) +PKGREL_LINE = re.compile(r"^pkgrel=([0-9]+)(\.[0-9]+)?$", re.M) + + +class RecipeError(Exception): + """The recipe does not have the shape this script knows how to change.""" + + +def single_match(pattern, text, what): + matches = list(pattern.finditer(text)) + if len(matches) != 1: + raise RecipeError(f"expected exactly one {what} line in PKGBUILD, found {len(matches)}") + return matches[0] + + +def rewrite(text, commit, revision): + """Return the rewritten PKGBUILD text and a record of what changed.""" + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise RecipeError(f"commit must be a full 40-character SHA, got {commit!r}") + if not re.fullmatch(r"[1-9][0-9]*", str(revision)): + raise RecipeError(f"revision must be a positive integer, got {revision!r}") + + old_commit = single_match(COMMIT_LINE, text, "_commit=") + old_pkgrel = single_match(PKGREL_LINE, text, "pkgrel=") + if old_pkgrel.group(2): + # pacman allows one subrelease; adding ours would not be a valid pkgrel. + raise RecipeError(f"pkgrel {old_pkgrel.group(0)!r} already has a subrelease") + if text.count(UPSTREAM_SOURCE) != 1: + raise RecipeError(f"expected the kernel source entry {UPSTREAM_SOURCE!r} exactly once") + + pkgrel = f"{old_pkgrel.group(1)}.{revision}" + text = COMMIT_LINE.sub(f"_commit={commit}", text, count=1) + text = PKGREL_LINE.sub(f"pkgrel={pkgrel}", text, count=1) + text = text.replace(UPSTREAM_SOURCE, LOCAL_SOURCE) + return text, { + "recipe_commit": old_commit.group(1), + "commit": commit, + "recipe_pkgrel": old_pkgrel.group(1), + "pkgrel": pkgrel, + "source": LOCAL_SOURCE, + } + + +def git(*args, cwd=None): + result = subprocess.run(["git", *args], cwd=cwd, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.returncode: + raise RecipeError(f"git {' '.join(args)} failed: {result.stderr.strip()}") + return result.stdout.strip() + + +def make_source(recipe, kernel, commit): + """Create a shallow bare repository holding only the commit under test.""" + source = recipe / SOURCE_DIR + if source.exists(): + raise RecipeError(f"{source} already exists; use a fresh recipe checkout") + git("init", "--quiet", "--bare", str(source)) + git("symbolic-ref", "HEAD", SOURCE_REF, cwd=source) + git("fetch", "--quiet", "--no-tags", "--depth=1", str(kernel.resolve()), + f"{commit}:{SOURCE_REF}", cwd=source) + fetched = git("rev-parse", "--verify", f"{SOURCE_REF}^{{commit}}", cwd=source) + if fetched != commit: + raise RecipeError(f"source repository holds {fetched}, expected {commit}") + return source + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--recipe", type=Path, required=True, + help="pkgbuilds/linux-aurora in an omarchy-pkgs checkout") + parser.add_argument("--kernel", type=Path, required=True, + help="kernel checkout that contains the commit") + parser.add_argument("--commit", required=True, help="full SHA of the commit to build") + parser.add_argument("--revision", required=True, + help="CI subrelease appended to pkgrel (e.g. the run number)") + parser.add_argument("--record", type=Path, + help="write what changed, as JSON, to this file") + args = parser.parse_args(argv) + + pkgbuild = args.recipe / "PKGBUILD" + try: + original = pkgbuild.read_text() + text, record = rewrite(original, args.commit, args.revision) + make_source(args.recipe, args.kernel, args.commit) + except (RecipeError, OSError) as error: + print(f"::error title=Recipe override::{error}", file=sys.stderr) + return 1 + pkgbuild.write_text(text) + + diff = "".join(difflib.unified_diff( + original.splitlines(keepends=True), text.splitlines(keepends=True), + "PKGBUILD (omarchy-pkgs)", "PKGBUILD (CI)")) + record["diff"] = diff + print(diff, end="") + if args.record: + args.record.write_text(json.dumps(record, indent=2) + "\n") + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a") as out: + out.write("### Recipe override\n\n" + f"Recipe pin `{record['recipe_commit']}` -> `{args.commit}`, " + f"pkgrel `{record['recipe_pkgrel']}` -> `{record['pkgrel']}`; " + "every other line is omarchy-pkgs' own.\n\n" + f"```diff\n{diff}```\n\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/result.py b/tools/aurora-ci/result.py new file mode 100755 index 0000000000000..cf58708d783d6 --- /dev/null +++ b/tools/aurora-ci/result.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Turn the kernel CI jobs' results into the one required check. + +Branch protection and the merge queue require a single check, so this job +decides what the run as a whole means. A job that was supposed to run must +have succeeded; a skipped job passes only when the plan skips it on purpose: + + - full-run jobs (package, kunit, boot) are skipped on a light run; + - kunit and boot are skipped when an earlier run already verified the same + build key (same kernel tree and packaging inputs); + - commit, compile and device tree checks are skipped for events without a + pull request or merge group range (push, schedule, manual runs). + +A light run that passes says only that the checks passed: the kernel itself +is built in the merge queue, and the summary says so. +""" + +import argparse +import json +import os +import sys + +LIGHT = ("host-tests", "self-test") +RANGE = ("commits", "compile", "devicetree") +FULL = ("package", "kunit", "boot") +RANGE_EVENTS = {"pull_request", "merge_group"} + + +def evaluate(needs, event_name): + """Return (ok, rows) where rows are (job, result, expected, verdict).""" + plan = needs.get("plan", {}) + if plan.get("result") != "success": + return False, [("plan", plan.get("result", "missing"), "success", "failed")] + outputs = plan.get("outputs", {}) + tier = outputs.get("tier") + verified = bool(outputs.get("verified_run")) + + expected = {job: "success" for job in LIGHT} + for job in RANGE: + expected[job] = "success" if event_name in RANGE_EVENTS else "skipped" + for job in FULL: + if tier != "full": + expected[job] = "skipped" + elif job != "package" and verified: + expected[job] = "skipped" + else: + expected[job] = "success" + + rows, ok = [("plan", "success", "success", "ok")], True + for job, want in expected.items(): + got = needs.get(job, {}).get("result", "missing") + good = got == want + ok &= good + rows.append((job, got, want, "ok" if good else "failed")) + return ok, rows + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--needs", default=os.environ.get("NEEDS_JSON"), + help="the workflow's toJSON(needs)") + parser.add_argument("--event-name", default=os.environ.get("GITHUB_EVENT_NAME")) + args = parser.parse_args(argv) + if not args.needs or not args.event_name: + parser.error("--needs and --event-name (or NEEDS_JSON and GITHUB_EVENT_NAME) are required") + + needs = json.loads(args.needs) + ok, rows = evaluate(needs, args.event_name) + outputs = needs.get("plan", {}).get("outputs", {}) + lines = [f"### Kernel CI: {'passed' if ok else 'FAILED'}", "", + f"{outputs.get('tier', '?')} run: {outputs.get('reason', '')}.", "", + "| Job | Result | Expected | |", "|---|---|---|---|"] + lines += [f"| {job} | {got} | {want} | {verdict} |" for job, got, want, verdict in rows] + if outputs.get("tier") == "light" and ok: + lines += ["", "The kernel was not built in this run; the merge queue builds and boots it."] + if outputs.get("verified_run"): + lines += ["", f"KUnit and the boot test passed for this build key in run {outputs['verified_run']}."] + text = "\n".join(lines) + "\n" + print(text) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(text) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a") as out: + out.write(f"ok={'true' if ok else 'false'}\n") + for job, got, want, verdict in rows: + if verdict != "ok": + print(f"::error title=Kernel CI::{job} {got}, expected {want}", file=sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/retired-dtbs.txt b/tools/aurora-ci/retired-dtbs.txt new file mode 100644 index 0000000000000..e3493f4b72498 --- /dev/null +++ b/tools/aurora-ci/retired-dtbs.txt @@ -0,0 +1,8 @@ +# Device trees the kernel no longer builds, on purpose. +# +# compare-packages.py fails a change that stops building a DTB the target +# branch ships: the boot bundle picks a Mac's DTB by name, so that Mac would +# not boot the new kernel. A deliberate retirement goes here, in the change +# that retires it, one per line: the file name, then the reason. +# +# t8103-j999.dtb merged into t8103-j998.dtb; m1n1 maps j999 to j998 diff --git a/tools/aurora-ci/run-kunit.py b/tools/aurora-ci/run-kunit.py new file mode 100755 index 0000000000000..cb755ab823bf7 --- /dev/null +++ b/tools/aurora-ci/run-kunit.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Build the Apple KUnit suites into one arm64 kernel, run it, check the results. + +aurora.kunitconfig selects every KUnit suite for this tree's Apple, Asahi and +Aurora code (drivers, their Rust code, and the kernel crate they use), with +the 16K pages and 48-bit VA the shipped kernel has. tools/testing/kunit/ +kunit.py builds it with LLVM and boots it in QEMU; it uses KVM when /dev/kvm +works and TCG otherwise. + +kunit.py's own verdict is not used: it fails on any failed case, expected or +not, and does not see kernel warnings inside cases that still pass. This +script compares every non-passing case and every splat with +kunit-expected.json instead, and fails on anything new, changed or missing. + + --install-toolchain (Ubuntu) LLVM 18 and QEMU from apt, Rust 1.85.0 with + rust-src and rustfmt, bindgen-cli 0.71.1 from crates.io +""" + +import argparse +import json +import os +import platform +import re +import shutil +import subprocess +import sys +from collections import Counter +from pathlib import Path + +HERE = Path(__file__).resolve().parent +KUNITCONFIG = HERE / "aurora.kunitconfig" +EXPECTED = HERE / "kunit-expected.json" +RUST = "1.85.0" +BINDGEN = "0.71.1" +LLVM_DIR = Path("/usr/lib/llvm-18") +APT = ["bc", "bison", "build-essential", "clang-18", "flex", "libclang-18-dev", "libelf-dev", + "libssl-dev", "lld-18", "llvm-18", "python3", "qemu-system-arm"] + + +def bindgen_root(): + return Path(os.environ.get("RUNNER_TEMP", Path.home() / ".cache")) / f"bindgen-{BINDGEN}" + + +def install_toolchain(): + subprocess.run(["sudo", "apt-get", "update", "-q"], check=True) + subprocess.run(["sudo", "apt-get", "install", "-y", "-q", "--no-install-recommends", *APT], check=True) + subprocess.run(["rustup", "toolchain", "install", RUST, "--profile", "minimal", + "--component", "rust-src", "--component", "rustfmt"], check=True) + subprocess.run(["cargo", f"+{RUST}", "install", "--locked", "bindgen-cli", "--version", BINDGEN, + "--root", str(bindgen_root())], check=True) + + +def kernel_env(): + env = dict(os.environ, RUSTUP_TOOLCHAIN=RUST, LIBCLANG_PATH=str(LLVM_DIR / "lib")) + # RUST_LOG makes bindgen print an extra version line, which the kernel's + # Rust availability check then misreads as a libclang mismatch. + env.pop("RUST_LOG", None) + return env + + +def make_options(): + return [f"LLVM={LLVM_DIR}/bin/", f"BINDGEN={bindgen_root() / 'bin/bindgen'}"] + + +# --- result checking ------------------------------------------------------- + +TIME = re.compile(r"^\[\s*\d+\.\d+\]\s?") +SUBTEST = re.compile(r"^(\s*)# Subtest: (\S+)") +RESULT = re.compile(r"^(\s*)(?:not )?ok \d+ (\S+)") +SPLATS = [ + (re.compile(r"^WARNING: (?:CPU: \d+ PID: \d+ at )?(\S+?)(?::\d+)? (?:at )?([A-Za-z0-9_.]+)\+"), + lambda m: f"{m.group(1)}:{m.group(2)}"), + (re.compile(r"^(kernel BUG at \S+?)(?::\d+)?!"), lambda m: m.group(1)), + (re.compile(r"^(BUG: [^\n]*?)(?: in | at |$)"), lambda m: m.group(1).strip()), + (re.compile(r"^(Unable to handle kernel [a-zA-Z ]+)"), lambda m: m.group(1).strip()), + (re.compile(r"^(Internal error: [^:]+)"), lambda m: m.group(1)), + (re.compile(r"^(UBSAN: [a-z-]+)"), lambda m: m.group(1)), + (re.compile(r"^(KASAN: [a-z-]+)"), lambda m: m.group(1)), + (re.compile(r"^(INFO: task \S+ blocked)"), lambda m: "INFO: task blocked"), +] + + +def flatten(group, path=(), results=None, suites=None): + """kunit.py --json tree -> {suite.test[.param]: status}, {suite: Counter}.""" + results = {} if results is None else results + suites = {} if suites is None else suites + for sub in group.get("sub_groups", []): + if not path: + suites.setdefault(sub["name"], Counter()) + flatten(sub, path + (sub["name"],), results, suites) + for case in group.get("test_cases", []): + results[".".join(path + (case["name"],))] = case["status"] + if path: + suites.setdefault(path[0], Counter())[case["status"]] += 1 + return results, suites + + +BOOT = "(during boot, before the first suite)" + + +def splats_by_test(log_text): + """Kernel splats in kunit.py's test.log, attributed to the case whose + result line follows them, or to BOOT when no suite has started yet: + {test id: Counter(call site)}.""" + found, pending, stack = {}, [], [] + for raw in log_text.splitlines(): + line = TIME.sub("", raw) + match = SUBTEST.match(line) + if match: + depth = len(match.group(1)) // 4 + del stack[depth:] + stack.extend([""] * (depth - len(stack))) + stack.append(match.group(2)) + continue + match = RESULT.match(line) + if match: + depth = len(match.group(1)) // 4 + if pending: + test = ".".join(stack[1:depth + 1] + [match.group(2)]) if depth else match.group(2) + found.setdefault(test, Counter()).update(pending) + pending = [] + continue + for pattern, site in SPLATS: + match = pattern.match(line) + if match: + if stack: + pending.append(site(match)) + else: + found.setdefault(BOOT, Counter())[site(match)] += 1 + break + if pending: + found.setdefault("(after the last result)", Counter()).update(pending) + return found + + +def compare(results, splats, expected): + """Every difference from kunit-expected.json, as text.""" + want, problems = expected["results"], [] + for test, status in sorted(results.items()): + entry = want.get(test) + if status == "PASS" and entry: + problems.append(f"now passes (expected {entry['status']}): {test}") + elif status != "PASS" and not entry: + problems.append(f"{status}: {test}") + elif status != "PASS" and entry["status"] != status: + problems.append(f"{status} (expected {entry['status']}): {test}") + problems += [f"did not run (expected {want[t]['status']}): {t}" for t in sorted(want) if t not in results] + warned = {t: {e["site"]: e["count"] for e in entries} for t, entries in expected.get("warnings", {}).items()} + for test in sorted(set(splats) | set(warned)): + seen, known = splats.get(test, Counter()), warned.get(test, {}) + for site in sorted(set(seen) | set(known)): + if seen.get(site, 0) != known.get(site, 0): + problems.append(f"kernel splat {site} in {test}: {seen.get(site, 0)} seen, " + f"{known.get(site, 0)} expected") + return problems + + +def summary(results, suites, splats, problems, expected): + totals = Counter() + for counts in suites.values(): + totals.update(counts) + lines = ["### KUnit (Apple suites)", "", + f"{len(suites)} suites, {len(results)} cases: {totals['PASS']} pass, {totals['FAIL']} fail, " + f"{totals['SKIP']} skip, {totals['ERROR']} error; " + f"{sum(sum(c.values()) for c in splats.values())} kernel splats inside cases.", ""] + if problems: + lines += ["**Differences from kunit-expected.json**", ""] + [f"- {p}" for p in problems] + [""] + known = [t for t, s in sorted(results.items()) if s != "PASS" and t in expected["results"]] + if known: + lines += ["
Expected non-passing cases", ""] + lines += [f"- `{t}` {results[t]}: {expected['results'][t]['reason']}" for t in known] + ["", "
", ""] + lines += ["
Per suite", "", "| Suite | Pass | Fail | Skip | Error |", + "|---|---:|---:|---:|---:|"] + lines += [f"| {name} | {c['PASS']} | {c['FAIL']} | {c['SKIP']} | {c['ERROR']} |" for name, c in sorted(suites.items())] + return "\n".join(lines + ["", "
", ""]) + "\n" + + +def check(results_json, log_path, expected): + tree = json.loads(Path(results_json).read_text()) + results, suites = flatten(tree) + if not results: + return results, suites, {}, ["no test results: the kernel did not build, boot or print KTAP"] + log_text = Path(log_path).read_text(errors="replace") if Path(log_path).exists() else "" + splats = splats_by_test(log_text) + return results, suites, splats, compare(results, splats, expected) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--install-toolchain", action="store_true", help="install the toolchain (Ubuntu) and exit") + parser.add_argument("--output", type=Path, help="build directory; results and logs land here") + parser.add_argument("--check-only", action="store_true", + help="only compare an earlier run's kunit-results.json and test.log in --output") + parser.add_argument("--jobs", type=int, default=os.cpu_count() or 1) + args = parser.parse_args(argv) + + if args.install_toolchain: + install_toolchain() + return 0 + if not args.output: + parser.error("--output is required") + args.output.mkdir(parents=True, exist_ok=True) + results_json, log = args.output / "kunit-results.json", args.output / "test.log" + + if not args.check_only: + env = kernel_env() + options = make_options() + tree = HERE.parent.parent + subprocess.run(["make", "ARCH=arm64", f"O={args.output}", *options, "rustavailable"], + cwd=tree, env=env, check=True) + command = [sys.executable, "tools/testing/kunit/kunit.py", "run", "--arch=arm64", + f"--build_dir={args.output}", f"--kunitconfig={KUNITCONFIG}", f"--jobs={args.jobs}", + "--timeout=1800", f"--json={results_json}", f"--junit={args.output / 'kunit-results.xml'}"] + for option in options: + command += ["--make_options", option] + if not (platform.machine() in ("aarch64", "arm64") and os.access("/dev/kvm", os.R_OK | os.W_OK)): + # kunit.py's arm64 default is -cpu max; under TCG a fixed model + # behaves the same on every QEMU (see boot-test.py, cpu_model). + command.append("--qemu_args=-cpu neoverse-n1") + results_json.unlink(missing_ok=True) + status = subprocess.run(command, cwd=tree, env=env).returncode + print(f"kunit.py exited with status {status}; checking against {EXPECTED.name}") + if not results_json.exists(): + print("::error title=KUnit::kunit.py wrote no results; see the build or boot output above", + file=sys.stderr) + return 1 + + expected = json.loads(EXPECTED.read_text()) + results, suites, splats, problems = check(results_json, log, expected) + text = summary(results, suites, splats, problems, expected) + print(text, end="") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as out: + out.write(text) + for problem in problems[:60]: + print(f"::error title=KUnit::{problem}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aurora-ci/tests/support.py b/tools/aurora-ci/tests/support.py new file mode 100644 index 0000000000000..7fad3a6b0c341 --- /dev/null +++ b/tools/aurora-ci/tests/support.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: GPL-2.0 +"""Helpers shared by the CI tools' unit tests.""" + +import contextlib +import importlib.util +import io +import json +import os +import subprocess +import tempfile +import unittest +import unittest.mock +from pathlib import Path + +TOOLS = Path(__file__).resolve().parent.parent +TREE = TOOLS.parent.parent + + +def load(script): + """Import tools/aurora-ci/