diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml index f9bd867971..1ebf41901b 100644 --- a/.github/workflows/_container.yml +++ b/.github/workflows/_container.yml @@ -105,7 +105,9 @@ jobs: type=ref,event=tag - name: package chart and push it + env: + REPO: ${{ github.repository_owner }} run: | helm dependencies update helm/blueapi helm package helm/blueapi --version ${GITHUB_REF##*/} --app-version ${GITHUB_REF##*/} -d /tmp/ - helm push /tmp/blueapi-${GITHUB_REF##*/}.tgz oci://ghcr.io/diamondlightsource/charts + helm push /tmp/blueapi-${GITHUB_REF##*/}.tgz oci://ghcr.io/${REPO@L}/charts diff --git a/.github/workflows/_debug_container.yml b/.github/workflows/_debug_container.yml deleted file mode 100644 index 31206c1c4c..0000000000 --- a/.github/workflows/_debug_container.yml +++ /dev/null @@ -1,49 +0,0 @@ -on: - workflow_call: - inputs: - publish: - type: boolean - description: If true, pushes image to container registry - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # Need this to get version number from last tag - fetch-depth: 0 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Docker Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create tags for publishing debug image - id: debug-meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository }} - tags: | - type=ref,event=tag,suffix=-debug - type=raw,value=latest-debug - - - name: Build and publish debug image to container registry - if: github.ref_type == 'tag' - uses: docker/build-push-action@v6 - env: - DOCKER_BUILD_RECORD_UPLOAD: false - with: - context: . - push: true - target: debug - tags: ${{ steps.debug-meta.outputs.tags }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9c4b99a0d..f9ebc6eeff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,15 +45,6 @@ jobs: contents: read packages: write - debug_container: - needs: [container, test] - uses: ./.github/workflows/_debug_container.yml - with: - publish: ${{ needs.test.result == 'success' }} - permissions: - contents: read - packages: write - docs: uses: ./.github/workflows/_docs.yml diff --git a/Dockerfile b/Dockerfile index 7bce161029..fb592f6c14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,51 +27,27 @@ RUN mkdir -p /.cache/pip; chmod o+wrX /.cache/pip # Requires buildkit 0.17.0 COPY --chmod=o+wrX . /workspaces/blueapi WORKDIR /workspaces/blueapi -RUN touch dev-requirements.txt && pip install --upgrade pip && pip install -c dev-requirements.txt . - - -FROM build AS debug - - -# Set origin to use ssh -RUN git remote set-url origin git@github.com:DiamondLightSource/blueapi.git - - -# For this pod to understand finding user information from LDAP -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive apt install libnss-ldapd -y -RUN sed -i 's/files/ldap files/g' /etc/nsswitch.conf - -# Make editable and debuggable -RUN pip install debugpy -RUN pip install -e . - -RUN groupadd -g 1000 blueapi && \ - useradd -m -u 1000 -g blueapi blueapi - -# Switch to the custom user -USER blueapi - -# Alternate entrypoint to allow devcontainer to attach -ENTRYPOINT [ "/bin/bash", "-c", "--" ] -CMD [ "while true; do sleep 30; done;" ] - +RUN touch dev-requirements.txt && pip install --upgrade pip && pip install debugpy && pip install -c dev-requirements.txt . # The runtime stage copies the built venv into a slim runtime container FROM python:${PYTHON_VERSION}-slim AS runtime # Add apt-get system dependecies for runtime here if needed -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y --no-install-recommends \ # Git required for installing packages at runtime git \ + # gdb required for attaching debugger + gdb \ + # May be required if attaching devcontainer + libnss-ldapd \ && rm -rf /var/lib/apt/lists/* + COPY --from=build --chmod=o+wrX /venv/ /venv/ COPY --from=build --chmod=o+wrX /.cache/pip /.cache/pip + ENV PATH=/venv/bin:$PATH ENV PYTHONPYCACHEPREFIX=/tmp/blueapi_pycache # For this pod to understand finding user information from LDAP -RUN apt update -RUN DEBIAN_FRONTEND=noninteractive apt install libnss-ldapd -y RUN sed -i 's/files/ldap files/g' /etc/nsswitch.conf # Set the MPLCONFIGDIR environment variable to a temporary directory to avoid @@ -81,9 +57,14 @@ RUN sed -i 's/files/ldap files/g' /etc/nsswitch.conf ENV MPLCONFIGDIR=/tmp/matplotlib +# Make a path to site-packages that is invariant with python version +# This allows our pathMapping in launch.jsons to always find build blueapi +WORKDIR /venv/lib +RUN ln -s python python${PYTHON_VERSION} + RUN groupadd -g 1000 blueapi && \ useradd -m -u 1000 -g blueapi blueapi - + # Switch to the custom user USER blueapi diff --git a/helm/blueapi/README.md b/helm/blueapi/README.md index 6a76fdae81..227d264a51 100644 --- a/helm/blueapi/README.md +++ b/helm/blueapi/README.md @@ -9,7 +9,9 @@ A Helm chart deploying a worker pod that runs Bluesky plans | Key | Type | Default | Description | |-----|------|---------|-------------| | affinity | object | `{}` | May be required to run on specific nodes (e.g. the control machine) | -| debug.enabled | bool | `false` | If enabled, disables liveness and readiness probes, and does not start the service on startup This allows connecting to the pod and starting the service manually to allow debugging on the cluster | +| debug.enabled | bool | `false` | If enabled, runs debugpy, allowing port-forwarding to expose port 5678 or attached vscode instance | +| debug.log_to_stderr | bool | `false` | If enabled configures debugpy to use the option `--log-to-stderr` | +| debug.suspend | bool | `false` | If enabled does not start the service on startup This allows connecting to the pod and starting the service manually to allow debugging on the cluster | | extraEnvVars | list | `[]` | Additional envVars to mount to the pod | | fullnameOverride | string | `""` | | | global | object | `{}` | Not used, but must be present for validation when using as a dependency of another chart | diff --git a/helm/blueapi/templates/statefulset.yaml b/helm/blueapi/templates/statefulset.yaml index 9f48c22035..37db3f5a96 100644 --- a/helm/blueapi/templates/statefulset.yaml +++ b/helm/blueapi/templates/statefulset.yaml @@ -72,7 +72,7 @@ spec: type: Directory {{- end }} {{- end }} - {{- if or .Values.debug.enabled (and .Values.initContainer.enabled .Values.initContainer.persistentVolume.enabled)}} + {{- if ne 1000.0 .Values.securityContext.runAsUser }} - name: home # Required for vscode to install plugins emptyDir: sizeLimit: 500Mi @@ -83,7 +83,7 @@ spec: {{- if .Values.initContainer.enabled }} initContainers: - name: setup-scratch - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}{{ ternary "-debug" "" .Values.debug.enabled }}" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} resources: {{- .Values.initResources | default .Values.resources | toYaml | nindent 12 }} @@ -116,7 +116,7 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}{{ ternary "-debug" "" .Values.debug.enabled }}" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: http @@ -151,17 +151,35 @@ spec: - name: venv mountPath: /venv {{- end }} - {{- if or .Values.debug.enabled (and .Values.initContainer.enabled .Values.initContainer.persistentVolume.enabled) }} + {{- if ne 1000.0 .Values.securityContext.runAsUser }} - mountPath: /home name: home - mountPath: /var/run/nslcd name: nslcd {{- end }} - {{- if not .Values.debug.enabled }} args: - "-c" - "/config/config.yaml" - "serve" + {{- if .Values.debug.enabled }} + command: + - "python" + - "-Xfrozen_modules=off" + - "-m" + - "debugpy" + - "--listen" + - "5678" + {{- if .Values.debug.log_to_stderr }} + - "--log-to-stderr" + {{- end }} + {{- if .Values.debug.suspend }} + - "--wait-for-client" + {{- end }} + - "--configure-subProcess" + - "true" + - "-m" + - "blueapi" + {{- end }} {{- with .Values.livenessProbe }} livenessProbe: {{- toYaml . | nindent 12 }} @@ -174,13 +192,12 @@ spec: startupProbe: {{- toYaml . | nindent 12 }} {{- end }} - {{- end }} envFrom: - configMapRef: name: {{ include "blueapi.fullname" . }}-otel-config env: {{- toYaml .Values.extraEnvVars | nindent 12 }} - {{- if or .Values.debug.enabled (and .Values.initContainer.persistentVolume.enabled .Values.initContainer.enabled )}} + {{- if ne 1000.0 .Values.securityContext.runAsUser }} - name: debug-account-sync image: ghcr.io/diamondlightsource/account-sync-sidecar:3.0.0 volumeMounts: diff --git a/helm/blueapi/values.schema.json b/helm/blueapi/values.schema.json index 250dc38f10..5747d10c0e 100644 --- a/helm/blueapi/values.schema.json +++ b/helm/blueapi/values.schema.json @@ -12,7 +12,15 @@ "type": "object", "properties": { "enabled": { - "description": "If enabled, disables liveness and readiness probes, and does not start the service on startup This allows connecting to the pod and starting the service manually to allow debugging on the cluster", + "description": "If enabled, runs debugpy, allowing port-forwarding to expose port 5678 or attached vscode instance", + "type": "boolean" + }, + "log_to_stderr": { + "description": "If enabled configures debugpy to use the option `--log-to-stderr`", + "type": "boolean" + }, + "suspend": { + "description": "If enabled does not start the service on startup This allows connecting to the pod and starting the service manually to allow debugging on the cluster", "type": "boolean" } } diff --git a/helm/blueapi/values.yaml b/helm/blueapi/values.yaml index c8eed552d9..9376bc524a 100644 --- a/helm/blueapi/values.yaml +++ b/helm/blueapi/values.yaml @@ -223,9 +223,13 @@ initContainer: existingClaimName: "" debug: - # -- If enabled, disables liveness and readiness probes, and does not start the service on startup - # This allows connecting to the pod and starting the service manually to allow debugging on the cluster + # -- If enabled, runs debugpy, allowing port-forwarding to expose port 5678 or attached vscode instance enabled: false + # -- If enabled does not start the service on startup + # This allows connecting to the pod and starting the service manually to allow debugging on the cluster + suspend: false + # -- If enabled configures debugpy to use the option `--log-to-stderr` + log_to_stderr: false # -- Not used, but must be present for validation when using as a dependency of another chart global: {} diff --git a/tests/unit_tests/test_helm_chart.py b/tests/unit_tests/test_helm_chart.py index 0a6e60aff3..631aed5470 100644 --- a/tests/unit_tests/test_helm_chart.py +++ b/tests/unit_tests/test_helm_chart.py @@ -391,10 +391,11 @@ def test_fluentd_ignore_false_when_graylog_disabled(): def render_persistent_volume_chart( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + init_container_enabled: bool = False, + persistent_volume_enabled: bool = False, + existing_claim_name: str | None = None, + debug_enabled: bool = False, + debug_suspend: bool = False, ): """Generated chart for this section of Values: ``` @@ -406,6 +407,7 @@ def render_persistent_volume_chart( debug: enabled: false + suspend: false ``` """ return render_chart( @@ -421,7 +423,7 @@ def render_persistent_volume_chart( else {} ), }, - "debug": {"enabled": debug_enabled}, + "debug": {"enabled": debug_enabled, "suspend": debug_suspend}, } ) @@ -504,9 +506,9 @@ def test_init_container_exists_conditions(init_container_enabled): @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_init_container_scratch_mount( - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, scratch_volume_mount, scratch_host_volume_mount, ): @@ -533,9 +535,9 @@ def test_init_container_scratch_mount( @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_init_container_init_config_mount( - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, init_config_volume_mount, ): manifests = render_persistent_volume_chart( @@ -556,9 +558,9 @@ def test_init_container_init_config_mount( @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_init_container_venv_volume_mount( - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, init_container_venv_volume_mount, ): manifests = render_persistent_volume_chart( @@ -580,10 +582,10 @@ def test_init_container_venv_volume_mount( @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_persistent_volume_claim_exists( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + init_container_enabled: bool, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, ): manifests = render_persistent_volume_chart( init_container_enabled, @@ -613,24 +615,11 @@ def test_persistent_volume_claim_exists( assert "PersistentVolumeClaim" not in manifests -@pytest.mark.parametrize("init_container_enabled", [True, False]) -@pytest.mark.parametrize("persistent_volume_enabled", [True, False]) -@pytest.mark.parametrize("existing_claim_name", [None, "foo"]) -@pytest.mark.parametrize("debug_enabled", [True, False]) -def test_debug_account_sync_exists( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, -): - manifests = render_persistent_volume_chart( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, - ) +@pytest.mark.parametrize("run_as_user", [0, 1, 1000, 1001, None]) +def test_ldap_account_sync_exists_for_non_default_user(run_as_user: int | None): + manifests = render_chart(values={"securityContext": {"runAsUser": run_as_user}}) - if debug_enabled or (init_container_enabled and persistent_volume_enabled): + if run_as_user != 1000: assert { "name": "debug-account-sync", "image": ANY, @@ -650,51 +639,15 @@ def test_debug_account_sync_exists( ) -@pytest.mark.parametrize("init_container_enabled", [True, False]) -@pytest.mark.parametrize("persistent_volume_enabled", [True, False]) -@pytest.mark.parametrize("existing_claim_name", [None, "foo"]) -@pytest.mark.parametrize("debug_enabled", [True, False]) -def test_container_image_has_debug_suffix( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, -): - manifests = render_persistent_volume_chart( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, - ) - - if debug_enabled: - image = manifests["StatefulSet"]["blueapi"]["spec"]["template"]["spec"][ - "containers" - ][0]["image"] - assert str(image).endswith("-debug") - - else: - image = manifests["StatefulSet"]["blueapi"]["spec"]["template"]["spec"][ - "containers" - ][0]["image"] - assert not str(image).endswith("-debug") - - if init_container_enabled: - init_image = manifests["StatefulSet"]["blueapi"]["spec"]["template"]["spec"][ - "initContainers" - ][0]["image"] - assert init_image == image - - @pytest.mark.parametrize("init_container_enabled", [True, False]) @pytest.mark.parametrize("persistent_volume_enabled", [True, False]) @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_main_container_scratch_mount( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + init_container_enabled: bool, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, scratch_volume_mount, scratch_host_volume_mount, ): @@ -725,10 +678,10 @@ def test_main_container_scratch_mount( @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_main_container_venv_volume_mount( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + init_container_enabled: bool, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, venv_volume_mount, ): manifests = render_persistent_volume_chart( @@ -748,30 +701,19 @@ def test_main_container_venv_volume_mount( assert venv_volume_mount not in volume_mounts -@pytest.mark.parametrize("init_container_enabled", [True, False]) -@pytest.mark.parametrize("persistent_volume_enabled", [True, False]) -@pytest.mark.parametrize("existing_claim_name", [None, "foo"]) -@pytest.mark.parametrize("debug_enabled", [True, False]) +@pytest.mark.parametrize("run_as_user", [0, 1, 1000, 1001, None]) def test_main_container_home_and_nslcd_volume_mounts( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + run_as_user: int | None, home_volume_mount, nslcd_volume_mount, ): - manifests = render_persistent_volume_chart( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, - ) + manifests = render_chart(values={"securityContext": {"runAsUser": run_as_user}}) volume_mounts = manifests["StatefulSet"]["blueapi"]["spec"]["template"]["spec"][ "containers" ][0]["volumeMounts"] - if debug_enabled or (init_container_enabled and persistent_volume_enabled): + if run_as_user != 1000: assert home_volume_mount in volume_mounts assert nslcd_volume_mount in volume_mounts else: @@ -779,47 +721,54 @@ def test_main_container_home_and_nslcd_volume_mounts( assert nslcd_volume_mount not in volume_mounts -@pytest.mark.parametrize("init_container_enabled", [True, False]) -@pytest.mark.parametrize("persistent_volume_enabled", [True, False]) -@pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) +@pytest.mark.parametrize("debug_suspend", [True, False]) def test_main_container_args( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, - home_volume_mount, - nslcd_volume_mount, + debug_enabled: bool, + debug_suspend: bool, ): manifests = render_persistent_volume_chart( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + debug_enabled=debug_enabled, + debug_suspend=debug_suspend, ) - if not debug_enabled: - assert manifests["StatefulSet"]["blueapi"]["spec"]["template"]["spec"][ - "containers" - ][0]["args"] == [ - "-c", - "/config/config.yaml", - "serve", + main_container = manifests["StatefulSet"]["blueapi"]["spec"]["template"]["spec"][ + "containers" + ][0] + + assert main_container["args"] == [ + "-c", + "/config/config.yaml", + "serve", + ] + + if debug_enabled: + expected_command = [ + "python", + "-Xfrozen_modules=off", + "-m", + "debugpy", + "--listen", + "5678", + "--configure-subProcess", + "true", + "-m", + "blueapi", ] + + if debug_suspend: + expected_command.insert(6, "--wait-for-client") + + assert main_container["command"] == expected_command else: - assert ( - "args" - not in manifests["StatefulSet"]["blueapi"]["spec"]["template"]["spec"][ - "containers" - ][0] - ) + assert "command" not in main_container @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_scratch_volume_uses_correct_name( - existing_claim_name, - debug_enabled, + existing_claim_name: str | None, + debug_enabled: bool, ): manifests = render_persistent_volume_chart( True, @@ -886,10 +835,10 @@ def nslcd_volume(): @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_worker_config_volume_declared( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + init_container_enabled: bool, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, worker_config_volume, ): manifests = render_persistent_volume_chart( @@ -910,10 +859,10 @@ def test_worker_config_volume_declared( @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_init_config_and_venv_volumes_declared( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + init_container_enabled: bool, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, init_config_volume, venv_volume, ): @@ -958,10 +907,10 @@ def test_init_config_and_venv_volumes_declared( @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_scratch_volume_declared( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + init_container_enabled: bool, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, scratch_volume, ): manifests = render_persistent_volume_chart( @@ -992,10 +941,10 @@ def test_scratch_volume_declared( @pytest.mark.parametrize("existing_claim_name", [None, "foo"]) @pytest.mark.parametrize("debug_enabled", [True, False]) def test_scratch_host_volume_declared( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + init_container_enabled: bool, + persistent_volume_enabled: bool, + existing_claim_name: str | None, + debug_enabled: bool, scratch_host_volume, ): manifests = render_persistent_volume_chart( @@ -1021,26 +970,15 @@ def test_scratch_host_volume_declared( ) -@pytest.mark.parametrize("init_container_enabled", [True, False]) -@pytest.mark.parametrize("persistent_volume_enabled", [True, False]) -@pytest.mark.parametrize("existing_claim_name", [None, "foo"]) -@pytest.mark.parametrize("debug_enabled", [True, False]) +@pytest.mark.parametrize("run_as_user", [0, 1, 1000, 1001, None]) def test_home_and_nslcd_volumes_declared( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, + run_as_user: int | None, home_volume, nslcd_volume, ): - manifests = render_persistent_volume_chart( - init_container_enabled, - persistent_volume_enabled, - existing_claim_name, - debug_enabled, - ) + manifests = render_chart(values={"securityContext": {"runAsUser": run_as_user}}) - if debug_enabled or (init_container_enabled and persistent_volume_enabled): + if run_as_user != 1000: assert ( home_volume in manifests["StatefulSet"]["blueapi"]["spec"]["template"]["spec"][