diff --git a/charms/kserve-llmisvc/src/charm.py b/charms/kserve-llmisvc/src/charm.py index 66d37358..b5f52b95 100644 --- a/charms/kserve-llmisvc/src/charm.py +++ b/charms/kserve-llmisvc/src/charm.py @@ -56,6 +56,7 @@ # Readiness relation consumed by the llm-integrator charm. This charm is the # PROVIDER: it publishes ready=true only once it reaches ActiveStatus. LLM_INTEGRATOR_SYNC_RELATION = "kserve-llmisvc" +LOGGING_RELATION = "logging" METRICS_PORT = 8080 METRICS_PROXY_CONTAINER = "metrics-proxy" METRICS_PROXY_PORT = 15090 @@ -146,6 +147,10 @@ def __init__(self, *args): self.on[LWS_SYNC_RELATION].relation_changed, self.on[LWS_SYNC_RELATION].relation_broken, self.on[LLM_INTEGRATOR_SYNC_RELATION].relation_changed, + self.on[LOGGING_RELATION].relation_joined, + self.on[LOGGING_RELATION].relation_changed, + self.on[LOGGING_RELATION].relation_departed, + self.on[LOGGING_RELATION].relation_broken, ]: self.framework.observe(event, self._on_event) self.framework.observe(self.on.remove, self._on_remove) @@ -185,8 +190,26 @@ def _scheduler_context(self): "scheduler_image": self.images_context.get("llm_scheduler", ""), "workload_image": self.images_context.get("llm_workload", ""), "llm_routing_sidecar": self.images_context.get("llm_routing_sidecar", ""), + "loki_url": self._loki_url, } + @property + def _loki_url(self) -> str: + """Return a Loki Push API endpoint published through the logging relation.""" + for relation in self.model.relations.get(LOGGING_RELATION, []): + for unit in sorted(relation.units, key=lambda remote_unit: remote_unit.name): + endpoint = relation.data[unit].get("endpoint") + if not endpoint: + continue + try: + url = json.loads(endpoint).get("url") + except (json.JSONDecodeError, TypeError): + log.warning("Ignoring malformed Loki endpoint from %s", unit.name) + continue + if isinstance(url, str) and url: + return url + return "" + @property def base_resource_handler(self): """K8s handler for core llmisvc resources.""" @@ -364,6 +387,7 @@ def _on_event(self, event): self.custom_images = parse_images_config(self.model.config["custom_images"]) self.images_context = self.get_images(DEFAULT_IMAGES, self.custom_images) + self._scheduler_config_resource_handler = None self.unit.status = MaintenanceStatus("Creating k8s resources") diff --git a/charms/kserve-llmisvc/src/default-custom-images.json b/charms/kserve-llmisvc/src/default-custom-images.json index 7e59dacc..108cdf91 100644 --- a/charms/kserve-llmisvc/src/default-custom-images.json +++ b/charms/kserve-llmisvc/src/default-custom-images.json @@ -2,6 +2,6 @@ "llm_scheduler": "docker.io/charmedkubeflow/llm-d-inference-scheduler:0.4.0-66449f9-20260724073351", "llm_workload": "ghcr.io/llm-d/llm-d-cuda:v0.4.0", "llm_routing_sidecar": "docker.io/charmedkubeflow/llm-d-routing-sidecar:0.4.0-26788a2-20260717115339", - "vllm": "docker.io/charmedkubeflow/vllm-cpu:0.19.0-5f4a278-20260825092922", + "vllm": "ghcr.io/welpaolo/vllm-cpu:0.19.0", "vllm_gpu": "docker.io/charmedkubeflow/vllm-cuda-gpu:0.19.0-e511371-20260702095505" } diff --git a/charms/kserve-llmisvc/src/templates/llmisvc_configs_manifests.yaml.j2 b/charms/kserve-llmisvc/src/templates/llmisvc_configs_manifests.yaml.j2 index ae44769e..44531ca3 100644 --- a/charms/kserve-llmisvc/src/templates/llmisvc_configs_manifests.yaml.j2 +++ b/charms/kserve-llmisvc/src/templates/llmisvc_configs_manifests.yaml.j2 @@ -26,6 +26,10 @@ spec: value: INFO - name: HF_HUB_CACHE value: /models +{% endraw %}{% if loki_url %}{% raw %} + - name: LOKI_URL + value: {% endraw %}{{ loki_url }}{% raw %} +{% endraw %}{% endif %}{% raw %} image: {% endraw %}{{ workload_image }}{% raw %} imagePullPolicy: IfNotPresent livenessProbe: @@ -256,6 +260,10 @@ spec: value: INFO - name: HF_HUB_CACHE value: /models +{% endraw %}{% if loki_url %}{% raw %} + - name: LOKI_URL + value: {% endraw %}{{ loki_url }}{% raw %} +{% endraw %}{% endif %}{% raw %} image: {% endraw %}{{ workload_image }}{% raw %} imagePullPolicy: IfNotPresent livenessProbe: @@ -485,6 +493,10 @@ spec: value: /models - name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS value: "1" +{% endraw %}{% if loki_url %}{% raw %} + - name: LOKI_URL + value: {% endraw %}{{ loki_url }}{% raw %} +{% endraw %}{% endif %}{% raw %} image: {% endraw %}{{ workload_image }}{% raw %} imagePullPolicy: IfNotPresent name: main @@ -555,6 +567,10 @@ spec: value: INFO - name: HF_HUB_CACHE value: /models +{% endraw %}{% if loki_url %}{% raw %} + - name: LOKI_URL + value: {% endraw %}{{ loki_url }}{% raw %} +{% endraw %}{% endif %}{% raw %} image: {% endraw %}{{ workload_image }}{% raw %} imagePullPolicy: IfNotPresent livenessProbe: @@ -736,6 +752,10 @@ spec: value: INFO - name: HF_HUB_CACHE value: /models +{% endraw %}{% if loki_url %}{% raw %} + - name: LOKI_URL + value: {% endraw %}{{ loki_url }}{% raw %} +{% endraw %}{% endif %}{% raw %} image: {% endraw %}{{ workload_image }}{% raw %} imagePullPolicy: IfNotPresent livenessProbe: @@ -912,6 +932,10 @@ spec: value: INFO - name: HF_HUB_CACHE value: /models +{% endraw %}{% if loki_url %}{% raw %} + - name: LOKI_URL + value: {% endraw %}{{ loki_url }}{% raw %} +{% endraw %}{% endif %}{% raw %} image: {% endraw %}{{ workload_image }}{% raw %} imagePullPolicy: IfNotPresent name: main @@ -1159,6 +1183,10 @@ spec: value: INFO - name: HF_HUB_CACHE value: /models +{% endraw %}{% if loki_url %}{% raw %} + - name: LOKI_URL + value: {% endraw %}{{ loki_url }}{% raw %} +{% endraw %}{% endif %}{% raw %} image: {% endraw %}{{ workload_image }}{% raw %} imagePullPolicy: IfNotPresent livenessProbe: @@ -1339,6 +1367,10 @@ spec: value: INFO - name: HF_HUB_CACHE value: /models +{% endraw %}{% if loki_url %}{% raw %} + - name: LOKI_URL + value: {% endraw %}{{ loki_url }}{% raw %} +{% endraw %}{% endif %}{% raw %} image: {% endraw %}{{ workload_image }}{% raw %} imagePullPolicy: IfNotPresent livenessProbe: @@ -1566,6 +1598,10 @@ spec: value: INFO - name: HF_HUB_CACHE value: /models +{% endraw %}{% if loki_url %}{% raw %} + - name: LOKI_URL + value: {% endraw %}{{ loki_url }}{% raw %} +{% endraw %}{% endif %}{% raw %} image: {% endraw %}{{ workload_image }}{% raw %} imagePullPolicy: IfNotPresent name: main diff --git a/charms/kserve-llmisvc/tests/unit/test_loki.py b/charms/kserve-llmisvc/tests/unit/test_loki.py new file mode 100644 index 00000000..2cd1489e --- /dev/null +++ b/charms/kserve-llmisvc/tests/unit/test_loki.py @@ -0,0 +1,107 @@ +# Copyright 2026 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Tests for forwarding vLLM workload logs to Loki.""" + +import json +from pathlib import Path +from unittest.mock import patch + +from charmed_kubeflow_chisme.kubernetes import KubernetesResourceHandler +from jinja2 import Template +from ops.model import ActiveStatus +from ops.testing import Relation, State + +LOKI_URL = "http://loki-0.loki-endpoints:3100/loki/api/v1/push" +TEMPLATE_PATH = Path(__file__).parents[2] / "src/templates/llmisvc_configs_manifests.yaml.j2" + + +def _logging_relation(endpoint: str | None = None) -> Relation: + unit_data = {"endpoint": endpoint} if endpoint else {} + return Relation( + endpoint="logging", + interface="loki_push_api", + remote_app_name="loki", + remote_units_data={0: unit_data}, + ) + + +def _render_template(loki_url: str = "") -> str: + return Template(TEMPLATE_PATH.read_text()).render( + namespace="test-model", + scheduler_image="scheduler-image", + workload_image="workload-image", + llm_routing_sidecar="routing-sidecar-image", + loki_url=loki_url, + ) + + +def test_vllm_templates_render_loki_url(): + """Every vLLM workload template should receive the COS Loki endpoint.""" + rendered = _render_template(LOKI_URL) + print(rendered) + assert rendered.count("name: LOKI_URL") == 9 + assert rendered.count(f"value: {LOKI_URL}") == 9 + assert rendered.count("vllm\n - serve") == 2 + + +def test_vllm_templates_omit_loki_url_without_cos(): + """vLLM workloads should not configure log forwarding without a Loki endpoint.""" + assert "LOKI_URL" not in _render_template() + + +def test_logging_relation_passes_loki_url_to_scheduler_context( + ctx, both_containers, controller_relation_ready, lws_relation_ready +): + """A ready logging relation should refresh the scheduler render context.""" + logging_relation = _logging_relation(json.dumps({"url": LOKI_URL})) + state_in = State( + leader=True, + containers=both_containers, + relations=[controller_relation_ready, lws_relation_ready, logging_relation], + ) + original_init = KubernetesResourceHandler.__init__ + + with patch.object(KubernetesResourceHandler, "__init__", autospec=True) as init: + init.side_effect = original_init + out = ctx.run(ctx.on.relation_changed(logging_relation), state_in) + + assert isinstance(out.unit_status, ActiveStatus) + scheduler_contexts = [ + call.kwargs["context"] + for call in init.call_args_list + if call.kwargs.get("template_files") == ["src/templates/llmisvc_configs_manifests.yaml.j2"] + ] + assert scheduler_contexts == [ + { + "namespace": out.model.name, + "scheduler_image": scheduler_contexts[0]["scheduler_image"], + "workload_image": scheduler_contexts[0]["workload_image"], + "llm_routing_sidecar": scheduler_contexts[0]["llm_routing_sidecar"], + "loki_url": LOKI_URL, + } + ] + + +def test_malformed_logging_endpoint_omits_loki_url( + ctx, both_containers, controller_relation_ready, lws_relation_ready +): + """Malformed logging relation data should not configure a vLLM forwarder.""" + logging_relation = _logging_relation("not-json") + state_in = State( + leader=True, + containers=both_containers, + relations=[controller_relation_ready, lws_relation_ready, logging_relation], + ) + original_init = KubernetesResourceHandler.__init__ + + with patch.object(KubernetesResourceHandler, "__init__", autospec=True) as init: + init.side_effect = original_init + ctx.run(ctx.on.relation_changed(logging_relation), state_in) + + scheduler_context = next( + call.kwargs["context"] + for call in init.call_args_list + if call.kwargs.get("template_files") == ["src/templates/llmisvc_configs_manifests.yaml.j2"] + ) + assert scheduler_context["loki_url"] == "" diff --git a/tests/integration/helpers/cos.py b/tests/integration/helpers/cos.py index eec5e435..410be8c9 100644 --- a/tests/integration/helpers/cos.py +++ b/tests/integration/helpers/cos.py @@ -163,6 +163,30 @@ def _workload_pod_name(isvc_name: str, namespace: str = NAMESPACE_DEFAULT) -> st raise AssertionError(f"No running workload pod found for LLMInferenceService '{isvc_name}'") +def workload_environment_value( + isvc_name: str, + environment_name: str, + namespace: str = NAMESPACE_DEFAULT, + container: str = "main", +) -> str | None: + """Return a named environment value from a running vLLM workload container.""" + from lightkube.resources.core_v1 import Pod + + pod_name = _workload_pod_name(isvc_name, namespace) + pod = get_client().get(Pod, name=pod_name, namespace=namespace) + containers = pod.spec.containers or [] + workload_container = next( + (candidate for candidate in containers if candidate.name == container), None + ) + if workload_container is None: + raise AssertionError(f"Container '{container}' not found in workload pod '{pod_name}'") + + for environment in workload_container.env or []: + if environment.name == environment_name: + return environment.value + return None + + def generate_inference_traffic( isvc_name: str, model_name: str, diff --git a/tests/integration/test_observability.py b/tests/integration/test_observability.py index fc387def..3fa39cf0 100644 --- a/tests/integration/test_observability.py +++ b/tests/integration/test_observability.py @@ -33,6 +33,7 @@ published_loki_logs, published_prometheus_alerts, query_prometheus, + workload_environment_value, ) from .helpers.deploy import deploy_serving_stack from .helpers.llmisvc_ops import apply_llmisvc_example @@ -67,7 +68,7 @@ IMAGE_CONTEXT = { "storage_initializer_image": KSERVE_CONTROLLER_IMAGES["configmap__storageInitializer"], - "vllm_image": KSERVE_LLMISVC_IMAGES["vllm"], + "vllm_image": "ghcr.io/welpaolo/vllm-cpu:0.19.0", "model_s3_uri": MODEL_S3_URI, "aws_access_key_id": AWS_ACCESS_KEY_ID, "aws_secret_access_key": AWS_SECRET_ACCESS_KEY, @@ -108,7 +109,7 @@ def _integrate(juju: jubilant.Juju, provider: str, requirer: str) -> None: @pytest.mark.abort_on_fail -def test_deploy_stack_and_model(juju: jubilant.Juju, request: pytest.FixtureRequest): +def test_deploy_stack(juju: jubilant.Juju, request: pytest.FixtureRequest): charms_path = request.config.getoption("--charms-path") if not charms_path: raise ValueError("--charms-path is required for the observability integration test") @@ -117,9 +118,6 @@ def test_deploy_stack_and_model(juju: jubilant.Juju, request: pytest.FixtureRequ deploy_serving_stack(juju, charms_path) - logger.info("Applying CPU LLMInferenceService example") - apply_llmisvc_example(manifest_path=str(CPU_EXAMPLE), context=IMAGE_CONTEXT, name=LLMISVC_NAME) - @pytest.mark.abort_on_fail def test_relate_to_cos(juju: jubilant.Juju): @@ -146,6 +144,13 @@ def test_relate_to_cos(juju: jubilant.Juju): juju.wait(jubilant.all_active, successes=1) +@pytest.mark.abort_on_fail +def test_deploy_model_with_loki_forwarding(): + """Create the workload only after its Loki endpoint is available in config.""" + logger.info("Applying CPU LLMInferenceService example with Loki forwarding enabled") + apply_llmisvc_example(manifest_path=str(CPU_EXAMPLE), context=IMAGE_CONTEXT, name=LLMISVC_NAME) + + @pytest.mark.abort_on_fail def test_generate_traffic(): generate_inference_traffic( @@ -181,3 +186,14 @@ def test_cos_data_published(juju: jubilant.Juju): logger.info("Checking charm logs reached Loki...") logs = published_loki_logs(juju, "juju_application", LLMISVC_APP_NAME) assert logs and logs.get("data", {}).get("result"), "no kserve-llmisvc logs in Loki" + + logger.info("Checking vLLM workload received the Loki endpoint...") + loki_url = workload_environment_value(LLMISVC_NAME, "LOKI_URL") + assert loki_url and loki_url.endswith("/loki/api/v1/push") + + logger.info("Checking vLLM workload logs reached Loki...") + workload_logs = published_loki_logs(juju, "app", "vllm") + logger.info("vLLM workload logs: %s", workload_logs) + assert workload_logs and workload_logs.get("data", {}).get( + "result" + ), "no vLLM workload logs in Loki"