[PRA-391] Add log forwarding feature

charms/kserve-llmisvc/src/charm.py

@@ -56,6 +56,7 @@
 # Readiness relation consumed by the llm-integrator charm. This charm is the
 # PROVIDER: it publishes ready=true only once it reaches ActiveStatus.
 LLM_INTEGRATOR_SYNC_RELATION = "kserve-llmisvc"
+LOGGING_RELATION = "logging"
 METRICS_PORT = 8080
 METRICS_PROXY_CONTAINER = "metrics-proxy"
 METRICS_PROXY_PORT = 15090
@@ -146,6 +147,10 @@ def __init__(self, *args):
             self.on[LWS_SYNC_RELATION].relation_changed,
             self.on[LWS_SYNC_RELATION].relation_broken,
             self.on[LLM_INTEGRATOR_SYNC_RELATION].relation_changed,
+            self.on[LOGGING_RELATION].relation_joined,
+            self.on[LOGGING_RELATION].relation_changed,
+            self.on[LOGGING_RELATION].relation_departed,
+            self.on[LOGGING_RELATION].relation_broken,
         ]:
             self.framework.observe(event, self._on_event)
         self.framework.observe(self.on.remove, self._on_remove)
@@ -185,8 +190,26 @@ def _scheduler_context(self):
             "scheduler_image": self.images_context.get("llm_scheduler", ""),
             "workload_image": self.images_context.get("llm_workload", ""),
             "llm_routing_sidecar": self.images_context.get("llm_routing_sidecar", ""),
+            "loki_url": self._loki_url,
         }
 
+    @property
+    def _loki_url(self) -> str:
+        """Return a Loki Push API endpoint published through the logging relation."""
+        for relation in self.model.relations.get(LOGGING_RELATION, []):
+            for unit in sorted(relation.units, key=lambda remote_unit: remote_unit.name):
+                endpoint = relation.data[unit].get("endpoint")
+                if not endpoint:
+                    continue
+                try:
+                    url = json.loads(endpoint).get("url")
+                except (json.JSONDecodeError, TypeError):
+                    log.warning("Ignoring malformed Loki endpoint from %s", unit.name)
+                    continue
+                if isinstance(url, str) and url:
+                    return url
+        return ""
+
     @property
     def base_resource_handler(self):
         """K8s handler for core llmisvc resources."""
@@ -364,6 +387,7 @@ def _on_event(self, event):
 
             self.custom_images = parse_images_config(self.model.config["custom_images"])
             self.images_context = self.get_images(DEFAULT_IMAGES, self.custom_images)
+            self._scheduler_config_resource_handler = None
 
             self.unit.status = MaintenanceStatus("Creating k8s resources")
 

charms/kserve-llmisvc/src/default-custom-images.json

@@ -2,6 +2,6 @@
     "llm_scheduler": "docker.io/charmedkubeflow/llm-d-inference-scheduler:0.4.0-66449f9-20260724073351",
     "llm_workload": "ghcr.io/llm-d/llm-d-cuda:v0.4.0",
     "llm_routing_sidecar": "docker.io/charmedkubeflow/llm-d-routing-sidecar:0.4.0-26788a2-20260717115339",
-    "vllm": "docker.io/charmedkubeflow/vllm-cpu:0.19.0-5f4a278-20260825092922",
+    "vllm": "ghcr.io/welpaolo/vllm-cpu:0.19.0",
     "vllm_gpu": "docker.io/charmedkubeflow/vllm-cuda-gpu:0.19.0-e511371-20260702095505"
 }

charms/kserve-llmisvc/src/templates/llmisvc_configs_manifests.yaml.j2

@@ -26,6 +26,10 @@ spec:
         value: INFO
       - name: HF_HUB_CACHE
         value: /models
+{% endraw %}{% if loki_url %}{% raw %}
+      - name: LOKI_URL
+        value: {% endraw %}{{ loki_url }}{% raw %}
+{% endraw %}{% endif %}{% raw %}
       image: {% endraw %}{{ workload_image }}{% raw %}
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -256,6 +260,10 @@ spec:
         value: INFO
       - name: HF_HUB_CACHE
         value: /models
+{% endraw %}{% if loki_url %}{% raw %}
+      - name: LOKI_URL
+        value: {% endraw %}{{ loki_url }}{% raw %}
+{% endraw %}{% endif %}{% raw %}
       image: {% endraw %}{{ workload_image }}{% raw %}
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -485,6 +493,10 @@ spec:
         value: /models
       - name: VLLM_RANDOMIZE_DP_DUMMY_INPUTS
         value: "1"
+{% endraw %}{% if loki_url %}{% raw %}
+      - name: LOKI_URL
+        value: {% endraw %}{{ loki_url }}{% raw %}
+{% endraw %}{% endif %}{% raw %}
       image: {% endraw %}{{ workload_image }}{% raw %}
       imagePullPolicy: IfNotPresent
       name: main
@@ -555,6 +567,10 @@ spec:
           value: INFO
         - name: HF_HUB_CACHE
           value: /models
+{% endraw %}{% if loki_url %}{% raw %}
+        - name: LOKI_URL
+          value: {% endraw %}{{ loki_url }}{% raw %}
+{% endraw %}{% endif %}{% raw %}
         image: {% endraw %}{{ workload_image }}{% raw %}
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -736,6 +752,10 @@ spec:
           value: INFO
         - name: HF_HUB_CACHE
           value: /models
+{% endraw %}{% if loki_url %}{% raw %}
+        - name: LOKI_URL
+          value: {% endraw %}{{ loki_url }}{% raw %}
+{% endraw %}{% endif %}{% raw %}
         image: {% endraw %}{{ workload_image }}{% raw %}
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -912,6 +932,10 @@ spec:
           value: INFO
         - name: HF_HUB_CACHE
           value: /models
+{% endraw %}{% if loki_url %}{% raw %}
+        - name: LOKI_URL
+          value: {% endraw %}{{ loki_url }}{% raw %}
+{% endraw %}{% endif %}{% raw %}
         image: {% endraw %}{{ workload_image }}{% raw %}
         imagePullPolicy: IfNotPresent
         name: main
@@ -1159,6 +1183,10 @@ spec:
         value: INFO
       - name: HF_HUB_CACHE
         value: /models
+{% endraw %}{% if loki_url %}{% raw %}
+      - name: LOKI_URL
+        value: {% endraw %}{{ loki_url }}{% raw %}
+{% endraw %}{% endif %}{% raw %}
       image: {% endraw %}{{ workload_image }}{% raw %}
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1339,6 +1367,10 @@ spec:
         value: INFO
       - name: HF_HUB_CACHE
         value: /models
+{% endraw %}{% if loki_url %}{% raw %}
+      - name: LOKI_URL
+        value: {% endraw %}{{ loki_url }}{% raw %}
+{% endraw %}{% endif %}{% raw %}
       image: {% endraw %}{{ workload_image }}{% raw %}
       imagePullPolicy: IfNotPresent
       livenessProbe:
@@ -1566,6 +1598,10 @@ spec:
         value: INFO
       - name: HF_HUB_CACHE
         value: /models
+{% endraw %}{% if loki_url %}{% raw %}
+      - name: LOKI_URL
+        value: {% endraw %}{{ loki_url }}{% raw %}
+{% endraw %}{% endif %}{% raw %}
       image: {% endraw %}{{ workload_image }}{% raw %}
       imagePullPolicy: IfNotPresent
       name: main

charms/kserve-llmisvc/tests/unit/test_loki.py

@@ -0,0 +1,107 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+"""Tests for forwarding vLLM workload logs to Loki."""
+
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+from charmed_kubeflow_chisme.kubernetes import KubernetesResourceHandler
+from jinja2 import Template
+from ops.model import ActiveStatus
+from ops.testing import Relation, State
+
+LOKI_URL = "http://loki-0.loki-endpoints:3100/loki/api/v1/push"
+TEMPLATE_PATH = Path(__file__).parents[2] / "src/templates/llmisvc_configs_manifests.yaml.j2"
+
+
+def _logging_relation(endpoint: str | None = None) -> Relation:
+    unit_data = {"endpoint": endpoint} if endpoint else {}
+    return Relation(
+        endpoint="logging",
+        interface="loki_push_api",
+        remote_app_name="loki",
+        remote_units_data={0: unit_data},
+    )
+
+
+def _render_template(loki_url: str = "") -> str:
+    return Template(TEMPLATE_PATH.read_text()).render(
+        namespace="test-model",
+        scheduler_image="scheduler-image",
+        workload_image="workload-image",
+        llm_routing_sidecar="routing-sidecar-image",
+        loki_url=loki_url,
+    )
+
+
+def test_vllm_templates_render_loki_url():
+    """Every vLLM workload template should receive the COS Loki endpoint."""
+    rendered = _render_template(LOKI_URL)
+    print(rendered)
+    assert rendered.count("name: LOKI_URL") == 9
+    assert rendered.count(f"value: {LOKI_URL}") == 9
+    assert rendered.count("vllm\n      - serve") == 2
+
+
+def test_vllm_templates_omit_loki_url_without_cos():
+    """vLLM workloads should not configure log forwarding without a Loki endpoint."""
+    assert "LOKI_URL" not in _render_template()
+
+
+def test_logging_relation_passes_loki_url_to_scheduler_context(
+    ctx, both_containers, controller_relation_ready, lws_relation_ready
+):
+    """A ready logging relation should refresh the scheduler render context."""
+    logging_relation = _logging_relation(json.dumps({"url": LOKI_URL}))
+    state_in = State(
+        leader=True,
+        containers=both_containers,
+        relations=[controller_relation_ready, lws_relation_ready, logging_relation],
+    )
+    original_init = KubernetesResourceHandler.__init__
+
+    with patch.object(KubernetesResourceHandler, "__init__", autospec=True) as init:
+        init.side_effect = original_init
+        out = ctx.run(ctx.on.relation_changed(logging_relation), state_in)
+
+    assert isinstance(out.unit_status, ActiveStatus)
+    scheduler_contexts = [
+        call.kwargs["context"]
+        for call in init.call_args_list
+        if call.kwargs.get("template_files") == ["src/templates/llmisvc_configs_manifests.yaml.j2"]
+    ]
+    assert scheduler_contexts == [
+        {
+            "namespace": out.model.name,
+            "scheduler_image": scheduler_contexts[0]["scheduler_image"],
+            "workload_image": scheduler_contexts[0]["workload_image"],
+            "llm_routing_sidecar": scheduler_contexts[0]["llm_routing_sidecar"],
+            "loki_url": LOKI_URL,
+        }
+    ]
+
+
+def test_malformed_logging_endpoint_omits_loki_url(
+    ctx, both_containers, controller_relation_ready, lws_relation_ready
+):
+    """Malformed logging relation data should not configure a vLLM forwarder."""
+    logging_relation = _logging_relation("not-json")
+    state_in = State(
+        leader=True,
+        containers=both_containers,
+        relations=[controller_relation_ready, lws_relation_ready, logging_relation],
+    )
+    original_init = KubernetesResourceHandler.__init__
+
+    with patch.object(KubernetesResourceHandler, "__init__", autospec=True) as init:
+        init.side_effect = original_init
+        ctx.run(ctx.on.relation_changed(logging_relation), state_in)
+
+    scheduler_context = next(
+        call.kwargs["context"]
+        for call in init.call_args_list
+        if call.kwargs.get("template_files") == ["src/templates/llmisvc_configs_manifests.yaml.j2"]
+    )
+    assert scheduler_context["loki_url"] == ""

tests/integration/helpers/cos.py

@@ -163,6 +163,30 @@ def _workload_pod_name(isvc_name: str, namespace: str = NAMESPACE_DEFAULT) -> st
     raise AssertionError(f"No running workload pod found for LLMInferenceService '{isvc_name}'")
 
 
+def workload_environment_value(
+    isvc_name: str,
+    environment_name: str,
+    namespace: str = NAMESPACE_DEFAULT,
+    container: str = "main",
+) -> str | None:
+    """Return a named environment value from a running vLLM workload container."""
+    from lightkube.resources.core_v1 import Pod
+
+    pod_name = _workload_pod_name(isvc_name, namespace)
+    pod = get_client().get(Pod, name=pod_name, namespace=namespace)
+    containers = pod.spec.containers or []
+    workload_container = next(
+        (candidate for candidate in containers if candidate.name == container), None
+    )
+    if workload_container is None:
+        raise AssertionError(f"Container '{container}' not found in workload pod '{pod_name}'")
+
+    for environment in workload_container.env or []:
+        if environment.name == environment_name:
+            return environment.value
+    return None
+
+
 def generate_inference_traffic(
     isvc_name: str,
     model_name: str,

tests/integration/test_observability.py

@@ -33,6 +33,7 @@
     published_loki_logs,
     published_prometheus_alerts,
     query_prometheus,
+    workload_environment_value,
 )
 from .helpers.deploy import deploy_serving_stack
 from .helpers.llmisvc_ops import apply_llmisvc_example
@@ -67,7 +68,7 @@
 
 IMAGE_CONTEXT = {
     "storage_initializer_image": KSERVE_CONTROLLER_IMAGES["configmap__storageInitializer"],
-    "vllm_image": KSERVE_LLMISVC_IMAGES["vllm"],
+    "vllm_image": "ghcr.io/welpaolo/vllm-cpu:0.19.0",
     "model_s3_uri": MODEL_S3_URI,
     "aws_access_key_id": AWS_ACCESS_KEY_ID,
     "aws_secret_access_key": AWS_SECRET_ACCESS_KEY,
@@ -108,7 +109,7 @@ def _integrate(juju: jubilant.Juju, provider: str, requirer: str) -> None:
 
 
 @pytest.mark.abort_on_fail
-def test_deploy_stack_and_model(juju: jubilant.Juju, request: pytest.FixtureRequest):
+def test_deploy_stack(juju: jubilant.Juju, request: pytest.FixtureRequest):
     charms_path = request.config.getoption("--charms-path")
     if not charms_path:
         raise ValueError("--charms-path is required for the observability integration test")
@@ -117,9 +118,6 @@ def test_deploy_stack_and_model(juju: jubilant.Juju, request: pytest.FixtureRequ
 
     deploy_serving_stack(juju, charms_path)
 
-    logger.info("Applying CPU LLMInferenceService example")
-    apply_llmisvc_example(manifest_path=str(CPU_EXAMPLE), context=IMAGE_CONTEXT, name=LLMISVC_NAME)
-
 
 @pytest.mark.abort_on_fail
 def test_relate_to_cos(juju: jubilant.Juju):
@@ -146,6 +144,13 @@ def test_relate_to_cos(juju: jubilant.Juju):
     juju.wait(jubilant.all_active, successes=1)
 
 
+@pytest.mark.abort_on_fail
+def test_deploy_model_with_loki_forwarding():
+    """Create the workload only after its Loki endpoint is available in config."""
+    logger.info("Applying CPU LLMInferenceService example with Loki forwarding enabled")
+    apply_llmisvc_example(manifest_path=str(CPU_EXAMPLE), context=IMAGE_CONTEXT, name=LLMISVC_NAME)
+
+
 @pytest.mark.abort_on_fail
 def test_generate_traffic():
     generate_inference_traffic(
@@ -181,3 +186,14 @@ def test_cos_data_published(juju: jubilant.Juju):
             logger.info("Checking charm logs reached Loki...")
             logs = published_loki_logs(juju, "juju_application", LLMISVC_APP_NAME)
             assert logs and logs.get("data", {}).get("result"), "no kserve-llmisvc logs in Loki"
+
+            logger.info("Checking vLLM workload received the Loki endpoint...")
+            loki_url = workload_environment_value(LLMISVC_NAME, "LOKI_URL")
+            assert loki_url and loki_url.endswith("/loki/api/v1/push")
+
+            logger.info("Checking vLLM workload logs reached Loki...")
+            workload_logs = published_loki_logs(juju, "app", "vllm")
+            logger.info("vLLM workload logs: %s", workload_logs)
+            assert workload_logs and workload_logs.get("data", {}).get(
+                "result"
+            ), "no vLLM workload logs in Loki"