From ed237debcec991b5035bf0f277c362169bbc2f54 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Wed, 21 Jan 2026 20:01:39 +0000 Subject: [PATCH 01/30] new daemonset for kubelet flag change --- modules/python/clusterloader2/utils.py | 67 ++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/modules/python/clusterloader2/utils.py b/modules/python/clusterloader2/utils.py index 8212b5ae7f..929f38dac9 100644 --- a/modules/python/clusterloader2/utils.py +++ b/modules/python/clusterloader2/utils.py @@ -196,3 +196,70 @@ def parse_xml_to_json(file_path, indent=0): # Convert the result dictionary to JSON json_result = json.dumps(result, indent=indent) return json_result + +def parse_json_to_daemonset(jsonString): + kubelet_daemonset = r""" +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-config-updater + namespace: kube-system +spec: + selector: + matchLabels: + app: kubelet-config-updater + template: + metadata: + labels: + app: kubelet-config-updater + spec: + hostPID: true + containers: + - name: kubelet-config-updater + image: ghcr.io/containerd/busybox:1.36 + securityContext: + privileged: true + command: + - /bin/sh + - -c + - | + echo "Updating kubelet configuration..." + export CUSTOM_EVICTION_FLAGS="{custom_kubelet_flags}" + export kubelet_flags_eof_key="--node-ip" + for kvp in $CUSTOM_EVICTION_FLAGS; do + key=${{kvp%%=*}} + value=${{kvp#*=}} + esc_key=$(printf '%s' "${{key}}" | sed -e 's/[][\/.^$*+?{{}}()|]/\\&/g') + esc_value=$(printf '%s' "${{value}}" | sed -e 's/[][\/.^$*+?{{}}()|]/\\&/g') + echo "Processing key: ${{esc_key}} with value: ${{esc_value}}" + if grep -q -E "(^|[[:space:]])${{esc_key}}=[^[:space:]]+" /etc/default/kubelet; then + echo "Found ${{esc_key}} in /etc/default/kubelet" + sed -i -r -E "s/(^|[[:space:]])(${{esc_key}})=[^[:space:]]*/\1\2=${{esc_value}}/g" "/etc/default/kubelet" + else + echo "Adding ${{esc_key}}=${{esc_value}} to /etc/default/kubelet" + repl=$(printf ' %s=%s' "${{esc_key}}" "${{esc_value}}") + sed -i -r -E "s/(${kubelet_flags_eof_key}}=[^[:space:]]+)/\1${repl}}/g" "/etc/default/kubelet" + export kubelet_flags_eof_key="${{esc_key}}" + fi + done + echo "Checking Updated kubelet configuration:" + cat /etc/default/kubelet + echo "Restarting kubelet..." + nsenter --mount=/proc/1/ns/mnt -- systemctl restart kubelet + echo "Done. Sleeping indefinitely to keep the pod running." + sleep infinity + volumeMounts: + - name: systemd + mountPath: /run/systemd + - name: kubelet-config + mountPath: /etc/default + volumes: + - name: kubelet-config + hostPath: + path: /etc/default + type: Directory + - name: systemd + hostPath: + path: /run/systemd + restartPolicy: Always""".format(custom_kubelet_flags=jsonString) + return kubelet_daemonset From e1d140c1becbc4e83f11bee83a426290f24fec74 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Wed, 21 Jan 2026 20:04:29 +0000 Subject: [PATCH 02/30] testing pipeline param --- .../CRI Benchmark/k8s-resource-pressure.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml index 1b601dc09f..29bdf66e55 100644 --- a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml +++ b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml @@ -1,5 +1,20 @@ trigger: none +parameters: + - name: myObject # Define a parameter named 'myObject' + type: object # The parameter type is object + default: # Default value is an object with nested properties + foo: FOO # Property 'foo' with value 'FOO' + bar: BAR # Property 'bar' with value 'BAR' + things: # Property 'things' is a list + - one + - two + - three + nested: # Property 'nested' is an object + one: apple # Property 'one' with value 'apple' + two: pear # Property 'two' with value 'pear' + count: 3 # Property 'count' with value 3 + variables: SCENARIO_TYPE: perf-eval SCENARIO_NAME: k8s-node-stress From 9dc534c01941f390148fa0486f8fc36e4ad5c568 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Fri, 23 Jan 2026 17:29:23 +0000 Subject: [PATCH 03/30] adding daemonset support with passed in string param --- modules/python/clusterloader2/cri/cri.py | 5 ++++- modules/python/clusterloader2/utils.py | 4 ++-- .../CRI Benchmark/k8s-resource-pressure.yml | 21 +++++++------------ 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/modules/python/clusterloader2/cri/cri.py b/modules/python/clusterloader2/cri/cri.py index 889d04c95d..7ace94b662 100644 --- a/modules/python/clusterloader2/cri/cri.py +++ b/modules/python/clusterloader2/cri/cri.py @@ -19,7 +19,10 @@ def override_config_clusterloader2( node_count, node_per_step, max_pods, repeats, operation_timeout, load_type, scale_enabled, pod_startup_latency_threshold, provider, - registry_endpoint, os_type, scrape_kubelets, scrape_containerd, containerd_scrape_interval, host_network, override_file): + registry_endpoint, os_type, scrape_kubelets, scrape_containerd, containerd_scrape_interval, host_network, override_file, use_custom_kubelet = False): + if use_custom_kubelet: + global MEMORY_SCALE_FACTOR + MEMORY_SCALE_FACTOR = 1.00 # Allow full memory access for load testing client = KubernetesClient(os.path.expanduser("~/.kube/config")) nodes = client.get_nodes(label_selector="cri-resource-consume=true") if len(nodes) == 0: diff --git a/modules/python/clusterloader2/utils.py b/modules/python/clusterloader2/utils.py index 929f38dac9..5fcd7998cf 100644 --- a/modules/python/clusterloader2/utils.py +++ b/modules/python/clusterloader2/utils.py @@ -197,7 +197,7 @@ def parse_xml_to_json(file_path, indent=0): json_result = json.dumps(result, indent=indent) return json_result -def parse_json_to_daemonset(jsonString): +def add_flags_to_daemonset(flags_string): kubelet_daemonset = r""" apiVersion: apps/v1 kind: DaemonSet @@ -261,5 +261,5 @@ def parse_json_to_daemonset(jsonString): - name: systemd hostPath: path: /run/systemd - restartPolicy: Always""".format(custom_kubelet_flags=jsonString) + restartPolicy: Always""".format(custom_kubelet_flags=flags_string) return kubelet_daemonset diff --git a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml index 29bdf66e55..7377b510c5 100644 --- a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml +++ b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml @@ -1,19 +1,14 @@ trigger: none parameters: - - name: myObject # Define a parameter named 'myObject' - type: object # The parameter type is object - default: # Default value is an object with nested properties - foo: FOO # Property 'foo' with value 'FOO' - bar: BAR # Property 'bar' with value 'BAR' - things: # Property 'things' is a list - - one - - two - - three - nested: # Property 'nested' is an object - one: apple # Property 'one' with value 'apple' - two: pear # Property 'two' with value 'pear' - count: 3 # Property 'count' with value 3 + - name: use_custom_kubelet + displayName: "Use Custom Kubelet Flags?" + type: boolean + default: true + - name: custom_kubelet_config + displayName: "Custom Kubelet Config Object" + type: string + default: "--eviction-hard=memory.available<200Mi,nodefs.available<10%,nodefs.inodesFree<5%,pid.available<2000 --eviction-soft=memory.available<400Mi,nodefs.available<15%,nodefs.inodesFree<10%,pid.available<3000 --eviction-soft-grace-period=memory.available=1m,nodefs.available=1m,nodefs.inodesFree=1m,pid.available=1m" variables: SCENARIO_TYPE: perf-eval From b388403c3ebc9826f22cd4c598430a5ec156018f Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Fri, 23 Jan 2026 20:21:49 +0000 Subject: [PATCH 04/30] full e2e workflow of getting kubelet flags and applying them to kubelet in engine steps --- modules/python/clusterloader2/cri/cri.py | 17 ++++- modules/python/clusterloader2/utils.py | 10 +-- .../CRI Benchmark/k8s-resource-pressure.yml | 8 +++ .../clusterloader2/cri-pressure/execute.yml | 69 +++++++++++++++++++ .../execute-clusterloader2.yml | 2 +- 5 files changed, 97 insertions(+), 9 deletions(-) create mode 100644 steps/engine/clusterloader2/cri-pressure/execute.yml diff --git a/modules/python/clusterloader2/cri/cri.py b/modules/python/clusterloader2/cri/cri.py index 7ace94b662..32710909ba 100644 --- a/modules/python/clusterloader2/cri/cri.py +++ b/modules/python/clusterloader2/cri/cri.py @@ -4,7 +4,7 @@ import math from datetime import datetime, timezone -from clusterloader2.utils import parse_xml_to_json, run_cl2_command, get_measurement +from clusterloader2.utils import parse_xml_to_json, run_cl2_command, get_measurement, add_flags_to_daemonset from clients.kubernetes_client import KubernetesClient, client as k8s_client from utils.logger_config import get_logger, setup_logging from utils.common import str2bool @@ -224,6 +224,9 @@ def collect_clusterloader2( with open(result_file, 'w', encoding='utf-8') as file: file.write(content) +def modify_kubelet_clusterloader2(custom_kubelet_config: str): + add_flags_to_daemonset(custom_kubelet_config) + def main(): parser = argparse.ArgumentParser(description="CRI Kubernetes resources.") subparsers = parser.add_subparsers(dest="command") @@ -380,6 +383,14 @@ def main(): "--registry_info", type=str, help="Container registry information scraped", ) + # Sub-command for modify-kubelet + parser_modify_kubelet = subparsers.add_parser( + "modify-kubelet", help="Add custom flags to kubelet and apply via daemonset" + ) + parser_modify_kubelet.add_argument( + "--custom_kubelet_config", type=str, help="Custom kubelet flags in string format" + ) + args = parser.parse_args() if args.command == "override": @@ -425,6 +436,10 @@ def main(): args.scrape_kubelets, args.registry_info ) + elif args.command == "modify-kubelet": + modify_kubelet_clusterloader2( + args.custom_kubelet_config + ) if __name__ == "__main__": main() diff --git a/modules/python/clusterloader2/utils.py b/modules/python/clusterloader2/utils.py index 5fcd7998cf..13a59c2cc4 100644 --- a/modules/python/clusterloader2/utils.py +++ b/modules/python/clusterloader2/utils.py @@ -1,12 +1,8 @@ from xml.dom import minidom import json import os -import docker -from clients.docker_client import DockerClient -from utils.logger_config import get_logger, setup_logging -setup_logging() -logger = get_logger(__name__) + POD_STARTUP_LATENCY_FILE_PREFIX_MEASUREMENT_MAP = { "PodStartupLatency_PodStartupLatency_": "PodStartupLatency_PodStartupLatency", @@ -238,7 +234,7 @@ def add_flags_to_daemonset(flags_string): else echo "Adding ${{esc_key}}=${{esc_value}} to /etc/default/kubelet" repl=$(printf ' %s=%s' "${{esc_key}}" "${{esc_value}}") - sed -i -r -E "s/(${kubelet_flags_eof_key}}=[^[:space:]]+)/\1${repl}}/g" "/etc/default/kubelet" + sed -i -r -E "s/(${{kubelet_flags_eof_key}}=[^[:space:]]+)/\1${{repl}}/g" "/etc/default/kubelet" export kubelet_flags_eof_key="${{esc_key}}" fi done @@ -262,4 +258,4 @@ def add_flags_to_daemonset(flags_string): hostPath: path: /run/systemd restartPolicy: Always""".format(custom_kubelet_flags=flags_string) - return kubelet_daemonset + print(kubelet_daemonset) diff --git a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml index 7377b510c5..08d976cded 100644 --- a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml +++ b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml @@ -39,6 +39,8 @@ stages: pod_startup_latency_threshold: 23s kubernetes_version: "1.34" scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ parameters.custom_kubelet_config }} n1-p60-memory-managed: node_count: 1 max_pods: 60 @@ -49,6 +51,8 @@ stages: kubernetes_version: "1.34" k8s_os_disk_type: Managed scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ parameters.custom_kubelet_config }} n1-p60-cpu-ephemeral: node_count: 1 max_pods: 60 @@ -58,6 +62,8 @@ stages: pod_startup_latency_threshold: 23s kubernetes_version: "1.34" scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ parameters.custom_kubelet_config }} n1-p60-cpu-managed: node_count: 1 max_pods: 60 @@ -68,6 +74,8 @@ stages: kubernetes_version: "1.34" k8s_os_disk_type: Managed scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ parameters.custom_kubelet_config }} max_parallel: 1 timeout_in_minutes: 120 credential_type: service_connection diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml new file mode 100644 index 0000000000..bc484f1ae1 --- /dev/null +++ b/steps/engine/clusterloader2/cri-pressure/execute.yml @@ -0,0 +1,69 @@ +parameters: + - name: cloud + type: string + default: "" + - name: engine_input + type: object + default: {} + - name: region + type: string + +steps: + - script: | + set -euo pipefail + set -x + + export DS=$(PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE modify-kubelet \ + --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG") + + printf '%s' "$DS" | kubectl apply -f - + + kubectl get pods -A -o wide + + workingDirectory: modules/python + env: + CLOUD: aks + REGION: ${{ parameters.region }} + PYTHON_SCRIPT_FILE: $(Pipeline.Workspace)/s/modules/python/clusterloader2/cri/cri.py + displayName: "Apply Custom Kubelet Config" + + - script: | + set -euo pipefail + + PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE override \ + --node_count $NODE_COUNT \ + --node_per_step ${NODE_PER_STEP:-$NODE_COUNT} \ + --max_pods $MAX_PODS \ + --repeats $REPEATS \ + --operation_timeout $OPERATION_TIMEOUT \ + --load_type $LOAD_TYPE \ + --scale_enabled ${SCALE_ENABLED:-False} \ + --pod_startup_latency_threshold ${POD_STARTUP_LATENCY_THRESHOLD:-15s} \ + --provider $CLOUD \ + --registry_endpoint ${REGISTRY_ENDPOINT:-"akscritelescope.azurecr.io"} \ + --os_type ${OS_TYPE:-linux} \ + --scrape_kubelets ${SCRAPE_KUBELETS:-False} \ + --scrape_containerd ${SCRAPE_CONTAINERD:-False} \ + --containerd_scrape_interval ${CONTAINERD_SCRAPE_INTERVAL:-15s} \ + --host_network ${HOST_NETWORK:-True} \ + --cl2_override_file ${CL2_CONFIG_DIR}/overrides.yaml + PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE execute \ + --cl2_image ${CL2_IMAGE} \ + --cl2_config_dir ${CL2_CONFIG_DIR} \ + --cl2_report_dir $CL2_REPORT_DIR \ + --kubeconfig ${HOME}/.kube/config \ + --provider $CLOUD \ + --scrape_kubelets ${SCRAPE_KUBELETS:-False} \ + --scrape_containerd ${SCRAPE_CONTAINERD:-False} + workingDirectory: modules/python + env: + ${{ if eq(parameters.cloud, 'azure') }}: + CLOUD: aks + ${{ else }}: + CLOUD: ${{ parameters.cloud }} + REGION: ${{ parameters.region }} + PYTHON_SCRIPT_FILE: $(Pipeline.Workspace)/s/modules/python/clusterloader2/cri/cri.py + CL2_IMAGE: ${{ parameters.engine_input.image }} + CL2_CONFIG_DIR: $(Pipeline.Workspace)/s/modules/python/clusterloader2/cri/config + CL2_REPORT_DIR: $(Pipeline.Workspace)/s/modules/python/clusterloader2/cri/results + displayName: "Run Benchmark" diff --git a/steps/topology/k8s-resource-pressure/execute-clusterloader2.yml b/steps/topology/k8s-resource-pressure/execute-clusterloader2.yml index fcdab04dbe..447f10ace9 100644 --- a/steps/topology/k8s-resource-pressure/execute-clusterloader2.yml +++ b/steps/topology/k8s-resource-pressure/execute-clusterloader2.yml @@ -10,7 +10,7 @@ parameters: default: {} steps: -- template: /steps/engine/clusterloader2/cri/execute.yml +- template: /steps/engine/clusterloader2/cri-pressure/execute.yml parameters: cloud: ${{ parameters.cloud }} engine_input: ${{ parameters.engine_input }} From 1e7f30c0782adc9a48a4c6b03e89667e35f48f2a Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Wed, 28 Jan 2026 16:15:37 +0000 Subject: [PATCH 05/30] testing convertToJson --- .../CRI Benchmark/k8s-resource-pressure.yml | 150 ++++++++++-------- 1 file changed, 86 insertions(+), 64 deletions(-) diff --git a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml index 08d976cded..4e315cb955 100644 --- a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml +++ b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml @@ -7,76 +7,98 @@ parameters: default: true - name: custom_kubelet_config displayName: "Custom Kubelet Config Object" - type: string - default: "--eviction-hard=memory.available<200Mi,nodefs.available<10%,nodefs.inodesFree<5%,pid.available<2000 --eviction-soft=memory.available<400Mi,nodefs.available<15%,nodefs.inodesFree<10%,pid.available<3000 --eviction-soft-grace-period=memory.available=1m,nodefs.available=1m,nodefs.inodesFree=1m,pid.available=1m" + type: object # The parameter type is object + default: # Default value is an object with nested properties + eviction-hard: + memory.available: "200Mi" + nodefs.available: "10%" + nodefs.inodesFree: "5%" + pid.available: "2000" + eviction-soft: + memory.available: "400Mi" + nodefs.available: "15%" + nodefs.inodesFree: "10%" + pid.available: "3000" + eviction-soft-grace-period: + memory.available: "1m" + nodefs.available: "1m" + nodefs.inodesFree: "1m" + pid.available: "1m" variables: SCENARIO_TYPE: perf-eval SCENARIO_NAME: k8s-node-stress + custom_kubelet_config_json: ${{ convertToJson(parameters.custom_kubelet_config) }} stages: - stage: azure_westus2_stress_baseline_1_34 dependsOn: [] jobs: - - template: /jobs/competitive-test.yml - parameters: - cloud: azure - regions: - - westus2 - terraform_input_file_mapping: - - westus2: "scenarios/perf-eval/k8s-node-stress/terraform-inputs/azure-ubuntu2204.tfvars" - topology: k8s-resource-pressure - engine: clusterloader2 - engine_input: - image: "ghcr.io/azure/clusterloader2:v20250513" - matrix: - n1-p60-memory-ephemeral: - node_count: 1 - max_pods: 60 - repeats: 1 - operation_timeout: 3m - load_type: memory - pod_startup_latency_threshold: 23s - kubernetes_version: "1.34" - scrape_kubelets: True - use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - custom_kubelet_config: ${{ parameters.custom_kubelet_config }} - n1-p60-memory-managed: - node_count: 1 - max_pods: 60 - repeats: 1 - operation_timeout: 3m - load_type: memory - pod_startup_latency_threshold: 23s - kubernetes_version: "1.34" - k8s_os_disk_type: Managed - scrape_kubelets: True - use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - custom_kubelet_config: ${{ parameters.custom_kubelet_config }} - n1-p60-cpu-ephemeral: - node_count: 1 - max_pods: 60 - repeats: 1 - operation_timeout: 3m - load_type: cpu - pod_startup_latency_threshold: 23s - kubernetes_version: "1.34" - scrape_kubelets: True - use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - custom_kubelet_config: ${{ parameters.custom_kubelet_config }} - n1-p60-cpu-managed: - node_count: 1 - max_pods: 60 - repeats: 1 - operation_timeout: 3m - load_type: cpu - pod_startup_latency_threshold: 23s - kubernetes_version: "1.34" - k8s_os_disk_type: Managed - scrape_kubelets: True - use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - custom_kubelet_config: ${{ parameters.custom_kubelet_config }} - max_parallel: 1 - timeout_in_minutes: 120 - credential_type: service_connection - ssh_key_enabled: false + - script: | + printf '%s' "$custom_kubelet_config_json" + env: + custom_kubelet_config_json: ${{ variables.custom_kubelet_config_json }} + displayName: "Intro Step" + + # - template: /jobs/competitive-test.yml + # parameters: + # cloud: azure + # regions: + # - westus2 + # terraform_input_file_mapping: + # - westus2: "scenarios/perf-eval/k8s-node-stress/terraform-inputs/azure-ubuntu2204.tfvars" + # topology: k8s-resource-pressure + # engine: clusterloader2 + # engine_input: + # image: "ghcr.io/azure/clusterloader2:v20250513" + # matrix: + # n1-p60-memory-ephemeral: + # node_count: 1 + # max_pods: 60 + # repeats: 1 + # operation_timeout: 3m + # load_type: memory + # pod_startup_latency_threshold: 23s + # kubernetes_version: "1.34" + # scrape_kubelets: True + # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + # n1-p60-memory-managed: + # node_count: 1 + # max_pods: 60 + # repeats: 1 + # operation_timeout: 3m + # load_type: memory + # pod_startup_latency_threshold: 23s + # kubernetes_version: "1.34" + # k8s_os_disk_type: Managed + # scrape_kubelets: True + # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + # n1-p60-cpu-ephemeral: + # node_count: 1 + # max_pods: 60 + # repeats: 1 + # operation_timeout: 3m + # load_type: cpu + # pod_startup_latency_threshold: 23s + # kubernetes_version: "1.34" + # scrape_kubelets: True + # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + # n1-p60-cpu-managed: + # node_count: 1 + # max_pods: 60 + # repeats: 1 + # operation_timeout: 3m + # load_type: cpu + # pod_startup_latency_threshold: 23s + # kubernetes_version: "1.34" + # k8s_os_disk_type: Managed + # scrape_kubelets: True + # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + # max_parallel: 1 + # timeout_in_minutes: 120 + # credential_type: service_connection + # ssh_key_enabled: false From 5f08dde3d7f62a17e5d4bdd6fb8f383e52d56c53 Mon Sep 17 00:00:00 2001 From: Peter Damianov <63809581+pdamianov-dev@users.noreply.github.com> Date: Wed, 28 Jan 2026 10:23:56 -0600 Subject: [PATCH 06/30] test fixes --- pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml index 4e315cb955..cf209cec93 100644 --- a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml +++ b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml @@ -34,6 +34,9 @@ stages: - stage: azure_westus2_stress_baseline_1_34 dependsOn: [] jobs: + - job: test + displayName: "Test" + steps: - script: | printf '%s' "$custom_kubelet_config_json" env: From f9b41ab70c4e0e37a0e820fc95f06ba0af4e34fe Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Wed, 28 Jan 2026 21:40:40 +0000 Subject: [PATCH 07/30] modify to pass in json from object in params --- modules/python/clusterloader2/utils.py | 29 +++- .../CRI Benchmark/k8s-resource-pressure.yml | 130 +++++++++--------- .../clusterloader2/cri-pressure/execute.yml | 2 +- 3 files changed, 90 insertions(+), 71 deletions(-) diff --git a/modules/python/clusterloader2/utils.py b/modules/python/clusterloader2/utils.py index 13a59c2cc4..5569a2317d 100644 --- a/modules/python/clusterloader2/utils.py +++ b/modules/python/clusterloader2/utils.py @@ -193,7 +193,8 @@ def parse_xml_to_json(file_path, indent=0): json_result = json.dumps(result, indent=indent) return json_result -def add_flags_to_daemonset(flags_string): +def add_flags_to_daemonset(flags_json_string): + kubelet_flags = parse_flags_from_json(flags_json_string) kubelet_daemonset = r""" apiVersion: apps/v1 kind: DaemonSet @@ -257,5 +258,29 @@ def add_flags_to_daemonset(flags_string): - name: systemd hostPath: path: /run/systemd - restartPolicy: Always""".format(custom_kubelet_flags=flags_string) + restartPolicy: Always""".format(custom_kubelet_flags=kubelet_flags) print(kubelet_daemonset) + +def parse_flags_from_json(json_string): + kubelet_flags = "" + try: + flags_dict = json.loads(json_string) + for key, value in flags_dict.items(): + if isinstance(value, dict) or isinstance(value, list): + if "eviction" in key: + delim = "<" + elif "reserved" in key: + delim = "=" + else: + raise ValueError(f"Unknown complex flag type for key: {key}") + value_str = "" + for key2, value2 in value.items(): + value_str += f"{key2}{delim}{value2}," + else: + value_str = str(value) + kubelet_flags += f"--{key}={value_str.rstrip(",")} " + kubelet_flags = kubelet_flags.strip() + + except Exception: + return "" + return kubelet_flags diff --git a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml index 4e315cb955..d4d182ea70 100644 --- a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml +++ b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml @@ -34,71 +34,65 @@ stages: - stage: azure_westus2_stress_baseline_1_34 dependsOn: [] jobs: - - script: | - printf '%s' "$custom_kubelet_config_json" - env: - custom_kubelet_config_json: ${{ variables.custom_kubelet_config_json }} - displayName: "Intro Step" - - # - template: /jobs/competitive-test.yml - # parameters: - # cloud: azure - # regions: - # - westus2 - # terraform_input_file_mapping: - # - westus2: "scenarios/perf-eval/k8s-node-stress/terraform-inputs/azure-ubuntu2204.tfvars" - # topology: k8s-resource-pressure - # engine: clusterloader2 - # engine_input: - # image: "ghcr.io/azure/clusterloader2:v20250513" - # matrix: - # n1-p60-memory-ephemeral: - # node_count: 1 - # max_pods: 60 - # repeats: 1 - # operation_timeout: 3m - # load_type: memory - # pod_startup_latency_threshold: 23s - # kubernetes_version: "1.34" - # scrape_kubelets: True - # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - # n1-p60-memory-managed: - # node_count: 1 - # max_pods: 60 - # repeats: 1 - # operation_timeout: 3m - # load_type: memory - # pod_startup_latency_threshold: 23s - # kubernetes_version: "1.34" - # k8s_os_disk_type: Managed - # scrape_kubelets: True - # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - # n1-p60-cpu-ephemeral: - # node_count: 1 - # max_pods: 60 - # repeats: 1 - # operation_timeout: 3m - # load_type: cpu - # pod_startup_latency_threshold: 23s - # kubernetes_version: "1.34" - # scrape_kubelets: True - # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - # n1-p60-cpu-managed: - # node_count: 1 - # max_pods: 60 - # repeats: 1 - # operation_timeout: 3m - # load_type: cpu - # pod_startup_latency_threshold: 23s - # kubernetes_version: "1.34" - # k8s_os_disk_type: Managed - # scrape_kubelets: True - # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - # max_parallel: 1 - # timeout_in_minutes: 120 - # credential_type: service_connection - # ssh_key_enabled: false + - template: /jobs/competitive-test.yml + parameters: + cloud: azure + regions: + - westus2 + terraform_input_file_mapping: + - westus2: "scenarios/perf-eval/k8s-node-stress/terraform-inputs/azure-ubuntu2204.tfvars" + topology: k8s-resource-pressure + engine: clusterloader2 + engine_input: + image: "ghcr.io/azure/clusterloader2:v20250513" + matrix: + n1-p60-memory-ephemeral: + node_count: 1 + max_pods: 60 + repeats: 1 + operation_timeout: 3m + load_type: memory + pod_startup_latency_threshold: 23s + kubernetes_version: "1.34" + scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + n1-p60-memory-managed: + node_count: 1 + max_pods: 60 + repeats: 1 + operation_timeout: 3m + load_type: memory + pod_startup_latency_threshold: 23s + kubernetes_version: "1.34" + k8s_os_disk_type: Managed + scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + n1-p60-cpu-ephemeral: + node_count: 1 + max_pods: 60 + repeats: 1 + operation_timeout: 3m + load_type: cpu + pod_startup_latency_threshold: 23s + kubernetes_version: "1.34" + scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + n1-p60-cpu-managed: + node_count: 1 + max_pods: 60 + repeats: 1 + operation_timeout: 3m + load_type: cpu + pod_startup_latency_threshold: 23s + kubernetes_version: "1.34" + k8s_os_disk_type: Managed + scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + max_parallel: 1 + timeout_in_minutes: 120 + credential_type: service_connection + ssh_key_enabled: false diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml index bc484f1ae1..ac9b8ac62d 100644 --- a/steps/engine/clusterloader2/cri-pressure/execute.yml +++ b/steps/engine/clusterloader2/cri-pressure/execute.yml @@ -16,7 +16,7 @@ steps: export DS=$(PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE modify-kubelet \ --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG") - printf '%s' "$DS" | kubectl apply -f - + printf '%s' "$DS" kubectl get pods -A -o wide From ae7f32a764ea48e6c433206088aa567fde59db15 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Wed, 28 Jan 2026 22:05:37 +0000 Subject: [PATCH 08/30] fix string syntax error in utils.py --- modules/python/clusterloader2/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/python/clusterloader2/utils.py b/modules/python/clusterloader2/utils.py index 5569a2317d..08cff0c92b 100644 --- a/modules/python/clusterloader2/utils.py +++ b/modules/python/clusterloader2/utils.py @@ -278,7 +278,7 @@ def parse_flags_from_json(json_string): value_str += f"{key2}{delim}{value2}," else: value_str = str(value) - kubelet_flags += f"--{key}={value_str.rstrip(",")} " + kubelet_flags += f"--{key}={value_str.rstrip(',')} " kubelet_flags = kubelet_flags.strip() except Exception: From 2c784475d1764e6da7dae08801e573b0aceaf42c Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 29 Jan 2026 14:40:12 +0000 Subject: [PATCH 09/30] fix unbound python path error --- steps/engine/clusterloader2/cri-pressure/execute.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml index ac9b8ac62d..83437fe1fe 100644 --- a/steps/engine/clusterloader2/cri-pressure/execute.yml +++ b/steps/engine/clusterloader2/cri-pressure/execute.yml @@ -13,8 +13,8 @@ steps: set -euo pipefail set -x - export DS=$(PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE modify-kubelet \ - --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG") + export DS="$(PYTHONPATH=${PYTHONPATH:-}:$(pwd) python3 $PYTHON_SCRIPT_FILE modify-kubelet \ + --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG")" printf '%s' "$DS" From 03c6fb5f1a6d6777522d6cdd0117e7871fc2cdcf Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 29 Jan 2026 15:26:38 +0000 Subject: [PATCH 10/30] moved pythonpath to separate line --- steps/engine/clusterloader2/cri-pressure/execute.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml index 83437fe1fe..842c8ed18b 100644 --- a/steps/engine/clusterloader2/cri-pressure/execute.yml +++ b/steps/engine/clusterloader2/cri-pressure/execute.yml @@ -13,7 +13,8 @@ steps: set -euo pipefail set -x - export DS="$(PYTHONPATH=${PYTHONPATH:-}:$(pwd) python3 $PYTHON_SCRIPT_FILE modify-kubelet \ + PYTHONPATH=$PYTHONPATH:$(pwd) + export DS="$(python3 $PYTHON_SCRIPT_FILE modify-kubelet \ --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG")" printf '%s' "$DS" From 2113fc6075f919e149a51578fe4ceadef17e7f02 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 29 Jan 2026 16:09:26 +0000 Subject: [PATCH 11/30] remove -u as its not anywhere else --- steps/engine/clusterloader2/cri-pressure/execute.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml index 842c8ed18b..7b11b7a90c 100644 --- a/steps/engine/clusterloader2/cri-pressure/execute.yml +++ b/steps/engine/clusterloader2/cri-pressure/execute.yml @@ -10,11 +10,10 @@ parameters: steps: - script: | - set -euo pipefail + set -eo pipefail set -x - PYTHONPATH=$PYTHONPATH:$(pwd) - export DS="$(python3 $PYTHON_SCRIPT_FILE modify-kubelet \ + export DS="$(PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE modify-kubelet \ --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG")" printf '%s' "$DS" @@ -29,7 +28,7 @@ steps: displayName: "Apply Custom Kubelet Config" - script: | - set -euo pipefail + set -eo pipefail PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE override \ --node_count $NODE_COUNT \ From 013ab432356e00d05e0580ad55c56a0eaf711f84 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 29 Jan 2026 16:25:09 +0000 Subject: [PATCH 12/30] re-add imports --- modules/python/clusterloader2/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/python/clusterloader2/utils.py b/modules/python/clusterloader2/utils.py index 08cff0c92b..56a821a8ac 100644 --- a/modules/python/clusterloader2/utils.py +++ b/modules/python/clusterloader2/utils.py @@ -1,8 +1,12 @@ from xml.dom import minidom import json import os +import docker +from clients.docker_client import DockerClient +from utils.logger_config import get_logger, setup_logging - +setup_logging() +logger = get_logger(__name__) POD_STARTUP_LATENCY_FILE_PREFIX_MEASUREMENT_MAP = { "PodStartupLatency_PodStartupLatency_": "PodStartupLatency_PodStartupLatency", From a6812f639a59eb91dedb7dd082506d6d6079c99d Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Fri, 30 Jan 2026 15:27:20 +0000 Subject: [PATCH 13/30] re add kubectl apply --- steps/engine/clusterloader2/cri-pressure/execute.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml index 7b11b7a90c..40b9fe1b19 100644 --- a/steps/engine/clusterloader2/cri-pressure/execute.yml +++ b/steps/engine/clusterloader2/cri-pressure/execute.yml @@ -16,7 +16,7 @@ steps: export DS="$(PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE modify-kubelet \ --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG")" - printf '%s' "$DS" + printf '%s' "$DS" | kubectl apply -f - kubectl get pods -A -o wide From 7b5526b6075404f727c2f6f68a08a904f141556e Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Fri, 30 Jan 2026 16:32:07 +0000 Subject: [PATCH 14/30] pylint errors --- modules/python/clusterloader2/cri/cri.py | 4 +--- modules/python/clusterloader2/utils.py | 10 +++++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/modules/python/clusterloader2/cri/cri.py b/modules/python/clusterloader2/cri/cri.py index 32710909ba..eccce8c71b 100644 --- a/modules/python/clusterloader2/cri/cri.py +++ b/modules/python/clusterloader2/cri/cri.py @@ -12,16 +12,14 @@ setup_logging() logger = get_logger(__name__) -MEMORY_SCALE_FACTOR = 0.95 # 95% of the total allocatable memory to account for error margin - # TODO: Refactor to use a config dataclass to reduce number of arguments # Reference: modules/python/clusterloader2/job_controller/job_controller.py def override_config_clusterloader2( node_count, node_per_step, max_pods, repeats, operation_timeout, load_type, scale_enabled, pod_startup_latency_threshold, provider, registry_endpoint, os_type, scrape_kubelets, scrape_containerd, containerd_scrape_interval, host_network, override_file, use_custom_kubelet = False): + MEMORY_SCALE_FACTOR = 0.95 # 95% of the total allocatable memory to account for error margin if use_custom_kubelet: - global MEMORY_SCALE_FACTOR MEMORY_SCALE_FACTOR = 1.00 # Allow full memory access for load testing client = KubernetesClient(os.path.expanduser("~/.kube/config")) nodes = client.get_nodes(label_selector="cri-resource-consume=true") diff --git a/modules/python/clusterloader2/utils.py b/modules/python/clusterloader2/utils.py index 56a821a8ac..a42bceec31 100644 --- a/modules/python/clusterloader2/utils.py +++ b/modules/python/clusterloader2/utils.py @@ -199,7 +199,7 @@ def parse_xml_to_json(file_path, indent=0): def add_flags_to_daemonset(flags_json_string): kubelet_flags = parse_flags_from_json(flags_json_string) - kubelet_daemonset = r""" + kubelet_daemonset = rf""" apiVersion: apps/v1 kind: DaemonSet metadata: @@ -225,7 +225,7 @@ def add_flags_to_daemonset(flags_json_string): - -c - | echo "Updating kubelet configuration..." - export CUSTOM_EVICTION_FLAGS="{custom_kubelet_flags}" + export CUSTOM_EVICTION_FLAGS="{kubelet_flags}" export kubelet_flags_eof_key="--node-ip" for kvp in $CUSTOM_EVICTION_FLAGS; do key=${{kvp%%=*}} @@ -262,7 +262,7 @@ def add_flags_to_daemonset(flags_json_string): - name: systemd hostPath: path: /run/systemd - restartPolicy: Always""".format(custom_kubelet_flags=kubelet_flags) + restartPolicy: Always""" print(kubelet_daemonset) def parse_flags_from_json(json_string): @@ -270,7 +270,7 @@ def parse_flags_from_json(json_string): try: flags_dict = json.loads(json_string) for key, value in flags_dict.items(): - if isinstance(value, dict) or isinstance(value, list): + if isinstance(value, (dict, list)): if "eviction" in key: delim = "<" elif "reserved" in key: @@ -284,7 +284,7 @@ def parse_flags_from_json(json_string): value_str = str(value) kubelet_flags += f"--{key}={value_str.rstrip(',')} " kubelet_flags = kubelet_flags.strip() - + except Exception: return "" return kubelet_flags From 1f97096e81432bdfa9c76350035f96656558b48a Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Fri, 30 Jan 2026 16:33:36 +0000 Subject: [PATCH 15/30] yaml lint errors --- pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml | 4 ++-- steps/engine/clusterloader2/cri-pressure/execute.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml index d4d182ea70..4abf0c2251 100644 --- a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml +++ b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml @@ -9,12 +9,12 @@ parameters: displayName: "Custom Kubelet Config Object" type: object # The parameter type is object default: # Default value is an object with nested properties - eviction-hard: + eviction-hard: memory.available: "200Mi" nodefs.available: "10%" nodefs.inodesFree: "5%" pid.available: "2000" - eviction-soft: + eviction-soft: memory.available: "400Mi" nodefs.available: "15%" nodefs.inodesFree: "10%" diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml index 40b9fe1b19..bff9b86b52 100644 --- a/steps/engine/clusterloader2/cri-pressure/execute.yml +++ b/steps/engine/clusterloader2/cri-pressure/execute.yml @@ -15,7 +15,7 @@ steps: export DS="$(PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE modify-kubelet \ --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG")" - + printf '%s' "$DS" | kubectl apply -f - kubectl get pods -A -o wide From 7ee9cfc0f73fe7a6414bd1765cc6579440d72abc Mon Sep 17 00:00:00 2001 From: Peter Damianov <63809581+pdamianov-dev@users.noreply.github.com> Date: Mon, 2 Feb 2026 12:46:59 -0500 Subject: [PATCH 16/30] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- steps/engine/clusterloader2/cri-pressure/execute.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml index bff9b86b52..d00e02cdca 100644 --- a/steps/engine/clusterloader2/cri-pressure/execute.yml +++ b/steps/engine/clusterloader2/cri-pressure/execute.yml @@ -17,6 +17,7 @@ steps: --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG")" printf '%s' "$DS" | kubectl apply -f - + kubectl rollout status ds/kubelet-config-updater -n kube-system --timeout=10m kubectl get pods -A -o wide From 534ad4201788dcf0dbc2ac30cb442adebd3cac73 Mon Sep 17 00:00:00 2001 From: Peter Damianov <63809581+pdamianov-dev@users.noreply.github.com> Date: Mon, 2 Feb 2026 14:26:19 -0500 Subject: [PATCH 17/30] Update execute.yml --- steps/engine/clusterloader2/cri-pressure/execute.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml index d00e02cdca..bff9b86b52 100644 --- a/steps/engine/clusterloader2/cri-pressure/execute.yml +++ b/steps/engine/clusterloader2/cri-pressure/execute.yml @@ -17,7 +17,6 @@ steps: --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG")" printf '%s' "$DS" | kubectl apply -f - - kubectl rollout status ds/kubelet-config-updater -n kube-system --timeout=10m kubectl get pods -A -o wide From bf219bfa1e3c2a8f669f874ad80d7e5a03cd3b3b Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Wed, 4 Feb 2026 20:12:32 +0000 Subject: [PATCH 18/30] use helm chart and jq for json processing in validation yml --- .../k8s-resource-pressure/chart/Chart.yaml | 6 ++ .../chart/templates/daemonset.yaml | 76 +++++++++++++++++++ .../k8s-resource-pressure/chart/values.yaml | 3 + .../validate-resources.yml | 29 +++++++ 4 files changed, 114 insertions(+) create mode 100644 steps/topology/k8s-resource-pressure/chart/Chart.yaml create mode 100644 steps/topology/k8s-resource-pressure/chart/templates/daemonset.yaml create mode 100644 steps/topology/k8s-resource-pressure/chart/values.yaml diff --git a/steps/topology/k8s-resource-pressure/chart/Chart.yaml b/steps/topology/k8s-resource-pressure/chart/Chart.yaml new file mode 100644 index 0000000000..b6fed55a84 --- /dev/null +++ b/steps/topology/k8s-resource-pressure/chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: kubelet-config-updater +description: Deploy a DaemonSet to update kubelet flags on nodes +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/steps/topology/k8s-resource-pressure/chart/templates/daemonset.yaml b/steps/topology/k8s-resource-pressure/chart/templates/daemonset.yaml new file mode 100644 index 0000000000..290cd2781d --- /dev/null +++ b/steps/topology/k8s-resource-pressure/chart/templates/daemonset.yaml @@ -0,0 +1,76 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kubelet-config-updater + namespace: kube-system + labels: + app: kubelet-config-updater +spec: + selector: + matchLabels: + app: kubelet-config-updater + template: + metadata: + labels: + app: kubelet-config-updater + spec: + hostPID: true + nodeSelector: + cri-resource-consume: "true" + tolerations: + - key: cri-resource-consume + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: cri-resource-consume + operator: "Equal" + value: "true" + effect: "NoExecute" + containers: + - name: kubelet-config-updater + image: ghcr.io/containerd/busybox:1.36 + securityContext: + privileged: true + command: + - /bin/sh + - -c + - | + echo "Updating kubelet configuration..." + export CUSTOM_EVICTION_FLAGS="{{ .Values.kubeletFlags }}" + export kubelet_flags_eof_key="--node-ip" + for kvp in $CUSTOM_EVICTION_FLAGS; do + key=${kvp%%=*} + value=${kvp#*=} + esc_key=$(printf '%s' "${key}" | sed -e 's/[][\/.^$*+?()|]/\\&/g') + esc_value=$(printf '%s' "${value}" | sed -e 's/[][\/.^$*+?()|]/\\&/g') + echo "Processing key: ${esc_key} with value: ${esc_value}" + if grep -q -E "(^|[[:space:]])${esc_key}=[^[:space:]]+" /etc/default/kubelet; then + echo "Found ${esc_key} in /etc/default/kubelet" + sed -i -r -E "s/(^|[[:space:]])(${esc_key})=[^[:space:]]*/\1\2=${esc_value}/g" "/etc/default/kubelet" + else + echo "Adding ${esc_key}=${esc_value} to /etc/default/kubelet" + repl=$(printf ' %s=%s' "${esc_key}" "${esc_value}") + sed -i -r -E "s/(${kubelet_flags_eof_key}=[^[:space:]]+)/\1${repl}/g" "/etc/default/kubelet" + export kubelet_flags_eof_key="${esc_key}" + fi + done + echo "Checking Updated kubelet configuration:" + cat /etc/default/kubelet + echo "Restarting kubelet..." + nsenter --mount=/proc/1/ns/mnt -- systemctl restart kubelet || true + echo "Done. Sleeping indefinitely to keep the pod running." + sleep infinity + volumeMounts: + - name: systemd + mountPath: /run/systemd + - name: kubelet-config + mountPath: /etc/default + volumes: + - name: kubelet-config + hostPath: + path: /etc/default + type: Directory + - name: systemd + hostPath: + path: /run/systemd + restartPolicy: Always diff --git a/steps/topology/k8s-resource-pressure/chart/values.yaml b/steps/topology/k8s-resource-pressure/chart/values.yaml new file mode 100644 index 0000000000..b85302e7d9 --- /dev/null +++ b/steps/topology/k8s-resource-pressure/chart/values.yaml @@ -0,0 +1,3 @@ +# Provide a single string with kubelet flags to add, example: +# "--eviction-hard=memory.available<100Mi --system-reserved=cpu=100m,memory=200Mi" +kubeletFlags: "" diff --git a/steps/topology/k8s-resource-pressure/validate-resources.yml b/steps/topology/k8s-resource-pressure/validate-resources.yml index e0efb27b4f..31abd2eaf5 100644 --- a/steps/topology/k8s-resource-pressure/validate-resources.yml +++ b/steps/topology/k8s-resource-pressure/validate-resources.yml @@ -14,3 +14,32 @@ steps: - template: /steps/engine/clusterloader2/large-cluster/validate.yml parameters: desired_nodes: 3 +- bash: | + set -euo pipefail + set -x + + flags_string=$(echo "$CUSTOM_EVICTION_FLAGS" | jq -r ' + def format_value(k; v): + if (v|type) == "object" then + (if (k|test("eviction")) then "<" + elif (k|test("reserved")) then "=" + else error("Unknown complex flag type for key: " + k) end) as $delim + | (v | to_entries | map(.key + $delim + (.value|tostring)) | join(",")) + elif (v|type) == "array" then + (v | map(tostring) | join(",")) + else + (v | tostring) + end; + + to_entries + | map("--" + .key + "=" + (format_value(.key; .value))) + | join(" ") + ') + + helm upgrade --install kubelet-config-updater ./steps/topology/k8s-resource-pressure/chart \ + --namespace kube-system --create-namespace \ + --set kubeletFlags=$flags_string + env: + CUSTOM_EVICTION_FLAGS: $(CUSTOM_KUBELET_CONFIG) + + displayName: "Validate Kubelet Custom Config Applied" From 59174918cf5932a2e96b62c8e82af5595698e7f4 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Wed, 4 Feb 2026 20:13:32 +0000 Subject: [PATCH 19/30] stop using cri-pressure engine --- steps/topology/k8s-resource-pressure/execute-clusterloader2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/steps/topology/k8s-resource-pressure/execute-clusterloader2.yml b/steps/topology/k8s-resource-pressure/execute-clusterloader2.yml index 447f10ace9..fcdab04dbe 100644 --- a/steps/topology/k8s-resource-pressure/execute-clusterloader2.yml +++ b/steps/topology/k8s-resource-pressure/execute-clusterloader2.yml @@ -10,7 +10,7 @@ parameters: default: {} steps: -- template: /steps/engine/clusterloader2/cri-pressure/execute.yml +- template: /steps/engine/clusterloader2/cri/execute.yml parameters: cloud: ${{ parameters.cloud }} engine_input: ${{ parameters.engine_input }} From 98a6b8afa3169f934ee43fed6aede5253ba3ef69 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 5 Feb 2026 03:36:31 +0000 Subject: [PATCH 20/30] ensure full string for flags --- steps/topology/k8s-resource-pressure/validate-resources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/steps/topology/k8s-resource-pressure/validate-resources.yml b/steps/topology/k8s-resource-pressure/validate-resources.yml index 31abd2eaf5..fd793cc123 100644 --- a/steps/topology/k8s-resource-pressure/validate-resources.yml +++ b/steps/topology/k8s-resource-pressure/validate-resources.yml @@ -38,7 +38,7 @@ steps: helm upgrade --install kubelet-config-updater ./steps/topology/k8s-resource-pressure/chart \ --namespace kube-system --create-namespace \ - --set kubeletFlags=$flags_string + --set kubeletFlags="$flags_string" env: CUSTOM_EVICTION_FLAGS: $(CUSTOM_KUBELET_CONFIG) From e5ad660b3a7ab9b85c7f7c4198e2f3107276ff16 Mon Sep 17 00:00:00 2001 From: Peter Damianov <63809581+pdamianov-dev@users.noreply.github.com> Date: Wed, 4 Feb 2026 21:37:48 -0600 Subject: [PATCH 21/30] temporary comment out for testing --- .../CRI Benchmark/k8s-resource-pressure.yml | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml index 4abf0c2251..3813287398 100644 --- a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml +++ b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml @@ -57,41 +57,41 @@ stages: scrape_kubelets: True use_custom_kubelet: ${{ parameters.use_custom_kubelet }} custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - n1-p60-memory-managed: - node_count: 1 - max_pods: 60 - repeats: 1 - operation_timeout: 3m - load_type: memory - pod_startup_latency_threshold: 23s - kubernetes_version: "1.34" - k8s_os_disk_type: Managed - scrape_kubelets: True - use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - n1-p60-cpu-ephemeral: - node_count: 1 - max_pods: 60 - repeats: 1 - operation_timeout: 3m - load_type: cpu - pod_startup_latency_threshold: 23s - kubernetes_version: "1.34" - scrape_kubelets: True - use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - n1-p60-cpu-managed: - node_count: 1 - max_pods: 60 - repeats: 1 - operation_timeout: 3m - load_type: cpu - pod_startup_latency_threshold: 23s - kubernetes_version: "1.34" - k8s_os_disk_type: Managed - scrape_kubelets: True - use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + # n1-p60-memory-managed: + # node_count: 1 + # max_pods: 60 + # repeats: 1 + # operation_timeout: 3m + # load_type: memory + # pod_startup_latency_threshold: 23s + # kubernetes_version: "1.34" + # k8s_os_disk_type: Managed + # scrape_kubelets: True + # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + # n1-p60-cpu-ephemeral: + # node_count: 1 + # max_pods: 60 + # repeats: 1 + # operation_timeout: 3m + # load_type: cpu + # pod_startup_latency_threshold: 23s + # kubernetes_version: "1.34" + # scrape_kubelets: True + # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + # n1-p60-cpu-managed: + # node_count: 1 + # max_pods: 60 + # repeats: 1 + # operation_timeout: 3m + # load_type: cpu + # pod_startup_latency_threshold: 23s + # kubernetes_version: "1.34" + # k8s_os_disk_type: Managed + # scrape_kubelets: True + # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} max_parallel: 1 timeout_in_minutes: 120 credential_type: service_connection From 855719140afaf4399b88865ef8577b2bf3273665 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 5 Feb 2026 04:05:22 +0000 Subject: [PATCH 22/30] more fixes --- steps/topology/k8s-resource-pressure/validate-resources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/steps/topology/k8s-resource-pressure/validate-resources.yml b/steps/topology/k8s-resource-pressure/validate-resources.yml index fd793cc123..323e2a3c98 100644 --- a/steps/topology/k8s-resource-pressure/validate-resources.yml +++ b/steps/topology/k8s-resource-pressure/validate-resources.yml @@ -32,7 +32,7 @@ steps: end; to_entries - | map("--" + .key + "=" + (format_value(.key; .value))) + | map("--" + .key + "='" + (format_value(.key; .value)) + "'") | join(" ") ') From e515461291243640228e095afdd9ea45ac01a0dc Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 5 Feb 2026 04:24:51 +0000 Subject: [PATCH 23/30] check pod status after helm upgrade --- steps/topology/k8s-resource-pressure/validate-resources.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/steps/topology/k8s-resource-pressure/validate-resources.yml b/steps/topology/k8s-resource-pressure/validate-resources.yml index 323e2a3c98..5bfee2b8ab 100644 --- a/steps/topology/k8s-resource-pressure/validate-resources.yml +++ b/steps/topology/k8s-resource-pressure/validate-resources.yml @@ -39,6 +39,9 @@ steps: helm upgrade --install kubelet-config-updater ./steps/topology/k8s-resource-pressure/chart \ --namespace kube-system --create-namespace \ --set kubeletFlags="$flags_string" + + kubectl get pods -A -o wide + env: CUSTOM_EVICTION_FLAGS: $(CUSTOM_KUBELET_CONFIG) From df0da52bb0f497ed946f48f44893d5502f27a033 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 5 Feb 2026 13:39:23 +0000 Subject: [PATCH 24/30] steps/topology/k8s-resource-pressure/validate-resources.yml --- steps/topology/k8s-resource-pressure/validate-resources.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/steps/topology/k8s-resource-pressure/validate-resources.yml b/steps/topology/k8s-resource-pressure/validate-resources.yml index 5bfee2b8ab..3d78844616 100644 --- a/steps/topology/k8s-resource-pressure/validate-resources.yml +++ b/steps/topology/k8s-resource-pressure/validate-resources.yml @@ -40,6 +40,8 @@ steps: --namespace kube-system --create-namespace \ --set kubeletFlags="$flags_string" + sleep 5 + kubectl get pods -A -o wide env: From 0d3333187d262906a7620ec58db6f8e329cf1e46 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 5 Feb 2026 16:16:44 +0000 Subject: [PATCH 25/30] try with file --- steps/topology/k8s-resource-pressure/validate-resources.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/steps/topology/k8s-resource-pressure/validate-resources.yml b/steps/topology/k8s-resource-pressure/validate-resources.yml index 3d78844616..8a26143a7f 100644 --- a/steps/topology/k8s-resource-pressure/validate-resources.yml +++ b/steps/topology/k8s-resource-pressure/validate-resources.yml @@ -32,13 +32,15 @@ steps: end; to_entries - | map("--" + .key + "='" + (format_value(.key; .value)) + "'") + | map("--" + .key + "=" + (format_value(.key; .value))) | join(" ") ') + printf '%s' "$flags_string" > "/steps/topology/k8s-resource-pressure/kubelet_flags.txt" + helm upgrade --install kubelet-config-updater ./steps/topology/k8s-resource-pressure/chart \ --namespace kube-system --create-namespace \ - --set kubeletFlags="$flags_string" + --set-file kubeletFlags="kubelet_flags.txt" sleep 5 From 5ba9069d63bc85f068d438db1b4e10579aef4b8d Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 5 Feb 2026 17:05:53 +0000 Subject: [PATCH 26/30] switch to rel path --- steps/topology/k8s-resource-pressure/validate-resources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/steps/topology/k8s-resource-pressure/validate-resources.yml b/steps/topology/k8s-resource-pressure/validate-resources.yml index 8a26143a7f..6459067f69 100644 --- a/steps/topology/k8s-resource-pressure/validate-resources.yml +++ b/steps/topology/k8s-resource-pressure/validate-resources.yml @@ -36,7 +36,7 @@ steps: | join(" ") ') - printf '%s' "$flags_string" > "/steps/topology/k8s-resource-pressure/kubelet_flags.txt" + printf '%s' "$flags_string" > "kubelet_flags.txt" helm upgrade --install kubelet-config-updater ./steps/topology/k8s-resource-pressure/chart \ --namespace kube-system --create-namespace \ From 4237ad25567b946f2f490b355dea8e17db4ca7d3 Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 5 Feb 2026 21:47:08 +0000 Subject: [PATCH 27/30] fixed kubelet flags issue --- steps/topology/k8s-resource-pressure/validate-resources.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/steps/topology/k8s-resource-pressure/validate-resources.yml b/steps/topology/k8s-resource-pressure/validate-resources.yml index 6459067f69..4a90926d5f 100644 --- a/steps/topology/k8s-resource-pressure/validate-resources.yml +++ b/steps/topology/k8s-resource-pressure/validate-resources.yml @@ -21,9 +21,8 @@ steps: flags_string=$(echo "$CUSTOM_EVICTION_FLAGS" | jq -r ' def format_value(k; v): if (v|type) == "object" then - (if (k|test("eviction")) then "<" - elif (k|test("reserved")) then "=" - else error("Unknown complex flag type for key: " + k) end) as $delim + (if (k == "eviction-hard") or (k == "eviction-soft") then "<" + else "=" end) as $delim | (v | to_entries | map(.key + $delim + (.value|tostring)) | join(",")) elif (v|type) == "array" then (v | map(tostring) | join(",")) From 1b4d651c6f5037c1e6a54aba220d4dae34a771fd Mon Sep 17 00:00:00 2001 From: Peter Damianov Date: Thu, 5 Feb 2026 22:10:52 +0000 Subject: [PATCH 28/30] revert all python changes --- modules/python/clusterloader2/cri/cri.py | 14 +-- modules/python/clusterloader2/utils.py | 91 ------------------- .../CRI Benchmark/k8s-resource-pressure.yml | 70 +++++++------- .../clusterloader2/cri-pressure/execute.yml | 69 -------------- 4 files changed, 36 insertions(+), 208 deletions(-) delete mode 100644 steps/engine/clusterloader2/cri-pressure/execute.yml diff --git a/modules/python/clusterloader2/cri/cri.py b/modules/python/clusterloader2/cri/cri.py index eccce8c71b..491f50042b 100644 --- a/modules/python/clusterloader2/cri/cri.py +++ b/modules/python/clusterloader2/cri/cri.py @@ -4,7 +4,7 @@ import math from datetime import datetime, timezone -from clusterloader2.utils import parse_xml_to_json, run_cl2_command, get_measurement, add_flags_to_daemonset +from clusterloader2.utils import parse_xml_to_json, run_cl2_command, get_measurement from clients.kubernetes_client import KubernetesClient, client as k8s_client from utils.logger_config import get_logger, setup_logging from utils.common import str2bool @@ -381,14 +381,6 @@ def main(): "--registry_info", type=str, help="Container registry information scraped", ) - # Sub-command for modify-kubelet - parser_modify_kubelet = subparsers.add_parser( - "modify-kubelet", help="Add custom flags to kubelet and apply via daemonset" - ) - parser_modify_kubelet.add_argument( - "--custom_kubelet_config", type=str, help="Custom kubelet flags in string format" - ) - args = parser.parse_args() if args.command == "override": @@ -434,10 +426,6 @@ def main(): args.scrape_kubelets, args.registry_info ) - elif args.command == "modify-kubelet": - modify_kubelet_clusterloader2( - args.custom_kubelet_config - ) if __name__ == "__main__": main() diff --git a/modules/python/clusterloader2/utils.py b/modules/python/clusterloader2/utils.py index a42bceec31..c5cdbf5e5d 100644 --- a/modules/python/clusterloader2/utils.py +++ b/modules/python/clusterloader2/utils.py @@ -197,94 +197,3 @@ def parse_xml_to_json(file_path, indent=0): json_result = json.dumps(result, indent=indent) return json_result -def add_flags_to_daemonset(flags_json_string): - kubelet_flags = parse_flags_from_json(flags_json_string) - kubelet_daemonset = rf""" -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kubelet-config-updater - namespace: kube-system -spec: - selector: - matchLabels: - app: kubelet-config-updater - template: - metadata: - labels: - app: kubelet-config-updater - spec: - hostPID: true - containers: - - name: kubelet-config-updater - image: ghcr.io/containerd/busybox:1.36 - securityContext: - privileged: true - command: - - /bin/sh - - -c - - | - echo "Updating kubelet configuration..." - export CUSTOM_EVICTION_FLAGS="{kubelet_flags}" - export kubelet_flags_eof_key="--node-ip" - for kvp in $CUSTOM_EVICTION_FLAGS; do - key=${{kvp%%=*}} - value=${{kvp#*=}} - esc_key=$(printf '%s' "${{key}}" | sed -e 's/[][\/.^$*+?{{}}()|]/\\&/g') - esc_value=$(printf '%s' "${{value}}" | sed -e 's/[][\/.^$*+?{{}}()|]/\\&/g') - echo "Processing key: ${{esc_key}} with value: ${{esc_value}}" - if grep -q -E "(^|[[:space:]])${{esc_key}}=[^[:space:]]+" /etc/default/kubelet; then - echo "Found ${{esc_key}} in /etc/default/kubelet" - sed -i -r -E "s/(^|[[:space:]])(${{esc_key}})=[^[:space:]]*/\1\2=${{esc_value}}/g" "/etc/default/kubelet" - else - echo "Adding ${{esc_key}}=${{esc_value}} to /etc/default/kubelet" - repl=$(printf ' %s=%s' "${{esc_key}}" "${{esc_value}}") - sed -i -r -E "s/(${{kubelet_flags_eof_key}}=[^[:space:]]+)/\1${{repl}}/g" "/etc/default/kubelet" - export kubelet_flags_eof_key="${{esc_key}}" - fi - done - echo "Checking Updated kubelet configuration:" - cat /etc/default/kubelet - echo "Restarting kubelet..." - nsenter --mount=/proc/1/ns/mnt -- systemctl restart kubelet - echo "Done. Sleeping indefinitely to keep the pod running." - sleep infinity - volumeMounts: - - name: systemd - mountPath: /run/systemd - - name: kubelet-config - mountPath: /etc/default - volumes: - - name: kubelet-config - hostPath: - path: /etc/default - type: Directory - - name: systemd - hostPath: - path: /run/systemd - restartPolicy: Always""" - print(kubelet_daemonset) - -def parse_flags_from_json(json_string): - kubelet_flags = "" - try: - flags_dict = json.loads(json_string) - for key, value in flags_dict.items(): - if isinstance(value, (dict, list)): - if "eviction" in key: - delim = "<" - elif "reserved" in key: - delim = "=" - else: - raise ValueError(f"Unknown complex flag type for key: {key}") - value_str = "" - for key2, value2 in value.items(): - value_str += f"{key2}{delim}{value2}," - else: - value_str = str(value) - kubelet_flags += f"--{key}={value_str.rstrip(',')} " - kubelet_flags = kubelet_flags.strip() - - except Exception: - return "" - return kubelet_flags diff --git a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml index 3813287398..4abf0c2251 100644 --- a/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml +++ b/pipelines/perf-eval/CRI Benchmark/k8s-resource-pressure.yml @@ -57,41 +57,41 @@ stages: scrape_kubelets: True use_custom_kubelet: ${{ parameters.use_custom_kubelet }} custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - # n1-p60-memory-managed: - # node_count: 1 - # max_pods: 60 - # repeats: 1 - # operation_timeout: 3m - # load_type: memory - # pod_startup_latency_threshold: 23s - # kubernetes_version: "1.34" - # k8s_os_disk_type: Managed - # scrape_kubelets: True - # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - # n1-p60-cpu-ephemeral: - # node_count: 1 - # max_pods: 60 - # repeats: 1 - # operation_timeout: 3m - # load_type: cpu - # pod_startup_latency_threshold: 23s - # kubernetes_version: "1.34" - # scrape_kubelets: True - # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} - # n1-p60-cpu-managed: - # node_count: 1 - # max_pods: 60 - # repeats: 1 - # operation_timeout: 3m - # load_type: cpu - # pod_startup_latency_threshold: 23s - # kubernetes_version: "1.34" - # k8s_os_disk_type: Managed - # scrape_kubelets: True - # use_custom_kubelet: ${{ parameters.use_custom_kubelet }} - # custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + n1-p60-memory-managed: + node_count: 1 + max_pods: 60 + repeats: 1 + operation_timeout: 3m + load_type: memory + pod_startup_latency_threshold: 23s + kubernetes_version: "1.34" + k8s_os_disk_type: Managed + scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + n1-p60-cpu-ephemeral: + node_count: 1 + max_pods: 60 + repeats: 1 + operation_timeout: 3m + load_type: cpu + pod_startup_latency_threshold: 23s + kubernetes_version: "1.34" + scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} + n1-p60-cpu-managed: + node_count: 1 + max_pods: 60 + repeats: 1 + operation_timeout: 3m + load_type: cpu + pod_startup_latency_threshold: 23s + kubernetes_version: "1.34" + k8s_os_disk_type: Managed + scrape_kubelets: True + use_custom_kubelet: ${{ parameters.use_custom_kubelet }} + custom_kubelet_config: ${{ variables.custom_kubelet_config_json }} max_parallel: 1 timeout_in_minutes: 120 credential_type: service_connection diff --git a/steps/engine/clusterloader2/cri-pressure/execute.yml b/steps/engine/clusterloader2/cri-pressure/execute.yml deleted file mode 100644 index bff9b86b52..0000000000 --- a/steps/engine/clusterloader2/cri-pressure/execute.yml +++ /dev/null @@ -1,69 +0,0 @@ -parameters: - - name: cloud - type: string - default: "" - - name: engine_input - type: object - default: {} - - name: region - type: string - -steps: - - script: | - set -eo pipefail - set -x - - export DS="$(PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE modify-kubelet \ - --custom_kubelet_config "$CUSTOM_KUBELET_CONFIG")" - - printf '%s' "$DS" | kubectl apply -f - - - kubectl get pods -A -o wide - - workingDirectory: modules/python - env: - CLOUD: aks - REGION: ${{ parameters.region }} - PYTHON_SCRIPT_FILE: $(Pipeline.Workspace)/s/modules/python/clusterloader2/cri/cri.py - displayName: "Apply Custom Kubelet Config" - - - script: | - set -eo pipefail - - PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE override \ - --node_count $NODE_COUNT \ - --node_per_step ${NODE_PER_STEP:-$NODE_COUNT} \ - --max_pods $MAX_PODS \ - --repeats $REPEATS \ - --operation_timeout $OPERATION_TIMEOUT \ - --load_type $LOAD_TYPE \ - --scale_enabled ${SCALE_ENABLED:-False} \ - --pod_startup_latency_threshold ${POD_STARTUP_LATENCY_THRESHOLD:-15s} \ - --provider $CLOUD \ - --registry_endpoint ${REGISTRY_ENDPOINT:-"akscritelescope.azurecr.io"} \ - --os_type ${OS_TYPE:-linux} \ - --scrape_kubelets ${SCRAPE_KUBELETS:-False} \ - --scrape_containerd ${SCRAPE_CONTAINERD:-False} \ - --containerd_scrape_interval ${CONTAINERD_SCRAPE_INTERVAL:-15s} \ - --host_network ${HOST_NETWORK:-True} \ - --cl2_override_file ${CL2_CONFIG_DIR}/overrides.yaml - PYTHONPATH=$PYTHONPATH:$(pwd) python3 $PYTHON_SCRIPT_FILE execute \ - --cl2_image ${CL2_IMAGE} \ - --cl2_config_dir ${CL2_CONFIG_DIR} \ - --cl2_report_dir $CL2_REPORT_DIR \ - --kubeconfig ${HOME}/.kube/config \ - --provider $CLOUD \ - --scrape_kubelets ${SCRAPE_KUBELETS:-False} \ - --scrape_containerd ${SCRAPE_CONTAINERD:-False} - workingDirectory: modules/python - env: - ${{ if eq(parameters.cloud, 'azure') }}: - CLOUD: aks - ${{ else }}: - CLOUD: ${{ parameters.cloud }} - REGION: ${{ parameters.region }} - PYTHON_SCRIPT_FILE: $(Pipeline.Workspace)/s/modules/python/clusterloader2/cri/cri.py - CL2_IMAGE: ${{ parameters.engine_input.image }} - CL2_CONFIG_DIR: $(Pipeline.Workspace)/s/modules/python/clusterloader2/cri/config - CL2_REPORT_DIR: $(Pipeline.Workspace)/s/modules/python/clusterloader2/cri/results - displayName: "Run Benchmark" From e4b9bd1c63dcd3b1c5f0b6b4d0ce8e50f31bcc8a Mon Sep 17 00:00:00 2001 From: Peter Damianov <63809581+pdamianov-dev@users.noreply.github.com> Date: Thu, 5 Feb 2026 17:11:44 -0500 Subject: [PATCH 29/30] Remove modify_kubelet_clusterloader2 function Removed the modify_kubelet_clusterloader2 function and its associated functionality. --- modules/python/clusterloader2/cri/cri.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/python/clusterloader2/cri/cri.py b/modules/python/clusterloader2/cri/cri.py index 491f50042b..6972a73b47 100644 --- a/modules/python/clusterloader2/cri/cri.py +++ b/modules/python/clusterloader2/cri/cri.py @@ -222,9 +222,6 @@ def collect_clusterloader2( with open(result_file, 'w', encoding='utf-8') as file: file.write(content) -def modify_kubelet_clusterloader2(custom_kubelet_config: str): - add_flags_to_daemonset(custom_kubelet_config) - def main(): parser = argparse.ArgumentParser(description="CRI Kubernetes resources.") subparsers = parser.add_subparsers(dest="command") From e3166e07c8d0c2bde0cfdcc6c50d326de3d0e7f3 Mon Sep 17 00:00:00 2001 From: Peter Damianov <63809581+pdamianov-dev@users.noreply.github.com> Date: Thu, 5 Feb 2026 17:12:08 -0500 Subject: [PATCH 30/30] Remove trailing newline from utils.py --- modules/python/clusterloader2/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/python/clusterloader2/utils.py b/modules/python/clusterloader2/utils.py index c5cdbf5e5d..8212b5ae7f 100644 --- a/modules/python/clusterloader2/utils.py +++ b/modules/python/clusterloader2/utils.py @@ -196,4 +196,3 @@ def parse_xml_to_json(file_path, indent=0): # Convert the result dictionary to JSON json_result = json.dumps(result, indent=indent) return json_result -