From 4496b0ef03799fa97c4af35a384fe326ec7c12c6 Mon Sep 17 00:00:00 2001 From: Larry Liu Date: Wed, 1 Apr 2026 16:41:49 -0700 Subject: [PATCH] update --- charts/prometheus-agent/Chart.yaml | 6 ++ .../prometheus-agent/templates/_helpers.tpl | 58 +++++++++++++++ .../templates/clusterrole.yaml | 22 ++++++ .../templates/clusterrolebinding.yaml | 14 ++++ .../prometheus-agent/templates/configmap.yaml | 66 +++++++++++++++++ .../templates/deployment.yaml | 73 +++++++++++++++++++ .../templates/serviceaccount.yaml | 8 ++ charts/prometheus-agent/values.yaml | 46 ++++++++++++ examples/validator-vfn/deploy.py | 61 ++++++++++++++++ 9 files changed, 354 insertions(+) create mode 100644 charts/prometheus-agent/Chart.yaml create mode 100644 charts/prometheus-agent/templates/_helpers.tpl create mode 100644 charts/prometheus-agent/templates/clusterrole.yaml create mode 100644 charts/prometheus-agent/templates/clusterrolebinding.yaml create mode 100644 charts/prometheus-agent/templates/configmap.yaml create mode 100644 charts/prometheus-agent/templates/deployment.yaml create mode 100644 charts/prometheus-agent/templates/serviceaccount.yaml create mode 100644 charts/prometheus-agent/values.yaml diff --git a/charts/prometheus-agent/Chart.yaml b/charts/prometheus-agent/Chart.yaml new file mode 100644 index 0000000..f9b11e4 --- /dev/null +++ b/charts/prometheus-agent/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: prometheus-agent +description: Prometheus Agent for scraping pods and pushing metrics to Mimir +type: application +version: 0.1.0 +appVersion: "2.51.0" diff --git a/charts/prometheus-agent/templates/_helpers.tpl b/charts/prometheus-agent/templates/_helpers.tpl new file mode 100644 index 0000000..51dc3b2 --- /dev/null +++ b/charts/prometheus-agent/templates/_helpers.tpl @@ -0,0 +1,58 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus-agent.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "prometheus-agent.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus-agent.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "prometheus-agent.labels" -}} +helm.sh/chart: {{ include "prometheus-agent.chart" . }} +{{ include "prometheus-agent.selectorLabels" . }} +app.kubernetes.io/version: {{ .Values.image.tag | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "prometheus-agent.selectorLabels" -}} +app.kubernetes.io/name: {{ include "prometheus-agent.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Service account name +*/}} +{{- define "prometheus-agent.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "prometheus-agent.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/prometheus-agent/templates/clusterrole.yaml b/charts/prometheus-agent/templates/clusterrole.yaml new file mode 100644 index 0000000..1e1afc8 --- /dev/null +++ b/charts/prometheus-agent/templates/clusterrole.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "prometheus-agent.fullname" . }} + labels: + {{- include "prometheus-agent.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: + - configmaps + verbs: ["get"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] diff --git a/charts/prometheus-agent/templates/clusterrolebinding.yaml b/charts/prometheus-agent/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..68a9209 --- /dev/null +++ b/charts/prometheus-agent/templates/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "prometheus-agent.fullname" . }} + labels: + {{- include "prometheus-agent.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "prometheus-agent.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ include "prometheus-agent.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/prometheus-agent/templates/configmap.yaml b/charts/prometheus-agent/templates/configmap.yaml new file mode 100644 index 0000000..2a166c2 --- /dev/null +++ b/charts/prometheus-agent/templates/configmap.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "prometheus-agent.fullname" . }} + labels: + {{- include "prometheus-agent.labels" . | nindent 4 }} +data: + prometheus.yml: | + global: + scrape_interval: {{ .Values.scrape.interval }} + scrape_timeout: {{ .Values.scrape.timeout }} + external_labels: + cluster: {{ .Values.externalLabels.cluster | quote }} + environment: {{ .Values.externalLabels.environment | quote }} + region: {{ .Values.externalLabels.region | quote }} + + remote_write: + - url: {{ .Values.remoteWrite.url | quote }} + basic_auth: + username: {{ .Values.remoteWrite.basicAuth.username | quote }} + password: {{ .Values.remoteWrite.basicAuth.password | quote }} + + scrape_configs: + # Scrape movement node pods with prometheus.io annotations + - job_name: 'movement-nodes' + kubernetes_sd_configs: + - role: pod + {{- if .Values.scrape.namespaces }} + namespaces: + names: + {{- range (splitList "," .Values.scrape.namespaces) }} + - {{ . | trim | quote }} + {{- end }} + {{- end }} + relabel_configs: + # Only scrape pods with prometheus.io/scrape=true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + # Use prometheus.io/port annotation for port + - source_labels: [__meta_kubernetes_pod_ip, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: (.+);(.+) + replacement: $1:$2 + # Use prometheus.io/path annotation for path + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + # Add pod metadata as labels + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + target_label: app + # Add node_type label from pod label (movementnetwork.xyz/node_type) + - source_labels: [__meta_kubernetes_pod_label_movementnetwork_xyz_node_type] + target_label: node_type + # Add node_name label from pod label (movementnetwork.xyz/node) + - source_labels: [__meta_kubernetes_pod_label_movementnetwork_xyz_node] + target_label: node_name + # Add network label + - source_labels: [__meta_kubernetes_pod_label_movementnetwork_xyz_network] + target_label: network diff --git a/charts/prometheus-agent/templates/deployment.yaml b/charts/prometheus-agent/templates/deployment.yaml new file mode 100644 index 0000000..38846da --- /dev/null +++ b/charts/prometheus-agent/templates/deployment.yaml @@ -0,0 +1,73 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "prometheus-agent.fullname" . }} + labels: + {{- include "prometheus-agent.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "prometheus-agent.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "prometheus-agent.selectorLabels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + spec: + serviceAccountName: {{ include "prometheus-agent.serviceAccountName" . }} + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + containers: + - name: prometheus + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--enable-feature=agent" + - "--storage.agent.path=/prometheus" + - "--web.enable-lifecycle" + ports: + - name: http + containerPort: 9090 + protocol: TCP + livenessProbe: + httpGet: + path: /-/healthy + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + readinessProbe: + httpGet: + path: /-/ready + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: storage + mountPath: /prometheus + volumes: + - name: config + configMap: + name: {{ include "prometheus-agent.fullname" . }} + - name: storage + emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/prometheus-agent/templates/serviceaccount.yaml b/charts/prometheus-agent/templates/serviceaccount.yaml new file mode 100644 index 0000000..ff425bc --- /dev/null +++ b/charts/prometheus-agent/templates/serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "prometheus-agent.serviceAccountName" . }} + labels: + {{- include "prometheus-agent.labels" . | nindent 4 }} +{{- end }} diff --git a/charts/prometheus-agent/values.yaml b/charts/prometheus-agent/values.yaml new file mode 100644 index 0000000..7f547c7 --- /dev/null +++ b/charts/prometheus-agent/values.yaml @@ -0,0 +1,46 @@ +# Prometheus Agent - scrapes pods, pushes to Mimir +# Deployed per-cluster to collect metrics from all movement nodes + +image: + repository: prom/prometheus + tag: v2.51.0 + pullPolicy: IfNotPresent + +# External labels added to all metrics (REQUIRED) +# These identify which cluster/environment metrics come from +externalLabels: + cluster: "" # e.g., "validator-08" + environment: "testnet" + region: "" # e.g., "us-west-2" + +# Remote write to Mimir (REQUIRED) +remoteWrite: + url: "" # e.g., "https://mimir.us-west-2.testnet.movementinfra.xyz/api/v1/push" + basicAuth: + username: "" + password: "" + +# Scrape configuration +scrape: + interval: 15s + timeout: 10s + # Scrape pods in these namespaces (comma-separated or empty for all) + namespaces: "movement-l1" + +# Resources +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +# Service account for pod discovery +serviceAccount: + create: true + name: "" + +nodeSelector: {} +tolerations: [] +affinity: {} diff --git a/examples/validator-vfn/deploy.py b/examples/validator-vfn/deploy.py index 85be94e..dc127a5 100644 --- a/examples/validator-vfn/deploy.py +++ b/examples/validator-vfn/deploy.py @@ -24,6 +24,7 @@ SCRIPT_DIR = Path(__file__).resolve().parent ROOT_DIR = SCRIPT_DIR.parents[1] CHART_DIR = ROOT_DIR / "charts" / "movement-node" +PROMETHEUS_AGENT_CHART_DIR = ROOT_DIR / "charts" / "prometheus-agent" # Default VFN connection keys (validator's fullnode-network identity) # These are used for the internal V ↔ VFN private connection @@ -393,6 +394,54 @@ def deploy_node( success(f"{node_type.upper()} '{node_name}' deployed successfully") +def deploy_prometheus_agent( + env_vars: dict, + namespace: str, + cluster_name: str, +) -> None: + """Deploy Prometheus Agent for metrics collection and push to Mimir.""" + from tools.helm import HelmManager + + info("Deploying Prometheus Agent for metrics push") + + # Get Mimir configuration + mimir_url = env_vars.get("MIMIR_URL", "").strip() + mimir_username = env_vars.get("MIMIR_USERNAME", "").strip() + mimir_password = env_vars.get("MIMIR_PASSWORD", "").strip() + + if not mimir_url: + error("MIMIR_URL not set, skipping Prometheus Agent deployment") + return + + if not mimir_username or not mimir_password: + error("MIMIR_USERNAME or MIMIR_PASSWORD not set, skipping Prometheus Agent deployment") + return + + region = env_vars.get("AWS_REGION", "us-east-1") + environment = env_vars.get("NETWORK_NAME", "testnet") + + set_values = { + "externalLabels.cluster": cluster_name, + "externalLabels.environment": environment, + "externalLabels.region": region, + "remoteWrite.url": mimir_url, + "remoteWrite.basicAuth.username": mimir_username, + "remoteWrite.basicAuth.password": mimir_password, + "scrape.namespaces": namespace, + } + + helm = HelmManager(PROMETHEUS_AGENT_CHART_DIR) + helm.upgrade_install( + release_name="prometheus-agent", + namespace=namespace, + create_namespace=False, + reset_values=True, + set_values=set_values, + ) + + success("Prometheus Agent deployed successfully") + + def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Path | None = None) -> None: """Deploy validator cluster with intelligent topology handling.""" tf_dir = terraform_dir or SCRIPT_DIR @@ -548,6 +597,15 @@ def deploy(env_vars: dict, force_create: bool, validate: bool, terraform_dir: Pa vfn_service=vfn_name if deploy_vfn else None, ) + # Deploy Prometheus Agent if monitoring is enabled + deploy_monitoring = env_vars.get("DEPLOY_MONITORING", "false").lower() in ("true", "1", "yes") + if deploy_monitoring: + deploy_prometheus_agent( + env_vars=env_vars, + namespace=namespace, + cluster_name=validator_name, + ) + # Step 3: Validation (if requested) if validate: from tools.validation import validate_deployment, wait_for_pods_ready @@ -627,6 +685,9 @@ def destroy(env_vars: dict, terraform_dir: Path | None = None) -> None: deploy_fullnode = env_vars.get("DEPLOY_FULLNODE", "false").lower() in ("true", "1", "yes") # Uninstall Helm releases in reverse order + deploy_monitoring = env_vars.get("DEPLOY_MONITORING", "false").lower() in ("true", "1", "yes") + if deploy_monitoring: + helm.uninstall("prometheus-agent", namespace) if deploy_fullnode: helm.uninstall(fullnode_name, namespace) if deploy_vfn: