Skip to content

Commit 7670886

Browse files
uplsh580uplsh580
committed
Apply review of @jscheffi - Add deploymentOverride in dagBundleConfigList
1 parent be1f8f2 commit 7670886

5 files changed

Lines changed: 131 additions & 139 deletions

File tree

‎chart/templates/dag-processor/dag-processor-deployment.yaml‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,12 @@ Expects context with:
3838
{{- $containerSecurityContextLogGroomerSidecar := include "containerSecurityContext" (list $mergedConfig.logGroomerSidecar .Values) }}
3939
{{- $containerSecurityContextWaitForMigrations := include "containerSecurityContext" (list $mergedConfig.waitForMigrations .Values) }}
4040
{{- $containerLifecycleHooks := or $mergedConfig.containerLifecycleHooks .Values.containerLifecycleHooks }}
41-
{{- /* Prepare args: use per-bundle args if enabled, otherwise use default args */}}
41+
{{- /* Prepare args: append --bundle-name to the base args for per-bundle deployments */}}
4242
{{- $args := $mergedConfig.args }}
43-
{{- if and $deployPerBundle .Values.dagProcessor.deployPerBundle.args }}
44-
{{- $args = list }}
45-
{{- range .Values.dagProcessor.deployPerBundle.args }}
46-
{{- $arg := . | toString | replace "{{ bundleName }}" $bundleName }}
47-
{{- $args = append $args $arg }}
48-
{{- end }}
43+
{{- if $deployPerBundle }}
44+
{{- $lastIdx := sub (len $args) 1 }}
45+
{{- $lastArg := printf "%s --bundle-name %s" (index $args (int $lastIdx)) $bundleName }}
46+
{{- $args = mustInitial $args | append $lastArg }}
4947
{{- end }}
5048
{{- /* Determine deployment name suffix */}}
5149
{{- $nameSuffix := "" }}
@@ -323,7 +321,7 @@ spec:
323321
{{- if eq $enabled nil}}
324322
{{ $enabled = ternary true false (semverCompare ">=3.0.0" .Values.airflowVersion) }}
325323
{{- end }}
326-
{{- $deployPerBundle := .Values.dagProcessor.deployPerBundle.enabled | default false }}
324+
{{- $deployPerBundle := .Values.dagProcessor.deployPerBundle | default false }}
327325
{{- if $enabled }}
328326
{{- if not $deployPerBundle }}
329327
{{- /* Single deployment mode: use base dagProcessor config */}}
@@ -334,7 +332,7 @@ spec:
334332
{{- $firstBundle := true }}
335333
{{- range $bundle := .Values.dagProcessor.dagBundleConfigList }}
336334
{{- $bundleName := $bundle.name }}
337-
{{- $bundleOverrides := index $.Values.dagProcessor.deployPerBundle.bundleOverrides $bundleName | default dict | deepCopy }}
335+
{{- $bundleOverrides := $bundle.deploymentOverride | default dict | deepCopy }}
338336
{{- $baseConfig := $.Values.dagProcessor | deepCopy }}
339337
{{- $mergedConfig := mergeOverwrite $baseConfig $bundleOverrides }}
340338
{{- $_ := set $mergedConfig "dagBundleConfigList" $.Values.dagProcessor.dagBundleConfigList }}

‎chart/templates/dag-processor/dag-processor-poddisruptionbudget.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ Expects context with:
2323
- .Values.dagProcessor: dag processor configuration (may be merged with bundle overrides)
2424
- .bundleName: bundle name (empty string if not per-bundle mode)
2525
- .deployPerBundle: boolean indicating if per-bundle mode is enabled
26+
- .bundleOverrides: raw bundle deploymentOverride dict (for PDB config resolution)
2627
*/}}
2728
{{- define "dag-processor.poddisruptionbudget" }}
2829
{{- $bundleName := .bundleName | default "" }}
@@ -82,7 +83,7 @@ spec:
8283
{{- if eq $enabled nil}}
8384
{{ $enabled = ternary true false (semverCompare ">=3.0.0" .Values.airflowVersion) }}
8485
{{- end }}
85-
{{- $deployPerBundle := .Values.dagProcessor.deployPerBundle.enabled | default false }}
86+
{{- $deployPerBundle := .Values.dagProcessor.deployPerBundle | default false }}
8687
{{- if and $enabled .Values.dagProcessor.podDisruptionBudget.enabled }}
8788
{{- if not $deployPerBundle }}
8889
{{- /* Single PDB mode: use base dagProcessor config */}}
@@ -93,7 +94,7 @@ spec:
9394
{{- $firstPDB := true }}
9495
{{- range $bundle := .Values.dagProcessor.dagBundleConfigList }}
9596
{{- $bundleName := $bundle.name }}
96-
{{- $bundleOverrides := index $.Values.dagProcessor.deployPerBundle.bundleOverrides $bundleName | default dict | deepCopy }}
97+
{{- $bundleOverrides := $bundle.deploymentOverride | default dict | deepCopy }}
9798
{{- $baseConfig := $.Values.dagProcessor | deepCopy }}
9899
{{- $mergedConfig := mergeOverwrite $baseConfig $bundleOverrides }}
99100
{{- $_ := set $mergedConfig "dagBundleConfigList" $.Values.dagProcessor.dagBundleConfigList }}

‎chart/values.schema.json‎

Lines changed: 8 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -4607,7 +4607,6 @@
46074607
],
46084608
"items": {
46094609
"type": "object",
4610-
"additionalProperties": false,
46114610
"required": [
46124611
"name",
46134612
"classpath"
@@ -4633,51 +4632,19 @@
46334632
"null"
46344633
]
46354634
}
4635+
},
4636+
"deploymentOverride": {
4637+
"description": "Per-bundle deployment overrides. Only used when deployPerBundle is true. Overrides base dagProcessor settings for this bundle.",
4638+
"type": "object",
4639+
"additionalProperties": {}
46364640
}
46374641
}
46384642
}
46394643
},
46404644
"deployPerBundle": {
4641-
"description": "Per-bundle deployment option. When enabled, creates a separate deployment for each bundle in `dagBundleConfigList`.",
4642-
"type": "object",
4643-
"additionalProperties": false,
4644-
"properties": {
4645-
"enabled": {
4646-
"description": "Enable per-bundle deployments. When true, creates a separate deployment for each bundle.",
4647-
"type": "boolean",
4648-
"default": false
4649-
},
4650-
"args": {
4651-
"description": "Command args template for per-bundle deployments. `{{ bundleName }}` will be replaced with the actual bundle name.",
4652-
"type": "array",
4653-
"default": [
4654-
"bash",
4655-
"-c",
4656-
"exec airflow dag-processor --bundle-name {{ bundleName }}"
4657-
],
4658-
"items": {
4659-
"type": "string"
4660-
}
4661-
},
4662-
"bundleOverrides": {
4663-
"description": "Per-bundle specific overrides. Each key should match a bundle name from `dagBundleConfigList`.",
4664-
"type": "object",
4665-
"additionalProperties": {
4666-
"type": "object",
4667-
"additionalProperties": {}
4668-
},
4669-
"default": {}
4670-
}
4671-
},
4672-
"default": {
4673-
"enabled": false,
4674-
"args": [
4675-
"bash",
4676-
"-c",
4677-
"exec airflow dag-processor --bundle-name {{ bundleName }}"
4678-
],
4679-
"bundleOverrides": {}
4680-
}
4645+
"description": "When true, creates a separate deployment for each bundle in dagBundleConfigList.",
4646+
"type": "boolean",
4647+
"default": false
46814648
},
46824649
"livenessProbe": {
46834650
"description": "Liveness probe configuration for dag processor.",

‎chart/values.yaml‎

Lines changed: 22 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2361,6 +2361,25 @@ dagProcessor:
23612361
# subdir: "dags"
23622362
# tracking_ref: "main"
23632363
# refresh_interval: 60
2364+
# # Per-bundle deployment overrides (only used when deployPerBundle is true)
2365+
# deploymentOverride:
2366+
# replicas: 3
2367+
# resources:
2368+
# requests:
2369+
# memory: "2Gi"
2370+
# cpu: "1000m"
2371+
# limits:
2372+
# memory: "4Gi"
2373+
# cpu: "2000m"
2374+
# nodeSelector:
2375+
# workload-type: production
2376+
# env:
2377+
# - name: SOME_ENV_VAR
2378+
# value: "20"
2379+
# podDisruptionBudget:
2380+
# enabled: true
2381+
# config:
2382+
# minAvailable: 1
23642383
# - name: bundle2
23652384
# classpath: "airflow.providers.git.bundles.git.GitDagBundle"
23662385
# kwargs:
@@ -2373,37 +2392,9 @@ dagProcessor:
23732392
# kwargs: {}
23742393

23752394
# Per-bundle deployment option
2376-
# When enabled, creates a separate deployment for each bundle in `dagBundleConfigList`
2377-
deployPerBundle:
2378-
enabled: false
2379-
# Command args template for per-bundle deployments
2380-
# `{{ bundleName }}` will be replaced with the actual bundle name
2381-
args: ["bash", "-c", "exec airflow dag-processor --bundle-name {{ bundleName }}"]
2382-
2383-
# Per-bundle specific overrides (optional)
2384-
# Each key should match a bundle name from `dagBundleConfigList`
2385-
# These settings override the base dagProcessor configuration for each bundle
2386-
bundleOverrides: {}
2387-
# Example: Override settings for a specific bundle
2388-
# bundleOverrides:
2389-
# bundle1:
2390-
# replicas: 3
2391-
# resources:
2392-
# requests:
2393-
# memory: "2Gi"
2394-
# cpu: "1000m"
2395-
# limits:
2396-
# memory: "4Gi"
2397-
# cpu: "2000m"
2398-
# nodeSelector:
2399-
# workload-type: production
2400-
# env:
2401-
# - name: SOME_ENV_VAR
2402-
# value: "20"
2403-
# podDisruptionBudget:
2404-
# enabled: true
2405-
# config:
2406-
# minAvailable: 1
2395+
# When enabled, creates a separate deployment for each bundle in `dagBundleConfigList`.
2396+
# Bundle-specific deployment overrides can be set via `deploymentOverride` in each bundle entry.
2397+
deployPerBundle: false
24072398

24082399
# Number of airflow dag processors in the deployment
24092400
replicas: 1

0 commit comments

Comments
 (0)