From ebbc42b896cfd4018ccd02a67cbc1e1ac6644688 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 22 Sep 2025 13:21:33 +0000
Subject: [PATCH 1/4] Initial plan


From 453714a9477811dd8ac5b80184c5e03323cf93d9 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 22 Sep 2025 13:34:16 +0000
Subject: [PATCH 2/4] Implement logical Thor cluster feature

Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com>
---
 helm/hpcc/docs/thor-logical-clusters.md       | 149 ++++++++++++++++++
 helm/hpcc/templates/_helpers.tpl              |  33 ++++
 helm/hpcc/templates/thor.yaml                 |  35 +++-
 helm/hpcc/values.schema.json                  |  18 +++
 helm/hpcc/values.yaml                         |  11 ++
 .../errtests/invalid-cluster-division.yaml    |   8 +
 testing/helm/tests/logical-thor-cluster.yaml  |   8 +
 testing/helm/tests/mixed-thor-config.yaml     |  12 ++
 8 files changed, 273 insertions(+), 1 deletion(-)
 create mode 100644 helm/hpcc/docs/thor-logical-clusters.md
 create mode 100644 testing/helm/errtests/invalid-cluster-division.yaml
 create mode 100644 testing/helm/tests/logical-thor-cluster.yaml
 create mode 100644 testing/helm/tests/mixed-thor-config.yaml

diff --git a/helm/hpcc/docs/thor-logical-clusters.md b/helm/hpcc/docs/thor-logical-clusters.md
new file mode 100644
index 00000000000..d42c015b477
--- /dev/null
+++ b/helm/hpcc/docs/thor-logical-clusters.md
@@ -0,0 +1,149 @@
+# Thor Logical Clusters
+
+This document describes the logical Thor cluster feature, which allows you to define a single logical Thor configuration that gets expanded into multiple physical Thor instances.
+
+## Problem Statement
+
+Previously, to create multiple Thor targets that share a common queue (using auxQueues), you needed to:
+
+1. Define multiple Thor configurations manually
+2. Manually calculate and divide maxJobs and maxGraphs between instances
+3. Ensure each instance had the same configuration except for limits
+
+This was error-prone and required conscious configuration steps to subdivide the total resources.
+
+## Solution: Logical Clusters
+
+The logical cluster feature allows you to define a single Thor configuration that automatically expands into multiple physical instances with properly divided limits.
+
+## Configuration
+
+### Basic Example
+
+```yaml
+thor:
+- name: thorcluster
+  cluster:
+    instances: 3
+    instanceTemplate: "{name}-az{instance}"
+  numWorkers: 2
+  maxJobs: 6      # Will be divided: 2 per instance
+  maxGraphs: 3    # Will be divided: 1 per instance
+```
+
+This configuration will generate:
+- `thorcluster-az1` with maxJobs=2, maxGraphs=1, auxQueues=[thorcluster]
+- `thorcluster-az2` with maxJobs=2, maxGraphs=1, auxQueues=[thorcluster]  
+- `thorcluster-az3` with maxJobs=2, maxGraphs=1, auxQueues=[thorcluster]
+- A logical queue named `thorcluster` that routes to all instances
+
+### Configuration Options
+
+#### cluster.instances (required)
+- Type: integer
+- Minimum: 2
+- Description: Number of physical Thor instances to create
+
+#### cluster.instanceTemplate (optional)
+- Type: string
+- Default: "{name}-{instance}"
+- Description: Template for generating instance names
+- Placeholders:
+  - `{name}`: Replaced with the logical cluster name
+  - `{instance}`: Replaced with instance number (1, 2, 3, ...)
+
+### Validation Rules
+
+1. **maxJobs** and **maxGraphs** must be evenly divisible by the number of instances
+2. All other configuration (numWorkers, resources, etc.) is copied to each instance
+3. Instance names must not conflict with existing Thor names
+4. The `cluster` configuration is removed from generated instances
+
+## Use Cases
+
+### Multi-AZ Deployment
+```yaml
+thor:
+- name: thor-prod
+  cluster:
+    instances: 3
+    instanceTemplate: "{name}-az{instance}"
+  numWorkers: 4
+  maxJobs: 12
+  maxGraphs: 6
+  # Each instance gets maxJobs=4, maxGraphs=2
+```
+
+### Custom Instance Naming
+```yaml
+thor:
+- name: thor-batch
+  cluster:
+    instances: 2
+    instanceTemplate: "{name}-node{instance}"
+  maxJobs: 8
+  maxGraphs: 4
+  # Generates: thor-batch-node1, thor-batch-node2
+```
+
+## Generated Queue Configuration
+
+When a logical cluster is expanded:
+
+1. Each physical instance gets its own individual queue
+2. Each instance is configured with `auxQueues: [<logical-name>]`
+3. An auxiliary queue with the logical cluster name is created
+4. Jobs submitted to the logical queue are distributed among all instances
+
+## Backward Compatibility
+
+The logical cluster feature is fully backward compatible:
+- Existing Thor configurations continue to work unchanged
+- Existing auxQueues configurations work as before
+- No changes to existing templates or configurations are required
+
+## Migration from Manual auxQueues
+
+### Before (Manual Configuration)
+```yaml
+thor:
+- name: thor-az1
+  auxQueues: [thor-prod]
+  maxJobs: 2
+  maxGraphs: 1
+- name: thor-az2  
+  auxQueues: [thor-prod]
+  maxJobs: 2
+  maxGraphs: 1
+- name: thor-az3
+  auxQueues: [thor-prod]
+  maxJobs: 2
+  maxGraphs: 1
+```
+
+### After (Logical Cluster)
+```yaml
+thor:
+- name: thor-prod
+  cluster:
+    instances: 3
+    instanceTemplate: "{name}-az{instance}"
+  maxJobs: 6
+  maxGraphs: 3
+```
+
+## Error Handling
+
+The system validates configurations and provides clear error messages:
+
+```
+Thor cluster 'thorcluster': maxJobs (5) and maxGraphs (3) must be evenly divisible by instances (3)
+```
+
+## Implementation Details
+
+The logical cluster expansion happens during Helm template processing:
+1. Templates detect Thor configurations with `cluster` properties
+2. Validation ensures proper divisibility
+3. Multiple Thor instances are generated with appropriate limits
+4. Queue configurations are updated to include both individual and auxiliary queues
\ No newline at end of file
diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl
index 8e5728fc2db..e59b699d804 100644
--- a/helm/hpcc/templates/_helpers.tpl
+++ b/helm/hpcc/templates/_helpers.tpl
@@ -1264,7 +1264,40 @@ Generate instance queue names
 {{ end -}}
 {{- $stdThorQueues := dict -}}
 {{- $stdThorQueuesWithAux := dict -}}
+{{- /* Expand logical clusters into physical instances */ -}}
+{{- $expandedThors := list -}}
 {{- range $.Values.thor -}}
+ {{- if not .disabled -}}
+  {{- if hasKey . "cluster" -}}
+   {{- $thorConfig := . -}}
+   {{- $cluster := .cluster -}}
+   {{- $instances := $cluster.instances | int -}}
+   {{- $instanceTemplate := $cluster.instanceTemplate | default "{name}-{instance}" -}}
+   
+   {{- /* Validate that maxJobs and maxGraphs are divisible by instances */ -}}
+   {{- $maxJobsPerInstance := div $thorConfig.maxJobs $instances -}}
+   {{- $maxGraphsPerInstance := div $thorConfig.maxGraphs $instances -}}
+   {{- if or (ne (mul $maxJobsPerInstance $instances | int) ($thorConfig.maxJobs | int)) (ne (mul $maxGraphsPerInstance $instances | int) ($thorConfig.maxGraphs | int)) -}}
+    {{- $_ := fail (printf "Thor cluster '%s': maxJobs (%d) and maxGraphs (%d) must be evenly divisible by instances (%d)" $thorConfig.name $thorConfig.maxJobs $thorConfig.maxGraphs $instances) -}}
+   {{- end -}}
+   
+   {{- /* Generate each instance */ -}}
+   {{- range $i := untilStep 1 (int (add1 $instances)) 1 -}}
+    {{- $instanceName := $instanceTemplate | replace "{name}" $thorConfig.name | replace "{instance}" (toString $i) -}}
+    {{- $instance := deepCopy $thorConfig -}}
+    {{- $_ := set $instance "name" $instanceName -}}
+    {{- $_ := set $instance "maxJobs" $maxJobsPerInstance -}}
+    {{- $_ := set $instance "maxGraphs" $maxGraphsPerInstance -}}
+    {{- $_ := set $instance "auxQueues" (list $thorConfig.name) -}}
+    {{- $_ := unset $instance "cluster" -}}
+    {{- $expandedThors = append $expandedThors $instance -}}
+   {{- end -}}
+  {{- else -}}
+   {{- $expandedThors = append $expandedThors . -}}
+  {{- end -}}
+ {{- end -}}
+{{- end -}}
+{{- range $expandedThors -}}
  {{- if not .disabled -}}
   {{- $queueItem := dict "name" .name "type" "thor" -}}
   {{- if hasKey . "prefix" -}}
diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml
index 1df58c21d3e..8a6cd4c5d7b 100644
--- a/helm/hpcc/templates/thor.yaml
+++ b/helm/hpcc/templates/thor.yaml
@@ -346,7 +346,40 @@ data:
               job: "_HPCC_JOBNAME_"
 {{- end -}}
 
-{{ range $.Values.thor -}}
+{{- /* Expand logical clusters into physical instances */ -}}
+{{- $expandedThors := list -}}
+{{- range $.Values.thor -}}
+ {{- if not .disabled -}}
+  {{- if hasKey . "cluster" -}}
+   {{- $thorConfig := . -}}
+   {{- $cluster := .cluster -}}
+   {{- $instances := $cluster.instances | int -}}
+   {{- $instanceTemplate := $cluster.instanceTemplate | default "{name}-{instance}" -}}
+   
+   {{- /* Validate that maxJobs and maxGraphs are divisible by instances */ -}}
+   {{- $maxJobsPerInstance := div $thorConfig.maxJobs $instances -}}
+   {{- $maxGraphsPerInstance := div $thorConfig.maxGraphs $instances -}}
+   {{- if or (ne (mul $maxJobsPerInstance $instances | int) ($thorConfig.maxJobs | int)) (ne (mul $maxGraphsPerInstance $instances | int) ($thorConfig.maxGraphs | int)) -}}
+    {{- $_ := fail (printf "Thor cluster '%s': maxJobs (%d) and maxGraphs (%d) must be evenly divisible by instances (%d)" $thorConfig.name $thorConfig.maxJobs $thorConfig.maxGraphs $instances) -}}
+   {{- end -}}
+   
+   {{- /* Generate each instance */ -}}
+   {{- range $i := untilStep 1 (int (add1 $instances)) 1 -}}
+    {{- $instanceName := $instanceTemplate | replace "{name}" $thorConfig.name | replace "{instance}" (toString $i) -}}
+    {{- $instance := deepCopy $thorConfig -}}
+    {{- $_ := set $instance "name" $instanceName -}}
+    {{- $_ := set $instance "maxJobs" $maxJobsPerInstance -}}
+    {{- $_ := set $instance "maxGraphs" $maxGraphsPerInstance -}}
+    {{- $_ := set $instance "auxQueues" (list $thorConfig.name) -}}
+    {{- $_ := unset $instance "cluster" -}}
+    {{- $expandedThors = append $expandedThors $instance -}}
+   {{- end -}}
+  {{- else -}}
+   {{- $expandedThors = append $expandedThors . -}}
+  {{- end -}}
+ {{- end -}}
+{{- end -}}
+{{ range $expandedThors -}}
 {{- if not .disabled -}}
 {{- $env := concat ($.Values.global.env | default list) (.env | default list) -}}
 {{- $secretsCategories := list "system" "eclUser" "ecl" "storage" }}
diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json
index 05dad707b15..dbc605c001f 100644
--- a/helm/hpcc/values.schema.json
+++ b/helm/hpcc/values.schema.json
@@ -2728,6 +2728,24 @@
           "items": { "type": "string" },
           "description": "[Optional] Auxiliary queue names to listen to. Multiple Thor's can listen to the same queue using this mechanism"
         },
+        "cluster": {
+          "type": "object",
+          "description": "[Optional] Logical cluster configuration that generates multiple Thor instances",
+          "properties": {
+            "instances": {
+              "type": "integer",
+              "description": "Number of physical Thor instances to create from this logical configuration",
+              "minimum": 2
+            },
+            "instanceTemplate": {
+              "type": "string",
+              "description": "Template for instance names (e.g., 'thor-az{instance}' generates thor-az1, thor-az2, etc.)",
+              "default": "{name}-{instance}"
+            }
+          },
+          "required": ["instances"],
+          "additionalProperties": false
+        },
         "image": {
           "$ref": "#/definitions/image"
         },
diff --git a/helm/hpcc/values.yaml b/helm/hpcc/values.yaml
index 04157c42b0c..5114c135fc5 100644
--- a/helm/hpcc/values.yaml
+++ b/helm/hpcc/values.yaml
@@ -883,6 +883,17 @@ thor:
   maxGraphs: 2
   #maxGraphStartupTime: 600
   #numWorkersPerPod: 1
+  
+  # Logical cluster configuration (optional)
+  # Expands a single Thor definition into multiple physical instances
+  #cluster:
+  #  instances: 3                            # Number of instances to create
+  #  instanceTemplate: "{name}-az{instance}" # Name template for instances
+  # When using logical clusters:
+  # - maxJobs and maxGraphs will be divided evenly among instances
+  # - Each instance will listen to the main queue via auxQueues
+  # - A logical queue with the cluster name will be created
+  
   #managerResources:
   #  cpu: "1"
   #  memory: "2G"
diff --git a/testing/helm/errtests/invalid-cluster-division.yaml b/testing/helm/errtests/invalid-cluster-division.yaml
new file mode 100644
index 00000000000..9913b3c5b39
--- /dev/null
+++ b/testing/helm/errtests/invalid-cluster-division.yaml
@@ -0,0 +1,8 @@
+thor:
+- name: thorcluster
+  cluster:
+    instances: 3
+    instanceTemplate: "{name}-az{instance}"
+  numWorkers: 2
+  maxJobs: 5  # Not divisible by 3
+  maxGraphs: 3
\ No newline at end of file
diff --git a/testing/helm/tests/logical-thor-cluster.yaml b/testing/helm/tests/logical-thor-cluster.yaml
new file mode 100644
index 00000000000..d5eddcdf4ac
--- /dev/null
+++ b/testing/helm/tests/logical-thor-cluster.yaml
@@ -0,0 +1,8 @@
+thor:
+- name: thorcluster
+  cluster:
+    instances: 3
+    instanceTemplate: "{name}-az{instance}"
+  numWorkers: 2
+  maxJobs: 6
+  maxGraphs: 3
\ No newline at end of file
diff --git a/testing/helm/tests/mixed-thor-config.yaml b/testing/helm/tests/mixed-thor-config.yaml
new file mode 100644
index 00000000000..6c6f5f0258a
--- /dev/null
+++ b/testing/helm/tests/mixed-thor-config.yaml
@@ -0,0 +1,12 @@
+thor:
+- name: thor-prod
+  cluster:
+    instances: 2
+    instanceTemplate: "{name}-node{instance}"
+  numWorkers: 4
+  maxJobs: 8
+  maxGraphs: 4
+- name: thor-dev
+  numWorkers: 1
+  maxJobs: 2
+  maxGraphs: 1
\ No newline at end of file

From 6703d18ee65bd3c5439a586efec85cc3a85e1bd8 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Mon, 22 Sep 2025 13:36:24 +0000
Subject: [PATCH 3/4] Implement logical Thor cluster feature for simplified
 multi-instance configuration

Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com>
---
 testing/helm/tests/comprehensive-thor-cluster.yaml | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 testing/helm/tests/comprehensive-thor-cluster.yaml

diff --git a/testing/helm/tests/comprehensive-thor-cluster.yaml b/testing/helm/tests/comprehensive-thor-cluster.yaml
new file mode 100644
index 00000000000..d2bc436f55f
--- /dev/null
+++ b/testing/helm/tests/comprehensive-thor-cluster.yaml
@@ -0,0 +1,13 @@
+thor:
+- name: production-thor
+  cluster:
+    instances: 3
+    instanceTemplate: "{name}-az{instance}"
+  numWorkers: 4
+  maxJobs: 9   # 3 per instance
+  maxGraphs: 6 # 2 per instance
+  prefix: thor
+  lingerPeriod: 60
+  # This demonstrates the exact use case from the problem statement
+  # Instead of manually defining 3 Thor instances with divided limits,
+  # we have a single configuration that expands automatically
\ No newline at end of file

From ac87eea5170aca53a7cf2030ec1c1955852920aa Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 23 Sep 2025 13:42:00 +0000
Subject: [PATCH 4/4] Address code review feedback: remove redundant test,
 improve docs, eliminate duplicate validation

Co-authored-by: jakesmith <902700+jakesmith@users.noreply.github.com>
---
 helm/hpcc/docs/thor-logical-clusters.md      | 17 ++++++++++-------
 helm/hpcc/templates/_helpers.tpl             |  5 +----
 testing/helm/tests/logical-thor-cluster.yaml |  8 --------
 3 files changed, 11 insertions(+), 19 deletions(-)
 delete mode 100644 testing/helm/tests/logical-thor-cluster.yaml

diff --git a/helm/hpcc/docs/thor-logical-clusters.md b/helm/hpcc/docs/thor-logical-clusters.md
index d42c015b477..e453a2acc74 100644
--- a/helm/hpcc/docs/thor-logical-clusters.md
+++ b/helm/hpcc/docs/thor-logical-clusters.md
@@ -74,16 +74,19 @@ thor:
   # Each instance gets maxJobs=4, maxGraphs=2
 ```
 
-### Custom Instance Naming
+### Custom Instance Naming with Regions
 ```yaml
 thor:
-- name: thor-batch
+- name: thor-global
   cluster:
-    instances: 2
-    instanceTemplate: "{name}-node{instance}"
-  maxJobs: 8
-  maxGraphs: 4
-  # Generates: thor-batch-node1, thor-batch-node2
+    instances: 4
+    instanceTemplate: "thor-region-{instance}-cluster"
+  maxJobs: 16
+  maxGraphs: 8
+  # Generates: thor-region-1-cluster, thor-region-2-cluster, 
+  #           thor-region-3-cluster, thor-region-4-cluster
+  # This pattern is useful when instances need specific naming for
+  # integration with external systems or monitoring tools
 ```
 
 ## Generated Queue Configuration
diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl
index e59b699d804..d58497f585f 100644
--- a/helm/hpcc/templates/_helpers.tpl
+++ b/helm/hpcc/templates/_helpers.tpl
@@ -1274,12 +1274,9 @@ Generate instance queue names
    {{- $instances := $cluster.instances | int -}}
    {{- $instanceTemplate := $cluster.instanceTemplate | default "{name}-{instance}" -}}
    
-   {{- /* Validate that maxJobs and maxGraphs are divisible by instances */ -}}
+   {{- /* Calculate divided limits */ -}}
    {{- $maxJobsPerInstance := div $thorConfig.maxJobs $instances -}}
    {{- $maxGraphsPerInstance := div $thorConfig.maxGraphs $instances -}}
-   {{- if or (ne (mul $maxJobsPerInstance $instances | int) ($thorConfig.maxJobs | int)) (ne (mul $maxGraphsPerInstance $instances | int) ($thorConfig.maxGraphs | int)) -}}
-    {{- $_ := fail (printf "Thor cluster '%s': maxJobs (%d) and maxGraphs (%d) must be evenly divisible by instances (%d)" $thorConfig.name $thorConfig.maxJobs $thorConfig.maxGraphs $instances) -}}
-   {{- end -}}
    
    {{- /* Generate each instance */ -}}
    {{- range $i := untilStep 1 (int (add1 $instances)) 1 -}}
diff --git a/testing/helm/tests/logical-thor-cluster.yaml b/testing/helm/tests/logical-thor-cluster.yaml
deleted file mode 100644
index d5eddcdf4ac..00000000000
--- a/testing/helm/tests/logical-thor-cluster.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
-thor:
-- name: thorcluster
-  cluster:
-    instances: 3
-    instanceTemplate: "{name}-az{instance}"
-  numWorkers: 2
-  maxJobs: 6
-  maxGraphs: 3
\ No newline at end of file