diff --git a/internal/api/manifest/utils.go b/internal/api/manifest/utils.go index 358804285..cf9bc6b27 100644 --- a/internal/api/manifest/utils.go +++ b/internal/api/manifest/utils.go @@ -6,6 +6,7 @@ import ( "math" "slices" + "github.com/berops/claudie/internal/nodepools" "github.com/berops/claudie/proto/pb/spec" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" @@ -333,6 +334,7 @@ func (ds *Manifest) CreateNodepools(pools []string, isControl bool) ([]*spec.Nod Labels: nodePool.Labels, Annotations: nodePool.Annotations, Taints: getTaints(nodePool.Taints), + SshPort: nodepools.NewSSHPort, // The nodes are left empty, as the desired state // in the manifest does not specify each of the nodes // individually, just the count of the nodes that the @@ -370,6 +372,7 @@ func (ds *Manifest) CreateNodepools(pools []string, isControl bool) ([]*spec.Nod Labels: nodePool.Labels, Annotations: nodePool.Annotations, Taints: taints, + SshPort: nodepools.NewSSHPort, Type: &spec.NodePool_StaticNodePool{ StaticNodePool: &spec.StaticNodePool{ NodeKeys: keys, diff --git a/internal/nodepools/nodepools.go b/internal/nodepools/nodepools.go index fbad4a4b5..52bfe1846 100644 --- a/internal/nodepools/nodepools.go +++ b/internal/nodepools/nodepools.go @@ -16,6 +16,34 @@ import ( "google.golang.org/protobuf/proto" ) +const ( + // DefaultSSHPort is the standard SSH port used by old/existing nodepools. + DefaultSSHPort = int32(22) + // NewSSHPort is the SSH port assigned to newly created nodepools. + NewSSHPort = int32(22522) +) + +// SSHPort returns the effective SSH port for a nodepool. +// Returns DefaultSSHPort (22) if not set (backwards compatibility with old nodepools). +func SSHPort(np *spec.NodePool) int32 { + if np.GetSshPort() == 0 { + return DefaultSSHPort + } + return np.GetSshPort() +} + +// NodeSSHPort returns the SSH port for a node based on its terraform output. +// Returns DefaultSSHPort (22) if not set. +func NodeSSHPort(node *spec.Node) int32 { + if node.GetSshPort() != "" { + var port int32 + if _, err := fmt.Sscanf(node.GetSshPort(), "%d", &port); err == nil && port > 0 { + return port + } + } + return DefaultSSHPort +} + type RegionNetwork struct { Region string ExternalNetwork string @@ -102,6 +130,7 @@ func PartialCopyWithNodeExclusion(np *spec.NodePool, nodes []string) *spec.NodeP Labels: np.Labels, Taints: np.Taints, Annotations: np.Annotations, + SshPort: np.SshPort, } for _, n := range np.Nodes { @@ -148,6 +177,7 @@ func PartialCopyWithNodeFilter(np *spec.NodePool, nodes []string) *spec.NodePool Labels: np.Labels, Taints: np.Taints, Annotations: np.Annotations, + SshPort: np.SshPort, } for _, n := range np.Nodes { @@ -195,6 +225,7 @@ func PartialCopyWithReplacedNodes(np *spec.NodePool, nodes []*spec.Node, nodeKey Labels: np.Labels, Taints: np.Taints, Annotations: np.Annotations, + SshPort: np.SshPort, } // To avoid issues with possible node counts, deep clone the node type itself. @@ -493,35 +524,37 @@ func RandomDynamicNode(nodepools iter.Seq[*spec.NodePool]) *spec.Node { return nodes[idx] } -// Returns a random node public Endpoint and a SSH key to connect to it. Nil if there is none. -func RandomNodePublicEndpoint(nps []*spec.NodePool) (string, string, string) { +// Returns a random node public Endpoint, SSH key, and SSH port to connect to it. +// Empty strings and 0 are returned if there are no nodepools/nodes. +func RandomNodePublicEndpoint(nps []*spec.NodePool) (username, endpoint, key string, sshPort int32) { if len(nps) == 0 { - return "", "", "" + return "", "", "", 0 } idx := rand.IntN(len(nps)) np := nps[idx] if len(np.Nodes) == 0 { - return "", "", "" + return "", "", "", 0 } idx = rand.IntN(len(np.Nodes)) node := np.Nodes[idx] - endpoint := node.Public - username := "root" + endpoint = node.Public + username = "root" if node.Username != "" && node.Username != username { username = node.Username } + sshPort = NodeSSHPort(node) - switch np := np.Type.(type) { + switch npt := np.Type.(type) { case *spec.NodePool_DynamicNodePool: - return username, endpoint, np.DynamicNodePool.PrivateKey + return username, endpoint, npt.DynamicNodePool.PrivateKey, sshPort case *spec.NodePool_StaticNodePool: - return username, endpoint, np.StaticNodePool.NodeKeys[node.Public] + return username, endpoint, npt.StaticNodePool.NodeKeys[node.Public], sshPort default: - return "", "", "" + return "", "", "", 0 } } diff --git a/internal/templateUtils/templates.go b/internal/templateUtils/templates.go index bfa6ac161..16c42c4e4 100644 --- a/internal/templateUtils/templates.go +++ b/internal/templateUtils/templates.go @@ -11,6 +11,8 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" + "github.com/berops/claudie/internal/nodepools" + "github.com/berops/claudie/proto/pb/spec" ) // directory - output directory @@ -65,6 +67,7 @@ func LoadTemplate(tplFile string) (*template.Template, error) { funcMap["replaceAll"] = strings.ReplaceAll funcMap["extractNetmaskFromCIDR"] = ExtractNetmaskFromCIDR funcMap["hasExtension"] = HasExtension + funcMap["nodeSshPort"] = NodeSSHPort tpl, err := template.New("").Funcs(funcMap).Parse(tplFile) if err != nil { @@ -73,6 +76,15 @@ func LoadTemplate(tplFile string) (*template.Template, error) { return tpl, nil } +// NodeSSHPort returns the SSH port for a node. +// Uses the port from terraform output if present, otherwise defaults to DefaultSSHPort -> 22. +func NodeSSHPort(node *spec.Node) string { + if node.SshPort != "" { + return node.SshPort + } + return fmt.Sprintf("%d", nodepools.DefaultSSHPort) +} + // ExtractNetmaskFromCIDR extracts the netmask from the CIDR notation. func ExtractNetmaskFromCIDR(cidr string) string { _, n, err := net.ParseCIDR(cidr) diff --git a/manifests/claudie/kustomization.yaml b/manifests/claudie/kustomization.yaml index d2f690234..d2c1eec69 100644 --- a/manifests/claudie/kustomization.yaml +++ b/manifests/claudie/kustomization.yaml @@ -56,16 +56,16 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: ghcr.io/berops/claudie/ansibler - newTag: 24da72e-3952 + newTag: dfdf0f9-3961 - name: ghcr.io/berops/claudie/autoscaler-adapter - newTag: 24da72e-3952 + newTag: dfdf0f9-3961 - name: ghcr.io/berops/claudie/claudie-operator - newTag: 24da72e-3952 + newTag: dfdf0f9-3961 - name: ghcr.io/berops/claudie/kube-eleven - newTag: 24da72e-3952 + newTag: dfdf0f9-3961 - name: ghcr.io/berops/claudie/kuber - newTag: 24da72e-3952 + newTag: dfdf0f9-3961 - name: ghcr.io/berops/claudie/manager - newTag: 24da72e-3952 + newTag: dfdf0f9-3961 - name: ghcr.io/berops/claudie/terraformer - newTag: 24da72e-3952 + newTag: dfdf0f9-3961 diff --git a/manifests/testing-framework/kustomization.yaml b/manifests/testing-framework/kustomization.yaml index 7c29f355d..832fce4bf 100644 --- a/manifests/testing-framework/kustomization.yaml +++ b/manifests/testing-framework/kustomization.yaml @@ -89,4 +89,4 @@ secretGenerator: images: - name: ghcr.io/berops/claudie/testing-framework - newTag: 24da72e-3952 + newTag: dfdf0f9-3961 diff --git a/proto/pb/spec/nodepool.pb.go b/proto/pb/spec/nodepool.pb.go index 8ee1fec30..5c3209c72 100644 --- a/proto/pb/spec/nodepool.pb.go +++ b/proto/pb/spec/nodepool.pb.go @@ -203,7 +203,11 @@ type NodePool struct { // User defined taints. Taints []*Taint `protobuf:"bytes,7,rep,name=taints,proto3" json:"taints,omitempty"` // User definded annotations. - Annotations map[string]string `protobuf:"bytes,8,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Annotations map[string]string `protobuf:"bytes,8,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // SSH port used to connect to nodes in this nodepool. + // 0 means default (port 22) for backwards compatibility with old nodepools. + // New nodepools are assigned port 22522. + SshPort int32 `protobuf:"varint,9,opt,name=sshPort,proto3" json:"sshPort,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -305,6 +309,13 @@ func (x *NodePool) GetAnnotations() map[string]string { return nil } +func (x *NodePool) GetSshPort() int32 { + if x != nil { + return x.SshPort + } + return 0 +} + type isNodePool_Type interface { isNodePool_Type() } @@ -399,7 +410,9 @@ type Node struct { // Username of a user with root privileges. Also used in SSH connection Username string `protobuf:"bytes,5,opt,name=username,proto3" json:"username,omitempty"` // Status of the node. - Status NodeStatus `protobuf:"varint,6,opt,name=status,proto3,enum=spec.NodeStatus" json:"status,omitempty"` + Status NodeStatus `protobuf:"varint,6,opt,name=status,proto3,enum=spec.NodeStatus" json:"status,omitempty"` + // Ssh port + SshPort string `protobuf:"bytes,7,opt,name=sshPort,proto3" json:"sshPort,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -476,6 +489,13 @@ func (x *Node) GetStatus() NodeStatus { return NodeStatus_Preparing } +func (x *Node) GetSshPort() string { + if x != nil { + return x.SshPort + } + return "" +} + // DynamicNodePool represents dynamic node pool used in cluster. type DynamicNodePool struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -818,7 +838,7 @@ var File_spec_nodepool_proto protoreflect.FileDescriptor const file_spec_nodepool_proto_rawDesc = "" + "\n" + - "\x13spec/nodepool.proto\x12\x04spec\x1a\x13spec/provider.proto\"\x80\x04\n" + + "\x13spec/nodepool.proto\x12\x04spec\x1a\x13spec/provider.proto\"\x9a\x04\n" + "\bNodePool\x12A\n" + "\x0fdynamicNodePool\x18\x01 \x01(\v2\x15.spec.DynamicNodePoolH\x00R\x0fdynamicNodePool\x12>\n" + "\x0estaticNodePool\x18\x02 \x01(\v2\x14.spec.StaticNodePoolH\x00R\x0estaticNodePool\x12\x12\n" + @@ -828,7 +848,8 @@ const file_spec_nodepool_proto_rawDesc = "" + "\tisControl\x18\x05 \x01(\bR\tisControl\x122\n" + "\x06labels\x18\x06 \x03(\v2\x1a.spec.NodePool.LabelsEntryR\x06labels\x12#\n" + "\x06taints\x18\a \x03(\v2\v.spec.TaintR\x06taints\x12A\n" + - "\vannotations\x18\b \x03(\v2\x1f.spec.NodePool.AnnotationsEntryR\vannotations\x1a9\n" + + "\vannotations\x18\b \x03(\v2\x1f.spec.NodePool.AnnotationsEntryR\vannotations\x12\x18\n" + + "\asshPort\x18\t \x01(\x05R\asshPort\x1a9\n" + "\vLabelsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1a>\n" + @@ -839,14 +860,15 @@ const file_spec_nodepool_proto_rawDesc = "" + "\x05Taint\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value\x12\x16\n" + - "\x06effect\x18\x03 \x01(\tR\x06effect\"\xbe\x01\n" + + "\x06effect\x18\x03 \x01(\tR\x06effect\"\xd8\x01\n" + "\x04Node\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n" + "\aprivate\x18\x02 \x01(\tR\aprivate\x12\x16\n" + "\x06public\x18\x03 \x01(\tR\x06public\x12*\n" + "\bnodeType\x18\x04 \x01(\x0e2\x0e.spec.NodeTypeR\bnodeType\x12\x1a\n" + "\busername\x18\x05 \x01(\tR\busername\x12(\n" + - "\x06status\x18\x06 \x01(\x0e2\x10.spec.NodeStatusR\x06status\"\xda\x03\n" + + "\x06status\x18\x06 \x01(\x0e2\x10.spec.NodeStatusR\x06status\x12\x18\n" + + "\asshPort\x18\a \x01(\tR\asshPort\"\xda\x03\n" + "\x0fDynamicNodePool\x12\x1e\n" + "\n" + "serverType\x18\x01 \x01(\tR\n" + diff --git a/proto/spec/nodepool.proto b/proto/spec/nodepool.proto index 96747b6d3..31c89627f 100644 --- a/proto/spec/nodepool.proto +++ b/proto/spec/nodepool.proto @@ -24,6 +24,10 @@ message NodePool { repeated Taint taints = 7; // User definded annotations. map annotations = 8; + // SSH port used to connect to nodes in this nodepool. + // 0 means default (port 22) for backwards compatibility with old nodepools. + // New nodepools are assigned port 22522. + int32 sshPort = 9; } // Taint defines a custom defined taint for the node pools. @@ -51,6 +55,8 @@ message Node { // Status of the node. NodeStatus status = 6; + // Ssh port + string sshPort = 7; } // NodeType specifies the type of the node. diff --git a/services/ansibler/templates/all-node-inventory.goini b/services/ansibler/templates/all-node-inventory.goini index a32eee69a..0e6fadf60 100644 --- a/services/ansibler/templates/all-node-inventory.goini +++ b/services/ansibler/templates/all-node-inventory.goini @@ -2,7 +2,7 @@ {{- range $nodepoolInfo := .NodepoolsInfo }} {{- range $nodepool := $nodepoolInfo.Nodepools.Dynamic }} {{- range $node := $nodepool.Nodes }} -{{ trimPrefix (printf "%s-" $nodepoolInfo.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} private_ip={{ $node.Private }} netmask={{ extractNetmaskFromCIDR $nodepoolInfo.ClusterNetwork }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ trimPrefix (printf "%s-" $nodepoolInfo.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} netmask={{ extractNetmaskFromCIDR $nodepoolInfo.ClusterNetwork }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} @@ -11,7 +11,7 @@ {{- range $nodepoolInfo := .NodepoolsInfo }} {{- range $nodepool := $nodepoolInfo.Nodepools.Static }} {{- range $node := $nodepool.Nodes }} -{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} private_ip={{ $node.Private }} netmask={{ extractNetmaskFromCIDR $nodepoolInfo.ClusterNetwork }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} netmask={{ extractNetmaskFromCIDR $nodepoolInfo.ClusterNetwork }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} diff --git a/services/ansibler/templates/k8s-inventory.goini b/services/ansibler/templates/k8s-inventory.goini index 576c78c3e..0095ae4a7 100644 --- a/services/ansibler/templates/k8s-inventory.goini +++ b/services/ansibler/templates/k8s-inventory.goini @@ -2,7 +2,7 @@ {{- range $nodepool := .K8sNodepools.Dynamic }} {{- if $nodepool.IsControl }} {{- range $node := $nodepool.Nodes }} -{{ trimPrefix (printf "%s-" $.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} private_ip={{ $node.Private }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ trimPrefix (printf "%s-" $.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} @@ -10,7 +10,7 @@ {{- range $nodepool := .K8sNodepools.Static }} {{- if $nodepool.IsControl }} {{- range $node := $nodepool.Nodes }} -{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} private_ip={{ $node.Private }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} @@ -19,7 +19,7 @@ {{- range $nodepool := .K8sNodepools.Dynamic }} {{- if not $nodepool.IsControl }} {{- range $node := $nodepool.Nodes }} -{{ trimPrefix (printf "%s-" $.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} private_ip={{ $node.Private }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ trimPrefix (printf "%s-" $.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} @@ -27,7 +27,7 @@ {{- range $nodepool := .K8sNodepools.Static }} {{- if not $nodepool.IsControl }} {{- range $node := $nodepool.Nodes }} -{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} private_ip={{ $node.Private }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} diff --git a/services/ansibler/templates/lb-inventory.goini b/services/ansibler/templates/lb-inventory.goini index dccbf0abe..cdeaddd34 100644 --- a/services/ansibler/templates/lb-inventory.goini +++ b/services/ansibler/templates/lb-inventory.goini @@ -2,13 +2,13 @@ {{- range $lbNodepool := $.LBnodepools.Dynamic }} {{- range $lbNode := $lbNodepool.Nodes }} {{/*key.pem is taken from a directory where ansible-playbook is called, thus it does not need to specify path relative to inventory.ini*/}} -{{ trimPrefix (printf "%s-%s-" $.Name $.Hash) $lbNode.Name }} ansible_user=root ansible_host={{ $lbNode.Public }} private_ip={{ $lbNode.Private }} ansible_ssh_private_key_file={{ $lbNodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ trimPrefix (printf "%s-%s-" $.Name $.Hash) $lbNode.Name }} ansible_user=root ansible_host={{ $lbNode.Public }} ansible_port={{ nodeSshPort $lbNode }} private_ip={{ $lbNode.Private }} ansible_ssh_private_key_file={{ $lbNodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- range $lbNodepool := $.LBnodepools.Static }} {{- range $lbNode := $lbNodepool.Nodes }} {{/*key.pem is taken from a directory where ansible-playbook is called, thus it does not need to specify path relative to inventory.ini*/}} -{{ $lbNode.Name }} ansible_user={{ $lbNode.Username }} ansible_host={{ $lbNode.Public }} private_ip={{ $lbNode.Private }} ansible_ssh_private_key_file={{ $lbNode.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ $lbNode.Name }} ansible_user={{ $lbNode.Username }} ansible_host={{ $lbNode.Public }} ansible_port={{ nodeSshPort $lbNode }} private_ip={{ $lbNode.Private }} ansible_ssh_private_key_file={{ $lbNode.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} diff --git a/services/ansibler/templates/proxy-envs.goini b/services/ansibler/templates/proxy-envs.goini index c814a9353..ba3b46c7f 100644 --- a/services/ansibler/templates/proxy-envs.goini +++ b/services/ansibler/templates/proxy-envs.goini @@ -2,14 +2,14 @@ {{- range $nodepool := .K8sNodepools.Dynamic }} {{- if $nodepool.IsControl }} {{- range $node := $nodepool.Nodes }} -{{ trimPrefix (printf "%s-" $.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} private_ip={{ $node.Private }} no_proxy_list={{ $.NoProxyList }} http_proxy_url={{ $.HttpProxyUrl }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ trimPrefix (printf "%s-" $.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} no_proxy_list={{ $.NoProxyList }} http_proxy_url={{ $.HttpProxyUrl }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} {{- range $nodepool := .K8sNodepools.Static }} {{- if $nodepool.IsControl }} {{- range $node := $nodepool.Nodes }} -{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} private_ip={{ $node.Private }} no_proxy_list={{ $.NoProxyList }} http_proxy_url={{ $.HttpProxyUrl }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} no_proxy_list={{ $.NoProxyList }} http_proxy_url={{ $.HttpProxyUrl }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} @@ -18,14 +18,14 @@ {{- range $nodepool := .K8sNodepools.Dynamic }} {{- if not $nodepool.IsControl }} {{- range $node := $nodepool.Nodes }} -{{ trimPrefix (printf "%s-" $.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} private_ip={{ $node.Private }} no_proxy_list={{ $.NoProxyList }} http_proxy_url={{ $.HttpProxyUrl }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ trimPrefix (printf "%s-" $.ClusterID) $node.Name }} ansible_user=root ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} no_proxy_list={{ $.NoProxyList }} http_proxy_url={{ $.HttpProxyUrl }} ansible_ssh_private_key_file={{ $nodepool.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} {{- range $nodepool := .K8sNodepools.Static }} {{- if not $nodepool.IsControl }} {{- range $node := $nodepool.Nodes }} -{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} private_ip={{ $node.Private }} no_proxy_list={{ $.NoProxyList }} http_proxy_url={{ $.HttpProxyUrl }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" +{{ $node.Name }} ansible_user={{ $node.Username }} ansible_host={{ $node.Public }} ansible_port={{ nodeSshPort $node }} private_ip={{ $node.Private }} no_proxy_list={{ $.NoProxyList }} http_proxy_url={{ $.HttpProxyUrl }} ansible_ssh_private_key_file={{ $node.Name }}.pem ansible_ssh_extra_args="-o IdentitiesOnly=yes" {{- end }} {{- end }} {{- end }} diff --git a/services/kube-eleven/templates/kubeone.tpl b/services/kube-eleven/templates/kubeone.tpl index c6e97495a..bb195a0f7 100644 --- a/services/kube-eleven/templates/kubeone.tpl +++ b/services/kube-eleven/templates/kubeone.tpl @@ -36,6 +36,7 @@ controlPlane: {{- if ge $nodeInfo.Node.NodeType 1}} - publicAddress: '{{ $nodeInfo.Node.Public }}' privateAddress: '{{ $nodeInfo.Node.Private }}' + sshPort: {{ nodeSshPort $nodeInfo.Node }} {{- if $nodepool.IsDynamic }} sshUsername: root sshPrivateKeyFile: '{{ $nodepool.NodepoolName }}.pem' @@ -61,6 +62,7 @@ staticWorkers: {{- if eq $nodeInfo.Node.NodeType 0}} - publicAddress: '{{ $nodeInfo.Node.Public }}' privateAddress: '{{ $nodeInfo.Node.Private }}' + sshPort: {{ nodeSshPort $nodeInfo.Node }} {{- if $nodepool.IsDynamic }} sshUsername: root sshPrivateKeyFile: '{{ $nodepool.NodepoolName }}.pem' diff --git a/services/manager/internal/service/existing_state.go b/services/manager/internal/service/existing_state.go index 0238936f2..37f04d7bd 100644 --- a/services/manager/internal/service/existing_state.go +++ b/services/manager/internal/service/existing_state.go @@ -570,6 +570,10 @@ outer: // name. continue outer } + // Transfer the SSH port from the current state. For old nodepools + // that predate the SSH port feature, SshPort will be 0 which + // resolves to port 22 at runtime for backwards compatibility. + desired.SshPort = current.SshPort transferDynamicNodePool(current, desired) case *spec.NodePool_StaticNodePool: if desired.GetStaticNodePool() == nil { @@ -579,6 +583,7 @@ outer: // name. continue outer } + desired.SshPort = current.SshPort transferStaticNodePool(current, desired) } } diff --git a/services/manager/internal/service/healthcheck.go b/services/manager/internal/service/healthcheck.go index 7ae9c62f2..664b87c57 100644 --- a/services/manager/internal/service/healthcheck.go +++ b/services/manager/internal/service/healthcheck.go @@ -18,10 +18,6 @@ import ( "golang.org/x/crypto/ssh" ) -// Port on which the ssh connection is established when healthchecking -// individual nodes. -const sshPort = "22" - // UnreachableNodesMap holds the nodepools and all of the nodes within // that nodepool that are unreachable via a Ping on the IPv4 public endpoint. type UnreachableIPv4Map = map[string][]string @@ -161,7 +157,7 @@ func healthCheckVPN(state *spec.Clusters) (bool, error) { nps = append(nps, lb.ClusterInfo.NodePools...) } - username, public, key := nodepools.RandomNodePublicEndpoint(nps) + username, public, key, port := nodepools.RandomNodePublicEndpoint(nps) if key == "" { // If there is no key, than there is no node. assume all is okay. return true, nil @@ -185,7 +181,7 @@ func healthCheckVPN(state *spec.Clusters) (bool, error) { Timeout: 2 * time.Second, } - endpoint := net.JoinHostPort(public, sshPort) + endpoint := net.JoinHostPort(public, fmt.Sprint(port)) client, err := ssh.Dial("tcp", endpoint, &cfg) if err != nil { return false, err diff --git a/services/terraformer/internal/worker/service/internal/cluster-builder/cluster_builder.go b/services/terraformer/internal/worker/service/internal/cluster-builder/cluster_builder.go index 500b40f14..e6a40fe83 100644 --- a/services/terraformer/internal/worker/service/internal/cluster-builder/cluster_builder.go +++ b/services/terraformer/internal/worker/service/internal/cluster-builder/cluster_builder.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "slices" comm "github.com/berops/claudie/internal/command" "github.com/berops/claudie/internal/fileutils" @@ -121,21 +122,26 @@ func (c ClusterBuilder) ReconcileNodePools() error { if err != nil { return fmt.Errorf("error while getting output from tofu for %s : %w", nodepool.Name, err) } - out, err := readIPs(output) + out, err := readNodeOutput(output) if err != nil { return fmt.Errorf("error while reading the tofu output for %s : %w", nodepool.Name, err) } for _, n := range nodepool.Nodes { var found bool - for target, ip := range generics.IterateMapInOrder(out.IPs) { + for target, val := range generics.IterateMapInOrder(out.Nodes) { if target != n.Name { continue } + ip, port, err := parseNodeOutput(val) + if err != nil { + return fmt.Errorf("node %q from nodepool %q: %w", n.Name, nodepool.Name, err) + } if ip == "" { return fmt.Errorf("node %q from nodepool %q has no public address assigned", n.Name, nodepool.Name) } found = true - n.Public = fmt.Sprint(ip) + n.Public = ip + n.SshPort = port break } if !found { @@ -243,6 +249,7 @@ func (c *ClusterBuilder) generateFiles(clusterDir string) error { Nodes: np.Nodes, Details: np.GetDynamicNodePool(), IsControl: np.IsControl, + SshPort: nodepools.SSHPort(np), }) if err := fileutils.CreateKey(dnp.GetPublicKey(), clusterDir, np.GetName()); err != nil { @@ -265,6 +272,15 @@ func (c *ClusterBuilder) generateFiles(clusterDir string) error { Fingerprint: hex.EncodeToString(hash.Digest128(filepath.Join(info.SpecName, path))), } + // Collect unique SSH ports from all nodepools for this provider. + var sshPorts []int32 + for _, np := range pools { + port := nodepools.SSHPort(np) + if !slices.Contains(sshPorts, port) { + sshPorts = append(sshPorts, port) + } + } + if err := g.GenerateNetworking(&templates.Networking{ ClusterData: clusterData, Provider: p, @@ -276,6 +292,7 @@ func (c *ClusterBuilder) generateFiles(clusterDir string) error { LBData: templates.LBData{ Roles: c.LBInfo.Roles, }, + SshPorts: sshPorts, }); err != nil { return fmt.Errorf("failed to generate networking_common template files: %w", err) } @@ -289,11 +306,42 @@ func (c *ClusterBuilder) generateFiles(clusterDir string) error { return nil } -// readIPs reads json output format from tofu and unmarshal it into map[string]map[string]string readable by Go. -func readIPs(data string) (templates.NodepoolIPs, error) { - var result templates.NodepoolIPs - // Unmarshal or Decode the JSON to the interface. - err := json.Unmarshal([]byte(data), &result.IPs) +// parseNodeOutput extracts the IP and optional SSH port from a terraform output value. +// The value can be either a string (just IP) or an array [IP, port]. +func parseNodeOutput(val any) (ip string, port string, err error) { + switch v := val.(type) { + case string: + return v, "", nil + case []any: + if len(v) == 0 { + return "", "", fmt.Errorf("empty output array") + } + ipStr, ok := v[0].(string) + if !ok { + return "", "", fmt.Errorf("expected string IP in output array, got %T", v[0]) + } + if len(v) >= 2 { + portStr, ok := v[1].(string) + if !ok { + // Handle numeric port from JSON (float64). + if portNum, ok := v[1].(float64); ok { + portStr = fmt.Sprintf("%d", int(portNum)) + } else { + return "", "", fmt.Errorf("expected string or number port in output array, got %T", v[1]) + } + } + return ipStr, portStr, nil + } + return ipStr, "", nil + default: + return fmt.Sprint(val), "", nil + } +} + +// readNodeOutput reads json output format from tofu and unmarshals it into a map of node names to their output values. +func readNodeOutput(data string) (templates.NodepoolOutput, error) { + var result templates.NodepoolOutput + err := json.Unmarshal([]byte(data), &result.Nodes) return result, err } diff --git a/services/terraformer/internal/worker/service/internal/cluster-builder/cluster_builder_test.go b/services/terraformer/internal/worker/service/internal/cluster-builder/cluster_builder_test.go index 0925648ca..ead5dc676 100644 --- a/services/terraformer/internal/worker/service/internal/cluster-builder/cluster_builder_test.go +++ b/services/terraformer/internal/worker/service/internal/cluster-builder/cluster_builder_test.go @@ -7,14 +7,14 @@ import ( "github.com/berops/claudie/services/terraformer/internal/worker/service/internal/templates" ) -func Test_readIPs(t *testing.T) { +func Test_readNodeOutput(t *testing.T) { type args struct { data string } tests := []struct { name string args args - want templates.NodepoolIPs + want templates.NodepoolOutput wantErr bool }{ { @@ -22,8 +22,8 @@ func Test_readIPs(t *testing.T) { args: args{ data: "{\"test-cluster-compute1\":\"0.0.0.65\",\n\"test-cluster-compute2\":\"0.0.0.512\", \"test-cluster-control1\":\"0.0.0.72\",\n\"test-cluster-control2\":\"0.0.0.65\"}", }, - want: templates.NodepoolIPs{ - IPs: map[string]any{ + want: templates.NodepoolOutput{ + Nodes: map[string]any{ "test-cluster-compute1": "0.0.0.65", "test-cluster-compute2": "0.0.0.512", "test-cluster-control1": "0.0.0.72", @@ -35,13 +35,68 @@ func Test_readIPs(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := readIPs(tt.args.data) + got, err := readNodeOutput(tt.args.data) if (err != nil) != tt.wantErr { - t.Errorf("readIPs() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("readNodeOutput() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(got, tt.want) { - t.Errorf("readIPs() got = %v, want %v", got, tt.want) + t.Errorf("readNodeOutput() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_parseNodeOutput(t *testing.T) { + tests := []struct { + name string + val any + wantIP string + wantPort string + wantErr bool + }{ + { + name: "string IP only", + val: "18.194.81.150", + wantIP: "18.194.81.150", + wantPort: "", + }, + { + name: "array with IP and port string", + val: []any{"18.194.81.150", "22"}, + wantIP: "18.194.81.150", + wantPort: "22", + }, + { + name: "array with IP and port number", + val: []any{"18.194.81.150", float64(22522)}, + wantIP: "18.194.81.150", + wantPort: "22522", + }, + { + name: "array with IP only", + val: []any{"18.194.81.150"}, + wantIP: "18.194.81.150", + wantPort: "", + }, + { + name: "empty array", + val: []any{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ip, port, err := parseNodeOutput(tt.val) + if (err != nil) != tt.wantErr { + t.Errorf("parseNodeOutput() error = %v, wantErr %v", err, tt.wantErr) + return + } + if ip != tt.wantIP { + t.Errorf("parseNodeOutput() ip = %v, want %v", ip, tt.wantIP) + } + if port != tt.wantPort { + t.Errorf("parseNodeOutput() port = %v, want %v", port, tt.wantPort) } }) } diff --git a/services/terraformer/internal/worker/service/internal/templates/structures.go b/services/terraformer/internal/worker/service/internal/templates/structures.go index 1886e8edc..7d80e2214 100644 --- a/services/terraformer/internal/worker/service/internal/templates/structures.go +++ b/services/terraformer/internal/worker/service/internal/templates/structures.go @@ -46,6 +46,9 @@ type ( // IsControl Specifies whether the nodepools is used as a control or compute nodepool within the cluster. // In the context of LB cluster, nodepools can only be compute or "worker" nodepools. IsControl bool + // SshPort is the SSH port used to connect to nodes in this nodepool. + // Old nodepools use 22, new nodepools use 22522. + SshPort int32 } ) @@ -174,6 +177,9 @@ type ( // for the firewall. // This data will be set if the ClusterType within ClusterData of this object is of type "LB". LBData LBData + // SshPorts holds the unique SSH ports used by the nodepools in this provider. + // Used to dynamically open firewall rules for the correct ports. + SshPorts []int32 } // Nodepools wraps all data related to generating terraform files to spawn VM instances as described @@ -242,24 +248,21 @@ type ( // All the following types grouped are passed in as "Outputs" from // using the generated template files. type ( - // NodepoolIPs wrap the output data that is acquired from using the - // generated template files. - NodepoolIPs struct { - // IPs holds the IPv4 addresses of the spawned VM instances from the - // generated templates files. It is expected that the template files - // in the nodepool directory that spawn the VM instances also - // expose the IP addresses of the Instances. - // For example (in the case of our own hetzner template files): - // - // output "{{ $nodepool.Name }}_{{ $specName }}_{{ $uniqueFingerPrint }}" { - // value = { - // {{- range $node := $nodepool.Nodes }} - // {{- $serverResourceName := printf "%s_%s" $node.Name $resourceSuffix }} - // "${hcloud_server.{{ $serverResourceName }}.name}" = hcloud_server.{{ $serverResourceName }}.ipv4_address - // {{- end }} - // } - //} - IPs map[string]any `json:"-"` + // NodepoolOutput wraps the output data that is acquired from using the + // generated template files. Each entry maps a node name to its output + // value, which can be a plain IP string or a [IP, port] array. + // For example (in the case of our own hetzner template files): + // + // output "{{ $nodepool.Name }}_{{ $specName }}_{{ $uniqueFingerPrint }}" { + // value = { + // {{- range $node := $nodepool.Nodes }} + // {{- $serverResourceName := printf "%s_%s" $node.Name $resourceSuffix }} + // "${hcloud_server.{{ $serverResourceName }}.name}" = hcloud_server.{{ $serverResourceName }}.ipv4_address, "{{ $nodepool.SshPort }}" + // {{- end }} + // } + //} + NodepoolOutput struct { + Nodes map[string]any `json:"-"` } // DNSDomain wrap the output data that is acquired from using the