diff --git a/build.gradle.kts b/build.gradle.kts index f3020143..5595abf1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -33,6 +33,7 @@ configurations.all { "com.google.code.gson:gson" -> useVersion("2.8.2") "org.jsoup:jsoup" -> useVersion("1.10.2") "com.jcraft:jzlib" -> useVersion("1.1.3") + "commons-codec:commons-codec" -> useVersion("1.11") } when (requested.group) { "org.jetbrains.kotlin" -> useVersion(kotlinVersion) @@ -43,8 +44,9 @@ configurations.all { } dependencies { - api("com.atlassian.performance.tools:infrastructure:[4.19.0,5.0.0)") - api("com.atlassian.performance.tools:aws-resources:[1.10.1, 2.0.0)") // 1.10.1 gives Ami.Builder.amiProvider + api(fileTree(mapOf("dir" to "lib", "include" to "*.jar"))) + api("com.atlassian.performance.tools:infrastructure:[4.12.2,5.0.0)") + api("com.atlassian.performance.tools:aws-resources:[1.1.1,2.0.0)") api("com.atlassian.performance.tools:jira-actions:[2.0.0,4.0.0)") api("com.atlassian.performance.tools:ssh:[2.4.1,3.0.0)") api("com.atlassian.performance.tools:virtual-users:[3.3.0,4.0.0)") diff --git a/lib/infrastructure-4.26.2-SNAPSHOT.jar b/lib/infrastructure-4.26.2-SNAPSHOT.jar new file mode 100644 index 00000000..222cbc93 Binary files /dev/null and b/lib/infrastructure-4.26.2-SNAPSHOT.jar differ diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/Infrastructure.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/Infrastructure.kt index 6b01f25f..0ec415ff 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/Infrastructure.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/Infrastructure.kt @@ -15,8 +15,8 @@ import java.net.URI import java.nio.file.Path import java.util.concurrent.Executors -class Infrastructure( - val virtualUsers: T, +class Infrastructure( + val virtualUsers: V, val jira: Jira, private val resultsTransport: Storage, val sshKey: SshKey diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/database/AsyncInstallHook.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/database/AsyncInstallHook.kt new file mode 100644 index 00000000..6009fa4e --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/database/AsyncInstallHook.kt @@ -0,0 +1,38 @@ +package com.atlassian.performance.tools.awsinfrastructure.api.database + +import com.atlassian.performance.tools.concurrency.api.submitWithLogContext +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode +import com.atlassian.performance.tools.infrastructure.api.jira.install.InstalledJira +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PostInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHook +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.ssh.api.SshConnection +import java.util.concurrent.Executors +import java.util.concurrent.Future + +/** + * Begins to run [hook] during [PreInstallHooks] and finishes during [PostInstallHooks] + */ +class AsyncInstallHook( + private val hook: PreInstallHook +) : PreInstallHook { + + override fun call(ssh: SshConnection, http: HttpNode, hooks: PreInstallHooks, reports: Reports) { + val thread = Executors.newSingleThreadExecutor() + val future = thread.submitWithLogContext("async-hook") { + hook.call(ssh, http, hooks, reports) + } + hooks.postInstall.insert(FutureHook(future)) + } +} + +private class FutureHook( + private val future: Future<*> +) : PostInstallHook { + + override fun call(ssh: SshConnection, jira: InstalledJira, hooks: PostInstallHooks, reports: Reports) { + future.get() + } +} diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/LegacyAwsInfrastructure.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/LegacyAwsInfrastructure.kt new file mode 100644 index 00000000..46e3c4be --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/LegacyAwsInfrastructure.kt @@ -0,0 +1,336 @@ +package com.atlassian.performance.tools.awsinfrastructure.api.jira + +import com.amazonaws.services.cloudformation.model.Parameter +import com.amazonaws.services.ec2.model.* +import com.amazonaws.services.ec2.model.Tag +import com.atlassian.performance.tools.aws.api.* +import com.atlassian.performance.tools.awsinfrastructure.TemplateBuilder +import com.atlassian.performance.tools.awsinfrastructure.api.hardware.C4EightExtraLargeElastic +import com.atlassian.performance.tools.awsinfrastructure.api.hardware.Computer +import com.atlassian.performance.tools.awsinfrastructure.api.hardware.M4ExtraLargeElastic +import com.atlassian.performance.tools.awsinfrastructure.api.hardware.Volume +import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.LoadBalancerFormula +import com.atlassian.performance.tools.awsinfrastructure.api.network.Network +import com.atlassian.performance.tools.awsinfrastructure.api.network.NetworkFormula +import com.atlassian.performance.tools.awsinfrastructure.api.network.ProvisionedNetwork +import com.atlassian.performance.tools.awsinfrastructure.api.network.access.* +import com.atlassian.performance.tools.awsinfrastructure.aws.TokenScrollingEc2 +import com.atlassian.performance.tools.infrastructure.api.jira.JiraNodeConfig +import com.atlassian.performance.tools.infrastructure.api.jira.install.HttpNode +import com.atlassian.performance.tools.infrastructure.api.jira.install.TcpNode +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PreStartHooks +import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer +import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancerPlan +import com.atlassian.performance.tools.infrastructure.api.network.HttpServerRoom +import com.atlassian.performance.tools.infrastructure.api.network.Networked +import com.atlassian.performance.tools.infrastructure.api.network.TcpServerRoom +import com.atlassian.performance.tools.ssh.api.Ssh +import com.atlassian.performance.tools.ssh.api.SshHost +import com.atlassian.performance.tools.workspace.api.RootWorkspace +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import java.nio.file.Path +import java.time.Duration +import java.util.* +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.function.Supplier + +/** + * @param [networking] all machines in this infra have access to the VPC CIDR + */ +class LegacyAwsInfrastructure private constructor( + private val aws: Aws, + private val investment: Investment, + private val networking: Supplier, + private val workspace: Path, + private val jiraComputer: Computer, + private val jiraVolume: Volume, + private val jiraNodeConfigs: List, + private val databaseComputer: Computer, + private val databaseVolume: Volume, + private val provisioningTimout: Duration +) : Networked, AutoCloseable { + private val logger: Logger = LogManager.getLogger(this::class.java) + private val nonce = UUID.randomUUID().toString() + private val resources: Queue = ConcurrentLinkedQueue() + private val sshKey: SshKey by lazy { provisionKey() } + private val network: Network by lazy { provisionNetwork() } + private val stack: ProvisionedStack by lazy { provisionStack() } + + private fun provisionKey(): SshKey { + val key = SshKeyFormula(aws.ec2, workspace, nonce, investment.lifespan).provision() + resources.add(key.remote) + return key + } + + private fun provisionNetwork(): Network { + val provisionedNetwork = networking.get() + resources.add(provisionedNetwork.resource) + return provisionedNetwork.network + } + + private fun provisionStack(): ProvisionedStack { + logger.info("Setting up Jira stack...") + val template = TemplateBuilder("2-nodes-dc-hooks.yaml").adaptTo(jiraNodeConfigs) + val stack = StackFormula( + investment = investment, + cloudformationTemplate = template, + parameters = listOf( + Parameter() + .withParameterKey("KeyName") + .withParameterValue(sshKey.remote.name), + Parameter() + .withParameterKey("InstanceProfile") + .withParameterValue(aws.shortTermStorageAccess()), + Parameter() + .withParameterKey("Ami") + .withParameterValue(aws.defaultAmi), + Parameter() + .withParameterKey("JiraInstanceType") + .withParameterValue(jiraComputer.instanceType.toString()), + Parameter() + .withParameterKey("JiraVolumeSize") + .withParameterValue(jiraVolume.size.toString()), + Parameter() + .withParameterKey("DatabaseInstanceType") + .withParameterValue(databaseComputer.instanceType.toString()), + Parameter() + .withParameterKey("DatabaseVolumeSize") + .withParameterValue(databaseVolume.size.toString()), + Parameter() + .withParameterKey("Vpc") + .withParameterValue(network.vpc.vpcId), + Parameter() + .withParameterKey("Subnet") + .withParameterValue(network.subnet.subnetId), + Parameter() + .withParameterKey("AccessCidr") + .withParameterValue(network.vpc.cidrBlock) + ), + aws = aws, + pollingTimeout = provisioningTimout + ).provision() + resources.add(stack) + logger.info("Jira stack is provisioned, it will expire at ${stack.expiry}") + return stack + } + + override fun close() { + CompositeResource(resources.toList()).release().get() + } + + override fun subnet(): String = network.subnet.cidrBlock + + val jiraNodesServerRoom: HttpServerRoom = StackJiraNodes() + val databaseServerRoom: TcpServerRoom = StackDatabase() + val sharedHomeServerRoom: TcpServerRoom = StackSharedHome() + val balancerServerRoom: HttpServerRoom = Ec2Balancer() + + private fun listMachines() = stack.listMachines() + + private inner class Ec2Balancer : HttpServerRoom { + + private val port = 80 + + override fun serveHttp(name: String): HttpNode { + logger.info("Setting up Apache load balancer...") + val ec2 = aws.ec2 + val securityGroup = aws.awaitingEc2.allocateSecurityGroup( + investment, + CreateSecurityGroupRequest() + .withGroupName("${investment.reuseKey()}-HttpListener") + .withDescription("Load balancer security group") + .withVpcId(network.vpc.vpcId) + ) + val (ssh, resource, instance) = aws.awaitingEc2.allocateInstance( + investment = investment, + key = sshKey, + vpcId = network.vpc.vpcId, + customizeLaunch = { launch -> + launch + .withInstanceInitiatedShutdownBehavior(ShutdownBehavior.Terminate) + .withSecurityGroupIds(securityGroup.groupId) + .withSubnetId(network.subnet.subnetId) + .withInstanceType(InstanceType.M5Large) + .withIamInstanceProfile(IamInstanceProfileSpecification().withName(aws.shortTermStorageAccess())) + } + ) + resources.add( + DependentResources( + user = resource, + dependency = Ec2SecurityGroup(securityGroup, ec2) + ) + ) + sshKey.file.facilitateSsh(ssh.host.ipAddress) + val accessToBalancer = SecurityGroupIngressAccessProvider + .Builder(ec2 = aws.ec2, securityGroup = securityGroup, portRange = port..port) + .build() + grantAccessFromVpc(accessToBalancer) + grantSelfAccess(accessToBalancer, instance) + grantAccessFromLocal(accessToBalancer) + val tcp = TcpNode( + publicIp = instance.publicIpAddress, + privateIp = instance.privateIpAddress, + port = port, + name = name, + ssh = ssh + ) + return HttpNode( + tcp = tcp, + basePath = "/", + supportsTls = false + ) + } + + private fun grantAccessFromVpc(accessToBalancer: AccessProvider) { + accessToBalancer.provideAccess(network.vpc.cidrBlock) + } + + /** + * This was missed when reporting and fixing JPERF-790. + * + * For Jira pre 8.9.0 if the instance has no access to its own HTTP the dashboard view may freeze + * (in our case it was after log in, however it may be related to dataset config). + * + * This access to self is described as required in the [setup documentation](https://confluence.atlassian.com/jirakb/configure-linux-firewall-for-jira-applications-741933610.html) + * and it was missed in implementation of aws-infrastructure 2.24.0 + * + * > 4 - Allowing connections to JIRA from itself (to ensure you don't run into problems with + * > [gadget titles showing as __MSG_gadget](https://confluence.atlassian.com/jirakb/fix-gadget-titles-showing-as-__msg_gadget-in-jira-server-813697086.html)) + * + * > ```iptables -t nat -I OUTPUT -p tcp -o lo --dport 80 -j REDIRECT --to-ports 8080``` + */ + private fun grantSelfAccess(accessToBalancer: AccessProvider, instance: Instance) { + accessToBalancer.provideAccess(instance.publicIpAddress.ipToCidr()) + } + + /** + * Grant access from local laptop to facilitate investigations when provisioning goes wrong. + */ + private fun grantAccessFromLocal(accessToBalancer: AccessProvider) { + val localIp = LocalPublicIpv4Provider.Builder().build().get() + accessToBalancer.provideAccess(localIp.ipToCidr()) + } + } + + private inner class StackSharedHome : TcpServerRoom { + + override fun serveTcp(name: String): TcpNode { + val machine = listMachines().single { it.tags.contains(Tag("jpt-shared-home", "true")) } + val publicIp = machine.publicIpAddress + val ssh = Ssh(SshHost(publicIp, "ubuntu", sshKey.file.path), connectivityPatience = 4) + sshKey.file.facilitateSsh(publicIp) + ssh.newConnection().use { jiraComputer.setUp(it) } + return TcpNode( + publicIp, + machine.privateIpAddress, + 3306, + name, + ssh + ) + } + + override fun serveTcp(name: String, tcpPorts: List, udpPorts: List): TcpNode { + val ports = "TCP $tcpPorts and UDP $udpPorts" + throw Exception( + "It's unclear whether $ports are expected to be open to the public or privately." + + " All ports are open within the VPC." + ) + } + } + + private inner class StackDatabase : TcpServerRoom { + + override fun serveTcp(name: String, tcpPorts: List, udpPorts: List): TcpNode { + if (tcpPorts.singleOrNull() == 3306 && udpPorts.isEmpty()) { + return serveTcp(name) + } else { + throw Exception("The stack is not prepared for TCP $tcpPorts and UDP $udpPorts") + } + } + + override fun serveTcp(name: String): TcpNode { + val machine = listMachines().single { it.tags.contains(Tag("jpt-database", "true")) } + val publicIp = machine.publicIpAddress + val ssh = Ssh(SshHost(publicIp, "ubuntu", sshKey.file.path), connectivityPatience = 5) + sshKey.file.facilitateSsh(publicIp) + ssh.newConnection().use { databaseComputer.setUp(it) } + return TcpNode( + publicIp, + machine.privateIpAddress, + 3306, + name, + ssh + ) + } + } + + private inner class StackJiraNodes : HttpServerRoom { + + private val machines by lazy { + listMachines().filter { it.tags.contains(Tag("jpt-jira", "true")) } + } + private var nodesRequested = 0 + + override fun serveHttp(name: String): HttpNode { + val machine = + machines[nodesRequested++] // TODO looks like a yikes, relies on sync across `List` and `List` + val publicIp = machine.publicIpAddress + val ssh = Ssh(SshHost(publicIp, "ubuntu", sshKey.file.path), connectivityPatience = 5) + sshKey.file.facilitateSsh(publicIp) + ssh.newConnection().use { jiraComputer.setUp(it) } + val tcp = TcpNode( + publicIp, + machine.privateIpAddress, + 8080, + name, + ssh + ) + return HttpNode(tcp, "/", false) + } + } + + class Builder( + private var aws: Aws, + private var investment: Investment + ) { + private var networking: Supplier = + Supplier { NetworkFormula(investment, aws).provisionAsResource() } + private var jiraNodeConfigs: List = listOf(1, 2).map { JiraNodeConfig.Builder().build() } + private var jiraComputer: Computer = C4EightExtraLargeElastic() + private var jiraVolume: Volume = Volume(100) + private var databaseComputer: Computer = M4ExtraLargeElastic() + private var databaseVolume: Volume = Volume(100) + private var provisioningTimout: Duration = Duration.ofMinutes(30) + private var workspace: Path = RootWorkspace().currentTask.isolateTest(investment.reuseKey()).directory + + fun aws(aws: Aws) = apply { this.aws = aws } + fun networking(networking: Supplier) = apply { this.networking = networking } + fun investment(investment: Investment) = apply { this.investment = investment } + fun jiraNodeConfigs(jiraNodeConfigs: List) = + apply { this.jiraNodeConfigs = jiraNodeConfigs } + + fun jiraComputer(jiraComputer: Computer) = apply { this.jiraComputer = jiraComputer } + fun jiraVolume(jiraVolume: Volume) = apply { this.jiraVolume = jiraVolume } + fun databaseComputer(databaseComputer: Computer) = apply { this.databaseComputer = databaseComputer } + fun databaseVolume(databaseVolume: Volume) = apply { this.databaseVolume = databaseVolume } + fun provisioningTimout(provisioningTimout: Duration) = + apply { this.provisioningTimout = provisioningTimout } + + fun workspace(workspace: Path) = apply { this.workspace = workspace } + + fun build(): LegacyAwsInfrastructure = LegacyAwsInfrastructure( + aws = aws, + networking = networking, + investment = investment, + jiraNodeConfigs = jiraNodeConfigs, + jiraComputer = jiraComputer, + jiraVolume = jiraVolume, + databaseComputer = databaseComputer, + databaseVolume = databaseVolume, + provisioningTimout = provisioningTimout, + workspace = workspace + ) + } +} + diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheEc2LoadBalancerFormula.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheEc2LoadBalancerFormula.kt index 8c078371..3eb8a16c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheEc2LoadBalancerFormula.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/loadbalancer/ApacheEc2LoadBalancerFormula.kt @@ -49,7 +49,7 @@ class ApacheEc2LoadBalancerFormula : LoadBalancerFormula { ) key.file.facilitateSsh(ssh.host.ipAddress) val loadBalancer = ApacheProxyLoadBalancer.Builder(ssh) - .nodes(instances.map { URI("http://${it.publicIpAddress}:8080/") }) + .nodes(instances.map { URI("http://${it.privateIpAddress}:8080/") }) .ipAddress(instance.publicIpAddress) .build() loadBalancer.provision() diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/network/access/ForIpAccessRequester.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/network/access/ForIpAccessRequester.kt index 1456bcd1..51bf5c97 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/network/access/ForIpAccessRequester.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/network/access/ForIpAccessRequester.kt @@ -4,4 +4,6 @@ import java.util.function.Supplier class ForIpAccessRequester( ipProvider: Supplier -) : AccessRequester by ForCidrAccessRequester(cidrProvider = Supplier { "${ipProvider.get()}/32" }) +) : AccessRequester by ForCidrAccessRequester(cidrProvider = Supplier { ipProvider.get().ipToCidr() }) + +internal fun String.ipToCidr() = "$this/32" \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/aws/TokenScrollingEc2.kt b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/aws/TokenScrollingEc2.kt new file mode 100644 index 00000000..f5763e12 --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/awsinfrastructure/aws/TokenScrollingEc2.kt @@ -0,0 +1,49 @@ +package com.atlassian.performance.tools.awsinfrastructure.aws + +import com.amazonaws.services.ec2.AmazonEC2 +import com.amazonaws.services.ec2.model.DescribeInstancesRequest +import com.amazonaws.services.ec2.model.Filter +import com.amazonaws.services.ec2.model.Instance +import com.amazonaws.services.ec2.model.InstanceStateName.* +import com.atlassian.performance.tools.aws.api.ScrollingEc2 + +/** + * Scrolls through batches of AWS EC2 instances using "page token". + */ +internal class TokenScrollingEc2( + private val ec2: AmazonEC2 +) : ScrollingEc2 { + override fun scrollThroughInstances( + vararg filters: Filter, + batchAction: (List) -> Unit + ) { + var token: String? = null + do { + val response = ec2.describeInstances( + DescribeInstancesRequest() + .withFilters(filters.toList()) + .withNextToken(token) + ) + val batch = response + .reservations + .flatMap { it.instances } + batchAction(batch) + token = response.nextToken + } while (token != null) + } + + override fun findInstances( + vararg filters: Filter + ): List { + val instances = mutableListOf() + scrollThroughInstances(*filters) { batch -> + instances += batch + } + return instances + } + + override fun allocated(): Filter = Filter( + "instance-state-name", + listOf(Pending, Running, ShuttingDown).map { it.toString() } + ) +} \ No newline at end of file diff --git a/src/main/resources/aws/2-nodes-dc-hooks.yaml b/src/main/resources/aws/2-nodes-dc-hooks.yaml new file mode 100644 index 00000000..e0c73a8a --- /dev/null +++ b/src/main/resources/aws/2-nodes-dc-hooks.yaml @@ -0,0 +1,189 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Serves a Jira Data Center cluster with Jira nodes, shared home and a MySQL database, without a load balancer +Parameters: + KeyName: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instance + Type: AWS::EC2::KeyPair::KeyName + ConstraintDescription: must be the name of an existing EC2 KeyPair. + InstanceProfile: + Type: String + Ami: + Type: String + JiraInstanceType: + Type: String + JiraVolumeSize: + Type: Number + DatabaseInstanceType: + Type: String + DatabaseVolumeSize: + Type: Number + Vpc: + Type: String + Subnet: + Type: String + AccessCidr: + Type: String +Resources: + jira1: + Type: AWS::EC2::Instance + Properties: + SecurityGroupIds: + - Ref: SshSecurityGroup + - Ref: JiraNodeSecurityGroup + InstanceType: !Ref JiraInstanceType + SubnetId: + Ref: Subnet + KeyName: + Ref: KeyName + ImageId: !Ref Ami + InstanceInitiatedShutdownBehavior: terminate + Tags: + - + Key: jpt-jira + Value: true + IamInstanceProfile: !Ref InstanceProfile + BlockDeviceMappings: + - + DeviceName: /dev/sda1 + Ebs: + VolumeSize: !Ref JiraVolumeSize + VolumeType: gp2 + jira2: + Type: AWS::EC2::Instance + Properties: + SecurityGroupIds: + - Ref: SshSecurityGroup + - Ref: JiraNodeSecurityGroup + InstanceType: !Ref JiraInstanceType + SubnetId: + Ref: Subnet + KeyName: + Ref: KeyName + ImageId: !Ref Ami + InstanceInitiatedShutdownBehavior: terminate + Tags: + - + Key: jpt-jira + Value: true + IamInstanceProfile: !Ref InstanceProfile + BlockDeviceMappings: + - + DeviceName: /dev/sda1 + Ebs: + VolumeSize: !Ref JiraVolumeSize + VolumeType: gp2 + SharedHome: + Type: AWS::EC2::Instance + Properties: + SecurityGroupIds: + - Ref: SshSecurityGroup + - Ref: SharedHomeSecurityGroup + InstanceType: !Ref JiraInstanceType + SubnetId: + Ref: Subnet + KeyName: + Ref: KeyName + ImageId: !Ref Ami + InstanceInitiatedShutdownBehavior: terminate + Tags: + - + Key: jpt-shared-home + Value: true + IamInstanceProfile: !Ref InstanceProfile + BlockDeviceMappings: + - + DeviceName: /dev/sda1 + Ebs: + VolumeSize: !Ref JiraVolumeSize + VolumeType: gp2 + Database: + Type: AWS::EC2::Instance + Properties: + InstanceType: !Ref DatabaseInstanceType + SubnetId: + Ref: Subnet + SecurityGroupIds: + - Ref: SshSecurityGroup + - Ref: DatabaseSecurityGroup + KeyName: + Ref: KeyName + ImageId: !Ref Ami + InstanceInitiatedShutdownBehavior: terminate + Tags: + - + Key: jpt-database + Value: true + IamInstanceProfile: !Ref InstanceProfile + BlockDeviceMappings: + - + DeviceName: /dev/sda1 + Ebs: + VolumeSize: !Ref DatabaseVolumeSize + VolumeType: gp2 + SshSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Ref: Vpc + GroupDescription: For SSH enabled instances + SecurityGroupIngress: + - + IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + JiraNodeSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Ref: Vpc + GroupDescription: For Jira DC nodes + SharedHomeSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Ref: Vpc + GroupDescription: For Jira DC shared home + DatabaseSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Ref: Vpc + GroupDescription: For Jira DC database + JiraNodeAccess: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !Ref JiraNodeSecurityGroup + IpProtocol: tcp + FromPort: 8080 + ToPort: 8080 + CidrIp: !Ref AccessCidr + CacheReplicationRmiPortAccess: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !Ref JiraNodeSecurityGroup + IpProtocol: tcp + # For cache replication we only need ports 40001 and 40011, + # however before Jira 7.4.0 the 2nd RMI port was always random and unconfigurable. + # We decided to not break compatibility with Jira 7.2 yet and keep every port between Jira nodes open. + # If the communication between nodes needs to be restricted, feel free to break the compatibility. + FromPort: 0 + ToPort: 65535 + CidrIp: !Ref AccessCidr + SharedHomeMountAccess: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !Ref SharedHomeSecurityGroup + IpProtocol: tcp + FromPort: 2049 + ToPort: 2049 + CidrIp: !Ref AccessCidr + MySqlPortAccess: + Type: AWS::EC2::SecurityGroupIngress + Properties: + GroupId: !Ref DatabaseSecurityGroup + IpProtocol: tcp + FromPort: 3306 + ToPort: 3306 + CidrIp: !Ref AccessCidr + # External access to Web App HTTP, JVM debug, JMX and Splunk forwarder ports added dynamically via `com.atlassian.performance.tools.awsinfrastructure.network.access.AccessProvider` diff --git a/src/main/resources/aws/2-nodes-dc-no-db.yaml b/src/main/resources/aws/2-nodes-dc-no-db.yaml new file mode 100644 index 00000000..8a86c6b0 --- /dev/null +++ b/src/main/resources/aws/2-nodes-dc-no-db.yaml @@ -0,0 +1,146 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Serves a Jira Data Center cluster with Jira nodes and shared home +Parameters: + KeyName: + Description: Name of an existing EC2 KeyPair to enable SSH access to the instance + Type: AWS::EC2::KeyPair::KeyName + ConstraintDescription: must be the name of an existing EC2 KeyPair. + InstanceProfile: + Type: String + Ami: + Type: String + JiraInstanceType: + Type: String + JiraVolumeSize: + Type: Number + Vpc: + Type: String + Subnet: + Type: String +Resources: + jira1: + Type: AWS::EC2::Instance + Properties: + SecurityGroupIds: + - Ref: SshSecurityGroup + - Ref: RmiSecurityGroup + - Ref: TomcatSecurityGroup + - Ref: SubnetSecurityGroup + InstanceType: !Ref JiraInstanceType + SubnetId: + Ref: Subnet + KeyName: + Ref: KeyName + ImageId: !Ref Ami + Tags: + - Key: jpt-jira + Value: true + IamInstanceProfile: !Ref InstanceProfile + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: !Ref JiraVolumeSize + VolumeType: gp2 + jira2: + Type: AWS::EC2::Instance + Properties: + SecurityGroupIds: + - Ref: SshSecurityGroup + - Ref: RmiSecurityGroup + - Ref: TomcatSecurityGroup + - Ref: SubnetSecurityGroup + InstanceType: !Ref JiraInstanceType + SubnetId: + Ref: Subnet + KeyName: + Ref: KeyName + ImageId: !Ref Ami + Tags: + - Key: jpt-jira + Value: true + IamInstanceProfile: !Ref InstanceProfile + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: !Ref JiraVolumeSize + VolumeType: gp2 + SharedHome: + Type: AWS::EC2::Instance + Properties: + SecurityGroupIds: + - Ref: SshSecurityGroup + - Ref: SubnetSecurityGroup + InstanceType: !Ref JiraInstanceType + SubnetId: + Ref: Subnet + KeyName: + Ref: KeyName + ImageId: !Ref Ami + Tags: + - Key: jpt-shared-home + Value: true + IamInstanceProfile: !Ref InstanceProfile + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: !Ref JiraVolumeSize + VolumeType: gp2 + SshSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Ref: Vpc + GroupDescription: Enables SSH access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: 0.0.0.0/0 + RmiSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Ref: Vpc + GroupDescription: Enables RMI access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 40001 + ToPort: 40001 + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: 40011 + ToPort: 40011 + CidrIp: 0.0.0.0/0 + TomcatSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Ref: Vpc + GroupDescription: Enables Tomcat HTTP access + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 8080 + ToPort: 8080 + CidrIp: 0.0.0.0/0 + MountTargetSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Ref: Vpc + GroupDescription: Security group for mount target + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 2049 + ToPort: 2049 + CidrIp: 0.0.0.0/0 + SubnetSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + VpcId: + Ref: Vpc + GroupDescription: Enable communication between nodes + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 0 + ToPort: 65535 + CidrIp: 10.0.0.0/24 diff --git a/src/main/resources/aws/2-nodes-dc.yaml b/src/main/resources/aws/2-nodes-dc.yaml index 706f0e2a..33785bb3 100644 --- a/src/main/resources/aws/2-nodes-dc.yaml +++ b/src/main/resources/aws/2-nodes-dc.yaml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: Serves a 2 node Jira Data Center cluster without a load balancer +Description: Serves a Jira Data Center cluster with Jira nodes, shared home and a MySQL database, without a load balancer Parameters: KeyName: Description: Name of an existing EC2 KeyPair to enable SSH access to the instance diff --git a/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/HooksDataCenterFormulaIT.kt b/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/HooksDataCenterFormulaIT.kt new file mode 100644 index 00000000..5ef65856 --- /dev/null +++ b/src/test/kotlin/com/atlassian/performance/tools/awsinfrastructure/api/jira/HooksDataCenterFormulaIT.kt @@ -0,0 +1,185 @@ +package com.atlassian.performance.tools.awsinfrastructure.api.jira + +import com.atlassian.performance.tools.aws.api.Investment +import com.atlassian.performance.tools.awsinfrastructure.IntegrationTestRuntime +import com.atlassian.performance.tools.awsinfrastructure.api.hardware.M5ExtraLargeEphemeral +import com.atlassian.performance.tools.awsinfrastructure.api.hardware.Volume +import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.ApacheEc2LoadBalancerFormula +import com.atlassian.performance.tools.infrastructure.api.database.DockerMysqlServer +import com.atlassian.performance.tools.infrastructure.api.dataset.HttpDatasetPackage +import com.atlassian.performance.tools.infrastructure.api.distribution.PublicJiraSoftwareDistribution +import com.atlassian.performance.tools.infrastructure.api.jira.JiraHomePackage +import com.atlassian.performance.tools.infrastructure.api.jira.JiraLaunchTimeouts +import com.atlassian.performance.tools.infrastructure.api.jira.install.ParallelInstallation +import com.atlassian.performance.tools.infrastructure.api.jira.install.hook.PreInstallHooks +import com.atlassian.performance.tools.infrastructure.api.jira.instance.JiraDataCenterPlan +import com.atlassian.performance.tools.infrastructure.api.jira.instance.JiraInstance +import com.atlassian.performance.tools.infrastructure.api.jira.instance.JiraNodePlan +import com.atlassian.performance.tools.infrastructure.api.jira.instance.PreInstanceHooks +import com.atlassian.performance.tools.infrastructure.api.jira.report.Reports +import com.atlassian.performance.tools.infrastructure.api.jira.sharedhome.NfsSharedHome +import com.atlassian.performance.tools.infrastructure.api.jira.start.JiraLaunchScript +import com.atlassian.performance.tools.infrastructure.api.jira.start.StartedJira +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHook +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.PostStartHooks +import com.atlassian.performance.tools.infrastructure.api.jira.start.hook.RestUpgrade +import com.atlassian.performance.tools.infrastructure.api.jvm.AdoptOpenJDK +import com.atlassian.performance.tools.infrastructure.api.loadbalancer.ApacheProxyPlan +import com.atlassian.performance.tools.ssh.api.SshConnection +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.SoftAssertions +import org.junit.Test +import java.net.URI +import java.nio.file.Files +import java.nio.file.Path +import java.time.Duration +import java.util.stream.Collectors +import kotlin.streams.toList + +class HooksDataCenterFormulaIT { + private val jiraVersion = "9.4.9" + private val workspace = IntegrationTestRuntime.taskWorkspace.isolateTest(javaClass.simpleName).directory + // it needs to be instantiated after IntegrationTestRuntime constructor + private val logger: Logger = LogManager.getLogger(this::class.java) + private val s3Bucket = URI("https://s3-eu-central-1.amazonaws.com/") + .resolve("jpt-custom-datasets-storage-a008820-datasetbucket-1nrja8d1upind/") + .resolve("dataset-a533e558-e5c5-46e7-9398-5aeda84d793a/") + private val mysql = HttpDatasetPackage( + uri = s3Bucket.resolve("database.tar.bz2"), + downloadTimeout = Duration.ofMinutes(6) + ) + private val jiraHome = JiraHomePackage( + HttpDatasetPackage( + uri = s3Bucket.resolve("jirahome.tar.bz2"), + downloadTimeout = Duration.ofMinutes(6) + ) + ) + private val infra: LegacyAwsInfrastructure = LegacyAwsInfrastructure.Builder( + IntegrationTestRuntime.aws, + Investment("Test Server provisioning hook API", Duration.ofMinutes(30)) + ) + .databaseComputer(M5ExtraLargeEphemeral()) + .databaseVolume(Volume(100)) + .workspace(workspace) + .build() + + private fun makeFailureObservable(block: () -> Unit) { + try { + block() + } catch (e: Exception) { + val potentialJiraProblems = findPotentialJiraProblems() + logger.error("Failed to provision DC. Report downloaded to $workspace") + e.addSuppressed(RuntimeException("Potential Jira problems found in workspace: $potentialJiraProblems")) + throw e + } + } + + @Test + fun shouldProvisionDc() { + // given + val database = DockerMysqlServer.Builder(infra.databaseServerRoom, mysql) + .source( + HttpDatasetPackage( + uri = s3Bucket.resolve("database.tar.bz2"), + downloadTimeout = Duration.ofMinutes(6) + ) + ) + .setPassword("admin", "admin") + .resetCaptcha("admin") + .build() + val startingNodeLogHook = object : PostStartHook { + override fun call(ssh: SshConnection, jira: StartedJira, hooks: PostStartHooks, reports: Reports) { + logger.info("Jira node is starting at ${jira.installed.http.addressPublicly()}") + } + } + val upgrade = RestUpgrade(JiraLaunchTimeouts.Builder().build(), "admin", "admin") + val installation = ParallelInstallation(jiraHome, PublicJiraSoftwareDistribution(jiraVersion), AdoptOpenJDK()) + val nodePlans = (1..2).map { + JiraNodePlan.Builder(infra.jiraNodesServerRoom) + .installation(installation) + .start(JiraLaunchScript()) + .hooks(PreInstallHooks.default().also { + it.postStart.insert(startingNodeLogHook) + it.postStart.insert(upgrade) + }) + .build() + } + val loadBalancer = ApacheProxyPlan(infra.balancerServerRoom) + val dcPlan = JiraDataCenterPlan.Builder(nodePlans, loadBalancer) + .instanceHooks( + PreInstanceHooks.default() + .also { it.insert(database) } + .also { it.insert(NfsSharedHome(jiraHome, infra.sharedHomeServerRoom, infra)) } + ) + .build() + + // when + var dataCenter: JiraInstance? = null + makeFailureObservable { + dataCenter = dcPlan.materialize() + } + // then + makeFailureObservable { + dataCenter!!.nodes.forEach { node -> + val installed = node.installed + val serverXml = installed + .installation + .resolve("conf/server.xml") + .download(Files.createTempFile("downloaded-config", ".xml")) + assertThat(serverXml.readText()).contains("") + }.assertAll() + } + + private fun findPotentialJiraProblems(): Map> { + val jiraLogFileName = "atlassian-jira.log" + val keywords = setOf("fatal", "error", "exception", "timeout", "waiting", "fail", "unable", "lock", "block") + + val result = mutableMapOf>() + val files = findFiles(workspace, jiraLogFileName) + + for (file in files) { + val filteredLines = filterFile(file, keywords) + result.putIfAbsent(file, mutableListOf()) + result[file]!!.addAll(filteredLines) + } + return result + } + + private fun findFiles(start: Path, fileName: String): List { + return Files.walk(start) + .filter { path -> Files.isRegularFile(path) && path.fileName.toString() == fileName } + .collect(Collectors.toList()) + } + + private fun filterFile(file: Path, keywords: Set): List { + return file.toFile().bufferedReader().lines().filter { line -> + keywords.any { keyword -> line.contains(keyword, ignoreCase = true) } + }.toList() + } + +}