From 7abd49d194e16e3d80f80c525665ec2ae0b40b89 Mon Sep 17 00:00:00 2001 From: Marcin Masiorski Date: Tue, 21 Jan 2020 07:07:12 +0100 Subject: [PATCH 1/7] HL-1205 Postgres support --- .../api/database/PostgresDatabase.kt | 49 +++++++++++++++++++ .../api/dataset/FileArchiver.kt | 6 ++- 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt new file mode 100644 index 00000000..351189ce --- /dev/null +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt @@ -0,0 +1,49 @@ +package com.atlassian.performance.tools.infrastructure.api.database + +import com.atlassian.performance.tools.infrastructure.DockerImage +import com.atlassian.performance.tools.infrastructure.api.dataset.DatasetPackage +import com.atlassian.performance.tools.ssh.api.SshConnection +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import java.net.URI +import java.time.Duration + +class PostgresDatabase( + private val source: DatasetPackage, + private val maxConnections: Int, + private val dataBaseVersion: String +) : Database { + private val logger: Logger = LogManager.getLogger(this::class.java) + + private val image: DockerImage = DockerImage( + name = "postgres:$dataBaseVersion", + pullTimeout = Duration.ofMinutes(5) + ) + + constructor( + source: DatasetPackage, + dataBaseVersion: String = "9.6" + ) : this( + source = source, + maxConnections = 200, + dataBaseVersion = dataBaseVersion + ) + + override fun setup(ssh: SshConnection): String { + val pgData = source.download(ssh) + val containerName = image.run( + ssh = ssh, + parameters = "-p 3306:5432 -v `realpath $pgData`:/var/lib/postgresql/data", + arguments = "-c 'listen_addresses='*'' -c 'max_connections=$maxConnections'" + ) + Thread.sleep(Duration.ofSeconds(15).toMillis()) + return pgData + } + + override fun start(jira: URI, ssh: SshConnection) { + // TODO Check logs for the following entry + // LOG: database system is ready to accept connections + Thread.sleep(Duration.ofSeconds(15).toMillis()) + } + +} \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/FileArchiver.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/FileArchiver.kt index e42fb2a0..e1244b29 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/FileArchiver.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/FileArchiver.kt @@ -18,7 +18,11 @@ class FileArchiver { ) { ubuntu.install(connection, listOf("lbzip2")) time("unzip") { - connection.execute("tar -I lbzip2 -xf $archive", timeout) + val cmd = if (archive.contains("SCALED_ISSUES_UNIMODAL_PG")) + "tar -xzvf $archive" + else + "tar -I lbzip2 -xf $archive" + connection.execute(cmd, timeout) } } From 0687e30a6623e431cec4ad764de3fe76a37189fc Mon Sep 17 00:00:00 2001 From: Marcin Masiorski Date: Mon, 3 Feb 2020 10:52:53 +0100 Subject: [PATCH 2/7] HL-1205 Refactoring postgres support --- .../tools/infrastructure/api/database/PostgresDatabase.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt index 351189ce..01ddd90a 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt @@ -31,12 +31,11 @@ class PostgresDatabase( override fun setup(ssh: SshConnection): String { val pgData = source.download(ssh) - val containerName = image.run( + image.run( ssh = ssh, parameters = "-p 3306:5432 -v `realpath $pgData`:/var/lib/postgresql/data", arguments = "-c 'listen_addresses='*'' -c 'max_connections=$maxConnections'" ) - Thread.sleep(Duration.ofSeconds(15).toMillis()) return pgData } From eafddcef50da948d144691ff6fed373181f443e4 Mon Sep 17 00:00:00 2001 From: Marcin Masiorski Date: Mon, 3 Feb 2020 10:55:05 +0100 Subject: [PATCH 3/7] HL-1205 Changelog update --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2895959..77b94759 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,13 @@ Adding a requirement of a major version of a dependency is breaking a contract. Dropping a requirement of a major version of a dependency is a new contract. ## [Unreleased] -[Unreleased]: https://github.com/atlassian/infrastructure/compare/release-4.14.4...master +[Unreleased]: https://github.com/atlassian/infrastructure/compare/release-4.14.5...master + +## [4.14.5] - 2020-02-03 +[4.14.5]: https://github.com/atlassian/infrastructure/compare/release-4.14.4...release-4.14.5 + +### Added +- Add support for postgres database. ## [4.14.4] - 2019-12-09 [4.14.4]: https://github.com/atlassian/infrastructure/compare/release-4.14.3...release-4.14.4 From 0fdbc058664ef25d97ca7256af844e95ec353bb5 Mon Sep 17 00:00:00 2001 From: Marcin Masiorski Date: Tue, 4 Feb 2020 08:47:51 +0100 Subject: [PATCH 4/7] HL-1205 Fix unzip --- .../tools/infrastructure/api/dataset/FileArchiver.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/FileArchiver.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/FileArchiver.kt index e1244b29..95da1d80 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/FileArchiver.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/dataset/FileArchiver.kt @@ -39,7 +39,11 @@ class FileArchiver { ) { ubuntu.install(connection, listOf("lbzip2")) time("unzip") { - connection.execute("tar -I lbzip2 -xf $archive -C $destination", timeout).output.splitToSequence("\n").asIterable() + val cmd = if (archive.contains("SCALED_ISSUES_UNIMODAL_PG")) + "tar -xzvf $archive -C $destination" + else + "tar -I lbzip2 -xf $archive -C $destination" + connection.execute(cmd, timeout).output.splitToSequence("\n").asIterable() } } From 8659d8f0dbae69f64fa6a62b05877721df8219d2 Mon Sep 17 00:00:00 2001 From: Marcin Masiorski Date: Thu, 13 Feb 2020 09:28:12 +0100 Subject: [PATCH 5/7] HL-1205 PR comments --- CHANGELOG.md | 1 - .../tools/infrastructure/api/database/PostgresDatabase.kt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b94759..57843860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,6 @@ Dropping a requirement of a major version of a dependency is a new contract. ## [Unreleased] [Unreleased]: https://github.com/atlassian/infrastructure/compare/release-4.14.5...master -## [4.14.5] - 2020-02-03 [4.14.5]: https://github.com/atlassian/infrastructure/compare/release-4.14.4...release-4.14.5 ### Added diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt index 01ddd90a..987f320d 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt @@ -22,7 +22,7 @@ class PostgresDatabase( constructor( source: DatasetPackage, - dataBaseVersion: String = "9.6" + dataBaseVersion: String ) : this( source = source, maxConnections = 200, From ed64ea77e7c4d6688308f91d8033b6ad223648f7 Mon Sep 17 00:00:00 2001 From: Marcin Masiorski Date: Thu, 13 Feb 2020 14:00:31 +0100 Subject: [PATCH 6/7] HL-1205 Changelog update --- CHANGELOG.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57843860..d2895959 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,12 +24,7 @@ Adding a requirement of a major version of a dependency is breaking a contract. Dropping a requirement of a major version of a dependency is a new contract. ## [Unreleased] -[Unreleased]: https://github.com/atlassian/infrastructure/compare/release-4.14.5...master - -[4.14.5]: https://github.com/atlassian/infrastructure/compare/release-4.14.4...release-4.14.5 - -### Added -- Add support for postgres database. +[Unreleased]: https://github.com/atlassian/infrastructure/compare/release-4.14.4...master ## [4.14.4] - 2019-12-09 [4.14.4]: https://github.com/atlassian/infrastructure/compare/release-4.14.3...release-4.14.4 From beab8586f4cc114c1ecdfa85c7b45c9c0c6a4a44 Mon Sep 17 00:00:00 2001 From: Marcin Masiorski Date: Tue, 18 Feb 2020 09:07:06 +0100 Subject: [PATCH 7/7] HL-1205 Return database type in order to distinguish databases --- .../tools/infrastructure/api/database/Database.kt | 5 +++++ .../infrastructure/api/database/LicenseOverridingMysql.kt | 5 ++--- .../tools/infrastructure/api/database/MySqlDatabase.kt | 2 ++ .../tools/infrastructure/api/database/PostgresDatabase.kt | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/Database.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/Database.kt index 85871586..dc958c91 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/Database.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/Database.kt @@ -14,4 +14,9 @@ interface Database { fun setup(ssh: SshConnection): String fun start(jira: URI, ssh: SshConnection) + + /** + * @return Database type e.g. mysql + */ + fun type(): String } diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/LicenseOverridingMysql.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/LicenseOverridingMysql.kt index e159901b..0a491e5c 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/LicenseOverridingMysql.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/LicenseOverridingMysql.kt @@ -7,9 +7,6 @@ import org.apache.logging.log4j.Logger import java.io.File import java.net.URI import java.nio.file.Files -import java.io.FileWriter -import java.io.BufferedWriter - /** * Removes all licenses from [database] and adds [licenses] instead. @@ -80,6 +77,8 @@ class LicenseOverridingMysql private constructor( ) } } + + override fun type(): String = "mysql" } internal fun createTempLicenseFile(license: String): File { diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MySqlDatabase.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MySqlDatabase.kt index 66bed4d1..9e678d84 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MySqlDatabase.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/MySqlDatabase.kt @@ -61,4 +61,6 @@ class MySqlDatabase( Thread.sleep(Duration.ofSeconds(10).toMillis()) } } + + override fun type(): String = "mysql" } \ No newline at end of file diff --git a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt index 987f320d..e4649aa3 100644 --- a/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt +++ b/src/main/kotlin/com/atlassian/performance/tools/infrastructure/api/database/PostgresDatabase.kt @@ -45,4 +45,5 @@ class PostgresDatabase( Thread.sleep(Duration.ofSeconds(15).toMillis()) } + override fun type(): String = "postgres" } \ No newline at end of file