diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 008b1a0..5c35162 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -58,9 +58,7 @@ Each call is measured from the app, round trip included. ## Reproduce -```bash -./scripts/stage-jdk-mobile.sh .jdk-mobile/stage --release jdk-mobile-27b35.3 -``` +The runtimes are jdk-mobile's latest release, which the plugin downloads. ```bash ./gradlew :examples:benchmark-facade:runHostBenchmark diff --git a/README.md b/README.md index 3e15f91..b94aba8 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,8 @@ kotlin { } nucleusJvm { facade(project(":search-facade")) - jdkMobile = layout.projectDirectory.dir("jdk-mobile") // see "jdk-mobile" below jvmOptions.addAll("-Xmx512m") + // The JVM's runtimes: jdk-mobile's latest release, downloaded (see "jdk-mobile" below) } ``` @@ -151,16 +151,24 @@ interpreter) suits occasional calls, not CPU-bound work; an empty call costs 2.4 ## jdk-mobile -The plugin reads jdk-mobile's runtimes from `nucleusJvm.jdkMobile`, one directory per target (`android-aarch64`, -`android-x64`, `ios-simulator-aarch64`, `ios-device-aarch64`), each with `static/`, `include/` and `lib/` (java.home). -[jdk-mobile](https://github.com/NucleusFramework/openjdk-mobile)'s releases carry all of them (from `jdk-mobile-27b35.3`, -whose Android runtimes include the static libraries built with `inprocess.patch`): +The JVM's runtimes are [jdk-mobile](https://github.com/NucleusFramework/openjdk-mobile)'s releases (from +`jdk-mobile-27b35.3`, whose Android runtimes include the static libraries built with `inprocess.patch`). The plugin +downloads those the module needs (the Android ABIs it ships, its iOS targets), checks them against the `.sha256` +published with them, keeps them in Gradle's cache (`~/.gradle/caches/nja/jdk-mobile`: a `clean` downloads nothing) and +extracts them into `build/nja/jdkMobile`; the tasks reading them depend on it (`njaJdkMobile`). -```bash -./scripts/stage-jdk-mobile.sh .jdk-mobile/stage --release jdk-mobile-27b35.3 +```kotlin +nucleusJvm { + jdkMobileRelease = "jdk-mobile-27b35.3" // default: latest (resolved once a day; offline, the last one) + jdkMobileRepository = "NucleusFramework/openjdk-mobile" // the default + jdkMobile = layout.projectDirectory.dir("my-jdk") // instead: runtimes of one's own, used as they are +} ``` -It downloads the four runtimes, checks their SHA-256 and lays them out. +With `latest`, a new jdk-mobile release reaches the next build: pin `jdkMobileRelease` for builds that must not move. +Runtimes of one's own are one directory per target (`android-aarch64`, `android-x64`, `ios-simulator-aarch64`, +`ios-device-aarch64`), each with `static/`, `include/` and `lib/` (java.home); `scripts/stage-jdk-mobile.sh` lays out +a local OpenJDK build that way. ### How jdk-mobile is built diff --git a/examples/benchmark/build.gradle.kts b/examples/benchmark/build.gradle.kts index 884987c..1e78200 100644 --- a/examples/benchmark/build.gradle.kts +++ b/examples/benchmark/build.gradle.kts @@ -36,6 +36,5 @@ kotlin { nucleusJvm { facade(project(":examples:benchmark-facade")) - jdkMobile = rootProject.layout.projectDirectory.dir(".jdk-mobile/stage") jvmOptions.addAll("-Xmx512m", "-Xss2m") } diff --git a/examples/calculator/build.gradle.kts b/examples/calculator/build.gradle.kts index df23c75..3d07b8a 100644 --- a/examples/calculator/build.gradle.kts +++ b/examples/calculator/build.gradle.kts @@ -42,6 +42,5 @@ nucleusJvm { facade(project(":examples:calculator-facade")) // A second façade in the same JVM: NJA's own stress tests facade(project(":examples:torture-facade")) - jdkMobile = rootProject.layout.projectDirectory.dir(".jdk-mobile/stage") jvmOptions.addAll("-Xss2m", "-Xmx512m") } diff --git a/plugin-build/plugin/src/main/kotlin/dev/nucleusframework/nja/plugin/NjaPlugin.kt b/plugin-build/plugin/src/main/kotlin/dev/nucleusframework/nja/plugin/NjaPlugin.kt index 5867b99..78470ce 100644 --- a/plugin-build/plugin/src/main/kotlin/dev/nucleusframework/nja/plugin/NjaPlugin.kt +++ b/plugin-build/plugin/src/main/kotlin/dev/nucleusframework/nja/plugin/NjaPlugin.kt @@ -5,6 +5,7 @@ import com.android.build.api.variant.AndroidComponentsExtension import com.android.build.api.variant.HasDeviceTests import dev.nucleusframework.nja.plugin.tasks.NjaAndroidLinkTask import dev.nucleusframework.nja.plugin.tasks.NjaAndroidRuntimeTask +import dev.nucleusframework.nja.plugin.tasks.NjaJdkMobileTask import dev.nucleusframework.nja.plugin.tasks.PACKAGES_FILE import org.gradle.api.Plugin import org.gradle.api.Project @@ -15,8 +16,10 @@ import org.gradle.api.attributes.Usage import org.gradle.api.attributes.java.TargetJvmEnvironment import org.gradle.api.file.DirectoryProperty import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property import org.gradle.api.provider.SetProperty import org.gradle.api.tasks.Copy +import org.gradle.api.tasks.TaskProvider import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType import org.jetbrains.kotlin.gradle.plugin.mpp.Executable @@ -29,9 +32,19 @@ import java.io.File import java.util.Properties abstract class NjaExtension(private val project: Project) { + /** + * The jdk-mobile release the runtimes come from: `latest` (the default) or a tag (`jdk-mobile-27b35.3`), downloaded + * from [jdkMobileRepository]'s releases, checked against their `.sha256` and kept in Gradle's cache. + */ + abstract val jdkMobileRelease: Property + + /** The GitHub repository of jdk-mobile's releases. */ + abstract val jdkMobileRepository: Property + /** * jdk-mobile's runtimes, one directory per target (`android-aarch64`, `android-x64`, `ios-simulator-aarch64`, - * `ios-device-aarch64`), each with `static/` (the VM and the JDK's natives), `include/` and `lib/` (java.home). + * `ios-device-aarch64`), each with `static/` (the VM and the JDK's natives), `include/` and `lib/` (java.home). By + * default the ones of [jdkMobileRelease], downloaded; set, a runtime of one's own, as it is. */ abstract val jdkMobile: DirectoryProperty @@ -57,6 +70,25 @@ class NjaPlugin : Plugin { val extension = project.extensions.create("nucleusJvm", NjaExtension::class.java, project) extension.androidAbis.convention(setOf("arm64-v8a")) extension.jvmOptions.convention(emptyList()) + extension.jdkMobileRelease.convention(NjaJdkMobileTask.LATEST) + extension.jdkMobileRepository.convention("NucleusFramework/openjdk-mobile") + val staged = project.layout.buildDirectory.dir("nja/jdkMobile") + extension.jdkMobile.convention(staged) + // One task per runtime, which the tasks reading it depend on: they download only when jdkMobile is the default + val stagedPath = staged.map { it.asFile.path } + val jdkMobileTasks = JDK_MOBILE_TARGETS.associateWith { target -> + project.tasks.register("njaJdkMobile${target.split('-').joinToString("") { it.replaceFirstChar(Char::uppercaseChar) }}", NjaJdkMobileTask::class.java) { + description = "Downloads and extracts jdk-mobile's runtime for $target." + repository.set(extension.jdkMobileRepository) + release.set(extension.jdkMobileRelease) + this.target.set(target) + cacheDir.set(project.gradle.gradleUserHomeDir.resolve("caches/nja/jdk-mobile")) + offline.set(project.gradle.startParameter.isOffline) + outputDir.set(staged.map { it.dir(target) }) + val chosen = extension.jdkMobile.map { it.asFile.path } + onlyIf("jdkMobile is the default (else a runtime of one's own)") { chosen.get() == stagedPath.get() } + } + } val facades = project.configurations.dependencyScope(FACADES).get() val clientSources = project.configurations.resolvable("njaClientSources") { @@ -73,12 +105,12 @@ class NjaPlugin : Plugin { kotlin.sourceSets.getByName("commonMain").kotlin.srcDir(clientSources.map { it.incoming.files }) project.dependencies.add("commonMainApi", "dev.nucleusframework.nja:nja-client:$NJA_VERSION") kotlin.targets.withType(KotlinNativeTarget::class.java).configureEach { - if (konanTarget.family == Family.IOS) configureIos(project, extension, classPath.get(), this) + if (konanTarget.family == Family.IOS) configureIos(project, extension, classPath.get(), this, jdkMobileTasks) } kotlin.targets.withType(KotlinJvmTarget::class.java).configureEach { configureJvm(project, this, facades, clientSources.get()) } } listOf("com.android.kotlin.multiplatform.library", "com.android.application", "com.android.library").forEach { id -> - project.pluginManager.withPlugin(id) { configureAndroid(project, extension, classPath.get()) } + project.pluginManager.withPlugin(id) { configureAndroid(project, extension, classPath.get(), jdkMobileTasks) } } } @@ -92,12 +124,20 @@ class NjaPlugin : Plugin { } } - private fun configureAndroid(project: Project, extension: NjaExtension, classPath: Configuration) { + private fun configureAndroid( + project: Project, + extension: NjaExtension, + classPath: Configuration, + jdkMobileTasks: Map>, + ) { val abiDirs = mapOf("arm64-v8a" to "android-aarch64", "x86_64" to "android-x64") + // The runtimes of the ABIs shipped + val staging = extension.androidAbis.map { abis -> abis.map { jdkMobileTasks.getValue(abiDirs[it] ?: error("NJA: unsupported ABI $it")) } } val runtimes = extension.androidAbis.zip(extension.jdkMobile) { abis, root -> abis.associateWith { abi -> root.asFile.resolve(abiDirs[abi] ?: error("NJA: unsupported ABI $abi")).path } } val runtime = project.tasks.register("njaAndroidRuntime", NjaAndroidRuntimeTask::class.java) { + dependsOn(staging) javaHomes.set(runtimes.map { m -> m.mapValues { File(it.value, "lib").path } }) javaHomeFiles.from(runtimes.map { m -> m.values.map { File(it, "lib") } }) this.classPath.from(classPath) @@ -114,6 +154,7 @@ class NjaPlugin : Plugin { } } val link = project.tasks.register("njaAndroidLink", NjaAndroidLinkTask::class.java) { + dependsOn(staging) this.runtimes.set(runtimes) runtimeFiles.from(runtimes.map { m -> m.values.flatMap { listOf(File(it, "static"), File(it, "include")) } }) bridgeSource.fileProvider(bridge.map { it.outputs.files.singleFile }) @@ -170,7 +211,13 @@ class NjaPlugin : Plugin { project.tasks.named(main.compileKotlinTaskName) { dependsOn(check) } } - private fun configureIos(project: Project, extension: NjaExtension, classPath: Configuration, target: KotlinNativeTarget) { + private fun configureIos( + project: Project, + extension: NjaExtension, + classPath: Configuration, + target: KotlinNativeTarget, + jdkMobileTasks: Map>, + ) { val dirName = when (target.konanTarget) { KonanTarget.IOS_SIMULATOR_ARM64 -> "ios-simulator-aarch64" KonanTarget.IOS_ARM64 -> "ios-device-aarch64" @@ -179,7 +226,9 @@ class NjaPlugin : Plugin { val runtime = extension.jdkMobile.dir(dirName) val name = target.name.replaceFirstChar { it.uppercaseChar() } // What the app's bundle carries: java.home (the VM sets it to /lib), the class path and options + val staging = jdkMobileTasks.getValue(dirName) val bundle = project.tasks.register("njaIosBundle$name", Copy::class.java) { + dependsOn(staging) into(project.layout.buildDirectory.dir("nja/ios/${target.name}")) // jdk-mobile's files are read-only: copies must stay writable to be refreshed filePermissions { user { read = true; write = true } } @@ -195,6 +244,7 @@ class NjaPlugin : Plugin { val binary = this // Read when the link task is configured: the jdk-mobile directory is set by the module's build script by then binary.linkTaskProvider.configure { + dependsOn(staging) val static = runtime.get().asFile.resolve("static") binary.linkerOpts += listOf("jvm", "java", "nio", "net", "zip", "jimage", "verify", "extnet", "fallbackLinker", "syslookup") .flatMap { listOf("-force_load", static.resolve("lib$it.a").path) } + @@ -217,5 +267,8 @@ class NjaPlugin : Plugin { companion object { const val FACADES = "njaFacades" + + /** jdk-mobile's targets, as its releases name their runtimes. */ + val JDK_MOBILE_TARGETS = listOf("android-aarch64", "android-x64", "ios-simulator-aarch64", "ios-device-aarch64") } } diff --git a/plugin-build/plugin/src/main/kotlin/dev/nucleusframework/nja/plugin/tasks/NjaJdkMobileTask.kt b/plugin-build/plugin/src/main/kotlin/dev/nucleusframework/nja/plugin/tasks/NjaJdkMobileTask.kt new file mode 100644 index 0000000..4663fed --- /dev/null +++ b/plugin-build/plugin/src/main/kotlin/dev/nucleusframework/nja/plugin/tasks/NjaJdkMobileTask.kt @@ -0,0 +1,153 @@ +package dev.nucleusframework.nja.plugin.tasks + +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.file.ArchiveOperations +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.FileSystemOperations +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.TaskAction +import org.gradle.work.DisableCachingByDefault +import java.io.File +import java.net.HttpURLConnection +import java.net.URI +import java.security.MessageDigest +import java.util.Properties +import javax.inject.Inject + +/** + * jdk-mobile's runtime for [target] (`android-aarch64`, `ios-simulator-aarch64`...), from a release of [repository]: + * [release] (`latest` by default), its archive checked against the `.sha256` published with it, kept in Gradle's cache + * ([cacheDir]) and extracted into [outputDir] in the layout the plugin reads (`static/`, `include/`, `lib/`). + * + * `latest` is resolved through the release page's redirect (no API, no rate limit) and kept a day; offline, the last + * one resolved. + */ +@DisableCachingByDefault(because = "Downloads; the archive is kept in Gradle's cache") +abstract class NjaJdkMobileTask : DefaultTask() { + @get:Inject abstract val archives: ArchiveOperations + @get:Inject abstract val files: FileSystemOperations + + @get:Input abstract val repository: Property + @get:Input abstract val release: Property + @get:Input abstract val target: Property + @get:Internal abstract val cacheDir: DirectoryProperty + @get:Internal abstract val offline: Property + @get:OutputDirectory abstract val outputDir: DirectoryProperty + + init { + // latest may have moved: checked every build (cheaply: the resolution is cached, the archive too) + outputs.upToDateWhen { false } + } + + @TaskAction + fun stage() { + val repo = repository.get() + val tag = resolve(repo, release.get()) + val asset = assetName(tag, target.get()) + val out = outputDir.get().asFile + val marker = File(out, ".nja-release") + if (marker.isFile && marker.readText() == "$repo $tag") return + + val archive = download(repo, tag, asset) + files.sync { + from(archives.tarTree(archive)) + into(out) + // jdk-mobile's files are read-only: the next sync must be able to replace them + filePermissions { user { read = true; write = true } } + } + marker.writeText("$repo $tag") + logger.lifecycle("nja: ${target.get()} from $repo $tag") + } + + private fun resolve(repo: String, release: String): String { + if (release != LATEST) return release + val cache = File(cacheDir.get().asFile, "latest-${repo.replace('/', '_')}.properties") + val known = Properties().apply { if (cache.isFile) cache.reader().use { load(it) } } + val knownTag = known.getProperty("tag") + val fresh = System.currentTimeMillis() - (known.getProperty("at")?.toLongOrNull() ?: 0L) < DAY_MILLIS + if (knownTag != null && (fresh || offline.get())) return knownTag + if (offline.get()) throw GradleException("NJA: offline, and $repo's latest jdk-mobile release was never resolved") + val tag = try { + latestTag(repo) + } catch (e: Exception) { + // The network down: the last one resolved, if any + knownTag?.also { logger.warn("nja: can't resolve $repo's latest release (${e.message}), using $it") } ?: throw e + } + cache.parentFile.mkdirs() + Properties().apply { + setProperty("tag", tag) + setProperty("at", System.currentTimeMillis().toString()) + cache.writer().use { store(it, null) } + } + return tag + } + + // github.com//releases/latest redirects to .../releases/tag/ + private fun latestTag(repo: String): String { + val connection = URI("https://github.com/$repo/releases/latest").toURL().openConnection() as HttpURLConnection + connection.instanceFollowRedirects = false + connection.requestMethod = "HEAD" + try { + val location = connection.getHeaderField("Location") + ?: throw GradleException("NJA: no latest release of $repo (HTTP ${connection.responseCode})") + return location.substringAfterLast("/releases/tag/").takeIf { it != location && it.isNotEmpty() } + ?: throw GradleException("NJA: unexpected latest release of $repo: $location") + } finally { + connection.disconnect() + } + } + + // In Gradle's cache, by release: checked against the .sha256 published with it + private fun download(repo: String, tag: String, asset: String): File { + val dir = File(cacheDir.get().asFile, "$repo/$tag").apply { mkdirs() } + val archive = File(dir, asset) + val checksum = File(dir, "$asset.sha256") + val base = "https://github.com/$repo/releases/download/$tag/$asset" + if (!checksum.isFile) fetch("$base.sha256", checksum) + val expected = checksum.readText().trim().substringBefore(' ').lowercase() + if (archive.isFile && sha256(archive) == expected) return archive + if (offline.get()) throw GradleException("NJA: offline, and $asset of $tag isn't in Gradle's cache") + fetch(base, archive) + val actual = sha256(archive) + if (actual != expected) { + archive.delete() + throw GradleException("NJA: unexpected $asset from $base (SHA-256 $actual, its .sha256 says $expected)") + } + return archive + } + + private fun fetch(url: String, file: File) { + val partial = File(file.parentFile, "${file.name}.part") + URI(url).toURL().openStream().use { input -> partial.outputStream().use { input.copyTo(it) } } + if (!partial.renameTo(file)) throw GradleException("NJA: can't write $file") + } + + private fun sha256(file: File): String { + val digest = MessageDigest.getInstance("SHA-256") + file.inputStream().use { input -> + val buffer = ByteArray(1 shl 16) + while (true) { + val read = input.read(buffer) + if (read < 0) break + digest.update(buffer, 0, read) + } + } + return digest.digest().joinToString("") { "%02x".format(it) } + } + + companion object { + const val LATEST = "latest" + private const val DAY_MILLIS = 24L * 60 * 60 * 1000 + + /** jdk-mobile-b. -> jdk-mobile---b..tar.gz */ + fun assetName(tag: String, target: String): String { + val match = Regex("^jdk-mobile-(.+?)b([0-9]+\\.[0-9]+)$").matchEntire(tag) + ?: throw GradleException("NJA: $tag isn't a jdk-mobile release (jdk-mobile-b.)") + return "jdk-mobile-${match.groupValues[1]}-$target-b${match.groupValues[2]}.tar.gz" + } + } +}