From 17ff789504c585719dd551ff37a09c25f0e8a4a4 Mon Sep 17 00:00:00 2001 From: Minseok Kim Date: Tue, 11 Aug 2026 11:35:51 +0900 Subject: [PATCH] Serve the JDBC metastore's rows again, for as long as it exists #418 removed `MetastoreInspector` and its dump endpoints as dead code, which they were. Something calls them now: `jdbc-metastore remaining` (#487) measures how much room each metadata scan window has left, and this is the only view of the rows such a scan actually reads. A `DELETE` rewrites its row with `Active.INACTIVE` instead of removing it, and `scanStorage` applies its `LIMIT` before anything filters on active, so every path above this one hides exactly what needs measuring. Scaffolding for the move to EdgeIndex, so it is named for the store it reads - `JdbcMetastoreInspector`, not a name claiming the general concept - and goes when that store does. Narrower than what was removed. No `prefix`: it built `k LIKE '%'` unescaped over base64url keys, whose `_` is a LIKE wildcard. No `sort`: `id ASC` is the primary key, so paging walks an index that already exists. No `/local`: no backing store for it on `main`. The row is `id`, `k` and four decoded fields. The request shape still matches 0.4.x, so one client reads both. Blocking JDBC work goes to `boundedElastic`, which the removed version did not do. Co-Authored-By: Claude Opus 5 (1M context) --- .../metastore/JdbcMetastoreInspector.kt | 88 ++++++++++++++ .../metastore/JdbcMetastoreInspectorTest.kt | 110 ++++++++++++++++++ .../v2/metastore/JdbcMetastoreController.kt | 35 ++++++ .../configuration/GraphConfiguration.kt | 4 + .../filter/ReadOnlyRequestFilterTest.kt | 1 + 5 files changed, 238 insertions(+) create mode 100644 engine/src/main/kotlin/com/kakao/actionbase/v2/engine/metastore/JdbcMetastoreInspector.kt create mode 100644 engine/src/test/kotlin/com/kakao/actionbase/v2/engine/metastore/JdbcMetastoreInspectorTest.kt create mode 100644 server/src/main/kotlin/com/kakao/actionbase/server/api/graph/v2/metastore/JdbcMetastoreController.kt diff --git a/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/metastore/JdbcMetastoreInspector.kt b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/metastore/JdbcMetastoreInspector.kt new file mode 100644 index 000000000..10f0b536c --- /dev/null +++ b/engine/src/main/kotlin/com/kakao/actionbase/v2/engine/metastore/JdbcMetastoreInspector.kt @@ -0,0 +1,88 @@ +package com.kakao.actionbase.v2.engine.metastore + +import com.kakao.actionbase.v2.core.code.DecodedEdge +import com.kakao.actionbase.v2.core.code.KeyValue +import com.kakao.actionbase.v2.engine.GraphDefaults +import com.kakao.actionbase.v2.engine.storage.jdbc.MetadataTable + +import org.jetbrains.exposed.sql.Database +import org.jetbrains.exposed.sql.SortOrder +import org.jetbrains.exposed.sql.selectAll +import org.jetbrains.exposed.sql.transactions.transaction + +import reactor.core.publisher.Mono +import reactor.core.scheduler.Schedulers + +/** + * Reads the JDBC metastore's rows as they sit on disk, tombstones included. No read path above this + * one shows them, and a scan key crowded with tombstones truncates silently. + */ +class JdbcMetastoreInspector( + private val metastore: Database, + private val table: MetadataTable, +) { + /** Ordered by `id`, the primary key, so paging walks an index that already exists. */ + fun dump( + limit: Int, + offset: Long, + ): Mono> = + Mono + .fromCallable { + transaction(metastore) { + table + .selectAll() + .orderBy(table.id, SortOrder.ASC) + .limit(limit, offset) + .map { row -> + JdbcMetastoreRow( + id = row[table.id].value, + k = row[table.k], + decoded = decode(row[table.k], row[table.v]), + ) + } + } + }.subscribeOn(Schedulers.boundedElastic()) + + fun count(): Mono = + Mono + .fromCallable { transaction(metastore) { table.selectAll().count() } } + .subscribeOn(Schedulers.boundedElastic()) + + // Null when the codec cannot read the row, which still fills its window and is still reported. + // Exception rather than runCatching, which would also swallow an Error and pass a broken JVM off + // as one more undecodable row. + private fun decode( + k: String, + v: String, + ): JdbcMetastoreEdge? = + try { + val edge = DecodedEdge.fromMetastore(KeyValue(k, v), emptyMap()) + JdbcMetastoreEdge(active = edge.isActive, src = edge.src, tgt = edge.tgt, labelId = edge.labelId) + } catch (e: Exception) { + null + } + + companion object { + // From the graph, not its parts: the server module cannot name Exposed's Database. + fun of(graph: GraphDefaults): JdbcMetastoreInspector = JdbcMetastoreInspector(graph.metastore, graph.metadataTable) + } +} + +/** `v` is not carried: it is the largest column and nothing reads it. */ +data class JdbcMetastoreRow( + val id: Long, + val k: String, + val decoded: JdbcMetastoreEdge?, +) + +/** + * `src` and `labelId` are the scan key `DdlService.getAll` builds, so they name the window whose + * limit is at stake; `active` says whether the row is a tombstone. Nothing else is needed. `src` and + * `tgt` stay `Any?` because the codec types them as `Object`. + */ +data class JdbcMetastoreEdge( + val active: Boolean, + val src: Any?, + val tgt: Any?, + val labelId: Int, +) diff --git a/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/metastore/JdbcMetastoreInspectorTest.kt b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/metastore/JdbcMetastoreInspectorTest.kt new file mode 100644 index 000000000..6ad97b49f --- /dev/null +++ b/engine/src/test/kotlin/com/kakao/actionbase/v2/engine/metastore/JdbcMetastoreInspectorTest.kt @@ -0,0 +1,110 @@ +package com.kakao.actionbase.v2.engine.metastore + +import com.kakao.actionbase.v2.engine.Graph +import com.kakao.actionbase.v2.engine.entity.EntityName +import com.kakao.actionbase.v2.engine.service.ddl.ServiceCreateRequest +import com.kakao.actionbase.v2.engine.service.ddl.ServiceDeleteRequest +import com.kakao.actionbase.v2.engine.service.ddl.ServiceUpdateRequest +import com.kakao.actionbase.v2.engine.storage.jdbc.BaseTableConstants +import com.kakao.actionbase.v2.engine.test.GraphFixtures + +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +import org.jetbrains.exposed.sql.insert +import org.jetbrains.exposed.sql.transactions.transaction +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class JdbcMetastoreInspectorTest { + private lateinit var graph: Graph + private lateinit var inspector: JdbcMetastoreInspector + + @BeforeEach + fun setup() { + // withTestData: a bare fixture graph keeps its metadata in the local store, leaving the + // JDBC table empty. + graph = GraphFixtures.create(withTestData = true) + inspector = JdbcMetastoreInspector(graph.metastore, graph.metadataTable) + } + + @AfterEach + fun teardown() { + graph.close() + } + + private fun dump( + limit: Int, + offset: Long, + ): List = inspector.dump(limit, offset).block()!! + + @Test + fun `counts every row the metastore holds`() { + val count = inspector.count().block()!! + val all = dump(count.toInt() + 10, 0) + + assertTrue(count > 0, "the fixture writes metadata to the metastore") + assertEquals(count, all.size.toLong(), "the count and the rows must agree") + } + + @Test + fun `pages without dropping or repeating a row`() { + val all = dump(1000, 0) + val paged = (0 until all.size step 2).flatMap { dump(2, it.toLong()) } + + assertEquals(all.map { it.id }, paged.map { it.id }, "paging must cover the table exactly once") + } + + @Test + fun `orders by id ascending, whatever the caller asks`() { + val ids = dump(1000, 0).map { it.id } + + assertEquals(ids.sorted(), ids, "id ASC is the only order this serves") + } + + @Test + fun `decodes a row into the window it belongs to`() { + val decoded = dump(1000, 0).mapNotNull { it.decoded } + + assertTrue(decoded.isNotEmpty(), "metadata rows decode") + assertTrue(decoded.all { it.src != null }, "src names the scan window and must survive the decode") + assertTrue(decoded.any { it.active }, "a bootstrapped graph has live metadata") + } + + @Test + fun `keeps a row the codec cannot read, so it still occupies its window`() { + transaction(graph.metastore) { + graph.metadataTable.insert { + it[k] = "not-an-encoded-key" + it[v] = "not-an-encoded-value" + it[createdBy] = javaClass.canonicalName.take(BaseTableConstants.MAX_LENGTH) + it[modifiedBy] = javaClass.canonicalName.take(BaseTableConstants.MAX_LENGTH) + } + } + + val undecodable = dump(1000, 0).single { it.k == "not-an-encoded-key" } + + assertNull(undecodable.decoded, "the codec cannot read it") + assertNotNull(undecodable.k, "but the row is still reported, because it still fills a window") + } + + /** The premise: DELETE rewrites the row with `Active.INACTIVE` rather than removing it. */ + @Test + fun `reports a deleted row as a tombstone still occupying its window`() { + val name = EntityName.fromOrigin("doomed") + graph.serviceDdl.create(name, ServiceCreateRequest(desc = "doomed")).block() + val before = inspector.count().block()!! + + graph.serviceDdl.update(name, ServiceUpdateRequest(active = false, desc = null)).block() + graph.serviceDdl.delete(name, ServiceDeleteRequest()).block() + + val row = dump(1000, 0).single { it.decoded?.tgt == "doomed" } + assertEquals(false, row.decoded?.active, "a deleted row is a tombstone, not a removal") + assertEquals(before, inspector.count().block()!!, "and it still takes up its place in the table") + } +} diff --git a/server/src/main/kotlin/com/kakao/actionbase/server/api/graph/v2/metastore/JdbcMetastoreController.kt b/server/src/main/kotlin/com/kakao/actionbase/server/api/graph/v2/metastore/JdbcMetastoreController.kt new file mode 100644 index 000000000..53a939983 --- /dev/null +++ b/server/src/main/kotlin/com/kakao/actionbase/server/api/graph/v2/metastore/JdbcMetastoreController.kt @@ -0,0 +1,35 @@ +package com.kakao.actionbase.server.api.graph.v2.metastore + +import com.kakao.actionbase.v2.engine.metastore.JdbcMetastoreInspector +import com.kakao.actionbase.v2.engine.metastore.JdbcMetastoreRow + +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.Pageable +import org.springframework.data.web.PageableDefault +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RestController + +import reactor.core.publisher.Mono + +/** + * Pages the JDBC metastore's rows, tombstones included, so `jdbc-metastore remaining` can measure how + * much room each scan window has left. + * + * Narrower than the endpoint #418 removed: no `prefix` (it built `k LIKE '%'` unescaped over + * base64url keys, whose `_` is a LIKE wildcard), no `sort`, no `/local`. The request shape still + * matches 0.4.x so one client reads both. + */ +@RestController +class JdbcMetastoreController( + private val metastoreInspector: JdbcMetastoreInspector, +) { + @GetMapping("/graph/v2/metastore/global") + fun metastoreGlobal( + @PageableDefault(size = 100, page = 0) pageable: Pageable, + ): Mono> = + metastoreInspector + .dump(pageable.pageSize, pageable.offset) + .zipWith(metastoreInspector.count()) + .map { PageImpl(it.t1, pageable, it.t2) } +} diff --git a/server/src/main/kotlin/com/kakao/actionbase/server/configuration/GraphConfiguration.kt b/server/src/main/kotlin/com/kakao/actionbase/server/configuration/GraphConfiguration.kt index 012b6668a..6f4a53a5e 100644 --- a/server/src/main/kotlin/com/kakao/actionbase/server/configuration/GraphConfiguration.kt +++ b/server/src/main/kotlin/com/kakao/actionbase/server/configuration/GraphConfiguration.kt @@ -12,6 +12,7 @@ import com.kakao.actionbase.v2.engine.cdc.CdcFactory import com.kakao.actionbase.v2.engine.cdc.DefaultCdcFactory import com.kakao.actionbase.v2.engine.client.kafka.KafkaClientFactory import com.kakao.actionbase.v2.engine.client.web.WebClientFactory +import com.kakao.actionbase.v2.engine.metastore.JdbcMetastoreInspector import com.kakao.actionbase.v2.engine.util.getLogger import com.kakao.actionbase.v2.engine.v3.V2BackedEngine import com.kakao.actionbase.v2.engine.wal.DefaultWalFactory @@ -133,6 +134,9 @@ class GraphConfiguration { return graph } + @Bean + fun provideJdbcMetastoreInspector(graph: Graph): JdbcMetastoreInspector = JdbcMetastoreInspector.of(graph) + @Bean fun provideV2BackedEngine(graph: Graph): V2BackedEngine = V2BackedEngine(graph) diff --git a/server/src/test/kotlin/com/kakao/actionbase/server/filter/ReadOnlyRequestFilterTest.kt b/server/src/test/kotlin/com/kakao/actionbase/server/filter/ReadOnlyRequestFilterTest.kt index 645114feb..37ffb5e8c 100644 --- a/server/src/test/kotlin/com/kakao/actionbase/server/filter/ReadOnlyRequestFilterTest.kt +++ b/server/src/test/kotlin/com/kakao/actionbase/server/filter/ReadOnlyRequestFilterTest.kt @@ -148,6 +148,7 @@ class ReadOnlyRequestFilterTest { "GET /graph/v2/admin/metadata/service/{service}/label", "GET /graph/v2/admin/metadata/storage", "GET /graph/v2/admin//migration/{name}", + "GET /graph/v2/metastore/global", "GET /graph/v2/service", "GET /graph/v2/service/{service}", "GET /graph/v2/service/{service}/alias",