From 6b8d2b3236d28ee84fb2ab6610ffd2688d6a3b0f Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 25 May 2026 04:59:21 +0300 Subject: [PATCH 01/50] Add disabled-by-default Iceberg REST catalog listener skeleton --- .../hmsproxy/app/HmsProxyApplication.java | 2 + .../hmsproxy/config/ProxyConfig.java | 7 +- .../hmsproxy/config/ProxyConfigLoader.java | 4 + .../config/restcatalog/RestCatalogConfig.java | 13 ++ .../restcatalog/RestCatalogConfigParser.java | 32 +++++ .../restcatalog/RestCatalogServer.java | 130 ++++++++++++++++++ .../resources/hms-proxy-example.properties | 9 ++ .../RestCatalogConfigParserTest.java | 108 +++++++++++++++ .../restcatalog/RestCatalogServerTest.java | 101 ++++++++++++++ 9 files changed, 405 insertions(+), 1 deletion(-) create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfig.java create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParser.java create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java create mode 100644 src/test/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParserTest.java create mode 100644 src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java index 41960d0..e332947 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java @@ -6,6 +6,7 @@ import io.github.mmalykhin.hmsproxy.frontend.HortonworksFrontendExtension; import io.github.mmalykhin.hmsproxy.observability.ProxyObservability; import io.github.mmalykhin.hmsproxy.federation.FederationLayer; +import io.github.mmalykhin.hmsproxy.restcatalog.RestCatalogServer; import io.github.mmalykhin.hmsproxy.routing.CatalogRouter; import io.github.mmalykhin.hmsproxy.routing.RoutingMetaStoreProxy; import io.github.mmalykhin.hmsproxy.security.FrontDoorSecurity; @@ -45,6 +46,7 @@ public static void main(String[] args) throws Exception { try (frontDoorSecurity; CatalogRouter router = CatalogRouter.open(config, observability.metrics()); ManagementHttpServer managementServer = ManagementHttpServer.open(config, router, observability); + RestCatalogServer restServer = RestCatalogServer.open(config); RoutingMetaStoreProxy handler = new RoutingMetaStoreProxy(config, router, new FederationLayer(config, router), frontDoorSecurity, observability)) { diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/config/ProxyConfig.java b/src/main/java/io/github/mmalykhin/hmsproxy/config/ProxyConfig.java index f16c5f3..c8b74d7 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/config/ProxyConfig.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/config/ProxyConfig.java @@ -13,6 +13,7 @@ import io.github.mmalykhin.hmsproxy.config.listener.AdditionalFrontendConfig; import io.github.mmalykhin.hmsproxy.config.management.ManagementConfig; import io.github.mmalykhin.hmsproxy.config.ratelimit.RateLimitConfig; +import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfig; import io.github.mmalykhin.hmsproxy.config.routing.BackendConfig; import io.github.mmalykhin.hmsproxy.config.routing.LatencyRoutingConfig; import io.github.mmalykhin.hmsproxy.config.security.SecurityConfig; @@ -30,6 +31,7 @@ public record ProxyConfig( FederationConfig federation, TransactionalDdlGuardConfig transactionalDdlGuard, ManagementConfig management, + RestCatalogConfig restCatalog, SyntheticReadLockStoreConfig syntheticReadLockStore, RateLimitConfig rateLimit, LatencyRoutingConfig latencyRouting, @@ -50,6 +52,7 @@ public record ProxyConfig( management = management == null ? new ManagementConfig(false, server.bindHost(), server.port() + 1000) : management; + restCatalog = restCatalog == null ? RestCatalogConfig.disabled() : restCatalog; Objects.requireNonNull(syntheticReadLockStore, "syntheticReadLockStore must be set explicitly: use SyntheticReadLockStoreConfig.inMemory() " + "for single-instance deployments or a ZOOKEEPER-backed config for HA."); @@ -73,6 +76,7 @@ public static final class Builder { private FederationConfig federation; private TransactionalDdlGuardConfig transactionalDdlGuard; private ManagementConfig management; + private RestCatalogConfig restCatalog; private SyntheticReadLockStoreConfig syntheticReadLockStore; private RateLimitConfig rateLimit; private LatencyRoutingConfig latencyRouting; @@ -88,6 +92,7 @@ public static final class Builder { public Builder federation(FederationConfig federation) { this.federation = federation; return this; } public Builder transactionalDdlGuard(TransactionalDdlGuardConfig guard) { this.transactionalDdlGuard = guard; return this; } public Builder management(ManagementConfig management) { this.management = management; return this; } + public Builder restCatalog(RestCatalogConfig restCatalog) { this.restCatalog = restCatalog; return this; } public Builder syntheticReadLockStore(SyntheticReadLockStoreConfig store) { this.syntheticReadLockStore = store; return this; } public Builder rateLimit(RateLimitConfig rateLimit) { this.rateLimit = rateLimit; return this; } public Builder latencyRouting(LatencyRoutingConfig latencyRouting) { this.latencyRouting = latencyRouting; return this; } @@ -98,7 +103,7 @@ public Builder additionalFrontends(List additionalFron public ProxyConfig build() { return new ProxyConfig(server, security, catalogDbSeparator, defaultCatalog, catalogs, - backend, compatibility, federation, transactionalDdlGuard, management, + backend, compatibility, federation, transactionalDdlGuard, management, restCatalog, syntheticReadLockStore, rateLimit, latencyRouting, additionalFrontends); } } diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/config/ProxyConfigLoader.java b/src/main/java/io/github/mmalykhin/hmsproxy/config/ProxyConfigLoader.java index 71d7344..1d4dd37 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/config/ProxyConfigLoader.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/config/ProxyConfigLoader.java @@ -22,6 +22,8 @@ import io.github.mmalykhin.hmsproxy.config.management.ManagementConfigParser; import io.github.mmalykhin.hmsproxy.config.ratelimit.RateLimitConfig; import io.github.mmalykhin.hmsproxy.config.ratelimit.RateLimitConfigParser; +import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfig; +import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfigParser; import io.github.mmalykhin.hmsproxy.config.routing.BackendConfig; import io.github.mmalykhin.hmsproxy.config.routing.LatencyRoutingConfig; import io.github.mmalykhin.hmsproxy.config.routing.LatencyRoutingConfigParser; @@ -57,6 +59,7 @@ public static ProxyConfig load(Path path) throws IOException { TransactionalDdlGuardConfig transactionalDdlGuard = TransactionalDdlGuardConfigParser.parse(reader); ManagementConfig management = ManagementConfigParser.parse(reader, server); + RestCatalogConfig restCatalog = RestCatalogConfigParser.parse(reader, server); SyntheticReadLockStoreConfig syntheticReadLockStore = SyntheticReadLockStoreConfigParser.parse(reader); RateLimitConfig rateLimit = RateLimitConfigParser.parse(reader, catalogs); @@ -76,6 +79,7 @@ public static ProxyConfig load(Path path) throws IOException { .federation(federation) .transactionalDdlGuard(transactionalDdlGuard) .management(management) + .restCatalog(restCatalog) .syntheticReadLockStore(syntheticReadLockStore) .rateLimit(rateLimit) .latencyRouting(latencyRouting) diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfig.java b/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfig.java new file mode 100644 index 0000000..ffa1b25 --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfig.java @@ -0,0 +1,13 @@ +package io.github.mmalykhin.hmsproxy.config.restcatalog; + +public record RestCatalogConfig( + boolean enabled, + String bindHost, + int port, + int minWorkerThreads, + int maxWorkerThreads +) { + public static RestCatalogConfig disabled() { + return new RestCatalogConfig(false, "0.0.0.0", 8181, 8, 64); + } +} diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParser.java b/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParser.java new file mode 100644 index 0000000..308183a --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParser.java @@ -0,0 +1,32 @@ +package io.github.mmalykhin.hmsproxy.config.restcatalog; + +import io.github.mmalykhin.hmsproxy.config.PropertyReader; +import io.github.mmalykhin.hmsproxy.config.server.ServerConfig; + +public final class RestCatalogConfigParser { + private static final int DEFAULT_PORT_OFFSET = 100; + private static final int DEFAULT_MIN_THREADS = 8; + private static final int DEFAULT_MAX_THREADS = 64; + + private RestCatalogConfigParser() { + } + + public static RestCatalogConfig parse(PropertyReader reader, ServerConfig server) { + boolean portConfigured = reader.has("rest-catalog.port"); + boolean enabled = reader.getBoolean("rest-catalog.enabled", portConfigured); + int port = reader.getInt("rest-catalog.port", server.port() + DEFAULT_PORT_OFFSET); + if (enabled && (port < 1 || port > 65535)) { + throw new IllegalArgumentException( + "rest-catalog.port must be between 1 and 65535, got: " + port); + } + String bindHost = reader.get("rest-catalog.bind-host", server.bindHost()); + int minWorkerThreads = reader.getPositiveInt("rest-catalog.min-worker-threads", DEFAULT_MIN_THREADS); + int maxWorkerThreads = reader.getInt("rest-catalog.max-worker-threads", DEFAULT_MAX_THREADS); + if (maxWorkerThreads < minWorkerThreads) { + throw new IllegalArgumentException( + "rest-catalog.max-worker-threads (" + maxWorkerThreads + + ") must be >= rest-catalog.min-worker-threads (" + minWorkerThreads + ")"); + } + return new RestCatalogConfig(enabled, bindHost, port, minWorkerThreads, maxWorkerThreads); + } +} diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java new file mode 100644 index 0000000..488b0c1 --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java @@ -0,0 +1,130 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import io.github.mmalykhin.hmsproxy.config.ProxyConfig; +import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfig; +import java.io.IOException; +import java.io.OutputStream; +import java.net.BindException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class RestCatalogServer implements AutoCloseable { + private static final Logger LOG = LoggerFactory.getLogger(RestCatalogServer.class); + private static final String JSON_CONTENT_TYPE = "application/json; charset=utf-8"; + private static final int STOP_TIMEOUT_SECONDS = 5; + + private final HttpServer server; + private final ThreadPoolExecutor executor; + + private RestCatalogServer(HttpServer server, ThreadPoolExecutor executor) { + this.server = server; + this.executor = executor; + } + + public static RestCatalogServer open(ProxyConfig config) throws IOException { + RestCatalogConfig restConfig = config.restCatalog(); + if (!restConfig.enabled()) { + return null; + } + + HttpServer server; + try { + server = HttpServer.create( + new InetSocketAddress(restConfig.bindHost(), restConfig.port()), 0); + } catch (BindException e) { + LOG.error("Failed to bind Iceberg REST catalog listener on {}:{} - {}", + restConfig.bindHost(), restConfig.port(), e.getMessage()); + throw e; + } + + server.createContext("/v1/config", new ConfigHandler()); + server.createContext("/", new NotFoundHandler()); + + ThreadPoolExecutor executor = new ThreadPoolExecutor( + restConfig.minWorkerThreads(), + restConfig.maxWorkerThreads(), + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + namedThreadFactory("hms-proxy-rest"), + new ThreadPoolExecutor.CallerRunsPolicy()); + server.setExecutor(executor); + server.start(); + LOG.info("Iceberg REST catalog listener started on {}:{} (threads: {}..{})", + restConfig.bindHost(), restConfig.port(), + restConfig.minWorkerThreads(), restConfig.maxWorkerThreads()); + return new RestCatalogServer(server, executor); + } + + public int boundPort() { + return server.getAddress().getPort(); + } + + @Override + public void close() { + server.stop(0); + executor.shutdown(); + try { + if (!executor.awaitTermination(STOP_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + executor.shutdownNow(); + LOG.warn("Iceberg REST catalog executor did not terminate within {}s after shutdown", + STOP_TIMEOUT_SECONDS); + } + } catch (InterruptedException e) { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + + private static ThreadFactory namedThreadFactory(String prefix) { + AtomicLong counter = new AtomicLong(); + return runnable -> { + Thread thread = new Thread(runnable); + thread.setName(prefix + "-" + counter.incrementAndGet()); + thread.setDaemon(true); + return thread; + }; + } + + private static void respond(HttpExchange exchange, int status, String contentType, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", contentType); + exchange.sendResponseHeaders(status, bytes.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(bytes); + } + } + + private static final class ConfigHandler implements HttpHandler { + @Override + public void handle(HttpExchange exchange) throws IOException { + String method = exchange.getRequestMethod(); + if (!"GET".equalsIgnoreCase(method) && !"HEAD".equalsIgnoreCase(method)) { + exchange.getResponseHeaders().set("Allow", "GET, HEAD"); + respond(exchange, 405, JSON_CONTENT_TYPE, + "{\"error\":{\"message\":\"Method not allowed\",\"type\":\"BadRequestException\",\"code\":405}}"); + return; + } + respond(exchange, 200, JSON_CONTENT_TYPE, "{\"defaults\":{},\"overrides\":{}}"); + } + } + + private static final class NotFoundHandler implements HttpHandler { + @Override + public void handle(HttpExchange exchange) throws IOException { + respond(exchange, 404, JSON_CONTENT_TYPE, + "{\"error\":{\"message\":\"Not implemented\",\"type\":\"NotImplementedException\",\"code\":404}}"); + } + } +} diff --git a/src/main/resources/hms-proxy-example.properties b/src/main/resources/hms-proxy-example.properties index 5c01651..3aff834 100644 --- a/src/main/resources/hms-proxy-example.properties +++ b/src/main/resources/hms-proxy-example.properties @@ -85,6 +85,15 @@ server.max-worker-threads=512 # additional-frontends.hdp.min-worker-threads=8 # additional-frontends.hdp.max-worker-threads=64 +# Optional Iceberg REST catalog listener. Runs in parallel with the Thrift HMS listener +# and exposes a subset of the Iceberg REST API; see docs for supported operations. +# Enabled automatically when rest-catalog.port is set, or explicitly via rest-catalog.enabled=true. +# rest-catalog.enabled=true +# rest-catalog.bind-host=0.0.0.0 +# rest-catalog.port=9183 +# rest-catalog.min-worker-threads=8 +# rest-catalog.max-worker-threads=64 + # ----------------------------------------------------------------------------- # Routing # ----------------------------------------------------------------------------- diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParserTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParserTest.java new file mode 100644 index 0000000..d39b09e --- /dev/null +++ b/src/test/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParserTest.java @@ -0,0 +1,108 @@ +package io.github.mmalykhin.hmsproxy.config.restcatalog; + +import io.github.mmalykhin.hmsproxy.config.ProxyConfig; +import io.github.mmalykhin.hmsproxy.config.ProxyConfigLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.Assert; +import org.junit.Test; + +public class RestCatalogConfigParserTest { + private static final String COMMON_PROPS = """ + synthetic-read-lock.store.mode=IN_MEMORY + catalogs=catalog1 + catalog.catalog1.conf.hive.metastore.uris=thrift://hms1:9083 + """; + + @Test + public void defaultsToDisabled() throws Exception { + RestCatalogConfig config = loadRestConfig("server.port=9083\n" + COMMON_PROPS); + Assert.assertFalse(config.enabled()); + Assert.assertEquals(9183, config.port()); + Assert.assertEquals("0.0.0.0", config.bindHost()); + Assert.assertEquals(8, config.minWorkerThreads()); + Assert.assertEquals(64, config.maxWorkerThreads()); + } + + @Test + public void enabledImplicitlyWhenPortIsSet() throws Exception { + RestCatalogConfig config = loadRestConfig( + "server.port=9083\nrest-catalog.port=8181\n" + COMMON_PROPS); + Assert.assertTrue(config.enabled()); + Assert.assertEquals(8181, config.port()); + } + + @Test + public void respectsExplicitlyDisabledEvenWithPortSet() throws Exception { + RestCatalogConfig config = loadRestConfig( + "server.port=9083\nrest-catalog.port=8181\nrest-catalog.enabled=false\n" + COMMON_PROPS); + Assert.assertFalse(config.enabled()); + } + + @Test + public void inheritsBindHostFromServerByDefault() throws Exception { + RestCatalogConfig config = loadRestConfig( + "server.port=9083\nserver.bind-host=127.0.0.1\n" + COMMON_PROPS); + Assert.assertEquals("127.0.0.1", config.bindHost()); + } + + @Test + public void allowsBindHostOverride() throws Exception { + RestCatalogConfig config = loadRestConfig( + "server.port=9083\nrest-catalog.bind-host=10.0.0.1\nrest-catalog.port=8181\n" + COMMON_PROPS); + Assert.assertEquals("10.0.0.1", config.bindHost()); + } + + @Test + public void rejectsInvalidPort() throws Exception { + try { + loadRestConfig("server.port=9083\nrest-catalog.port=70000\n" + COMMON_PROPS); + Assert.fail("expected IllegalArgumentException for port out of range"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("rest-catalog.port")); + } + } + + @Test + public void rejectsMinThreadsBelowOne() throws Exception { + try { + loadRestConfig( + "server.port=9083\nrest-catalog.port=8181\nrest-catalog.min-worker-threads=0\n" + COMMON_PROPS); + Assert.fail("expected IllegalArgumentException for non-positive min threads"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("rest-catalog.min-worker-threads")); + } + } + + @Test + public void rejectsMaxThreadsLessThanMin() throws Exception { + try { + loadRestConfig( + "server.port=9083\nrest-catalog.port=8181\n" + + "rest-catalog.min-worker-threads=16\nrest-catalog.max-worker-threads=4\n" + + COMMON_PROPS); + Assert.fail("expected IllegalArgumentException for max < min"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("rest-catalog.max-worker-threads")); + Assert.assertTrue(e.getMessage().contains("rest-catalog.min-worker-threads")); + } + } + + @Test + public void skipsPortValidationWhenDisabled() throws Exception { + RestCatalogConfig config = loadRestConfig( + "server.port=9083\nrest-catalog.enabled=false\n" + COMMON_PROPS); + Assert.assertFalse(config.enabled()); + } + + private static RestCatalogConfig loadRestConfig(String body) throws Exception { + Path file = Files.createTempFile("hms-proxy-rest", ".properties"); + try { + Files.writeString(file, body); + ProxyConfig config = ProxyConfigLoader.load(file); + return config.restCatalog(); + } finally { + Files.deleteIfExists(file); + } + } +} diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java new file mode 100644 index 0000000..b084c08 --- /dev/null +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java @@ -0,0 +1,101 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import io.github.mmalykhin.hmsproxy.config.ProxyConfig; +import io.github.mmalykhin.hmsproxy.config.catalog.CatalogAccessMode; +import io.github.mmalykhin.hmsproxy.config.catalog.CatalogConfig; +import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfig; +import io.github.mmalykhin.hmsproxy.config.routing.BackendConfig; +import io.github.mmalykhin.hmsproxy.config.server.ServerConfig; +import io.github.mmalykhin.hmsproxy.config.syntheticlock.SyntheticReadLockStoreConfig; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; + +public class RestCatalogServerTest { + private static final Duration HTTP_TIMEOUT = Duration.ofSeconds(5); + + @Test + public void returnsNullWhenDisabled() throws Exception { + ProxyConfig config = buildConfig(new RestCatalogConfig(false, "127.0.0.1", 0, 1, 4)); + RestCatalogServer server = RestCatalogServer.open(config); + Assert.assertNull(server); + } + + @Test + public void servesEmptyIcebergConfigOnGet() throws Exception { + ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)); + try (RestCatalogServer server = RestCatalogServer.open(config)) { + Assert.assertNotNull(server); + HttpResponse response = request(server, "/v1/config", "GET"); + Assert.assertEquals(200, response.statusCode()); + Assert.assertEquals("{\"defaults\":{},\"overrides\":{}}", response.body()); + Assert.assertTrue(response.headers().firstValue("Content-Type").orElse("") + .startsWith("application/json")); + } + } + + @Test + public void rejectsNonReadMethodsOnConfig() throws Exception { + ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)); + try (RestCatalogServer server = RestCatalogServer.open(config)) { + HttpResponse response = request(server, "/v1/config", "POST"); + Assert.assertEquals(405, response.statusCode()); + Assert.assertEquals("GET, HEAD", response.headers().firstValue("Allow").orElse("")); + } + } + + @Test + public void respondsWithNotFoundForUnknownPath() throws Exception { + ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)); + try (RestCatalogServer server = RestCatalogServer.open(config)) { + HttpResponse response = request(server, "/v1/namespaces", "GET"); + Assert.assertEquals(404, response.statusCode()); + Assert.assertTrue(response.body().contains("\"code\":404")); + } + } + + @Test + public void boundPortReflectsActualListener() throws Exception { + ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)); + try (RestCatalogServer server = RestCatalogServer.open(config)) { + Assert.assertTrue("port must be allocated", server.boundPort() > 0); + } + } + + private static HttpResponse request(RestCatalogServer server, String path, String method) throws Exception { + HttpClient client = HttpClient.newBuilder().connectTimeout(HTTP_TIMEOUT).build(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("http://127.0.0.1:" + server.boundPort() + path)) + .timeout(HTTP_TIMEOUT) + .method(method, HttpRequest.BodyPublishers.noBody()) + .build(); + return client.send(request, HttpResponse.BodyHandlers.ofString()); + } + + private static ProxyConfig buildConfig(RestCatalogConfig restCatalog) { + return ProxyConfig.builder() + .server(new ServerConfig("hms-proxy-test", "127.0.0.1", 9083, 1, 4)) + .catalogDbSeparator(".") + .defaultCatalog("catalog1") + .catalogs(Map.of("catalog1", new CatalogConfig( + "catalog1", + null, + null, + false, + CatalogAccessMode.READ_WRITE, + List.of(), + null, + null, + Map.of("hive.metastore.uris", "thrift://hms-test:9083")))) + .backend(new BackendConfig(Map.of())) + .restCatalog(restCatalog) + .syntheticReadLockStore(SyntheticReadLockStoreConfig.inMemory()) + .build(); + } +} From f3e88b2548e28c9b702b0d1fc904083fa496fe25 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 25 May 2026 05:48:31 +0300 Subject: [PATCH 02/50] Wire read-only Iceberg REST endpoints to routing layer --- pom.xml | 67 ++ .../hmsproxy/app/HmsProxyApplication.java | 7 +- .../restcatalog/IcebergHttpHandler.java | 226 ++++++ .../restcatalog/IcebergRestMapper.java | 33 + .../restcatalog/IcebergRestService.java | 73 ++ .../restcatalog/RestCatalogServer.java | 10 +- .../restcatalog/RoutingClientPool.java | 29 + .../restcatalog/RoutingHiveCatalog.java | 72 ++ .../restcatalog/RoutingMetaStoreClient.java | 115 +++ .../iceberg/rest/RESTCatalogAdapter.java | 662 ++++++++++++++++++ .../IcebergRestEndpointIntegrationTest.java | 144 ++++ .../restcatalog/RecordingThriftIface.java | 115 +++ .../restcatalog/RoutingHiveCatalogTest.java | 55 ++ .../RoutingMetaStoreClientTest.java | 94 +++ 14 files changed, 1699 insertions(+), 3 deletions(-) create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestMapper.java create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingClientPool.java create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingHiveCatalog.java create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java create mode 100644 src/main/java/org/apache/iceberg/rest/RESTCatalogAdapter.java create mode 100644 src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java create mode 100644 src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java create mode 100644 src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingHiveCatalogTest.java create mode 100644 src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java diff --git a/pom.xml b/pom.xml index ce5c253..54e1c3b 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,7 @@ 2.12.0 3.1.3 2.6.5 + 1.5.2 4.13.2 + + org.apache.iceberg + iceberg-bundled-guava + ${iceberg.version} + + + org.apache.iceberg + iceberg-hive-metastore + ${iceberg.version} + + + org.apache.hive + hive-metastore + + + org.apache.hive + hive-serde + + + org.apache.avro + avro + + + org.slf4j + slf4j-log4j12 + + + com.google.code.findbugs + jsr305 + + + org.pentaho + pentaho-aggdesigner-algorithm + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-core + + + org.apache.curator curator-framework diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java index e332947..448065a 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java @@ -6,6 +6,7 @@ import io.github.mmalykhin.hmsproxy.frontend.HortonworksFrontendExtension; import io.github.mmalykhin.hmsproxy.observability.ProxyObservability; import io.github.mmalykhin.hmsproxy.federation.FederationLayer; +import io.github.mmalykhin.hmsproxy.restcatalog.IcebergRestService; import io.github.mmalykhin.hmsproxy.restcatalog.RestCatalogServer; import io.github.mmalykhin.hmsproxy.routing.CatalogRouter; import io.github.mmalykhin.hmsproxy.routing.RoutingMetaStoreProxy; @@ -46,7 +47,6 @@ public static void main(String[] args) throws Exception { try (frontDoorSecurity; CatalogRouter router = CatalogRouter.open(config, observability.metrics()); ManagementHttpServer managementServer = ManagementHttpServer.open(config, router, observability); - RestCatalogServer restServer = RestCatalogServer.open(config); RoutingMetaStoreProxy handler = new RoutingMetaStoreProxy(config, router, new FederationLayer(config, router), frontDoorSecurity, observability)) { @@ -56,7 +56,10 @@ public static void main(String[] args) throws Exception { handler, HortonworksFrontendExtension.class); try (AdditionalFrontendThriftServers extras = - AdditionalFrontendThriftServers.open(config, proxy, frontDoorSecurity)) { + AdditionalFrontendThriftServers.open(config, proxy, frontDoorSecurity); + IcebergRestService restService = + config.restCatalog().enabled() ? new IcebergRestService(config, proxy) : null; + RestCatalogServer restServer = RestCatalogServer.open(config, restService)) { MetastoreThriftServer server = new MetastoreThriftServer(config, proxy, frontDoorSecurity); installShutdownHook(server, teardownComplete, config.server().shutdownTimeoutSeconds()); LOG.info("Starting HMS proxy '{}' on {}:{}", config.server().name(), diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java new file mode 100644 index 0000000..3980239 --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java @@ -0,0 +1,226 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.iceberg.exceptions.RESTException; +import org.apache.iceberg.rest.RESTCatalogAdapter.HTTPMethod; +import org.apache.iceberg.rest.RESTCatalogAdapter.Route; +import org.apache.iceberg.rest.RESTRequest; +import org.apache.iceberg.rest.RESTResponse; +import org.apache.iceberg.rest.responses.ConfigResponse; +import org.apache.iceberg.rest.responses.ErrorResponse; +import org.apache.iceberg.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Maps JDK HttpExchange requests to {@link IcebergRestService} dispatches. + * URL form: /v1/{prefix}/... The {prefix} segment must equal the proxy's + * default catalog name; any other prefix returns 404. + */ +final class IcebergHttpHandler implements HttpHandler { + private static final Logger LOG = LoggerFactory.getLogger(IcebergHttpHandler.class); + private static final String JSON_CONTENT_TYPE = "application/json; charset=utf-8"; + private static final String V1_PREFIX = "/v1/"; + private static final String CONFIG_PATH = "v1/config"; + + private final IcebergRestService service; + + IcebergHttpHandler(IcebergRestService service) { + this.service = service; + } + + @Override + public void handle(HttpExchange exchange) throws IOException { + try { + String rawPath = exchange.getRequestURI().getPath(); + if (!rawPath.startsWith(V1_PREFIX) && !rawPath.equals("/v1")) { + writeError(exchange, 404, "NotImplementedException", "Path not handled by Iceberg REST endpoint"); + return; + } + HTTPMethod method; + try { + method = HTTPMethod.valueOf(exchange.getRequestMethod().toUpperCase()); + } catch (IllegalArgumentException e) { + writeError(exchange, 405, "BadRequestException", + "Method not allowed: " + exchange.getRequestMethod()); + return; + } + + Map queryParams = parseQueryString(exchange.getRequestURI().getRawQuery()); + String relativePath = stripPrefixSegment(rawPath); + if (relativePath == null) { + writeError(exchange, 404, "NoSuchCatalogException", + "Unknown catalog prefix in URL: " + rawPath); + return; + } + + if (CONFIG_PATH.equals(relativePath)) { + ConfigResponse cfg = service.loadConfig(); + writeJson(exchange, 200, IcebergRestMapper.mapper().writeValueAsString(cfg)); + return; + } + + Pair> routeAndVars = Route.from(method, relativePath); + if (routeAndVars == null) { + writeError(exchange, 404, "NotImplementedException", + "Route not supported: " + method + " " + relativePath); + return; + } + Route route = routeAndVars.first(); + Object body = readBody(exchange, route); + Class responseType = route.responseClass(); + + ErrorResponse[] capturedError = new ErrorResponse[1]; + RESTResponse response; + Class effectiveResponseType = + responseType == null ? RESTResponse.class : responseType; + try { + response = dispatchInternal(method, relativePath, queryParams, body, + effectiveResponseType, err -> capturedError[0] = err); + } catch (RESTException e) { + // RESTCatalogAdapter always rethrows after invoking the error handler, + // so an error response is already captured when we get here. + if (capturedError[0] != null) { + writeErrorResponse(exchange, capturedError[0]); + } else { + writeError(exchange, 500, e.getClass().getSimpleName(), e.getMessage()); + } + return; + } + + if (capturedError[0] != null) { + writeErrorResponse(exchange, capturedError[0]); + return; + } + + if (responseType == null || response == null) { + exchange.sendResponseHeaders(204, -1); + exchange.getResponseBody().close(); + return; + } + writeJson(exchange, 200, IcebergRestMapper.mapper().writeValueAsString(response)); + } catch (Exception e) { + LOG.warn("Unhandled error serving {} {}", + exchange.getRequestMethod(), exchange.getRequestURI(), e); + writeError(exchange, 500, e.getClass().getSimpleName(), + e.getMessage() == null ? "internal error" : e.getMessage()); + } + } + + private T dispatchInternal( + HTTPMethod method, + String relativePath, + Map queryParams, + Object body, + Class responseType, + java.util.function.Consumer errorHandler) { + return service.dispatch(method, relativePath, queryParams, body, responseType, + java.util.Map.of(), errorHandler); + } + + private String stripPrefixSegment(String rawPath) { + String trimmed = rawPath.substring(V1_PREFIX.length()); + int slash = trimmed.indexOf('/'); + if (slash < 0) { + String prefixOnly = trimmed; + if ("config".equals(prefixOnly)) { + return CONFIG_PATH; + } + if (!service.supportsPrefix(prefixOnly)) { + return null; + } + return "v1"; + } + String prefix = trimmed.substring(0, slash); + if ("config".equals(prefix) && slash == trimmed.length() - 1) { + return CONFIG_PATH; + } + if (!service.supportsPrefix(prefix)) { + return null; + } + String rest = trimmed.substring(slash + 1); + return rest.isEmpty() ? "v1" : "v1/" + rest; + } + + private Object readBody(HttpExchange exchange, Route route) throws IOException { + Class requestClass = route.requestClass(); + if (requestClass == null) { + drain(exchange.getRequestBody()); + return null; + } + try (InputStream input = exchange.getRequestBody()) { + return IcebergRestMapper.mapper().readValue(input, requestClass); + } + } + + private static void drain(InputStream input) throws IOException { + try (input) { + input.transferTo(OutputStream.nullOutputStream()); + } + } + + private static Map parseQueryString(String rawQuery) { + if (rawQuery == null || rawQuery.isEmpty()) { + return Map.of(); + } + Map result = new LinkedHashMap<>(); + for (String pair : rawQuery.split("&")) { + int eq = pair.indexOf('='); + String key = eq < 0 ? pair : pair.substring(0, eq); + String value = eq < 0 ? "" : pair.substring(eq + 1); + result.put( + URLDecoder.decode(key, StandardCharsets.UTF_8), + URLDecoder.decode(value, StandardCharsets.UTF_8)); + } + return result; + } + + private static void writeJson(HttpExchange exchange, int status, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", JSON_CONTENT_TYPE); + exchange.sendResponseHeaders(status, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + } + + private static void writeError(HttpExchange exchange, int status, String type, String message) throws IOException { + ErrorResponse error = ErrorResponse.builder() + .responseCode(status) + .withType(type) + .withMessage(message == null ? "" : message) + .withStackTrace(Arrays.asList()) + .build(); + writeErrorResponse(exchange, error); + } + + private static void writeErrorResponse(HttpExchange exchange, ErrorResponse error) throws IOException { + int status = error.code() > 0 ? error.code() : 500; + String body = IcebergRestMapper.mapper().writeValueAsString(error); + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", JSON_CONTENT_TYPE); + exchange.sendResponseHeaders(status, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + } + + private static int statusForException(RESTException e) { + String name = e.getClass().getSimpleName(); + if (name.contains("NoSuch")) return 404; + if (name.contains("AlreadyExists")) return 409; + if (name.contains("NotAuthorized")) return 401; + if (name.contains("Forbidden")) return 403; + if (name.contains("Validation")) return 400; + return 500; + } +} diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestMapper.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestMapper.java new file mode 100644 index 0000000..33193ae --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestMapper.java @@ -0,0 +1,33 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import org.apache.iceberg.rest.RESTSerializers; + +/** + * Mirror of Iceberg's package-private RESTObjectMapper. We construct an equivalent + * Jackson ObjectMapper here so the handler does not need to live in + * org.apache.iceberg.rest. registerAll() is the only public touchpoint we rely on. + */ +final class IcebergRestMapper { + private static final ObjectMapper MAPPER; + + static { + ObjectMapper m = new ObjectMapper(); + m.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + m.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + m.setPropertyNamingStrategy(new PropertyNamingStrategy.KebabCaseStrategy()); + RESTSerializers.registerAll(m); + MAPPER = m; + } + + private IcebergRestMapper() { + } + + static ObjectMapper mapper() { + return MAPPER; + } +} diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java new file mode 100644 index 0000000..b14241b --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java @@ -0,0 +1,73 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import io.github.mmalykhin.hmsproxy.config.ProxyConfig; +import java.io.IOException; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.rest.RESTCatalogAdapter; +import org.apache.iceberg.rest.RESTCatalogAdapter.HTTPMethod; +import org.apache.iceberg.rest.RESTResponse; +import org.apache.iceberg.rest.responses.ConfigResponse; +import org.apache.iceberg.rest.responses.ErrorResponse; + +/** + * Bridges Iceberg REST calls to the proxy's ThriftHiveMetastore.Iface via a + * single shared RoutingHiveCatalog. The MVP supports the proxy's default catalog + * only; multi-catalog support requires namespace prefix rewriting on both + * request and response paths (planned for a follow-up step). + */ +public final class IcebergRestService implements AutoCloseable { + private static final String UNUSED_URI = "thrift://hms-proxy-loopback:0"; + + private final ProxyConfig config; + private final RoutingHiveCatalog catalog; + private final RESTCatalogAdapter adapter; + + public IcebergRestService(ProxyConfig config, ThriftHiveMetastore.Iface delegate) { + this.config = Objects.requireNonNull(config, "config"); + Objects.requireNonNull(delegate, "delegate"); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate); + this.catalog = new RoutingHiveCatalog(client, new Configuration()); + this.catalog.initialize( + config.defaultCatalog(), + Map.of(CatalogProperties.URI, UNUSED_URI)); + this.adapter = new RESTCatalogAdapter(catalog); + } + + /** True when the prefix segment in the request URL maps to a known catalog we can serve. */ + public boolean supportsPrefix(String prefix) { + return prefix != null && prefix.equals(config.defaultCatalog()); + } + + /** + * Returns the GET /v1/config response that Iceberg clients use for discovery. + * Setting overrides.prefix locks the client to the proxy's default catalog + * so all subsequent /v1/{prefix}/... requests land here. + */ + public ConfigResponse loadConfig() { + return ConfigResponse.builder() + .withOverride("prefix", config.defaultCatalog()) + .build(); + } + + public T dispatch( + HTTPMethod method, + String relativePath, + Map queryParams, + Object body, + Class responseType, + Map headers, + Consumer errorHandler) { + return adapter.execute(method, relativePath, queryParams, body, responseType, headers, errorHandler); + } + + @Override + public void close() throws IOException { + catalog.close(); + } +} diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java index 488b0c1..a12c5f0 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java @@ -33,6 +33,10 @@ private RestCatalogServer(HttpServer server, ThreadPoolExecutor executor) { } public static RestCatalogServer open(ProxyConfig config) throws IOException { + return open(config, null); + } + + public static RestCatalogServer open(ProxyConfig config, IcebergRestService service) throws IOException { RestCatalogConfig restConfig = config.restCatalog(); if (!restConfig.enabled()) { return null; @@ -48,7 +52,11 @@ public static RestCatalogServer open(ProxyConfig config) throws IOException { throw e; } - server.createContext("/v1/config", new ConfigHandler()); + if (service != null) { + server.createContext("/v1/", new IcebergHttpHandler(service)); + } else { + server.createContext("/v1/config", new ConfigHandler()); + } server.createContext("/", new NotFoundHandler()); ThreadPoolExecutor executor = new ThreadPoolExecutor( diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingClientPool.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingClientPool.java new file mode 100644 index 0000000..d51b5cf --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingClientPool.java @@ -0,0 +1,29 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import java.util.Objects; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.iceberg.ClientPool; +import org.apache.thrift.TException; + +/** + * Stateless ClientPool that always hands the same IMetaStoreClient back to + * HiveCatalog. The wrapped client is a Proxy over our ThriftHiveMetastore.Iface, + * so there is no connection to acquire, recycle, or close. + */ +public final class RoutingClientPool implements ClientPool { + private final IMetaStoreClient client; + + public RoutingClientPool(IMetaStoreClient client) { + this.client = Objects.requireNonNull(client, "client"); + } + + @Override + public R run(Action action) throws TException, InterruptedException { + return action.run(client); + } + + @Override + public R run(Action action, boolean retry) throws TException, InterruptedException { + return action.run(client); + } +} diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingHiveCatalog.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingHiveCatalog.java new file mode 100644 index 0000000..044b2b7 --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingHiveCatalog.java @@ -0,0 +1,72 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import java.lang.reflect.Field; +import java.util.Map; +import java.util.Objects; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.iceberg.ClientPool; +import org.apache.iceberg.hive.HiveCatalog; + +/** + * HiveCatalog that bypasses the standalone HiveMetaStoreClient and dispatches all + * IMetaStoreClient calls through the proxy's ThriftHiveMetastore.Iface (and therefore + * through RoutingMetaStoreProxy, federation, observability, etc.). + * + * Iceberg's HiveCatalog has no extension point for the client pool, so the private + * {@code clients} field is replaced via reflection right after {@link #initialize}. + * This is pinned to a specific Iceberg version (see pom.xml); upgrades must rerun + * RoutingHiveCatalogTest, which exercises the inject. + */ +public final class RoutingHiveCatalog extends HiveCatalog { + private static final String CLIENTS_FIELD_NAME = "clients"; + + private final IMetaStoreClient client; + + public RoutingHiveCatalog(IMetaStoreClient client, Configuration conf) { + this.client = Objects.requireNonNull(client, "client"); + setConf(Objects.requireNonNull(conf, "conf")); + } + + @Override + public void initialize(String name, Map properties) { + super.initialize(name, properties); + replaceClientPool(); + } + + private void replaceClientPool() { + try { + Field clientsField = HiveCatalog.class.getDeclaredField(CLIENTS_FIELD_NAME); + clientsField.setAccessible(true); + Object previous = clientsField.get(this); + clientsField.set(this, new RoutingClientPool(client)); + closeQuietly(previous); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException( + "Failed to inject RoutingClientPool into HiveCatalog; Iceberg version may be incompatible", + e); + } + } + + private static void closeQuietly(Object resource) { + if (resource instanceof AutoCloseable closeable) { + try { + closeable.close(); + } catch (Exception ignored) { + // Swallow: replaced pool may already be closed or never opened a connection. + } + } + } + + ClientPool activeClientPool() { + try { + Field clientsField = HiveCatalog.class.getDeclaredField(CLIENTS_FIELD_NAME); + clientsField.setAccessible(true); + @SuppressWarnings("unchecked") + ClientPool pool = (ClientPool) clientsField.get(this); + return pool; + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException("Cannot read HiveCatalog.clients", e); + } + } +} diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java new file mode 100644 index 0000000..5e0292b --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java @@ -0,0 +1,115 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.Objects; +import java.util.List; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; + +/** + * Bridges Iceberg's HiveCatalog (which requires IMetaStoreClient) to the proxy's + * ThriftHiveMetastore.Iface. Only methods used by HiveCatalog's read paths are + * implemented; write paths and unsupported methods throw UnsupportedOperationException + * until Phase 5 brings write support online. + */ +public final class RoutingMetaStoreClient { + private RoutingMetaStoreClient() { + } + + public static IMetaStoreClient create(ThriftHiveMetastore.Iface delegate) { + Objects.requireNonNull(delegate, "delegate"); + return (IMetaStoreClient) Proxy.newProxyInstance( + IMetaStoreClient.class.getClassLoader(), + new Class[]{IMetaStoreClient.class}, + new RoutingInvocationHandler(delegate)); + } + + private static final class RoutingInvocationHandler implements InvocationHandler { + private final ThriftHiveMetastore.Iface delegate; + + RoutingInvocationHandler(ThriftHiveMetastore.Iface delegate) { + this.delegate = delegate; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + String name = method.getName(); + Class[] paramTypes = method.getParameterTypes(); + switch (name) { + case "close": + case "reconnect": + case "flushCache": + return null; + case "isCompatibleWith": + case "isLocalMetaStore": + return false; + case "equals": + return proxy == args[0]; + case "hashCode": + return System.identityHashCode(proxy); + case "toString": + return "RoutingMetaStoreClient[delegate=" + delegate + "]"; + case "getDatabase": + return delegate.get_database((String) args[0]); + case "getAllDatabases": + return delegate.get_all_databases(); + case "getDatabases": + return delegate.get_databases((String) args[0]); + case "getAllTables": + return delegate.get_all_tables((String) args[0]); + case "getTables": + if (paramTypes.length == 2) { + return delegate.get_tables((String) args[0], (String) args[1]); + } + if (paramTypes.length == 3) { + return delegate.get_tables_by_type( + (String) args[0], (String) args[1], String.valueOf(args[2])); + } + break; + case "getTable": + if (paramTypes.length == 2 + && paramTypes[0] == String.class && paramTypes[1] == String.class) { + return delegate.get_table((String) args[0], (String) args[1]); + } + break; + case "getTableObjectsByName": + if (paramTypes.length == 2 + && paramTypes[0] == String.class && paramTypes[1] == List.class) { + @SuppressWarnings("unchecked") + List tableNames = (List) args[1]; + return delegate.get_table_objects_by_name((String) args[0], tableNames); + } + break; + case "tableExists": + if (paramTypes.length == 2) { + try { + delegate.get_table((String) args[0], (String) args[1]); + return true; + } catch (NoSuchObjectException e) { + return false; + } + } + break; + default: + break; + } + throw new UnsupportedOperationException( + "HMS proxy REST gateway does not support IMetaStoreClient." + name + + "(" + parameterSignature(paramTypes) + ")"); + } + + private static String parameterSignature(Class[] paramTypes) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < paramTypes.length; i++) { + if (i > 0) { + builder.append(','); + } + builder.append(paramTypes[i].getSimpleName()); + } + return builder.toString(); + } + } +} diff --git a/src/main/java/org/apache/iceberg/rest/RESTCatalogAdapter.java b/src/main/java/org/apache/iceberg/rest/RESTCatalogAdapter.java new file mode 100644 index 0000000..ca703c7 --- /dev/null +++ b/src/main/java/org/apache/iceberg/rest/RESTCatalogAdapter.java @@ -0,0 +1,662 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +// Vendored verbatim from apache-iceberg-1.5.2 (core/src/test/java/org/apache/iceberg/rest). +// In 1.5.x this class lives in test sources of iceberg-core; the copy here lets the +// proxy use it without depending on the tests classifier. When bumping the Iceberg +// version, diff this file against the matching upstream tag. +package org.apache.iceberg.rest; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.BaseTransaction; +import org.apache.iceberg.Table; +import org.apache.iceberg.Transaction; +import org.apache.iceberg.Transactions; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.SupportsNamespaces; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.iceberg.exceptions.NamespaceNotEmptyException; +import org.apache.iceberg.exceptions.NoSuchIcebergTableException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; +import org.apache.iceberg.exceptions.NotAuthorizedException; +import org.apache.iceberg.exceptions.RESTException; +import org.apache.iceberg.exceptions.UnprocessableEntityException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.base.Splitter; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.rest.requests.CommitTransactionRequest; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.requests.CreateViewRequest; +import org.apache.iceberg.rest.requests.RegisterTableRequest; +import org.apache.iceberg.rest.requests.RenameTableRequest; +import org.apache.iceberg.rest.requests.ReportMetricsRequest; +import org.apache.iceberg.rest.requests.UpdateNamespacePropertiesRequest; +import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.ConfigResponse; +import org.apache.iceberg.rest.responses.CreateNamespaceResponse; +import org.apache.iceberg.rest.responses.ErrorResponse; +import org.apache.iceberg.rest.responses.GetNamespaceResponse; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; +import org.apache.iceberg.rest.responses.OAuthTokenResponse; +import org.apache.iceberg.rest.responses.UpdateNamespacePropertiesResponse; +import org.apache.iceberg.util.Pair; +import org.apache.iceberg.util.PropertyUtil; + +/** Adaptor class to translate REST requests into {@link Catalog} API calls. */ +public class RESTCatalogAdapter implements RESTClient { + private static final Splitter SLASH = Splitter.on('/'); + + private static final Map, Integer> EXCEPTION_ERROR_CODES = + ImmutableMap., Integer>builder() + .put(IllegalArgumentException.class, 400) + .put(ValidationException.class, 400) + .put(NamespaceNotEmptyException.class, 400) // TODO: should this be more specific? + .put(NotAuthorizedException.class, 401) + .put(ForbiddenException.class, 403) + .put(NoSuchNamespaceException.class, 404) + .put(NoSuchTableException.class, 404) + .put(NoSuchViewException.class, 404) + .put(NoSuchIcebergTableException.class, 404) + .put(UnsupportedOperationException.class, 406) + .put(AlreadyExistsException.class, 409) + .put(CommitFailedException.class, 409) + .put(UnprocessableEntityException.class, 422) + .put(CommitStateUnknownException.class, 500) + .buildOrThrow(); + + private final Catalog catalog; + private final SupportsNamespaces asNamespaceCatalog; + private final ViewCatalog asViewCatalog; + + public RESTCatalogAdapter(Catalog catalog) { + this.catalog = catalog; + this.asNamespaceCatalog = + catalog instanceof SupportsNamespaces ? (SupportsNamespaces) catalog : null; + this.asViewCatalog = catalog instanceof ViewCatalog ? (ViewCatalog) catalog : null; + } + + // Patched from upstream: was package-private; promoted to public so callers outside + // org.apache.iceberg.rest (notably IcebergRestService) can dispatch by method. + public enum HTTPMethod { + GET, + HEAD, + POST, + DELETE + } + + // Patched from upstream: was package-private; promoted to public so HTTP front-ends + // outside this package can resolve route and request type for body deserialization. + public enum Route { + TOKENS(HTTPMethod.POST, "v1/oauth/tokens", null, OAuthTokenResponse.class), + SEPARATE_AUTH_TOKENS_URI( + HTTPMethod.POST, "https://auth-server.com/token", null, OAuthTokenResponse.class), + CONFIG(HTTPMethod.GET, "v1/config", null, ConfigResponse.class), + LIST_NAMESPACES(HTTPMethod.GET, "v1/namespaces", null, ListNamespacesResponse.class), + CREATE_NAMESPACE( + HTTPMethod.POST, + "v1/namespaces", + CreateNamespaceRequest.class, + CreateNamespaceResponse.class), + LOAD_NAMESPACE(HTTPMethod.GET, "v1/namespaces/{namespace}", null, GetNamespaceResponse.class), + DROP_NAMESPACE(HTTPMethod.DELETE, "v1/namespaces/{namespace}"), + UPDATE_NAMESPACE( + HTTPMethod.POST, + "v1/namespaces/{namespace}/properties", + UpdateNamespacePropertiesRequest.class, + UpdateNamespacePropertiesResponse.class), + LIST_TABLES(HTTPMethod.GET, "v1/namespaces/{namespace}/tables", null, ListTablesResponse.class), + CREATE_TABLE( + HTTPMethod.POST, + "v1/namespaces/{namespace}/tables", + CreateTableRequest.class, + LoadTableResponse.class), + LOAD_TABLE( + HTTPMethod.GET, "v1/namespaces/{namespace}/tables/{name}", null, LoadTableResponse.class), + REGISTER_TABLE( + HTTPMethod.POST, + "v1/namespaces/{namespace}/register", + RegisterTableRequest.class, + LoadTableResponse.class), + UPDATE_TABLE( + HTTPMethod.POST, + "v1/namespaces/{namespace}/tables/{name}", + UpdateTableRequest.class, + LoadTableResponse.class), + DROP_TABLE(HTTPMethod.DELETE, "v1/namespaces/{namespace}/tables/{name}"), + RENAME_TABLE(HTTPMethod.POST, "v1/tables/rename", RenameTableRequest.class, null), + REPORT_METRICS( + HTTPMethod.POST, + "v1/namespaces/{namespace}/tables/{name}/metrics", + ReportMetricsRequest.class, + null), + COMMIT_TRANSACTION( + HTTPMethod.POST, "v1/transactions/commit", CommitTransactionRequest.class, null), + LIST_VIEWS(HTTPMethod.GET, "v1/namespaces/{namespace}/views", null, ListTablesResponse.class), + LOAD_VIEW( + HTTPMethod.GET, "v1/namespaces/{namespace}/views/{name}", null, LoadViewResponse.class), + CREATE_VIEW( + HTTPMethod.POST, + "v1/namespaces/{namespace}/views", + CreateViewRequest.class, + LoadViewResponse.class), + UPDATE_VIEW( + HTTPMethod.POST, + "v1/namespaces/{namespace}/views/{name}", + UpdateTableRequest.class, + LoadViewResponse.class), + RENAME_VIEW(HTTPMethod.POST, "v1/views/rename", RenameTableRequest.class, null), + DROP_VIEW(HTTPMethod.DELETE, "v1/namespaces/{namespace}/views/{name}"); + + private final HTTPMethod method; + private final int requiredLength; + private final Map requirements; + private final Map variables; + private final Class requestClass; + private final Class responseClass; + + Route(HTTPMethod method, String pattern) { + this(method, pattern, null, null); + } + + Route( + HTTPMethod method, + String pattern, + Class requestClass, + Class responseClass) { + this.method = method; + + // parse the pattern into requirements and variables + List parts = SLASH.splitToList(pattern); + ImmutableMap.Builder requirementsBuilder = ImmutableMap.builder(); + ImmutableMap.Builder variablesBuilder = ImmutableMap.builder(); + for (int pos = 0; pos < parts.size(); pos += 1) { + String part = parts.get(pos); + if (part.startsWith("{") && part.endsWith("}")) { + variablesBuilder.put(pos, part.substring(1, part.length() - 1)); + } else { + requirementsBuilder.put(pos, part); + } + } + + this.requestClass = requestClass; + this.responseClass = responseClass; + + this.requiredLength = parts.size(); + this.requirements = requirementsBuilder.build(); + this.variables = variablesBuilder.build(); + } + + private boolean matches(HTTPMethod requestMethod, List requestPath) { + return method == requestMethod + && requiredLength == requestPath.size() + && requirements.entrySet().stream() + .allMatch( + requirement -> + requirement + .getValue() + .equalsIgnoreCase(requestPath.get(requirement.getKey()))); + } + + private Map variables(List requestPath) { + ImmutableMap.Builder vars = ImmutableMap.builder(); + variables.forEach((key, value) -> vars.put(value, requestPath.get(key))); + return vars.build(); + } + + public static Pair> from(HTTPMethod method, String path) { + List parts = SLASH.splitToList(path); + for (Route candidate : Route.values()) { + if (candidate.matches(method, parts)) { + return Pair.of(candidate, candidate.variables(parts)); + } + } + + return null; + } + + public Class requestClass() { + return requestClass; + } + + public Class responseClass() { + return responseClass; + } + } + + private static OAuthTokenResponse handleOAuthRequest(Object body) { + Map request = (Map) castRequest(Map.class, body); + String grantType = request.get("grant_type"); + switch (grantType) { + case "client_credentials": + return OAuthTokenResponse.builder() + .withToken("client-credentials-token:sub=" + request.get("client_id")) + .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token") + .withTokenType("Bearer") + .build(); + + case "urn:ietf:params:oauth:grant-type:token-exchange": + String actor = request.get("actor_token"); + String token = + String.format( + "token-exchange-token:sub=%s%s", + request.get("subject_token"), actor != null ? ",act=" + actor : ""); + return OAuthTokenResponse.builder() + .withToken(token) + .withIssuedTokenType("urn:ietf:params:oauth:token-type:access_token") + .withTokenType("Bearer") + .build(); + + default: + throw new UnsupportedOperationException("Unsupported grant_type: " + grantType); + } + } + + @SuppressWarnings({"MethodLength", "checkstyle:CyclomaticComplexity"}) + public T handleRequest( + Route route, Map vars, Object body, Class responseType) { + switch (route) { + case TOKENS: + return castResponse(responseType, handleOAuthRequest(body)); + + case CONFIG: + return castResponse(responseType, ConfigResponse.builder().build()); + + case LIST_NAMESPACES: + if (asNamespaceCatalog != null) { + Namespace ns; + if (vars.containsKey("parent")) { + ns = + Namespace.of( + RESTUtil.NAMESPACE_SPLITTER + .splitToStream(vars.get("parent")) + .toArray(String[]::new)); + } else { + ns = Namespace.empty(); + } + + return castResponse(responseType, CatalogHandlers.listNamespaces(asNamespaceCatalog, ns)); + } + break; + + case CREATE_NAMESPACE: + if (asNamespaceCatalog != null) { + CreateNamespaceRequest request = castRequest(CreateNamespaceRequest.class, body); + return castResponse( + responseType, CatalogHandlers.createNamespace(asNamespaceCatalog, request)); + } + break; + + case LOAD_NAMESPACE: + if (asNamespaceCatalog != null) { + Namespace namespace = namespaceFromPathVars(vars); + return castResponse( + responseType, CatalogHandlers.loadNamespace(asNamespaceCatalog, namespace)); + } + break; + + case DROP_NAMESPACE: + if (asNamespaceCatalog != null) { + CatalogHandlers.dropNamespace(asNamespaceCatalog, namespaceFromPathVars(vars)); + return null; + } + break; + + case UPDATE_NAMESPACE: + if (asNamespaceCatalog != null) { + Namespace namespace = namespaceFromPathVars(vars); + UpdateNamespacePropertiesRequest request = + castRequest(UpdateNamespacePropertiesRequest.class, body); + return castResponse( + responseType, + CatalogHandlers.updateNamespaceProperties(asNamespaceCatalog, namespace, request)); + } + break; + + case LIST_TABLES: + { + Namespace namespace = namespaceFromPathVars(vars); + return castResponse(responseType, CatalogHandlers.listTables(catalog, namespace)); + } + + case CREATE_TABLE: + { + Namespace namespace = namespaceFromPathVars(vars); + CreateTableRequest request = castRequest(CreateTableRequest.class, body); + request.validate(); + if (request.stageCreate()) { + return castResponse( + responseType, CatalogHandlers.stageTableCreate(catalog, namespace, request)); + } else { + return castResponse( + responseType, CatalogHandlers.createTable(catalog, namespace, request)); + } + } + + case DROP_TABLE: + { + if (PropertyUtil.propertyAsBoolean(vars, "purgeRequested", false)) { + CatalogHandlers.purgeTable(catalog, identFromPathVars(vars)); + } else { + CatalogHandlers.dropTable(catalog, identFromPathVars(vars)); + } + return null; + } + + case LOAD_TABLE: + { + TableIdentifier ident = identFromPathVars(vars); + return castResponse(responseType, CatalogHandlers.loadTable(catalog, ident)); + } + + case REGISTER_TABLE: + { + Namespace namespace = namespaceFromPathVars(vars); + RegisterTableRequest request = castRequest(RegisterTableRequest.class, body); + return castResponse( + responseType, CatalogHandlers.registerTable(catalog, namespace, request)); + } + + case UPDATE_TABLE: + { + TableIdentifier ident = identFromPathVars(vars); + UpdateTableRequest request = castRequest(UpdateTableRequest.class, body); + return castResponse(responseType, CatalogHandlers.updateTable(catalog, ident, request)); + } + + case RENAME_TABLE: + { + RenameTableRequest request = castRequest(RenameTableRequest.class, body); + CatalogHandlers.renameTable(catalog, request); + return null; + } + + case REPORT_METRICS: + { + // nothing to do here other than checking that we're getting the correct request + castRequest(ReportMetricsRequest.class, body); + return null; + } + + case COMMIT_TRANSACTION: + { + CommitTransactionRequest request = castRequest(CommitTransactionRequest.class, body); + commitTransaction(catalog, request); + return null; + } + + case LIST_VIEWS: + { + if (null != asViewCatalog) { + Namespace namespace = namespaceFromPathVars(vars); + return castResponse(responseType, CatalogHandlers.listViews(asViewCatalog, namespace)); + } + break; + } + + case CREATE_VIEW: + { + if (null != asViewCatalog) { + Namespace namespace = namespaceFromPathVars(vars); + CreateViewRequest request = castRequest(CreateViewRequest.class, body); + return castResponse( + responseType, CatalogHandlers.createView(asViewCatalog, namespace, request)); + } + break; + } + + case LOAD_VIEW: + { + if (null != asViewCatalog) { + TableIdentifier ident = identFromPathVars(vars); + return castResponse(responseType, CatalogHandlers.loadView(asViewCatalog, ident)); + } + break; + } + + case UPDATE_VIEW: + { + if (null != asViewCatalog) { + TableIdentifier ident = identFromPathVars(vars); + UpdateTableRequest request = castRequest(UpdateTableRequest.class, body); + return castResponse( + responseType, CatalogHandlers.updateView(asViewCatalog, ident, request)); + } + break; + } + + case RENAME_VIEW: + { + if (null != asViewCatalog) { + RenameTableRequest request = castRequest(RenameTableRequest.class, body); + CatalogHandlers.renameView(asViewCatalog, request); + return null; + } + break; + } + + case DROP_VIEW: + { + if (null != asViewCatalog) { + CatalogHandlers.dropView(asViewCatalog, identFromPathVars(vars)); + return null; + } + break; + } + + default: + if (responseType == OAuthTokenResponse.class) { + return castResponse(responseType, handleOAuthRequest(body)); + } + } + + return null; + } + + /** + * This is a very simplistic approach that only validates the requirements for each table and does + * not do any other conflict detection. Therefore, it does not guarantee true transactional + * atomicity, which is left to the implementation details of a REST server. + */ + private static void commitTransaction(Catalog catalog, CommitTransactionRequest request) { + List transactions = Lists.newArrayList(); + + for (UpdateTableRequest tableChange : request.tableChanges()) { + Table table = catalog.loadTable(tableChange.identifier()); + if (table instanceof BaseTable) { + Transaction transaction = + Transactions.newTransaction( + tableChange.identifier().toString(), ((BaseTable) table).operations()); + transactions.add(transaction); + + BaseTransaction.TransactionTable txTable = + (BaseTransaction.TransactionTable) transaction.table(); + + // this performs validations and makes temporary commits that are in-memory + CatalogHandlers.commit(txTable.operations(), tableChange); + } else { + throw new IllegalStateException("Cannot wrap catalog that does not produce BaseTable"); + } + } + + // only commit if validations passed previously + transactions.forEach(Transaction::commitTransaction); + } + + public T execute( + HTTPMethod method, + String path, + Map queryParams, + Object body, + Class responseType, + Map headers, + Consumer errorHandler) { + ErrorResponse.Builder errorBuilder = ErrorResponse.builder(); + Pair> routeAndVars = Route.from(method, path); + if (routeAndVars != null) { + try { + ImmutableMap.Builder vars = ImmutableMap.builder(); + if (queryParams != null) { + vars.putAll(queryParams); + } + vars.putAll(routeAndVars.second()); + + return handleRequest(routeAndVars.first(), vars.build(), body, responseType); + + } catch (RuntimeException e) { + configureResponseFromException(e, errorBuilder); + } + + } else { + errorBuilder + .responseCode(400) + .withType("BadRequestException") + .withMessage(String.format("No route for request: %s %s", method, path)); + } + + ErrorResponse error = errorBuilder.build(); + errorHandler.accept(error); + + // if the error handler doesn't throw an exception, throw a generic one + throw new RESTException("Unhandled error: %s", error); + } + + @Override + public T delete( + String path, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.DELETE, path, null, null, responseType, headers, errorHandler); + } + + @Override + public T delete( + String path, + Map queryParams, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.DELETE, path, queryParams, null, responseType, headers, errorHandler); + } + + @Override + public T post( + String path, + RESTRequest body, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.POST, path, null, body, responseType, headers, errorHandler); + } + + @Override + public T get( + String path, + Map queryParams, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.GET, path, queryParams, null, responseType, headers, errorHandler); + } + + @Override + public void head(String path, Map headers, Consumer errorHandler) { + execute(HTTPMethod.HEAD, path, null, null, null, headers, errorHandler); + } + + @Override + public T postForm( + String path, + Map formData, + Class responseType, + Map headers, + Consumer errorHandler) { + return execute(HTTPMethod.POST, path, null, formData, responseType, headers, errorHandler); + } + + @Override + public void close() throws IOException { + // The calling test is responsible for closing the underlying catalog backing this REST catalog + // so that the underlying backend catalog is not closed and reopened during the REST catalog's + // initialize method when fetching the server configuration. + } + + private static class BadResponseType extends RuntimeException { + private BadResponseType(Class responseType, Object response) { + super( + String.format("Invalid response object, not a %s: %s", responseType.getName(), response)); + } + } + + private static class BadRequestType extends RuntimeException { + private BadRequestType(Class requestType, Object request) { + super(String.format("Invalid request object, not a %s: %s", requestType.getName(), request)); + } + } + + public static T castRequest(Class requestType, Object request) { + if (requestType.isInstance(request)) { + return requestType.cast(request); + } + + throw new BadRequestType(requestType, request); + } + + public static T castResponse(Class responseType, Object response) { + if (responseType.isInstance(response)) { + return responseType.cast(response); + } + + throw new BadResponseType(responseType, response); + } + + public static void configureResponseFromException( + Exception exc, ErrorResponse.Builder errorBuilder) { + errorBuilder + .responseCode(EXCEPTION_ERROR_CODES.getOrDefault(exc.getClass(), 500)) + .withType(exc.getClass().getSimpleName()) + .withMessage(exc.getMessage()) + .withStackTrace(exc); + } + + private static Namespace namespaceFromPathVars(Map pathVars) { + return RESTUtil.decodeNamespace(pathVars.get("namespace")); + } + + private static TableIdentifier identFromPathVars(Map pathVars) { + return TableIdentifier.of( + namespaceFromPathVars(pathVars), RESTUtil.decodeString(pathVars.get("name"))); + } +} diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java new file mode 100644 index 0000000..6bab1b6 --- /dev/null +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java @@ -0,0 +1,144 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import io.github.mmalykhin.hmsproxy.config.ProxyConfig; +import io.github.mmalykhin.hmsproxy.config.catalog.CatalogAccessMode; +import io.github.mmalykhin.hmsproxy.config.catalog.CatalogConfig; +import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfig; +import io.github.mmalykhin.hmsproxy.config.routing.BackendConfig; +import io.github.mmalykhin.hmsproxy.config.server.ServerConfig; +import io.github.mmalykhin.hmsproxy.config.syntheticlock.SyntheticReadLockStoreConfig; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class IcebergRestEndpointIntegrationTest { + private static final String CATALOG_NAME = "catalog1"; + private static final Duration HTTP_TIMEOUT = Duration.ofSeconds(5); + + private RecordingThriftIface delegate; + private IcebergRestService service; + private RestCatalogServer server; + + @Before + public void setUp() throws Exception { + delegate = new RecordingThriftIface(); + delegate.allDatabases = List.of("sales", "marketing"); + delegate.databases.put("sales", RecordingThriftIface.database("sales")); + delegate.databases.put("marketing", RecordingThriftIface.database("marketing")); + delegate.tablesByDatabase.put("sales", List.of("orders")); + delegate.tables.put("sales.orders", RecordingThriftIface.table("sales", "orders")); + + ProxyConfig config = buildConfig(); + service = new IcebergRestService(config, delegate.iface); + server = RestCatalogServer.open(config, service); + Assert.assertNotNull("server must start", server); + } + + @After + public void tearDown() throws Exception { + if (server != null) { + server.close(); + } + if (service != null) { + service.close(); + } + } + + @Test + public void configEndpointReturnsDefaultCatalogPrefix() throws Exception { + HttpResponse response = get("/v1/config"); + Assert.assertEquals("body: " + response.body(), 200, response.statusCode()); + Assert.assertTrue("response: " + response.body(), + response.body().contains("\"prefix\"")); + Assert.assertTrue("response: " + response.body(), + response.body().contains("\"" + CATALOG_NAME + "\"")); + } + + @Test + public void listNamespacesReturnsAllDatabases() throws Exception { + HttpResponse response = get("/v1/" + CATALOG_NAME + "/namespaces"); + Assert.assertEquals("body: " + response.body(), 200, response.statusCode()); + String body = response.body(); + Assert.assertTrue(body, body.contains("\"sales\"")); + Assert.assertTrue(body, body.contains("\"marketing\"")); + } + + @Test + public void loadNamespaceReturnsDatabaseMetadata() throws Exception { + HttpResponse response = get("/v1/" + CATALOG_NAME + "/namespaces/sales"); + Assert.assertEquals("body: " + response.body(), 200, response.statusCode()); + Assert.assertTrue(response.body(), response.body().contains("\"sales\"")); + } + + @Test + public void loadMissingNamespaceReturns404() throws Exception { + HttpResponse response = get("/v1/" + CATALOG_NAME + "/namespaces/missing"); + Assert.assertEquals("body: " + response.body(), 404, response.statusCode()); + } + + @Test + public void listTablesReturnsTableNames() throws Exception { + HttpResponse response = get("/v1/" + CATALOG_NAME + "/namespaces/sales/tables"); + Assert.assertEquals("body: " + response.body(), 200, response.statusCode()); + Assert.assertTrue(response.body(), response.body().contains("\"orders\"")); + } + + @Test + public void unknownCatalogPrefixReturns404() throws Exception { + HttpResponse response = get("/v1/unknown-catalog/namespaces"); + Assert.assertEquals(404, response.statusCode()); + } + + @Test + public void rootPathReturns404() throws Exception { + HttpResponse response = get("/"); + Assert.assertEquals(404, response.statusCode()); + } + + @Test + public void loadingNonIcebergTableReturnsErrorResponse() throws Exception { + HttpResponse response = get("/v1/" + CATALOG_NAME + "/namespaces/sales/tables/orders"); + Assert.assertTrue("expected 4xx for non-iceberg table, got " + response.statusCode() + + " body: " + response.body(), + response.statusCode() >= 400 && response.statusCode() < 500); + } + + private HttpResponse get(String path) throws Exception { + HttpClient client = HttpClient.newBuilder().connectTimeout(HTTP_TIMEOUT).build(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("http://127.0.0.1:" + server.boundPort() + path)) + .timeout(HTTP_TIMEOUT) + .GET() + .build(); + return client.send(request, HttpResponse.BodyHandlers.ofString()); + } + + private static ProxyConfig buildConfig() { + return ProxyConfig.builder() + .server(new ServerConfig("hms-proxy-test", "127.0.0.1", 9083, 1, 4)) + .catalogDbSeparator(".") + .defaultCatalog(CATALOG_NAME) + .catalogs(Map.of(CATALOG_NAME, new CatalogConfig( + CATALOG_NAME, + null, + null, + false, + CatalogAccessMode.READ_WRITE, + List.of(), + null, + null, + Map.of("hive.metastore.uris", "thrift://hms-test:9083")))) + .backend(new BackendConfig(Map.of())) + .restCatalog(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)) + .syntheticReadLockStore(SyntheticReadLockStoreConfig.inMemory()) + .build(); + } +} diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java new file mode 100644 index 0000000..425b543 --- /dev/null +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java @@ -0,0 +1,115 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; + +/** In-memory fake of ThriftHiveMetastore.Iface for unit tests in the restcatalog package. */ +final class RecordingThriftIface { + final List calls = new ArrayList<>(); + final Map databases = new HashMap<>(); + final Map tables = new HashMap<>(); + final Map> tablesByDatabase = new HashMap<>(); + List allDatabases = Collections.emptyList(); + final ThriftHiveMetastore.Iface iface; + + RecordingThriftIface() { + this.iface = (ThriftHiveMetastore.Iface) Proxy.newProxyInstance( + ThriftHiveMetastore.Iface.class.getClassLoader(), + new Class[]{ThriftHiveMetastore.Iface.class}, + this::handle); + } + + static Database database(String name) { + Database db = new Database(); + db.setName(name); + db.setDescription(""); + db.setLocationUri("file:///tmp/" + name); + db.setParameters(new HashMap<>()); + return db; + } + + static Table table(String db, String name) { + Table t = new Table(); + t.setDbName(db); + t.setTableName(name); + t.setOwner("test"); + t.setTableType("EXTERNAL_TABLE"); + Map params = new HashMap<>(); + // HiveCatalog filters listTables() by this marker; without it, Iceberg-aware + // clients see an empty namespace even though the underlying HMS row exists. + params.put("table_type", "ICEBERG"); + t.setParameters(params); + return t; + } + + private Object handle(Object proxy, Method method, Object[] args) throws Throwable { + String name = method.getName(); + switch (name) { + case "get_database": { + String db = (String) args[0]; + calls.add("get_database:" + db); + Database value = databases.get(db); + if (value == null) { + throw new NoSuchObjectException("no database " + db); + } + return value; + } + case "get_all_databases": + calls.add("get_all_databases"); + return allDatabases; + case "get_databases": { + String pattern = (String) args[0]; + calls.add("get_databases:" + pattern); + return allDatabases; + } + case "get_all_tables": { + String db = (String) args[0]; + calls.add("get_all_tables:" + db); + return tablesByDatabase.getOrDefault(db, Collections.emptyList()); + } + case "get_tables": { + String db = (String) args[0]; + String pattern = (String) args[1]; + calls.add("get_tables:" + db + ":" + pattern); + return tablesByDatabase.getOrDefault(db, Collections.emptyList()); + } + case "get_table": { + String db = (String) args[0]; + String tbl = (String) args[1]; + calls.add("get_table:" + db + ":" + tbl); + Table value = tables.get(db + "." + tbl); + if (value == null) { + throw new NoSuchObjectException("no table " + db + "." + tbl); + } + return value; + } + case "get_table_objects_by_name": { + String db = (String) args[0]; + @SuppressWarnings("unchecked") + List tableNames = (List) args[1]; + calls.add("get_table_objects_by_name:" + db + ":" + tableNames); + List result = new java.util.ArrayList<>(); + for (String t : tableNames) { + Table value = tables.get(db + "." + t); + if (value != null) { + result.add(value); + } + } + return result; + } + default: + throw new UnsupportedOperationException("unexpected Iface call: " + name + + " args=" + Arrays.toString(args)); + } + } +} diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingHiveCatalogTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingHiveCatalogTest.java new file mode 100644 index 0000000..49b5117 --- /dev/null +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingHiveCatalogTest.java @@ -0,0 +1,55 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.iceberg.ClientPool; +import org.apache.iceberg.catalog.Namespace; +import org.junit.Assert; +import org.junit.Test; + +public class RoutingHiveCatalogTest { + @Test + public void reflectionInjectReplacesClientPool() { + RecordingThriftIface delegate = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + RoutingHiveCatalog catalog = new RoutingHiveCatalog(client, new Configuration()); + + catalog.initialize("test", Map.of()); + + ClientPool activePool = catalog.activeClientPool(); + Assert.assertTrue( + "Expected RoutingClientPool after initialize, got: " + activePool.getClass().getName(), + activePool instanceof RoutingClientPool); + } + + @Test + public void namespaceExistsRoutesThroughDelegate() { + RecordingThriftIface delegate = new RecordingThriftIface(); + delegate.databases.put("sales", RecordingThriftIface.database("sales")); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + RoutingHiveCatalog catalog = new RoutingHiveCatalog(client, new Configuration()); + catalog.initialize("test", Map.of()); + + boolean exists = catalog.namespaceExists(Namespace.of("sales")); + + Assert.assertTrue(exists); + Assert.assertTrue("expected get_database call, recorded: " + delegate.calls, + delegate.calls.stream().anyMatch(c -> c.startsWith("get_database:sales"))); + } + + @Test + public void loadNamespaceMetadataRoutesThroughDelegate() { + RecordingThriftIface delegate = new RecordingThriftIface(); + delegate.databases.put("sales", RecordingThriftIface.database("sales")); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + RoutingHiveCatalog catalog = new RoutingHiveCatalog(client, new Configuration()); + catalog.initialize("test", Map.of()); + + Map metadata = catalog.loadNamespaceMetadata(Namespace.of("sales")); + + Assert.assertNotNull(metadata); + Assert.assertTrue("expected get_database call, recorded: " + delegate.calls, + delegate.calls.stream().anyMatch(c -> c.startsWith("get_database:sales"))); + } +} diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java new file mode 100644 index 0000000..dd6e105 --- /dev/null +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java @@ -0,0 +1,94 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import java.util.List; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Table; +import org.junit.Assert; +import org.junit.Test; + +public class RoutingMetaStoreClientTest { + @Test + public void getDatabaseDelegatesToGetDatabaseRpc() throws Exception { + RecordingThriftIface delegate = new RecordingThriftIface(); + delegate.databases.put("sales", RecordingThriftIface.database("sales")); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + Database db = client.getDatabase("sales"); + + Assert.assertEquals("sales", db.getName()); + Assert.assertEquals(List.of("get_database:sales"), delegate.calls); + } + + @Test + public void getAllDatabasesDelegates() throws Exception { + RecordingThriftIface delegate = new RecordingThriftIface(); + delegate.allDatabases = List.of("sales", "marketing"); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + Assert.assertEquals(List.of("sales", "marketing"), client.getAllDatabases()); + Assert.assertEquals(List.of("get_all_databases"), delegate.calls); + } + + @Test + public void getAllTablesDelegates() throws Exception { + RecordingThriftIface delegate = new RecordingThriftIface(); + delegate.tablesByDatabase.put("sales", List.of("orders", "customers")); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + Assert.assertEquals(List.of("orders", "customers"), client.getAllTables("sales")); + Assert.assertEquals(List.of("get_all_tables:sales"), delegate.calls); + } + + @Test + public void getTableDelegates() throws Exception { + RecordingThriftIface delegate = new RecordingThriftIface(); + delegate.tables.put("sales.orders", RecordingThriftIface.table("sales", "orders")); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + Table table = client.getTable("sales", "orders"); + Assert.assertEquals("orders", table.getTableName()); + Assert.assertEquals(List.of("get_table:sales:orders"), delegate.calls); + } + + @Test + public void tableExistsReturnsTrueWhenTableFound() throws Exception { + RecordingThriftIface delegate = new RecordingThriftIface(); + delegate.tables.put("sales.orders", RecordingThriftIface.table("sales", "orders")); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + Assert.assertTrue(client.tableExists("sales", "orders")); + } + + @Test + public void tableExistsReturnsFalseOnNoSuchObject() throws Exception { + RecordingThriftIface delegate = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + Assert.assertFalse(client.tableExists("sales", "missing")); + } + + @Test + public void unsupportedMethodThrowsUnsupportedOperationException() { + RecordingThriftIface delegate = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + try { + client.dropDatabase("anything"); + Assert.fail("expected UnsupportedOperationException"); + } catch (UnsupportedOperationException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("dropDatabase")); + } catch (Exception other) { + Assert.fail("expected UnsupportedOperationException, got " + other.getClass().getName()); + } + } + + @Test + public void closeIsNoop() { + RecordingThriftIface delegate = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + client.close(); + Assert.assertTrue(delegate.calls.isEmpty()); + } +} From e1cc69aeca4974b7bfac7c21116e3136e5c5c03d Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 25 May 2026 06:20:10 +0300 Subject: [PATCH 03/50] Add SPNEGO/Kerberos auth to Iceberg REST endpoint --- pom.xml | 6 + .../config/restcatalog/RestCatalogConfig.java | 10 +- .../restcatalog/RestCatalogConfigParser.java | 13 +- .../restcatalog/IcebergHttpHandler.java | 18 ++ .../restcatalog/RestCatalogServer.java | 23 ++- .../restcatalog/RestKerberosBootstrap.java | 31 +++ .../restcatalog/SpnegoAuthenticator.java | 127 +++++++++++++ .../resources/hms-proxy-example.properties | 7 + .../IcebergRestEndpointIntegrationTest.java | 2 +- .../restcatalog/RestCatalogServerTest.java | 10 +- .../restcatalog/SpnegoIntegrationTest.java | 179 ++++++++++++++++++ 11 files changed, 415 insertions(+), 11 deletions(-) create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestKerberosBootstrap.java create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoAuthenticator.java create mode 100644 src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoIntegrationTest.java diff --git a/pom.xml b/pom.xml index 54e1c3b..6907dc8 100644 --- a/pom.xml +++ b/pom.xml @@ -273,6 +273,12 @@ 2.3 test + + org.apache.hadoop + hadoop-minikdc + 3.3.6 + test + diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfig.java b/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfig.java index ffa1b25..f2fb1e7 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfig.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfig.java @@ -5,9 +5,15 @@ public record RestCatalogConfig( String bindHost, int port, int minWorkerThreads, - int maxWorkerThreads + int maxWorkerThreads, + String kerberosPrincipal, + String kerberosKeytab ) { public static RestCatalogConfig disabled() { - return new RestCatalogConfig(false, "0.0.0.0", 8181, 8, 64); + return new RestCatalogConfig(false, "0.0.0.0", 8181, 8, 64, null, null); + } + + public boolean kerberosEnabled() { + return kerberosPrincipal != null && kerberosKeytab != null; } } diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParser.java b/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParser.java index 308183a..8b41885 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParser.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/config/restcatalog/RestCatalogConfigParser.java @@ -1,5 +1,6 @@ package io.github.mmalykhin.hmsproxy.config.restcatalog; +import io.github.mmalykhin.hmsproxy.config.ConfigParsing; import io.github.mmalykhin.hmsproxy.config.PropertyReader; import io.github.mmalykhin.hmsproxy.config.server.ServerConfig; @@ -27,6 +28,16 @@ public static RestCatalogConfig parse(PropertyReader reader, ServerConfig server "rest-catalog.max-worker-threads (" + maxWorkerThreads + ") must be >= rest-catalog.min-worker-threads (" + minWorkerThreads + ")"); } - return new RestCatalogConfig(enabled, bindHost, port, minWorkerThreads, maxWorkerThreads); + String principal = reader.getOrNull("rest-catalog.kerberos.principal"); + String keytab = reader.getOrNull("rest-catalog.kerberos.keytab"); + if ((principal == null) != (keytab == null)) { + throw new IllegalArgumentException( + "rest-catalog.kerberos.principal and rest-catalog.kerberos.keytab must be set together"); + } + if (enabled && keytab != null) { + ConfigParsing.requireReadableFile(keytab, "rest-catalog.kerberos.keytab"); + } + return new RestCatalogConfig( + enabled, bindHost, port, minWorkerThreads, maxWorkerThreads, principal, keytab); } } diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java index 3980239..cb4c917 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java @@ -2,6 +2,8 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpPrincipal; +import io.github.mmalykhin.hmsproxy.security.ClientRequestContext; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -40,6 +42,22 @@ final class IcebergHttpHandler implements HttpHandler { @Override public void handle(HttpExchange exchange) throws IOException { + String remoteAddress = exchange.getRemoteAddress() != null + ? exchange.getRemoteAddress().getAddress().getHostAddress() + : null; + HttpPrincipal principal = exchange.getPrincipal(); + String remoteUser = principal != null ? principal.getUsername() : null; + String previousAddress = ClientRequestContext.setRemoteAddress(remoteAddress); + String previousUser = ClientRequestContext.setRemoteUser(remoteUser); + try { + doHandle(exchange); + } finally { + ClientRequestContext.restoreRemoteAddress(previousAddress); + ClientRequestContext.restoreRemoteUser(previousUser); + } + } + + private void doHandle(HttpExchange exchange) throws IOException { try { String rawPath = exchange.getRequestURI().getPath(); if (!rawPath.startsWith(V1_PREFIX) && !rawPath.equals("/v1")) { diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java index a12c5f0..cc93eda 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java @@ -1,5 +1,6 @@ package io.github.mmalykhin.hmsproxy.restcatalog; +import com.sun.net.httpserver.HttpContext; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; @@ -16,6 +17,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,10 +54,27 @@ public static RestCatalogServer open(ProxyConfig config, IcebergRestService serv throw e; } + SpnegoAuthenticator authenticator = null; + if (restConfig.kerberosEnabled()) { + try { + UserGroupInformation ugi = RestKerberosBootstrap.login(restConfig); + authenticator = new SpnegoAuthenticator(ugi, restConfig.kerberosPrincipal()); + LOG.info("Iceberg REST SPNEGO enabled for principal {}", restConfig.kerberosPrincipal()); + } catch (Exception e) { + server.stop(0); + throw new IOException( + "Failed to initialise SPNEGO authenticator for Iceberg REST listener: " + e.getMessage(), e); + } + } + + HttpContext v1Context; if (service != null) { - server.createContext("/v1/", new IcebergHttpHandler(service)); + v1Context = server.createContext("/v1/", new IcebergHttpHandler(service)); } else { - server.createContext("/v1/config", new ConfigHandler()); + v1Context = server.createContext("/v1/config", new ConfigHandler()); + } + if (authenticator != null) { + v1Context.setAuthenticator(authenticator); } server.createContext("/", new NotFoundHandler()); diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestKerberosBootstrap.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestKerberosBootstrap.java new file mode 100644 index 0000000..27aeb82 --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestKerberosBootstrap.java @@ -0,0 +1,31 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfig; +import io.github.mmalykhin.hmsproxy.security.KerberosPrincipalUtil; +import java.io.IOException; +import org.apache.hadoop.security.UserGroupInformation; + +/** + * Logs in the SPNEGO service principal (HTTP/<host>@REALM) from its keytab + * without overwriting the global Hadoop login user, which is owned by + * FrontDoorSecurity for the Thrift listener. Returns a UGI usable as the + * Subject for {@code GSSContext.acceptSecContext()}. + */ +final class RestKerberosBootstrap { + private RestKerberosBootstrap() { + } + + static UserGroupInformation login(RestCatalogConfig config) throws IOException { + if (!config.kerberosEnabled()) { + throw new IllegalStateException( + "RestKerberosBootstrap.login called with Kerberos disabled in rest-catalog config"); + } + if (!UserGroupInformation.isSecurityEnabled()) { + throw new IllegalStateException( + "Iceberg REST SPNEGO requires Hadoop security to be enabled. Set security.mode=KERBEROS " + + "so FrontDoorSecurity installs a Kerberos UGI configuration before the REST listener starts."); + } + String principal = KerberosPrincipalUtil.resolveForLocalHost(config.kerberosPrincipal()); + return UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, config.kerberosKeytab()); + } +} diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoAuthenticator.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoAuthenticator.java new file mode 100644 index 0000000..37d72be --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoAuthenticator.java @@ -0,0 +1,127 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import com.sun.net.httpserver.Authenticator; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpPrincipal; +import io.github.mmalykhin.hmsproxy.security.KerberosPrincipalUtil; +import java.security.PrivilegedExceptionAction; +import java.util.Base64; +import java.util.Objects; +import org.apache.hadoop.security.UserGroupInformation; +import org.ietf.jgss.GSSContext; +import org.ietf.jgss.GSSCredential; +import org.ietf.jgss.GSSManager; +import org.ietf.jgss.GSSName; +import org.ietf.jgss.Oid; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * SPNEGO Authenticator backed by JDK GSSAPI. Validates the inbound + * Authorization: Negotiate header against credentials acquired from the REST + * service keytab. On success the authenticated principal is exposed via + * {@link HttpExchange#getPrincipal()} so the downstream handler can stamp + * ClientRequestContext. + * + * Thread safety: serverCredentials is acquired once and reused. GSSContext is + * not thread-safe but a new one is created per request. + */ +final class SpnegoAuthenticator extends Authenticator { + private static final Logger LOG = LoggerFactory.getLogger(SpnegoAuthenticator.class); + private static final String NEGOTIATE = "Negotiate"; + private static final Oid SPNEGO_OID = oid("1.3.6.1.5.5.2"); + private static final Oid KERBEROS_OID = oid("1.2.840.113554.1.2.2"); + + private final UserGroupInformation ugi; + private final String realm; + private final GSSCredential serverCredentials; + + SpnegoAuthenticator(UserGroupInformation ugi, String principalName) throws Exception { + this.ugi = Objects.requireNonNull(ugi, "ugi"); + String resolved = KerberosPrincipalUtil.resolveForLocalHost( + Objects.requireNonNull(principalName, "principalName")); + this.realm = extractRealm(resolved); + this.serverCredentials = acquireServerCredentials(ugi, resolved); + } + + @Override + public Result authenticate(HttpExchange exchange) { + String header = exchange.getRequestHeaders().getFirst("Authorization"); + if (header == null || !header.regionMatches(true, 0, NEGOTIATE, 0, NEGOTIATE.length())) { + exchange.getResponseHeaders().set("WWW-Authenticate", NEGOTIATE); + return new Retry(401); + } + String tokenPart = header.length() > NEGOTIATE.length() + ? header.substring(NEGOTIATE.length()).trim() + : ""; + if (tokenPart.isEmpty()) { + exchange.getResponseHeaders().set("WWW-Authenticate", NEGOTIATE); + return new Retry(401); + } + + byte[] token; + try { + token = Base64.getDecoder().decode(tokenPart); + } catch (IllegalArgumentException e) { + LOG.debug("Malformed SPNEGO token (not base64)", e); + return new Failure(400); + } + + try { + return ugi.doAs((PrivilegedExceptionAction) () -> acceptToken(exchange, token)); + } catch (Exception e) { + LOG.warn("SPNEGO authentication failed: {}", e.getMessage()); + return new Failure(401); + } + } + + private Result acceptToken(HttpExchange exchange, byte[] token) throws Exception { + GSSManager manager = GSSManager.getInstance(); + GSSContext context = manager.createContext(serverCredentials); + try { + byte[] outToken = context.acceptSecContext(token, 0, token.length); + if (outToken != null && outToken.length > 0) { + exchange.getResponseHeaders().set( + "WWW-Authenticate", + NEGOTIATE + " " + Base64.getEncoder().encodeToString(outToken)); + } + if (!context.isEstablished()) { + // Multi-leg SPNEGO is rare; treat as auth-incomplete. + return new Retry(401); + } + String clientName = context.getSrcName().toString(); + return new Success(new HttpPrincipal(clientName, realm)); + } finally { + try { + context.dispose(); + } catch (Exception ignored) { + // best effort + } + } + } + + private static GSSCredential acquireServerCredentials(UserGroupInformation ugi, String principal) throws Exception { + return ugi.doAs((PrivilegedExceptionAction) () -> { + GSSManager manager = GSSManager.getInstance(); + GSSName serverName = manager.createName(principal, GSSName.NT_USER_NAME, KERBEROS_OID); + return manager.createCredential( + serverName, + GSSCredential.INDEFINITE_LIFETIME, + new Oid[]{SPNEGO_OID, KERBEROS_OID}, + GSSCredential.ACCEPT_ONLY); + }); + } + + private static String extractRealm(String principal) { + int at = principal.lastIndexOf('@'); + return at >= 0 ? principal.substring(at + 1) : ""; + } + + private static Oid oid(String value) { + try { + return new Oid(value); + } catch (Exception e) { + throw new IllegalStateException("Cannot construct OID " + value, e); + } + } +} diff --git a/src/main/resources/hms-proxy-example.properties b/src/main/resources/hms-proxy-example.properties index 3aff834..6904b74 100644 --- a/src/main/resources/hms-proxy-example.properties +++ b/src/main/resources/hms-proxy-example.properties @@ -93,6 +93,13 @@ server.max-worker-threads=512 # rest-catalog.port=9183 # rest-catalog.min-worker-threads=8 # rest-catalog.max-worker-threads=64 +# Optional SPNEGO/Kerberos protection for the REST listener. Requires security.mode=KERBEROS +# (the Thrift front-door must already have installed a Kerberos UGI). The REST principal +# is a separate HTTP/@REALM (SPNEGO RFC requirement); it is NOT the same as +# security.server-principal (which is hms/@REALM for Thrift). Both principals can +# live in the same keytab or in two separate keytab files. +# rest-catalog.kerberos.principal=HTTP/_HOST@EXAMPLE.COM +# rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab # ----------------------------------------------------------------------------- # Routing diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java index 6bab1b6..57487c3 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java @@ -137,7 +137,7 @@ private static ProxyConfig buildConfig() { null, Map.of("hive.metastore.uris", "thrift://hms-test:9083")))) .backend(new BackendConfig(Map.of())) - .restCatalog(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)) + .restCatalog(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)) .syntheticReadLockStore(SyntheticReadLockStoreConfig.inMemory()) .build(); } diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java index b084c08..a7adcc4 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java @@ -22,14 +22,14 @@ public class RestCatalogServerTest { @Test public void returnsNullWhenDisabled() throws Exception { - ProxyConfig config = buildConfig(new RestCatalogConfig(false, "127.0.0.1", 0, 1, 4)); + ProxyConfig config = buildConfig(new RestCatalogConfig(false, "127.0.0.1", 0, 1, 4, null, null)); RestCatalogServer server = RestCatalogServer.open(config); Assert.assertNull(server); } @Test public void servesEmptyIcebergConfigOnGet() throws Exception { - ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)); + ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)); try (RestCatalogServer server = RestCatalogServer.open(config)) { Assert.assertNotNull(server); HttpResponse response = request(server, "/v1/config", "GET"); @@ -42,7 +42,7 @@ public void servesEmptyIcebergConfigOnGet() throws Exception { @Test public void rejectsNonReadMethodsOnConfig() throws Exception { - ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)); + ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)); try (RestCatalogServer server = RestCatalogServer.open(config)) { HttpResponse response = request(server, "/v1/config", "POST"); Assert.assertEquals(405, response.statusCode()); @@ -52,7 +52,7 @@ public void rejectsNonReadMethodsOnConfig() throws Exception { @Test public void respondsWithNotFoundForUnknownPath() throws Exception { - ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)); + ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)); try (RestCatalogServer server = RestCatalogServer.open(config)) { HttpResponse response = request(server, "/v1/namespaces", "GET"); Assert.assertEquals(404, response.statusCode()); @@ -62,7 +62,7 @@ public void respondsWithNotFoundForUnknownPath() throws Exception { @Test public void boundPortReflectsActualListener() throws Exception { - ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4)); + ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)); try (RestCatalogServer server = RestCatalogServer.open(config)) { Assert.assertTrue("port must be allocated", server.boundPort() > 0); } diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoIntegrationTest.java new file mode 100644 index 0000000..e890699 --- /dev/null +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoIntegrationTest.java @@ -0,0 +1,179 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import io.github.mmalykhin.hmsproxy.config.ProxyConfig; +import io.github.mmalykhin.hmsproxy.config.catalog.CatalogAccessMode; +import io.github.mmalykhin.hmsproxy.config.catalog.CatalogConfig; +import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfig; +import io.github.mmalykhin.hmsproxy.config.routing.BackendConfig; +import io.github.mmalykhin.hmsproxy.config.server.ServerConfig; +import io.github.mmalykhin.hmsproxy.config.syntheticlock.SyntheticReadLockStoreConfig; +import java.io.File; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.security.PrivilegedExceptionAction; +import java.time.Duration; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.ietf.jgss.GSSContext; +import org.ietf.jgss.GSSManager; +import org.ietf.jgss.GSSName; +import org.ietf.jgss.Oid; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class SpnegoIntegrationTest { + private static final Duration HTTP_TIMEOUT = Duration.ofSeconds(10); + private static final String SPNEGO_OID = "1.3.6.1.5.5.2"; + + private static MiniKdc kdc; + private static File workDir; + private static File serverKeytab; + private static File clientKeytab; + private static String realm; + private static String serverPrincipal; + private static String clientPrincipal; + + @BeforeClass + public static void startKdc() throws Exception { + workDir = Files.createTempDirectory("hms-proxy-spnego-test").toFile(); + Properties kdcProps = MiniKdc.createConf(); + kdc = new MiniKdc(kdcProps, workDir); + kdc.start(); + realm = kdc.getRealm(); + serverPrincipal = "HTTP/localhost@" + realm; + clientPrincipal = "alice@" + realm; + + serverKeytab = new File(workDir, "server.keytab"); + kdc.createPrincipal(serverKeytab, "HTTP/localhost"); + clientKeytab = new File(workDir, "alice.keytab"); + kdc.createPrincipal(clientKeytab, "alice"); + + // Krb5 Config caches the previously-loaded krb5.conf; force a reload so + // UGI.setConfiguration picks up MiniKdc's freshly-written file when other + // tests in the suite already touched the JGSS singleton. + Class configClass = Class.forName("sun.security.krb5.Config"); + configClass.getDeclaredMethod("refresh").invoke(null); + + Configuration conf = new Configuration(); + conf.set("hadoop.security.authentication", "kerberos"); + // KerberosName.rules is JVM-global; other tests in the suite may have + // installed custom rules. Explicit per-component rules (instead of bare + // DEFAULT) reliably match HTTP/@REALM and alice@REALM regardless of + // what previously ran in this JVM. + String rules = String.join("\n", + "RULE:[1:$1@$0](.*@" + realm + ")s/@.*//", + "RULE:[2:$1@$0](.*@" + realm + ")s/@.*//", + "DEFAULT"); + conf.set("hadoop.security.auth_to_local", rules); + UserGroupInformation.setConfiguration(conf); + KerberosName.setRules(rules); + } + + @AfterClass + public static void stopKdc() { + if (kdc != null) { + kdc.stop(); + } + Configuration simple = new Configuration(); + simple.set("hadoop.security.authentication", "simple"); + UserGroupInformation.setConfiguration(simple); + } + + @Test + public void unauthenticatedRequestGetsNegotiateChallenge() throws Exception { + try (RestCatalogServer server = RestCatalogServer.open(buildProxyConfig(), null)) { + HttpResponse response = HttpClient.newBuilder().connectTimeout(HTTP_TIMEOUT).build() + .send(HttpRequest.newBuilder() + .uri(URI.create("http://127.0.0.1:" + server.boundPort() + "/v1/config")) + .timeout(HTTP_TIMEOUT) + .GET() + .build(), + HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals(401, response.statusCode()); + Assert.assertEquals("Negotiate", + response.headers().firstValue("WWW-Authenticate").orElse("")); + } + } + + @Test + public void invalidNegotiateTokenIsRejected() throws Exception { + try (RestCatalogServer server = RestCatalogServer.open(buildProxyConfig(), null)) { + HttpResponse response = HttpClient.newBuilder().connectTimeout(HTTP_TIMEOUT).build() + .send(HttpRequest.newBuilder() + .uri(URI.create("http://127.0.0.1:" + server.boundPort() + "/v1/config")) + .timeout(HTTP_TIMEOUT) + .header("Authorization", "Negotiate not-a-real-base64!@#") + .GET() + .build(), + HttpResponse.BodyHandlers.ofString()); + Assert.assertTrue("expected 4xx, got " + response.statusCode(), + response.statusCode() == 400 || response.statusCode() == 401); + } + } + + @Test + public void authenticatedRequestSucceeds() throws Exception { + try (RestCatalogServer server = RestCatalogServer.open(buildProxyConfig(), null)) { + UserGroupInformation clientUgi = UserGroupInformation + .loginUserFromKeytabAndReturnUGI(clientPrincipal, clientKeytab.getAbsolutePath()); + String token = clientUgi.doAs((PrivilegedExceptionAction) () -> { + GSSManager manager = GSSManager.getInstance(); + Oid spnegoOid = new Oid(SPNEGO_OID); + GSSName serverName = manager.createName(serverPrincipal, GSSName.NT_USER_NAME); + GSSContext context = manager.createContext(serverName, spnegoOid, null, GSSContext.DEFAULT_LIFETIME); + try { + byte[] outToken = context.initSecContext(new byte[0], 0, 0); + return Base64.getEncoder().encodeToString(outToken); + } finally { + context.dispose(); + } + }); + + HttpResponse response = HttpClient.newBuilder().connectTimeout(HTTP_TIMEOUT).build() + .send(HttpRequest.newBuilder() + .uri(URI.create("http://127.0.0.1:" + server.boundPort() + "/v1/config")) + .timeout(HTTP_TIMEOUT) + .header("Authorization", "Negotiate " + token) + .GET() + .build(), + HttpResponse.BodyHandlers.ofString()); + Assert.assertEquals("body: " + response.body(), 200, response.statusCode()); + Assert.assertTrue("expected ConfigResponse skeleton, got " + response.body(), + response.body().contains("\"defaults\"") && response.body().contains("\"overrides\"")); + } + } + + private static ProxyConfig buildProxyConfig() { + RestCatalogConfig rest = new RestCatalogConfig( + true, "127.0.0.1", 0, 1, 4, serverPrincipal, serverKeytab.getAbsolutePath()); + return ProxyConfig.builder() + .server(new ServerConfig("hms-proxy-spnego-test", "127.0.0.1", 9083, 1, 4)) + .catalogDbSeparator(".") + .defaultCatalog("catalog1") + .catalogs(Map.of("catalog1", new CatalogConfig( + "catalog1", + null, + null, + false, + CatalogAccessMode.READ_WRITE, + List.of(), + null, + null, + Map.of("hive.metastore.uris", "thrift://hms-test:9083")))) + .backend(new BackendConfig(Map.of())) + .restCatalog(rest) + .syntheticReadLockStore(SyntheticReadLockStoreConfig.inMemory()) + .build(); + } +} From 00ee5b3a6e06838027003ca94476de71df46ab54 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 25 May 2026 06:24:16 +0300 Subject: [PATCH 04/50] Document Iceberg REST frontend in README and CHANGELOG --- CHANGELOG.md | 16 +++++++++++ CHANGELOG.ru.md | 16 +++++++++++ README.md | 73 ++++++++++++++++++++++++++++++++++++++++++++++++ README.ru.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a665cbb..8d63c5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -275,6 +275,22 @@ For a Russian version, see [CHANGELOG.ru.md](CHANGELOG.ru.md). - `APACHE_4_1_0` enum value in `FrontendProfile` and `MetastoreRuntimeProfile`. The latter rejects being used as a backend (`BackendAdapterFactory` throws) — Hive 4 is supported as a front-door profile only. +- Iceberg REST Catalog frontend (experimental, read-only). A parallel HTTP + listener configured via `rest-catalog.*` properties exposes a subset of the + Iceberg REST Catalog spec — `GET /v1/config`, namespace list/load, and table + list/load — backed by the same routing/federation pipeline as the Thrift HMS + front door via an in-process `IMetaStoreClient` proxy. Only the proxy's + `routing.default-catalog` is exposed (multi-catalog REST is planned). +- SPNEGO/Kerberos protection for the REST endpoint. The listener uses a + separate `HTTP/@REALM` principal (`rest-catalog.kerberos.principal` + + `.keytab`); the authenticated principal is propagated into + `ClientRequestContext.remoteUser` so audit logs match the user. Requires + `security.mode=KERBEROS` on the front door. + +### Tests + +- `hadoop-minikdc` test dependency was added so the SPNEGO handshake can be + validated end-to-end inside a single JVM (`SpnegoIntegrationTest`). ## 2026-05-19 diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 29e9de1..86310ec 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -280,6 +280,22 @@ English version: [CHANGELOG.md](CHANGELOG.md). - `APACHE_4_1_0` enum value в `FrontendProfile` и `MetastoreRuntimeProfile`. Последний запрещает использовать себя как backend (`BackendAdapterFactory` throws) — Hive 4 поддержан только как front-door profile. +- Iceberg REST Catalog frontend (экспериментально, read-only). Параллельный + HTTP listener, настраиваемый через `rest-catalog.*`, открывает подмножество + Iceberg REST Catalog spec — `GET /v1/config`, list/load namespace, list/load + table — поверх того же routing/federation pipeline, что и Thrift HMS front + door, через in-process `IMetaStoreClient` proxy. Доступен только + `routing.default-catalog` (multi-catalog REST — на следующую итерацию). +- SPNEGO/Kerberos защита REST endpoint'а. Listener использует отдельный + principal `HTTP/@REALM` (`rest-catalog.kerberos.principal` + + `.keytab`); аутентифицированный principal пробрасывается в + `ClientRequestContext.remoteUser`, чтобы audit log соответствовал + пользователю. Требует `security.mode=KERBEROS` на front door. + +### Тесты + +- Добавлена test-dependency `hadoop-minikdc` для валидации SPNEGO handshake + end-to-end внутри одного JVM (`SpnegoIntegrationTest`). ## 2026-05-19 diff --git a/README.md b/README.md index fbf4a1c..765522f 100644 --- a/README.md +++ b/README.md @@ -974,6 +974,79 @@ SELECT * FROM `catalog2__sales`.orders LIMIT 10; If you keep the default separator `.`, older Hive SQL clients can treat `catalog.db.table` ambiguously, so `__` is usually the safer choice. +## Iceberg REST Catalog frontend + +The proxy can also run a parallel HTTP listener that speaks the Iceberg REST +Catalog spec, backed by the same routing/federation pipeline as the Thrift HMS +front door. Status: **experimental, read-only**. Iceberg clients (PyIceberg, +Spark `iceberg-rest`, Trino `iceberg-rest`) can discover and load Iceberg +tables stored in HMS via the standard `metadata_location` table parameter; +writes, commits, view operations, and multi-catalog routing are NOT supported +in this iteration. + +Enable it via: + +```properties +rest-catalog.enabled=true +rest-catalog.port=9183 +# Optional but recommended for production: SPNEGO. Requires security.mode=KERBEROS. +rest-catalog.kerberos.principal=HTTP/_HOST@EXAMPLE.COM +rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab +``` + +### Supported endpoints + +| Endpoint | Status | +| ----------------------------------------------------- | ------------------------------- | +| `GET /v1/config` | supported | +| `GET /v1/{prefix}/namespaces` | supported | +| `GET /v1/{prefix}/namespaces/{ns}` | supported | +| `GET /v1/{prefix}/namespaces/{ns}/tables` | supported (Iceberg tables only) | +| `GET /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | supported (Iceberg tables only) | +| `POST`, `DELETE`, view operations, commits | unsupported | + +`{prefix}` must equal the proxy's `routing.default-catalog`. Multi-catalog REST +(prefix-per-catalog with namespace prefix rewriting on both request and +response paths) is a planned follow-up. + +### SPNEGO setup + +SPNEGO is an HTTP RFC requirement: the principal must be `HTTP/@REALM`. +This is a **separate** principal from `security.server-principal` (which is +usually `hms/@REALM` for the Thrift listener). Both can live in the same +keytab file or in two separate keytabs. The REST listener calls +`UserGroupInformation.loginUserFromKeytabAndReturnUGI` to acquire its own UGI +without overwriting the Thrift one, so they coexist in the same JVM. + +### Client examples + +PyIceberg: + +```python +from pyiceberg.catalog.rest import RestCatalog +catalog = RestCatalog("my-catalog", **{ + "uri": "http://hms-proxy:9183", +}) +``` + +Spark: + +```properties +spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog +spark.sql.catalog.my_catalog.catalog-impl=org.apache.iceberg.rest.RESTCatalog +spark.sql.catalog.my_catalog.uri=http://hms-proxy:9183 +``` + +### Caveats + +- Iceberg REST is **Iceberg-spec only** by design. Non-Iceberg Hive tables + (parquet/orc/text without `metadata_location`) are filtered out by HiveCatalog + and stay invisible through REST. Continue to use the Thrift listener for + native Hive tables. +- `RoutingHiveCatalog` uses reflection on Iceberg's private `HiveCatalog.clients` + field, pinned to Iceberg `1.5.2`. Bumping the Iceberg version requires + running `RoutingHiveCatalogTest` to confirm the inject still works. + ## Security ### Without Kerberos diff --git a/README.ru.md b/README.ru.md index c444185..15969be 100644 --- a/README.ru.md +++ b/README.ru.md @@ -971,6 +971,80 @@ catalog.catalog2.conf.hms.proxy.external-table-drop-purge.allowed-prefixes=hdfs: данные уже удалены — результат purge смотри в логе proxy. При остановке proxy запущенные purge дожидаются завершения; оставшиеся в очереди логируются и пропускаются. +## Iceberg REST Catalog frontend + +Proxy дополнительно умеет поднять параллельный HTTP listener со спецификацией +Iceberg REST Catalog, использующий тот же routing/federation pipeline что и +Thrift HMS front door. Статус: **экспериментально, read-only**. Iceberg-клиенты +(PyIceberg, Spark `iceberg-rest`, Trino `iceberg-rest`) могут discover и load +Iceberg-таблицы, хранящиеся в HMS через стандартный параметр +`metadata_location`. Writes, commits, view-операции и multi-catalog routing в +этой итерации НЕ поддерживаются. + +Включается так: + +```properties +rest-catalog.enabled=true +rest-catalog.port=9183 +# Опционально, но рекомендуется для прода: SPNEGO. Требует security.mode=KERBEROS. +rest-catalog.kerberos.principal=HTTP/_HOST@EXAMPLE.COM +rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab +``` + +### Поддерживаемые endpoint'ы + +| Endpoint | Статус | +| ----------------------------------------------------- | --------------------------------------- | +| `GET /v1/config` | поддержан | +| `GET /v1/{prefix}/namespaces` | поддержан | +| `GET /v1/{prefix}/namespaces/{ns}` | поддержан | +| `GET /v1/{prefix}/namespaces/{ns}/tables` | поддержан (только Iceberg-таблицы) | +| `GET /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | поддержан (только Iceberg-таблицы) | +| `POST`, `DELETE`, view-операции, commits | не поддержан | + +`{prefix}` должен совпадать с `routing.default-catalog` прокси. Multi-catalog +REST (prefix-per-catalog с rewriting namespace prefix на request и response +путях) — запланированный follow-up. + +### Настройка SPNEGO + +SPNEGO по RFC требует principal вида `HTTP/@REALM`. Это **отдельный** +principal от `security.server-principal` (обычно `hms/@REALM` для +Thrift listener). Оба могут лежать в одном keytab или в двух разных. REST +listener делает `UserGroupInformation.loginUserFromKeytabAndReturnUGI`, чтобы +получить отдельный UGI и не перезаписать Thrift'овый — оба сосуществуют в +одном JVM. + +### Примеры клиентов + +PyIceberg: + +```python +from pyiceberg.catalog.rest import RestCatalog +catalog = RestCatalog("my-catalog", **{ + "uri": "http://hms-proxy:9183", +}) +``` + +Spark: + +```properties +spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog +spark.sql.catalog.my_catalog.catalog-impl=org.apache.iceberg.rest.RESTCatalog +spark.sql.catalog.my_catalog.uri=http://hms-proxy:9183 +``` + +### Особенности и ограничения + +- Iceberg REST по дизайну работает **только с Iceberg-таблицами**. Native + Hive-таблицы (parquet/orc/text без `metadata_location`) HiveCatalog + отфильтровывает, и через REST их не видно. Для native Hive продолжайте + использовать Thrift listener. +- `RoutingHiveCatalog` использует reflection на private поле + `HiveCatalog.clients`, привязанное к Iceberg `1.5.2`. При апгрейде Iceberg + обязательно прогнать `RoutingHiveCatalogTest`, чтобы убедиться, что inject + ещё работает. + ## Безопасность ### Без Kerberos From 21cdf195988a428ec5de4a1538916434398965b3 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 16:18:08 +0300 Subject: [PATCH 05/50] Cover the Iceberg REST front door with a stand smoke scenario --scenario rest drives the listener with curl: config discovery (the advertised prefix is reused for every later path), namespace and table listings, a table load asserting metadata-location comes back, and the negative shapes - an unknown prefix, an unknown table and a write route all have to fail cleanly. Plain Hive tables of the same database must stay invisible through REST, which pins down the HiveCatalog filter. The step joins --scenario all and is skipped when HMS_SMOKE_REST_URL is not set. The stand enables the listener on its plain profile (host port 19183) and registers a minimal Iceberg table for the load check: a hand-written format-version-2 metadata.json in HDFS plus a Hive table shell carrying table_type=ICEBERG and metadata_location. A passing load therefore proves the whole chain - REST route, HiveCatalog, the proxy's routing layer, HMS, HDFS. The Kerberos profile keeps the listener off: SPNEGO would need a GSS-enabled curl in-network and is already covered by SpnegoIntegrationTest. Recorded as section G in TEST-MATRIX; both README/CHANGELOG locales updated. --- CHANGELOG.md | 8 ++ CHANGELOG.ru.md | 8 ++ ...real-installation-smoke.simple.env.example | 10 ++ scripts/run-real-installation-smoke.sh | 108 +++++++++++++++++- smoke-stand/README.md | 39 +++++++ smoke-stand/README.ru.md | 40 +++++++ smoke-stand/TEST-MATRIX.md | 19 +++ smoke-stand/TEST-MATRIX.ru.md | 19 +++ smoke-stand/docker-compose.yml | 1 + smoke-stand/env/simple.env | 9 ++ smoke-stand/proxy/hms-proxy.properties | 6 + 11 files changed, 264 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d63c5d..d85bb90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,14 @@ For a Russian version, see [CHANGELOG.ru.md](CHANGELOG.ru.md). statement never reaches: `HMS_SMOKE_SQL_RUN_CROSS_CATALOG_JOIN` (default `true`, read-only) and `HMS_SMOKE_SQL_RUN_CROSS_DATABASE_JOIN` (default `false`, since it creates a database). +- `--scenario rest` in the smoke runners drives the Iceberg REST catalog + front door with curl: config discovery, namespace and table listings, a + table load (asserting `metadata-location` comes back), the invisibility of + plain Hive tables, and clean failures for an unknown prefix, an unknown + table and a write route. Configured via `HMS_SMOKE_REST_*`; skipped in + `--scenario all` when `HMS_SMOKE_REST_URL` is unset. The local stand enables + the listener on its plain profile (host port 19183) and registers a minimal + Iceberg table for the load check. ### Fixed diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 86310ec..73884d5 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -27,6 +27,14 @@ English version: [CHANGELOG.md](CHANGELOG.md). запросом в одном namespace: `HMS_SMOKE_SQL_RUN_CROSS_CATALOG_JOIN` (по умолчанию `true`, только чтение) и `HMS_SMOKE_SQL_RUN_CROSS_DATABASE_JOIN` (по умолчанию `false`, так как создаёт базу). +- `--scenario rest` в smoke-раннерах гоняет Iceberg REST catalog front door + curl'ом: discovery конфигурации, листинги namespace и таблиц, load таблицы + (с проверкой, что вернулся `metadata-location`), невидимость обычных + Hive-таблиц и чистые отказы на неизвестный prefix, неизвестную таблицу и + write-роут. Настраивается через `HMS_SMOKE_REST_*`; в `--scenario all` + пропускается, если `HMS_SMOKE_REST_URL` не задан. Локальный стенд включает + listener в plain-профиле (host-порт 19183) и регистрирует минимальную + Iceberg-таблицу для проверки load. ### Исправлено diff --git a/scripts/hms-real-installation-smoke.simple.env.example b/scripts/hms-real-installation-smoke.simple.env.example index 7e4a074..33d1e44 100644 --- a/scripts/hms-real-installation-smoke.simple.env.example +++ b/scripts/hms-real-installation-smoke.simple.env.example @@ -93,3 +93,13 @@ HMS_SMOKE_PARTITION_LOCK_TRANSACTIONAL=false # HMS_SMOKE_NOTIFICATION_NEGATIVE_DB=apache__default # HMS_SMOKE_NOTIFICATION_NEGATIVE_TABLE=smoke_txn_tbl # HMS_SMOKE_HDP_STANDALONE_METASTORE_JAR=/opt/hms-proxy/hive-metastore/hive-standalone-metastore-3.1.0.3.1.0.0-78.jar + +# Optional Iceberg REST catalog smoke (read-only HTTP endpoint, driven with curl). If left unset, +# the REST step is skipped in --scenario all. The prefix must equal the proxy's +# routing.default-catalog; the Iceberg table must carry table_type=ICEBERG and a readable +# metadata_location, and the non-Iceberg table proves plain Hive tables stay invisible over REST. +# HMS_SMOKE_REST_URL=http://proxy-host:9183 +# HMS_SMOKE_REST_PREFIX=hdp +# HMS_SMOKE_REST_NAMESPACE=default +# HMS_SMOKE_REST_ICEBERG_TABLE=smoke_iceberg_tbl +# HMS_SMOKE_REST_NON_ICEBERG_TABLE=smoke_read_tbl diff --git a/scripts/run-real-installation-smoke.sh b/scripts/run-real-installation-smoke.sh index ff79bec..a4cc363 100755 --- a/scripts/run-real-installation-smoke.sh +++ b/scripts/run-real-installation-smoke.sh @@ -14,7 +14,7 @@ ENV_FILE="" usage() { cat <&1 | grep 'requires a Hortonworks backend runtime' ``` +## Iceberg REST catalog front door + +The plain profile also enables the proxy's Iceberg REST listener (`rest-catalog.*` in +`proxy/hms-proxy.properties`, host port 19183). It is read-only and serves the default +catalog (`hdp`) only. `--scenario rest` (or the REST step of `--scenario all`) drives it +with curl from the host: config discovery, namespace and table listings, a table load, and +the negative shapes — unknown prefix, unknown table, and a write route, all of which must +fail cleanly. + +The load-table check needs a real Iceberg table. The stand registers a minimal one by hand — +a hand-written `metadata.json` in HDFS plus a Hive table shell that points at it: + +```bash +# 1. Put a minimal Iceberg table metadata file onto the hdp catalog's cluster +docker cp stand-namenode:/tmp/00000-smoke.metadata.json +docker exec stand-namenode bash -c \ + 'hdfs dfs -mkdir -p /warehouse/hdp/smoke_iceberg_tbl/metadata && + hdfs dfs -put -f /tmp/00000-smoke.metadata.json /warehouse/hdp/smoke_iceberg_tbl/metadata/' + +# 2. Register the table in the hdp catalog with the two properties HiveCatalog keys on +docker exec stand-hs2 bash -c "java -cp '/opt/hs2/conf:/opt/hs2/lib/*' org.apache.hive.beeline.BeeLine \ + -u 'jdbc:hive2://localhost:10000/default' -n hive --silent=true \ + -e \"create external table if not exists hdp__default.smoke_iceberg_tbl (id int, ds string) + stored as parquet + location 'hdfs://namenode:8020/warehouse/hdp/smoke_iceberg_tbl' + tblproperties ( + 'table_type'='ICEBERG', + 'metadata_location'='hdfs://namenode:8020/warehouse/hdp/smoke_iceberg_tbl/metadata/00000-smoke.metadata.json');\"" +``` + +The proxy reads the metadata file from HDFS itself (HadoopFileIO with a bare `Configuration`), +so a passing load proves the whole chain: REST route → HiveCatalog → the proxy's own routing +layer → HMS → HDFS. Plain Hive tables of the same database (`smoke_read_hdp`, +`smoke_txn_tbl`) must stay invisible through REST — the smoke asserts that too. + +The Kerberos profile leaves the REST listener off: SPNEGO needs a GSS-enabled curl inside +the network, and the handshake itself is already covered end-to-end by +`SpnegoIntegrationTest` on hadoop-minikdc. + ## Hortonworks HiveServer2 (`--profile hdp`) A real HDP HiveServer2 that connects to the Hortonworks front door, so that listener is driven by diff --git a/smoke-stand/README.ru.md b/smoke-stand/README.ru.md index e926498..565ed85 100644 --- a/smoke-stand/README.ru.md +++ b/smoke-stand/README.ru.md @@ -179,6 +179,46 @@ door. docker logs stand-proxy 2>&1 | grep 'requires a Hortonworks backend runtime' ``` +## Iceberg REST catalog front door + +Plain-профиль включает и Iceberg REST listener прокси (`rest-catalog.*` в +`proxy/hms-proxy.properties`, host-порт 19183). Он read-only и обслуживает только +default-каталог (`hdp`). `--scenario rest` (или REST-шаг `--scenario all`) гоняет его +curl'ом с хоста: discovery конфигурации, листинги namespace и таблиц, load таблицы и +негативные формы — неизвестный prefix, неизвестная таблица и write-роут; все должны +падать чисто. + +Проверке load-table нужна настоящая Iceberg-таблица. Стенд регистрирует минимальную +вручную — написанный руками `metadata.json` в HDFS плюс Hive-оболочка таблицы, которая +на него указывает: + +```bash +# 1. Положить минимальный файл table metadata Iceberg на кластер каталога hdp +docker cp stand-namenode:/tmp/00000-smoke.metadata.json +docker exec stand-namenode bash -c \ + 'hdfs dfs -mkdir -p /warehouse/hdp/smoke_iceberg_tbl/metadata && + hdfs dfs -put -f /tmp/00000-smoke.metadata.json /warehouse/hdp/smoke_iceberg_tbl/metadata/' + +# 2. Зарегистрировать таблицу в каталоге hdp с двумя свойствами, по которым её узнаёт HiveCatalog +docker exec stand-hs2 bash -c "java -cp '/opt/hs2/conf:/opt/hs2/lib/*' org.apache.hive.beeline.BeeLine \ + -u 'jdbc:hive2://localhost:10000/default' -n hive --silent=true \ + -e \"create external table if not exists hdp__default.smoke_iceberg_tbl (id int, ds string) + stored as parquet + location 'hdfs://namenode:8020/warehouse/hdp/smoke_iceberg_tbl' + tblproperties ( + 'table_type'='ICEBERG', + 'metadata_location'='hdfs://namenode:8020/warehouse/hdp/smoke_iceberg_tbl/metadata/00000-smoke.metadata.json');\"" +``` + +Файл метаданных прокси читает из HDFS сам (HadoopFileIO с голой `Configuration`), поэтому +прошедший load доказывает всю цепочку: REST-роут → HiveCatalog → собственный routing-слой +прокси → HMS → HDFS. Обычные Hive-таблицы той же базы (`smoke_read_hdp`, `smoke_txn_tbl`) +через REST видны быть не должны — smoke проверяет и это. + +Kerberos-профиль оставляет REST listener выключенным: SPNEGO требует curl с GSS внутри +сети, а сам handshake уже покрыт end-to-end тестом `SpnegoIntegrationTest` на +hadoop-minikdc. + ## Hortonworks HiveServer2 (`--profile hdp`) Настоящий HDP HiveServer2, подключённый к Hortonworks front door: этот listener наконец проверяется diff --git a/smoke-stand/TEST-MATRIX.md b/smoke-stand/TEST-MATRIX.md index 07d8166..cff3a24 100644 --- a/smoke-stand/TEST-MATRIX.md +++ b/smoke-stand/TEST-MATRIX.md @@ -79,6 +79,22 @@ only path that covers the Hortonworks front door with a real client. | E2 | Readiness probe does not disturb SASL (15 × `/readyz`, then a Kerberos smoke run) | ✅ | | E3 | `hms_proxy_lock_request_split_total{catalog}` counts lock-request splits | ✅ | +## G. Iceberg REST catalog front door (host port 19183) + +Plain profile only; the Kerberos profile keeps the listener off (SPNEGO is covered in-JVM by +`SpnegoIntegrationTest`). Driven by `--scenario rest` with curl from the host; the loaded +table is the hand-registered `smoke_iceberg_tbl` (see the stand README). + +| # | Check | plain | kerberos | +| --- | --- | --- | --- | +| G1 | `GET /v1/config` advertises `prefix=hdp` (the default catalog) | ✅ | n/a | +| G2 | Namespace list and load (`default`) | ✅ | n/a | +| G3 | Table listing shows the Iceberg table and hides plain Hive tables of the same database | ✅ | n/a | +| G4 | Table load returns `metadata-location` and full metadata read from HDFS by the proxy itself | ✅ | n/a | +| G5 | Unknown prefix → clean 404 `NoSuchCatalogException` | ✅ | n/a | +| G6 | Unknown table → clean 404 | ✅ | n/a | +| G7 | Write route (`DELETE` table) refused, non-2xx | ✅ | n/a | + ## F. Not covered, and why | Area | Reason | @@ -105,6 +121,9 @@ executed is claimed; a row not listed was not repeated and its ✅ stands on the `show functions like` matching a bare name that Hive 3.1.3 registers qualified, and the runner's cleanup `RETURN` trap re-fired in the enclosing function after a two-pass run and killed it under `set -u` after every assertion had already passed. + Later the same day the branch's Iceberg REST listener was enabled on the plain profile and + section G was run for the first time (`--scenario rest`, and again as the REST step of a + full green `--scenario all`). ## Two caveats on faithfulness diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index 1cdd52c..5baaa70 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -79,6 +79,22 @@ HDP-клиент не может пользоваться Apache-listener — Th | E2 | Readiness-проба не ломает SASL (15 × `/readyz`, следом Kerberos-смоук) | ✅ | | E3 | `hms_proxy_lock_request_split_total{catalog}` считает расщепления lock-запросов | ✅ | +## G. Iceberg REST catalog front door (host-порт 19183) + +Только plain-профиль; Kerberos-профиль держит listener выключенным (SPNEGO покрыт in-JVM +тестом `SpnegoIntegrationTest`). Гоняется через `--scenario rest` curl'ом с хоста; +загружаемая таблица — зарегистрированная вручную `smoke_iceberg_tbl` (см. README стенда). + +| # | Проверка | plain | kerberos | +| --- | --- | --- | --- | +| G1 | `GET /v1/config` объявляет `prefix=hdp` (default-каталог) | ✅ | n/a | +| G2 | Листинг и load namespace (`default`) | ✅ | n/a | +| G3 | Листинг таблиц показывает Iceberg-таблицу и прячет обычные Hive-таблицы той же базы | ✅ | n/a | +| G4 | Load таблицы возвращает `metadata-location` и полные метаданные, прочитанные из HDFS самим прокси | ✅ | n/a | +| G5 | Неизвестный prefix → чистый 404 `NoSuchCatalogException` | ✅ | n/a | +| G6 | Неизвестная таблица → чистый 404 | ✅ | n/a | +| G7 | Write-роут (`DELETE` таблицы) отклонён, не-2xx | ✅ | n/a | + ## F. Что не покрыто и почему | Область | Причина | @@ -106,6 +122,9 @@ HDP-клиент не может пользоваться Apache-listener — Th `show functions like` найдёт короткое имя, хотя Hive 3.1.3 регистрирует функцию квалифицированной, а cleanup-`trap RETURN` раннера срабатывал повторно в объемлющей функции после двухпроходного прогона и убивал его под `set -u` уже после всех пройденных проверок. + Позже в тот же день на plain-профиле был включён Iceberg REST listener ветки и впервые + прогнан раздел G (`--scenario rest`, затем ещё раз как REST-шаг полностью зелёного + `--scenario all`). ## Две оговорки честности diff --git a/smoke-stand/docker-compose.yml b/smoke-stand/docker-compose.yml index 48f41d7..deb7550 100644 --- a/smoke-stand/docker-compose.yml +++ b/smoke-stand/docker-compose.yml @@ -224,6 +224,7 @@ services: - "19085:9083" - "19086:9084" - "19090:9090" + - "19183:9183" volumes: - ./proxy/hms-proxy.properties:/opt/hms-proxy/hms-proxy.properties:ro - ./proxy/hms-proxy-kerberos.properties:/opt/hms-proxy/hms-proxy-kerberos.properties:ro diff --git a/smoke-stand/env/simple.env b/smoke-stand/env/simple.env index af3eab9..d25ec28 100644 --- a/smoke-stand/env/simple.env +++ b/smoke-stand/env/simple.env @@ -81,3 +81,12 @@ HMS_SMOKE_HDP_STANDALONE_METASTORE_JAR=hive-metastore/hive-standalone-metastore- # The permanent-UDF block works: UDFReverse lives in hive-exec, which both HiveServer2 images now # carry, so the default HMS_SMOKE_SQL_RUN_UDF=true is left alone. # HMS_SMOKE_SQL_RUN_CROSS_DATABASE_JOIN=true + +# Iceberg REST catalog front door (read-only). The smoke drives it with curl from the host. +# smoke_iceberg_tbl is a pre-registered Iceberg table (see smoke-stand/README.md); +# smoke_read_hdp is a plain Hive table that must stay invisible through REST. +HMS_SMOKE_REST_URL=http://localhost:19183 +HMS_SMOKE_REST_PREFIX=hdp +HMS_SMOKE_REST_NAMESPACE=default +HMS_SMOKE_REST_ICEBERG_TABLE=smoke_iceberg_tbl +HMS_SMOKE_REST_NON_ICEBERG_TABLE=smoke_read_hdp diff --git a/smoke-stand/proxy/hms-proxy.properties b/smoke-stand/proxy/hms-proxy.properties index afd165f..3b78217 100644 --- a/smoke-stand/proxy/hms-proxy.properties +++ b/smoke-stand/proxy/hms-proxy.properties @@ -23,6 +23,12 @@ additional-frontends.hdp.port=9084 additional-frontends.hdp.frontend-profile=HORTONWORKS_3_1_0_3_1_0_78 additional-frontends.hdp.standalone-metastore-jar=/opt/hms-proxy/hive-metastore/hive-standalone-metastore-3.1.0.3.1.0.0-78.jar +# Iceberg REST catalog front door (read-only, experimental). Plain profile only: the SPNEGO +# variant is covered by SpnegoIntegrationTest in-JVM, and a curl-based smoke would need a +# GSS-enabled curl inside the network. +rest-catalog.enabled=true +rest-catalog.port=9183 + routing.default-catalog=hdp routing.catalog-db-separator=__ From 375153a9f6b7cb66833353cf91d58e212e41c878 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 16:46:34 +0300 Subject: [PATCH 06/50] Add catalog name translation for the REST multi-catalog phase --- ...-07-27-iceberg-rest-phase2-multicatalog.md | 546 ++++++++++++++++++ ...iceberg-rest-phase2-multicatalog-design.md | 74 +++ .../restcatalog/CatalogNameTranslation.java | 38 ++ .../CatalogNameTranslationTest.java | 31 + 4 files changed, 689 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-27-iceberg-rest-phase2-multicatalog.md create mode 100644 docs/superpowers/specs/2026-07-27-iceberg-rest-phase2-multicatalog-design.md create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslation.java create mode 100644 src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslationTest.java diff --git a/docs/superpowers/plans/2026-07-27-iceberg-rest-phase2-multicatalog.md b/docs/superpowers/plans/2026-07-27-iceberg-rest-phase2-multicatalog.md new file mode 100644 index 0000000..35faba1 --- /dev/null +++ b/docs/superpowers/plans/2026-07-27-iceberg-rest-phase2-multicatalog.md @@ -0,0 +1,546 @@ +# Iceberg REST Phase 2: Multi-Catalog Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Expose every configured proxy catalog through the Iceberg REST listener as its own prefix (`/v1//...`), read-only, per `docs/superpowers/specs/2026-07-27-iceberg-rest-phase2-multicatalog-design.md`. + +**Architecture:** A per-catalog `IcebergRestService` registry built at startup; the default catalog keeps the phase-1 federated view (untranslated client), every other catalog gets a name-translating `IMetaStoreClient` layer so Iceberg responses are built from internal names and the proxy keeps seeing external ones. `GET /v1/config?warehouse=` selects the prefix. + +**Tech Stack:** Java 17, JUnit 4, existing `RecordingThriftIface` fake, Iceberg 1.5.2 vendored adapter. Build/test only with `JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19` and offline Maven (`mvn -o`). + +## Global Constraints + +- Java 17 compatibility; 2-space indent, explicit imports (AGENTS.md). +- No new dependencies. +- Config keys unchanged: no new `rest-catalog.*` keys in this phase. +- Docs are bilingual: every EN doc change lands with its RU counterpart in the same task. +- Commit messages in English, no Claude attribution footers. +- The user has pre-approved commits for this implementation (spec commits together with it); do NOT push without an explicit command. + +--- + +### Task 1: CatalogNameTranslation + +**Files:** +- Create: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslation.java` +- Test: `src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslationTest.java` + +**Interfaces:** +- Produces: `CatalogNameTranslation(String catalogName, String separator)`, `String toExternal(String internalDb)`, `String fromExternalOrNull(String externalDb)`, `List internalNames(List externalDbs)`. + +- [ ] **Step 1: Write the failing test** + +```java +package io.github.mmalykhin.hmsproxy.restcatalog; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.List; +import org.junit.Test; + +public class CatalogNameTranslationTest { + private final CatalogNameTranslation translation = new CatalogNameTranslation("apache", "__"); + + @Test + public void toExternalPrependsCatalogPrefix() { + assertEquals("apache__default", translation.toExternal("default")); + assertEquals("apache__*", translation.toExternal("*")); + } + + @Test + public void fromExternalStripsOwnPrefixOnly() { + assertEquals("default", translation.fromExternalOrNull("apache__default")); + assertNull(translation.fromExternalOrNull("default")); + assertNull(translation.fromExternalOrNull("hdp__default")); + assertNull(translation.fromExternalOrNull("apache__")); + } + + @Test + public void internalNamesFiltersAndStrips() { + assertEquals(List.of("default", "sales"), + translation.internalNames(List.of("default", "apache__default", "hdp__x", "apache__sales"))); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 mvn -o -Dtest=CatalogNameTranslationTest test` +Expected: compilation FAILURE — `CatalogNameTranslation` does not exist. + +- [ ] **Step 3: Write minimal implementation** + +```java +package io.github.mmalykhin.hmsproxy.restcatalog; + +import java.util.List; +import java.util.Objects; + +/** + * Maps database names between a non-default catalog's internal view ("default") + * and the proxy's external federated view ("apache__default"). The REST layer + * shows internal names; the proxy keeps seeing external ones, so federation, + * exposure rules and access modes stay untouched. + */ +final class CatalogNameTranslation { + private final String externalPrefix; + + CatalogNameTranslation(String catalogName, String separator) { + this.externalPrefix = Objects.requireNonNull(catalogName, "catalogName") + + Objects.requireNonNull(separator, "separator"); + } + + String toExternal(String internalDb) { + return externalPrefix + internalDb; + } + + String fromExternalOrNull(String externalDb) { + if (externalDb == null || !externalDb.startsWith(externalPrefix)) { + return null; + } + String internal = externalDb.substring(externalPrefix.length()); + return internal.isEmpty() ? null : internal; + } + + List internalNames(List externalDbs) { + return externalDbs.stream() + .map(this::fromExternalOrNull) + .filter(Objects::nonNull) + .toList(); + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `JAVA_HOME=... mvn -o -Dtest=CatalogNameTranslationTest test` +Expected: `Tests run: 3, Failures: 0` + +- [ ] **Step 5: Commit** (spec + plan + this task) + +```bash +git add docs/superpowers src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslation.java src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslationTest.java +git commit -m "Add catalog name translation for the REST multi-catalog phase" +``` + +--- + +### Task 2: Name-translating RoutingMetaStoreClient + +**Files:** +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java` +- Test: `src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java` (add cases) + +**Interfaces:** +- Consumes: `CatalogNameTranslation` from Task 1. +- Produces: `RoutingMetaStoreClient.create(ThriftHiveMetastore.Iface delegate)` (unchanged, no translation) and `RoutingMetaStoreClient.create(ThriftHiveMetastore.Iface delegate, CatalogNameTranslation translation)` (translation may be null = untranslated). + +- [ ] **Step 1: Add failing tests to RoutingMetaStoreClientTest** + +```java + @Test + public void scopedClientTranslatesDatabaseArguments() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + recording.databases.put("apache__default", RecordingThriftIface.database("apache__default")); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + Database db = client.getDatabase("default"); + assertEquals("default", db.getName()); + assertEquals(List.of("get_database:apache__default"), recording.calls); + } + + @Test + public void scopedClientFiltersAndStripsDatabaseListing() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + recording.allDatabases = List.of("default", "apache__default", "hdp__x"); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + assertEquals(List.of("default"), client.getAllDatabases()); + } + + @Test + public void scopedClientRewritesTableDbName() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + recording.tables.put("apache__default.t1", RecordingThriftIface.table("apache__default", "t1")); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + Table t = client.getTable("default", "t1"); + assertEquals("default", t.getDbName()); + } +``` + +(Match the existing test file's helpers: `RecordingThriftIface` keys tables by `db.name` — check its `get_table` handler and use the same key form.) + +- [ ] **Step 2: Run to verify failure** + +Run: `JAVA_HOME=... mvn -o -Dtest=RoutingMetaStoreClientTest test` +Expected: compilation FAILURE — no two-argument `create`. + +- [ ] **Step 3: Implement translation inside the invocation handler** + +In `RoutingMetaStoreClient`: + +```java + public static IMetaStoreClient create(ThriftHiveMetastore.Iface delegate) { + return create(delegate, null); + } + + public static IMetaStoreClient create( + ThriftHiveMetastore.Iface delegate, CatalogNameTranslation translation) { + Objects.requireNonNull(delegate, "delegate"); + return (IMetaStoreClient) Proxy.newProxyInstance( + IMetaStoreClient.class.getClassLoader(), + new Class[]{IMetaStoreClient.class}, + new RoutingInvocationHandler(delegate, translation)); + } +``` + +`RoutingInvocationHandler` gains a nullable `translation` field. Inside `invoke`: +- helper `String db(String internal)` returns `translation == null ? internal : translation.toExternal(internal)`; apply to every first-String db argument (`getDatabase`, `getDatabases` pattern, `getAllTables`, `getTables`, `getTable`, `getTableObjectsByName`, `tableExists`). +- `getAllDatabases`: `translation == null ? delegate.get_all_databases() : translation.internalNames(delegate.get_all_databases())`. +- `getDatabases`: same wrap of `delegate.get_databases(db(pattern))`. +- `getDatabase`: result rewritten when translated: `Database copy = new Database(result); copy.setName(translation.fromExternalOrNull(result.getName()) != null ? translation.fromExternalOrNull(result.getName()) : result.getName()); return copy;` — extract a small `Database rewriteDatabase(Database)` / `Table rewriteTable(Table)` pair of private helpers (thrift copy constructors, then `setName`/`setDbName` with the internal name when `fromExternalOrNull` is non-null). +- `getTable` and each element of `getTableObjectsByName`: `rewriteTable`. + +- [ ] **Step 4: Run to verify pass** + +Run: `JAVA_HOME=... mvn -o -Dtest=RoutingMetaStoreClientTest test` +Expected: all existing + 3 new tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java +git commit -m "Teach the REST metastore client to translate catalog-scoped names" +``` + +--- + +### Task 3: Per-catalog IcebergRestService + +**Files:** +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java` +- Modify (call sites): `src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java` (temporarily keeps compiling — final wiring in Task 5), tests that construct the service (`IcebergRestEndpointIntegrationTest`, `RestCatalogServerTest`, `SpnegoIntegrationTest` — check with `grep -rn "new IcebergRestService" src/`). + +**Interfaces:** +- Consumes: two-arg `RoutingMetaStoreClient.create` from Task 2. +- Produces: `IcebergRestService(String catalogName, ThriftHiveMetastore.Iface delegate, CatalogNameTranslation translationOrNull)`; `String catalogName()`; `supportsPrefix(String)` and `loadConfig()` now keyed on `catalogName`. + +- [ ] **Step 1: Change the constructor** + +```java + private final String catalogName; + + public IcebergRestService( + String catalogName, + ThriftHiveMetastore.Iface delegate, + CatalogNameTranslation translationOrNull) { + this.catalogName = Objects.requireNonNull(catalogName, "catalogName"); + Objects.requireNonNull(delegate, "delegate"); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate, translationOrNull); + this.catalog = new RoutingHiveCatalog(client, new Configuration()); + this.catalog.initialize(catalogName, Map.of(CatalogProperties.URI, UNUSED_URI)); + this.adapter = new RESTCatalogAdapter(catalog); + } + + public String catalogName() { + return catalogName; + } +``` + +Drop the `ProxyConfig config` field; `supportsPrefix` compares against `catalogName`, `loadConfig()` uses `.withOverride("prefix", catalogName)`. + +- [ ] **Step 2: Update every constructor call site** + +Old form `new IcebergRestService(config, proxy)` becomes +`new IcebergRestService(config.defaultCatalog(), proxy, null)` — in `HmsProxyApplication` and each test found by the grep. + +- [ ] **Step 3: Run the restcatalog tests** + +Run: `JAVA_HOME=... mvn -o -Dtest='io.github.mmalykhin.hmsproxy.restcatalog.*' test` +Expected: all pass (behavior unchanged for the default catalog). + +- [ ] **Step 4: Commit** + +```bash +git add -A src/main src/test +git commit -m "Key IcebergRestService on an explicit catalog name" +``` + +--- + +### Task 4: IcebergRestServices registry + +**Files:** +- Create: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java` +- Test: `src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServicesTest.java` + +**Interfaces:** +- Consumes: `IcebergRestService` (Task 3), `ProxyConfig` (`catalogNames()`, `defaultCatalog()`, `catalogDbSeparator()`). +- Produces: `static IcebergRestServices open(ProxyConfig config, ThriftHiveMetastore.Iface delegate)`; `IcebergRestService serviceFor(String prefix)` (null when unknown); `IcebergRestService byWarehouse(String warehouseOrNull)` (null warehouse → default service, unknown → null); `String defaultPrefix()`; `close()` closes every service. + +- [ ] **Step 1: Write the failing test** + +```java +package io.github.mmalykhin.hmsproxy.restcatalog; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +public class IcebergRestServicesTest { + @Test + public void registryServesEveryConfiguredCatalog() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + // build a two-catalog ProxyConfig the same way IcebergRestEndpointIntegrationTest.buildConfig does, + // with catalogs "hdp" (default) and "apache" + try (IcebergRestServices services = IcebergRestServices.open(buildTwoCatalogConfig(), recording.iface)) { + assertEquals("hdp", services.defaultPrefix()); + assertNotNull(services.serviceFor("hdp")); + assertNotNull(services.serviceFor("apache")); + assertNull(services.serviceFor("nope")); + assertEquals("hdp", services.byWarehouse(null).catalogName()); + assertEquals("apache", services.byWarehouse("apache").catalogName()); + assertNull(services.byWarehouse("nope")); + } + } +} +``` + +(Write `buildTwoCatalogConfig()` by copying `IcebergRestEndpointIntegrationTest.buildConfig()` and adding a second `CatalogConfig` entry; keep the same builder fields.) + +- [ ] **Step 2: Run to verify it fails** (class missing), then implement: + +```java +package io.github.mmalykhin.hmsproxy.restcatalog; + +import io.github.mmalykhin.hmsproxy.config.ProxyConfig; +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; + +/** + * Prefix -> per-catalog REST service registry. The default catalog keeps the + * phase-1 federated view (no name translation); every other catalog gets a + * clean, name-translated view. Built eagerly so a broken configuration fails + * the proxy start, not the first REST request. + */ +public final class IcebergRestServices implements AutoCloseable { + private final Map byPrefix; + private final String defaultPrefix; + + private IcebergRestServices(Map byPrefix, String defaultPrefix) { + this.byPrefix = byPrefix; + this.defaultPrefix = defaultPrefix; + } + + public static IcebergRestServices open(ProxyConfig config, ThriftHiveMetastore.Iface delegate) { + Map services = new LinkedHashMap<>(); + for (String catalog : config.catalogNames()) { + CatalogNameTranslation translation = catalog.equals(config.defaultCatalog()) + ? null + : new CatalogNameTranslation(catalog, config.catalogDbSeparator()); + services.put(catalog, new IcebergRestService(catalog, delegate, translation)); + } + return new IcebergRestServices(services, config.defaultCatalog()); + } + + public IcebergRestService serviceFor(String prefix) { + return byPrefix.get(prefix); + } + + public IcebergRestService byWarehouse(String warehouseOrNull) { + if (warehouseOrNull == null || warehouseOrNull.isEmpty()) { + return byPrefix.get(defaultPrefix); + } + return byPrefix.get(warehouseOrNull); + } + + public String defaultPrefix() { + return defaultPrefix; + } + + @Override + public void close() throws IOException { + for (IcebergRestService service : byPrefix.values()) { + service.close(); + } + } +} +``` + +- [ ] **Step 3: Run to verify pass** + +Run: `JAVA_HOME=... mvn -o -Dtest=IcebergRestServicesTest test` + +- [ ] **Step 4: Commit** + +```bash +git add src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServicesTest.java +git commit -m "Build a per-catalog registry of Iceberg REST services" +``` + +--- + +### Task 5: Multi-prefix HTTP routing and warehouse discovery + +**Files:** +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java` +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java` (`open(ProxyConfig, IcebergRestServices)`) +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java` +- Test: `src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java` + +**Interfaces:** +- Consumes: `IcebergRestServices` (Task 4). +- Produces: HTTP behavior per spec — clean second prefix, federated default prefix, `?warehouse=` selection, 400 unknown warehouse, 404 unknown prefix. + +- [ ] **Step 1: Extend the integration test with a second catalog** + +Rework `buildConfig()` into a two-catalog config (`catalog1` default + `catalog2`, separator `__`). Seed `RecordingThriftIface.allDatabases = List.of("default", "catalog2__default")` and tables for both views. Add tests: + +```java + @Test + public void configWithWarehouseSelectsCatalogPrefix() throws Exception { + HttpResponse response = get("/v1/config?warehouse=catalog2"); + assertEquals(200, response.statusCode()); + assertTrue(response.body().contains("\"prefix\":\"catalog2\"")); + } + + @Test + public void configWithUnknownWarehouseReturns400() throws Exception { + assertEquals(400, get("/v1/config?warehouse=nope").statusCode()); + } + + @Test + public void secondPrefixShowsCleanView() throws Exception { + HttpResponse response = get("/v1/catalog2/namespaces"); + assertEquals(200, response.statusCode()); + assertTrue(response.body().contains("[\"default\"]")); + assertFalse(response.body().contains("catalog2__default")); + } + + @Test + public void defaultPrefixKeepsFederatedView() throws Exception { + HttpResponse response = get("/v1/catalog1/namespaces"); + assertEquals(200, response.statusCode()); + assertTrue(response.body().contains("[\"catalog2__default\"]")); + } +``` + +- [ ] **Step 2: Run to verify failures** (constructor/type mismatches first — fix wiring as part of this task). + +- [ ] **Step 3: Implement** + +`IcebergHttpHandler` takes `IcebergRestServices services`; `doHandle` parses the first path segment itself: +- `config` segment → `handleConfig(exchange, queryParams)`: `IcebergRestService svc = services.byWarehouse(queryParams.get("warehouse"))`; null → `writeError(exchange, 400, "BadRequestException", "Unknown warehouse: " + ...)`; else serialize `svc.loadConfig()`. +- otherwise `IcebergRestService svc = services.serviceFor(prefix)`; null → existing 404 `NoSuchCatalogException`; else dispatch with the remaining relative path exactly as today (`stripPrefixSegment` logic folds into this parse; delete `supportsPrefix` usage — keep the method on the service or drop it if unused after the change; drop it from `IcebergRestService` only if no test references remain). + +`RestCatalogServer.open(ProxyConfig config, IcebergRestServices services)` — same null-check contract as today (`services == null` → skeleton `ConfigHandler`; that branch now only serves the never-enabled case, keep it). + +`HmsProxyApplication` try-with-resources: + +```java + try (AdditionalFrontendThriftServers extras = + AdditionalFrontendThriftServers.open(config, proxy, frontDoorSecurity); + IcebergRestServices restServices = + config.restCatalog().enabled() ? IcebergRestServices.open(config, proxy) : null; + RestCatalogServer restServer = RestCatalogServer.open(config, restServices)) { +``` + +- [ ] **Step 4: Run the full restcatalog suite** + +Run: `JAVA_HOME=... mvn -o -Dtest='io.github.mmalykhin.hmsproxy.restcatalog.*' test` +Expected: all pass, including Spnego tests (they only wrap the handler in an authenticator). + +- [ ] **Step 5: Full test run** + +Run: `JAVA_HOME=... mvn -o test` +Expected: `BUILD SUCCESS`, 0 failures, 0 skipped. + +- [ ] **Step 6: Commit** + +```bash +git add -A src/main src/test +git commit -m "Serve every configured catalog as an Iceberg REST prefix" +``` + +--- + +### Task 6: Documentation (both locales) + +**Files:** +- Modify: `README.md`, `README.ru.md` — Iceberg REST section: multi-catalog behavior table (`{prefix}` = any configured catalog; default prefix keeps the federated view, other prefixes are clean; `?warehouse=` discovery; 400 negative), drop the "multi-catalog is planned" caveat. +- Modify: `CHANGELOG.md`, `CHANGELOG.ru.md` — entry under 2026-07-27 describing phase 2 (multi-catalog prefixes, warehouse discovery, federated default view kept for compatibility). +- Modify: `src/main/resources/hms-proxy-example.properties` — only if it mentions single-catalog REST wording; adjust the comment. + +- [ ] **Step 1: Update all files listed above** — EN and RU in the same edit batch. +- [ ] **Step 2: Commit** + +```bash +git add README.md README.ru.md CHANGELOG.md CHANGELOG.ru.md src/main/resources/hms-proxy-example.properties +git commit -m "Document multi-catalog Iceberg REST prefixes" +``` + +--- + +### Task 7: Smoke coverage and stand run + +**Files:** +- Modify: `scripts/run-real-installation-smoke.sh` — in `run_rest_smoke`, after the existing checks add an optional second-prefix block driven by `HMS_SMOKE_REST_SECOND_PREFIX` (+ optional `HMS_SMOKE_REST_SECOND_ICEBERG_TABLE`): + +```bash + local second_prefix="${HMS_SMOKE_REST_SECOND_PREFIX:-}" + if [[ -n "${second_prefix}" ]]; then + code="$(rest_request GET "/v1/config?warehouse=${second_prefix}" "${body}")" + [[ "${code}" == "200" ]] || fail "config?warehouse=${second_prefix} returned HTTP ${code}: $(cat "${body}")" + grep -q "\"prefix\"[[:space:]]*:[[:space:]]*\"${second_prefix}\"" "${body}" \ + || fail "warehouse discovery did not advertise prefix '${second_prefix}': $(cat "${body}")" + + code="$(rest_request GET "/v1/config?warehouse=no_such_warehouse_smoke" "${body}")" + [[ "${code}" == "400" ]] || fail "unknown warehouse expected HTTP 400, got ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${second_prefix}/namespaces" "${body}")" + [[ "${code}" == "200" ]] || fail "GET /v1/${second_prefix}/namespaces returned HTTP ${code}: $(cat "${body}")" + grep -q "\[\"${namespace}\"\]" "${body}" \ + || fail "namespace '${namespace}' missing under prefix '${second_prefix}': $(cat "${body}")" + if grep -q "${second_prefix}__" "${body}"; then + fail "external names leaked into the clean view of '${second_prefix}': $(cat "${body}")" + fi + + if [[ -n "${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE:-}" ]]; then + code="$(rest_request GET "/v1/${second_prefix}/namespaces/${namespace}/tables/${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE}" "${body}")" + [[ "${code}" == "200" ]] || fail "REST load under '${second_prefix}' returned HTTP ${code}: $(cat "${body}")" + grep -q '"metadata-location"' "${body}" \ + || fail "second-prefix load carries no metadata-location: $(cat "${body}")" + fi + fi +``` + + Also document both variables in the usage text next to the other `HMS_SMOKE_REST_*` lines. +- Modify: `scripts/hms-real-installation-smoke.simple.env.example` — add the two commented keys. +- Modify: `smoke-stand/env/simple.env` — set `HMS_SMOKE_REST_SECOND_PREFIX=apache`, `HMS_SMOKE_REST_SECOND_ICEBERG_TABLE=smoke_iceberg_tbl_ap`. +- Modify: `smoke-stand/README.md` + `smoke-stand/README.ru.md` — extend the REST section: the second Iceberg table lives in the `apache` catalog on `namenode-b` (`hdfs://namenode-b:8020/warehouse/apache/smoke_iceberg_tbl_ap`), registered the same way as the first one. +- Modify after the run: `smoke-stand/TEST-MATRIX.md` + `.ru.md` — extend section G (multi-catalog rows G8+: warehouse discovery, clean second view, 400 negative, second-prefix load) and the revalidation log. + +- [ ] **Step 1: Implement the runner/env/README changes; `bash -n` the runner.** +- [ ] **Step 2: Bring the stand up** (`cd smoke-stand && docker compose up -d`; containers were only stopped, state is preserved). Rebuild + restage the fat jar first: `JAVA_HOME=... mvn -o -DskipTests package && cd smoke-stand && ./prepare.sh`, then `docker compose up -d --build` to pick the new jar. +- [ ] **Step 3: Register the second Iceberg table** — metadata.json (copy of the first with `location`/`table-uuid` adjusted, e.g. uuid `7a9d0e3f-5c8b-4d2e-af40-3b6c9d8e7f20`) onto `stand-namenode-b` under `/warehouse/apache/smoke_iceberg_tbl_ap/metadata/00000-smoke.metadata.json`, then Beeline `create external table if not exists apache__default.smoke_iceberg_tbl_ap ... location 'hdfs://namenode-b:8020/warehouse/apache/smoke_iceberg_tbl_ap' tblproperties ('table_type'='ICEBERG','metadata_location'='hdfs://namenode-b:8020/warehouse/apache/smoke_iceberg_tbl_ap/metadata/00000-smoke.metadata.json')`. +- [ ] **Step 4: Run** `scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple.env --scenario rest`, then `--scenario all`. Expected: `completed successfully` both times. +- [ ] **Step 5: Update TEST-MATRIX (EN+RU) with the observed results; commit everything.** + +```bash +git add scripts smoke-stand +git commit -m "Cover multi-catalog REST prefixes in the stand smoke" +``` + +--- + +## Self-Review + +- Spec coverage: prefixes/registry (T4, T5), federated default view (T2 null-translation + T5 test), clean views (T1, T2, T5), warehouse discovery + 400 (T5), unknown prefix 404 (unchanged, asserted in T5 suite), read-only unchanged (no write-path edits anywhere), no new config keys (none added), unit/integration/smoke/docs (T1–T7). No gaps found. +- Placeholders: none — every code step carries concrete code. +- Type consistency: `CatalogNameTranslation` (T1) used in T2/T4; two-arg `create` (T2) used in T3; `IcebergRestService(String, Iface, CatalogNameTranslation)` (T3) used in T4; `IcebergRestServices` (T4) used in T5. Names match. diff --git a/docs/superpowers/specs/2026-07-27-iceberg-rest-phase2-multicatalog-design.md b/docs/superpowers/specs/2026-07-27-iceberg-rest-phase2-multicatalog-design.md new file mode 100644 index 0000000..6f04650 --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-iceberg-rest-phase2-multicatalog-design.md @@ -0,0 +1,74 @@ +# Iceberg REST frontend, phase 2: multi-catalog — design + +Date: 2026-07-27 +Branch: `feature/iceberg-rest-fe-phase1` +Status: approved + +## Goal + +Expose every configured proxy catalog through the Iceberg REST listener as its own +prefix (`/v1//...`), instead of serving only `routing.default-catalog`. +The endpoint stays read-only (writes are phase 5). + +## Behavior + +- **Prefixes.** Every catalog name from the proxy configuration is a valid prefix. + An unknown prefix keeps returning 404 `NoSuchCatalogException`. +- **Default catalog keeps the federated view.** Under the default catalog's prefix + the listing stays exactly what phase 1 shows: the proxy's external names — the + default catalog's own databases plus `` names of the other + catalogs. Phase 1 clients see no change. +- **Non-default catalogs get clean views.** Under `/v1//` only that + catalog's databases are visible, under their internal names (`default`, not + `apache__default`). Tables of other catalogs do not exist under that prefix by + construction. +- **Discovery.** `GET /v1/config?warehouse=` answers + `overrides.prefix=`. No `warehouse` parameter → the default catalog + (phase 1 compatible). Unknown `warehouse` → 400 `BadRequestException`. +- **Reads only.** Write routes keep failing exactly as in phase 1 + (`UnsupportedOperationException` behind the adapter, non-2xx on the wire). + +## Architecture + +- `RestCatalogServer`, SPNEGO, and the `rest-catalog.*` config keys do not change. + No new configuration keys: all configured catalogs are exposed — they are already + visible through the Thrift federation, and the endpoint is read-only. +- A registry (prefix → per-catalog `IcebergRestService`) is built at startup from + `config.catalogNames()`, so a broken configuration fails the start, not the first + request. `IcebergHttpHandler` resolves the prefix segment against the registry; + `/v1/config` gains the `warehouse` lookup. +- The default catalog's service is built on the existing untranslated + `RoutingMetaStoreClient` — the federated view needs no name mapping. +- Each non-default catalog's service wraps the shared `ThriftHiveMetastore.Iface` + in a **name-translating client layer**: + - db arguments: internal → external (`default` → `apache__default`) before the call; + - `get_all_databases`: filter to the catalog's own external names, strip the prefix; + - `Database.name` / `Table.dbName` in results: external → internal. + The proxy itself keeps seeing ordinary external names, so routing, exposure rules + and catalog access modes apply unchanged, with no edits to the federation layer. + +## Error handling + +- Unknown prefix → 404 `NoSuchCatalogException` (unchanged). +- Unknown `warehouse` in `/v1/config` → 400 `BadRequestException`. +- A table of another catalog under a clean prefix → the ordinary 404 + `NoSuchTableException` path (the name simply does not resolve). + +## Testing + +- **Unit**: the translating layer — both mapping directions, listing filtration, + result-name rewriting — against the existing `RecordingThriftIface`. +- **Integration**: extend `IcebergRestEndpointIntegrationTest` — clean view under a + second prefix, federated view under the default prefix, `warehouse` discovery, + 400 on unknown warehouse, 404 on unknown prefix. +- **Smoke**: `--scenario rest` learns a second prefix + (`HMS_SMOKE_REST_SECOND_PREFIX`), the stand registers an Iceberg table in the + `apache` catalog too; run on the local stand after implementation. +- **Docs**: README/CHANGELOG in both locales; stand README and TEST-MATRIX after + the stand run. + +## Out of scope + +- Writes, commits, view and transaction endpoints (phase 5 and later). +- Pagination and the remaining read endpoints of the REST spec. +- Kerberos changes: SPNEGO stays as shipped in phase 1. diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslation.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslation.java new file mode 100644 index 0000000..0eb53f9 --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslation.java @@ -0,0 +1,38 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import java.util.List; +import java.util.Objects; + +/** + * Maps database names between a non-default catalog's internal view ("default") + * and the proxy's external federated view ("apache__default"). The REST layer + * shows internal names; the proxy keeps seeing external ones, so federation, + * exposure rules and access modes stay untouched. + */ +final class CatalogNameTranslation { + private final String externalPrefix; + + CatalogNameTranslation(String catalogName, String separator) { + this.externalPrefix = Objects.requireNonNull(catalogName, "catalogName") + + Objects.requireNonNull(separator, "separator"); + } + + String toExternal(String internalDb) { + return externalPrefix + internalDb; + } + + String fromExternalOrNull(String externalDb) { + if (externalDb == null || !externalDb.startsWith(externalPrefix)) { + return null; + } + String internal = externalDb.substring(externalPrefix.length()); + return internal.isEmpty() ? null : internal; + } + + List internalNames(List externalDbs) { + return externalDbs.stream() + .map(this::fromExternalOrNull) + .filter(Objects::nonNull) + .toList(); + } +} diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslationTest.java new file mode 100644 index 0000000..8d19c01 --- /dev/null +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/CatalogNameTranslationTest.java @@ -0,0 +1,31 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.List; +import org.junit.Test; + +public class CatalogNameTranslationTest { + private final CatalogNameTranslation translation = new CatalogNameTranslation("apache", "__"); + + @Test + public void toExternalPrependsCatalogPrefix() { + assertEquals("apache__default", translation.toExternal("default")); + assertEquals("apache__*", translation.toExternal("*")); + } + + @Test + public void fromExternalStripsOwnPrefixOnly() { + assertEquals("default", translation.fromExternalOrNull("apache__default")); + assertNull(translation.fromExternalOrNull("default")); + assertNull(translation.fromExternalOrNull("hdp__default")); + assertNull(translation.fromExternalOrNull("apache__")); + } + + @Test + public void internalNamesFiltersAndStrips() { + assertEquals(List.of("default", "sales"), + translation.internalNames(List.of("default", "apache__default", "hdp__x", "apache__sales"))); + } +} From 6862ef9ef26b74f522996214a10f05ddf6d88953 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 16:50:54 +0300 Subject: [PATCH 07/50] Teach the REST metastore client to translate catalog-scoped names RoutingMetaStoreClient.create gains an overload that accepts a CatalogNameTranslation. When present, db-name arguments are translated to the proxy's external federated form before delegating, and Database/Table results coming back are rewritten to the catalog's internal names. The untranslated single-arg create keeps delegating with a null translation, so existing behavior is unchanged. --- .../restcatalog/RoutingMetaStoreClient.java | 78 ++++++++++++++++--- .../RoutingMetaStoreClientTest.java | 30 +++++++ 2 files changed, 96 insertions(+), 12 deletions(-) diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java index 5e0292b..11605c0 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java @@ -3,10 +3,13 @@ import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.lang.reflect.Proxy; -import java.util.Objects; +import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; /** @@ -20,18 +23,56 @@ private RoutingMetaStoreClient() { } public static IMetaStoreClient create(ThriftHiveMetastore.Iface delegate) { + return create(delegate, null); + } + + public static IMetaStoreClient create( + ThriftHiveMetastore.Iface delegate, CatalogNameTranslation translation) { Objects.requireNonNull(delegate, "delegate"); return (IMetaStoreClient) Proxy.newProxyInstance( IMetaStoreClient.class.getClassLoader(), new Class[]{IMetaStoreClient.class}, - new RoutingInvocationHandler(delegate)); + new RoutingInvocationHandler(delegate, translation)); } private static final class RoutingInvocationHandler implements InvocationHandler { private final ThriftHiveMetastore.Iface delegate; + private final CatalogNameTranslation translation; - RoutingInvocationHandler(ThriftHiveMetastore.Iface delegate) { + RoutingInvocationHandler( + ThriftHiveMetastore.Iface delegate, CatalogNameTranslation translation) { this.delegate = delegate; + this.translation = translation; + } + + private String db(String internal) { + return translation == null ? internal : translation.toExternal(internal); + } + + private Database rewriteDatabase(Database result) { + if (translation == null || result == null) { + return result; + } + String internalName = translation.fromExternalOrNull(result.getName()); + if (internalName == null) { + return result; + } + Database copy = new Database(result); + copy.setName(internalName); + return copy; + } + + private Table rewriteTable(Table result) { + if (translation == null || result == null) { + return result; + } + String internalName = translation.fromExternalOrNull(result.getDbName()); + if (internalName == null) { + return result; + } + Table copy = new Table(result); + copy.setDbName(internalName); + return copy; } @Override @@ -53,26 +94,30 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl case "toString": return "RoutingMetaStoreClient[delegate=" + delegate + "]"; case "getDatabase": - return delegate.get_database((String) args[0]); + return rewriteDatabase(delegate.get_database(db((String) args[0]))); case "getAllDatabases": - return delegate.get_all_databases(); + return translation == null + ? delegate.get_all_databases() + : translation.internalNames(delegate.get_all_databases()); case "getDatabases": - return delegate.get_databases((String) args[0]); + return translation == null + ? delegate.get_databases((String) args[0]) + : translation.internalNames(delegate.get_databases(db((String) args[0]))); case "getAllTables": - return delegate.get_all_tables((String) args[0]); + return delegate.get_all_tables(db((String) args[0])); case "getTables": if (paramTypes.length == 2) { - return delegate.get_tables((String) args[0], (String) args[1]); + return delegate.get_tables(db((String) args[0]), (String) args[1]); } if (paramTypes.length == 3) { return delegate.get_tables_by_type( - (String) args[0], (String) args[1], String.valueOf(args[2])); + db((String) args[0]), (String) args[1], String.valueOf(args[2])); } break; case "getTable": if (paramTypes.length == 2 && paramTypes[0] == String.class && paramTypes[1] == String.class) { - return delegate.get_table((String) args[0], (String) args[1]); + return rewriteTable(delegate.get_table(db((String) args[0]), (String) args[1])); } break; case "getTableObjectsByName": @@ -80,13 +125,22 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl && paramTypes[0] == String.class && paramTypes[1] == List.class) { @SuppressWarnings("unchecked") List tableNames = (List) args[1]; - return delegate.get_table_objects_by_name((String) args[0], tableNames); + List
results = delegate.get_table_objects_by_name( + db((String) args[0]), tableNames); + if (translation == null) { + return results; + } + List
rewritten = new ArrayList<>(results.size()); + for (Table result : results) { + rewritten.add(rewriteTable(result)); + } + return rewritten; } break; case "tableExists": if (paramTypes.length == 2) { try { - delegate.get_table((String) args[0], (String) args[1]); + delegate.get_table(db((String) args[0]), (String) args[1]); return true; } catch (NoSuchObjectException e) { return false; diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java index dd6e105..befe096 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java @@ -91,4 +91,34 @@ public void closeIsNoop() { client.close(); Assert.assertTrue(delegate.calls.isEmpty()); } + + @Test + public void scopedClientTranslatesDatabaseArguments() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + recording.databases.put("apache__default", RecordingThriftIface.database("apache__default")); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + Database db = client.getDatabase("default"); + Assert.assertEquals("default", db.getName()); + Assert.assertEquals(List.of("get_database:apache__default"), recording.calls); + } + + @Test + public void scopedClientFiltersAndStripsDatabaseListing() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + recording.allDatabases = List.of("default", "apache__default", "hdp__x"); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + Assert.assertEquals(List.of("default"), client.getAllDatabases()); + } + + @Test + public void scopedClientRewritesTableDbName() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + recording.tables.put("apache__default.t1", RecordingThriftIface.table("apache__default", "t1")); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + Table t = client.getTable("default", "t1"); + Assert.assertEquals("default", t.getDbName()); + } } From 24726b83159f2fa403b43f4d191bbb3c2081ac27 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 16:56:06 +0300 Subject: [PATCH 08/50] Key IcebergRestService on an explicit catalog name --- .../hmsproxy/app/HmsProxyApplication.java | 3 +- .../restcatalog/IcebergRestService.java | 28 +++++++++++-------- .../IcebergRestEndpointIntegrationTest.java | 2 +- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java index 448065a..2432ab3 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java @@ -58,7 +58,8 @@ public static void main(String[] args) throws Exception { try (AdditionalFrontendThriftServers extras = AdditionalFrontendThriftServers.open(config, proxy, frontDoorSecurity); IcebergRestService restService = - config.restCatalog().enabled() ? new IcebergRestService(config, proxy) : null; + config.restCatalog().enabled() + ? new IcebergRestService(config.defaultCatalog(), proxy, null) : null; RestCatalogServer restServer = RestCatalogServer.open(config, restService)) { MetastoreThriftServer server = new MetastoreThriftServer(config, proxy, frontDoorSecurity); installShutdownHook(server, teardownComplete, config.server().shutdownTimeoutSeconds()); diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java index b14241b..6041f35 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java @@ -1,6 +1,5 @@ package io.github.mmalykhin.hmsproxy.restcatalog; -import io.github.mmalykhin.hmsproxy.config.ProxyConfig; import java.io.IOException; import java.util.Map; import java.util.Objects; @@ -24,34 +23,39 @@ public final class IcebergRestService implements AutoCloseable { private static final String UNUSED_URI = "thrift://hms-proxy-loopback:0"; - private final ProxyConfig config; + private final String catalogName; private final RoutingHiveCatalog catalog; private final RESTCatalogAdapter adapter; - public IcebergRestService(ProxyConfig config, ThriftHiveMetastore.Iface delegate) { - this.config = Objects.requireNonNull(config, "config"); + public IcebergRestService( + String catalogName, + ThriftHiveMetastore.Iface delegate, + CatalogNameTranslation translationOrNull) { + this.catalogName = Objects.requireNonNull(catalogName, "catalogName"); Objects.requireNonNull(delegate, "delegate"); - IMetaStoreClient client = RoutingMetaStoreClient.create(delegate); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate, translationOrNull); this.catalog = new RoutingHiveCatalog(client, new Configuration()); - this.catalog.initialize( - config.defaultCatalog(), - Map.of(CatalogProperties.URI, UNUSED_URI)); + this.catalog.initialize(catalogName, Map.of(CatalogProperties.URI, UNUSED_URI)); this.adapter = new RESTCatalogAdapter(catalog); } + public String catalogName() { + return catalogName; + } + /** True when the prefix segment in the request URL maps to a known catalog we can serve. */ public boolean supportsPrefix(String prefix) { - return prefix != null && prefix.equals(config.defaultCatalog()); + return prefix != null && prefix.equals(catalogName); } /** * Returns the GET /v1/config response that Iceberg clients use for discovery. - * Setting overrides.prefix locks the client to the proxy's default catalog - * so all subsequent /v1/{prefix}/... requests land here. + * Setting overrides.prefix locks the client to this service's catalog so all + * subsequent /v1/{prefix}/... requests land here. */ public ConfigResponse loadConfig() { return ConfigResponse.builder() - .withOverride("prefix", config.defaultCatalog()) + .withOverride("prefix", catalogName) .build(); } diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java index 57487c3..60eef6a 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java @@ -37,7 +37,7 @@ public void setUp() throws Exception { delegate.tables.put("sales.orders", RecordingThriftIface.table("sales", "orders")); ProxyConfig config = buildConfig(); - service = new IcebergRestService(config, delegate.iface); + service = new IcebergRestService(CATALOG_NAME, delegate.iface, null); server = RestCatalogServer.open(config, service); Assert.assertNotNull("server must start", server); } From edc3d1eefc68677edb2833b1378f5fe968488339 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 16:59:45 +0300 Subject: [PATCH 09/50] Build a per-catalog registry of Iceberg REST services Adds IcebergRestServices, an eager prefix -> IcebergRestService registry built from ProxyConfig: the default catalog keeps the untranslated view, every other catalog gets a CatalogNameTranslation. Backs multi-catalog prefix resolution for the REST endpoint (byWarehouse/serviceFor) ahead of wiring it into HTTP routing. --- .../restcatalog/IcebergRestServices.java | 56 ++++++++++++++++ .../restcatalog/IcebergRestServicesTest.java | 64 +++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java create mode 100644 src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServicesTest.java diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java new file mode 100644 index 0000000..daaa0c6 --- /dev/null +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java @@ -0,0 +1,56 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import io.github.mmalykhin.hmsproxy.config.ProxyConfig; +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Map; +import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; + +/** + * Prefix -> per-catalog REST service registry. The default catalog keeps the + * phase-1 federated view (no name translation); every other catalog gets a + * clean, name-translated view. Built eagerly so a broken configuration fails + * the proxy start, not the first REST request. + */ +public final class IcebergRestServices implements AutoCloseable { + private final Map byPrefix; + private final String defaultPrefix; + + private IcebergRestServices(Map byPrefix, String defaultPrefix) { + this.byPrefix = byPrefix; + this.defaultPrefix = defaultPrefix; + } + + public static IcebergRestServices open(ProxyConfig config, ThriftHiveMetastore.Iface delegate) { + Map services = new LinkedHashMap<>(); + for (String catalog : config.catalogNames()) { + CatalogNameTranslation translation = catalog.equals(config.defaultCatalog()) + ? null + : new CatalogNameTranslation(catalog, config.catalogDbSeparator()); + services.put(catalog, new IcebergRestService(catalog, delegate, translation)); + } + return new IcebergRestServices(services, config.defaultCatalog()); + } + + public IcebergRestService serviceFor(String prefix) { + return byPrefix.get(prefix); + } + + public IcebergRestService byWarehouse(String warehouseOrNull) { + if (warehouseOrNull == null || warehouseOrNull.isEmpty()) { + return byPrefix.get(defaultPrefix); + } + return byPrefix.get(warehouseOrNull); + } + + public String defaultPrefix() { + return defaultPrefix; + } + + @Override + public void close() throws IOException { + for (IcebergRestService service : byPrefix.values()) { + service.close(); + } + } +} diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServicesTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServicesTest.java new file mode 100644 index 0000000..1ca1385 --- /dev/null +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServicesTest.java @@ -0,0 +1,64 @@ +package io.github.mmalykhin.hmsproxy.restcatalog; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import io.github.mmalykhin.hmsproxy.config.ProxyConfig; +import io.github.mmalykhin.hmsproxy.config.catalog.CatalogAccessMode; +import io.github.mmalykhin.hmsproxy.config.catalog.CatalogConfig; +import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfig; +import io.github.mmalykhin.hmsproxy.config.routing.BackendConfig; +import io.github.mmalykhin.hmsproxy.config.server.ServerConfig; +import io.github.mmalykhin.hmsproxy.config.syntheticlock.SyntheticReadLockStoreConfig; +import java.util.List; +import java.util.Map; +import org.junit.Test; + +public class IcebergRestServicesTest { + @Test + public void registryServesEveryConfiguredCatalog() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + try (IcebergRestServices services = IcebergRestServices.open(buildTwoCatalogConfig(), recording.iface)) { + assertEquals("hdp", services.defaultPrefix()); + assertNotNull(services.serviceFor("hdp")); + assertNotNull(services.serviceFor("apache")); + assertNull(services.serviceFor("nope")); + assertEquals("hdp", services.byWarehouse(null).catalogName()); + assertEquals("apache", services.byWarehouse("apache").catalogName()); + assertNull(services.byWarehouse("nope")); + } + } + + private static ProxyConfig buildTwoCatalogConfig() { + return ProxyConfig.builder() + .server(new ServerConfig("hms-proxy-test", "127.0.0.1", 9083, 1, 4)) + .catalogDbSeparator(".") + .defaultCatalog("hdp") + .catalogs(Map.of( + "hdp", new CatalogConfig( + "hdp", + null, + null, + false, + CatalogAccessMode.READ_WRITE, + List.of(), + null, + null, + Map.of("hive.metastore.uris", "thrift://hms-test:9083")), + "apache", new CatalogConfig( + "apache", + null, + null, + false, + CatalogAccessMode.READ_WRITE, + List.of(), + null, + null, + Map.of("hive.metastore.uris", "thrift://hms-test:9084")))) + .backend(new BackendConfig(Map.of())) + .restCatalog(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)) + .syntheticReadLockStore(SyntheticReadLockStoreConfig.inMemory()) + .build(); + } +} From 64a9d2105c11fce867c0e79f6c05c5e88e45d545 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 17:07:40 +0300 Subject: [PATCH 10/50] Serve every configured catalog as an Iceberg REST prefix --- .../hmsproxy/app/HmsProxyApplication.java | 9 +-- .../restcatalog/IcebergHttpHandler.java | 70 ++++++++-------- .../restcatalog/IcebergRestService.java | 5 -- .../restcatalog/RestCatalogServer.java | 6 +- .../IcebergRestEndpointIntegrationTest.java | 79 +++++++++++++++---- 5 files changed, 101 insertions(+), 68 deletions(-) diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java index 2432ab3..816c857 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java @@ -6,7 +6,7 @@ import io.github.mmalykhin.hmsproxy.frontend.HortonworksFrontendExtension; import io.github.mmalykhin.hmsproxy.observability.ProxyObservability; import io.github.mmalykhin.hmsproxy.federation.FederationLayer; -import io.github.mmalykhin.hmsproxy.restcatalog.IcebergRestService; +import io.github.mmalykhin.hmsproxy.restcatalog.IcebergRestServices; import io.github.mmalykhin.hmsproxy.restcatalog.RestCatalogServer; import io.github.mmalykhin.hmsproxy.routing.CatalogRouter; import io.github.mmalykhin.hmsproxy.routing.RoutingMetaStoreProxy; @@ -57,10 +57,9 @@ public static void main(String[] args) throws Exception { HortonworksFrontendExtension.class); try (AdditionalFrontendThriftServers extras = AdditionalFrontendThriftServers.open(config, proxy, frontDoorSecurity); - IcebergRestService restService = - config.restCatalog().enabled() - ? new IcebergRestService(config.defaultCatalog(), proxy, null) : null; - RestCatalogServer restServer = RestCatalogServer.open(config, restService)) { + IcebergRestServices restServices = + config.restCatalog().enabled() ? IcebergRestServices.open(config, proxy) : null; + RestCatalogServer restServer = RestCatalogServer.open(config, restServices)) { MetastoreThriftServer server = new MetastoreThriftServer(config, proxy, frontDoorSecurity); installShutdownHook(server, teardownComplete, config.server().shutdownTimeoutSeconds()); LOG.info("Starting HMS proxy '{}' on {}:{}", config.server().name(), diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java index cb4c917..6678fc9 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java @@ -25,19 +25,20 @@ /** * Maps JDK HttpExchange requests to {@link IcebergRestService} dispatches. - * URL form: /v1/{prefix}/... The {prefix} segment must equal the proxy's - * default catalog name; any other prefix returns 404. + * URL form: /v1/{prefix}/... The {prefix} segment selects the target catalog + * via {@link IcebergRestServices#serviceFor(String)}; an unknown prefix returns 404. */ final class IcebergHttpHandler implements HttpHandler { private static final Logger LOG = LoggerFactory.getLogger(IcebergHttpHandler.class); private static final String JSON_CONTENT_TYPE = "application/json; charset=utf-8"; private static final String V1_PREFIX = "/v1/"; - private static final String CONFIG_PATH = "v1/config"; + private static final String CONFIG_SEGMENT = "config"; + private static final String WAREHOUSE_PARAM = "warehouse"; - private final IcebergRestService service; + private final IcebergRestServices services; - IcebergHttpHandler(IcebergRestService service) { - this.service = service; + IcebergHttpHandler(IcebergRestServices services) { + this.services = services; } @Override @@ -74,18 +75,23 @@ private void doHandle(HttpExchange exchange) throws IOException { } Map queryParams = parseQueryString(exchange.getRequestURI().getRawQuery()); - String relativePath = stripPrefixSegment(rawPath); - if (relativePath == null) { - writeError(exchange, 404, "NoSuchCatalogException", - "Unknown catalog prefix in URL: " + rawPath); + String trimmed = rawPath.equals("/v1") ? "" : rawPath.substring(V1_PREFIX.length()); + int slash = trimmed.indexOf('/'); + String firstSegment = slash < 0 ? trimmed : trimmed.substring(0, slash); + String remainder = slash < 0 ? "" : trimmed.substring(slash + 1); + + if (CONFIG_SEGMENT.equals(firstSegment) && remainder.isEmpty()) { + handleConfig(exchange, queryParams); return; } - if (CONFIG_PATH.equals(relativePath)) { - ConfigResponse cfg = service.loadConfig(); - writeJson(exchange, 200, IcebergRestMapper.mapper().writeValueAsString(cfg)); + IcebergRestService service = services.serviceFor(firstSegment); + if (service == null) { + writeError(exchange, 404, "NoSuchCatalogException", + "Unknown catalog prefix in URL: " + rawPath); return; } + String relativePath = remainder.isEmpty() ? "v1" : "v1/" + remainder; Pair> routeAndVars = Route.from(method, relativePath); if (routeAndVars == null) { @@ -102,7 +108,7 @@ private void doHandle(HttpExchange exchange) throws IOException { Class effectiveResponseType = responseType == null ? RESTResponse.class : responseType; try { - response = dispatchInternal(method, relativePath, queryParams, body, + response = dispatchInternal(service, method, relativePath, queryParams, body, effectiveResponseType, err -> capturedError[0] = err); } catch (RESTException e) { // RESTCatalogAdapter always rethrows after invoking the error handler, @@ -134,7 +140,19 @@ private void doHandle(HttpExchange exchange) throws IOException { } } + private void handleConfig(HttpExchange exchange, Map queryParams) throws IOException { + String warehouse = queryParams.get(WAREHOUSE_PARAM); + IcebergRestService service = services.byWarehouse(warehouse); + if (service == null) { + writeError(exchange, 400, "BadRequestException", "Unknown warehouse: " + warehouse); + return; + } + ConfigResponse cfg = service.loadConfig(); + writeJson(exchange, 200, IcebergRestMapper.mapper().writeValueAsString(cfg)); + } + private T dispatchInternal( + IcebergRestService service, HTTPMethod method, String relativePath, Map queryParams, @@ -145,30 +163,6 @@ private T dispatchInternal( java.util.Map.of(), errorHandler); } - private String stripPrefixSegment(String rawPath) { - String trimmed = rawPath.substring(V1_PREFIX.length()); - int slash = trimmed.indexOf('/'); - if (slash < 0) { - String prefixOnly = trimmed; - if ("config".equals(prefixOnly)) { - return CONFIG_PATH; - } - if (!service.supportsPrefix(prefixOnly)) { - return null; - } - return "v1"; - } - String prefix = trimmed.substring(0, slash); - if ("config".equals(prefix) && slash == trimmed.length() - 1) { - return CONFIG_PATH; - } - if (!service.supportsPrefix(prefix)) { - return null; - } - String rest = trimmed.substring(slash + 1); - return rest.isEmpty() ? "v1" : "v1/" + rest; - } - private Object readBody(HttpExchange exchange, Route route) throws IOException { Class requestClass = route.requestClass(); if (requestClass == null) { diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java index 6041f35..46a2cbf 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java @@ -43,11 +43,6 @@ public String catalogName() { return catalogName; } - /** True when the prefix segment in the request URL maps to a known catalog we can serve. */ - public boolean supportsPrefix(String prefix) { - return prefix != null && prefix.equals(catalogName); - } - /** * Returns the GET /v1/config response that Iceberg clients use for discovery. * Setting overrides.prefix locks the client to this service's catalog so all diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java index cc93eda..ca2a04e 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java @@ -38,7 +38,7 @@ public static RestCatalogServer open(ProxyConfig config) throws IOException { return open(config, null); } - public static RestCatalogServer open(ProxyConfig config, IcebergRestService service) throws IOException { + public static RestCatalogServer open(ProxyConfig config, IcebergRestServices services) throws IOException { RestCatalogConfig restConfig = config.restCatalog(); if (!restConfig.enabled()) { return null; @@ -68,8 +68,8 @@ public static RestCatalogServer open(ProxyConfig config, IcebergRestService serv } HttpContext v1Context; - if (service != null) { - v1Context = server.createContext("/v1/", new IcebergHttpHandler(service)); + if (services != null) { + v1Context = server.createContext("/v1/", new IcebergHttpHandler(services)); } else { v1Context = server.createContext("/v1/config", new ConfigHandler()); } diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java index 60eef6a..47af1de 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java @@ -21,24 +21,31 @@ public class IcebergRestEndpointIntegrationTest { private static final String CATALOG_NAME = "catalog1"; + private static final String CATALOG2_NAME = "catalog2"; private static final Duration HTTP_TIMEOUT = Duration.ofSeconds(5); private RecordingThriftIface delegate; - private IcebergRestService service; + private IcebergRestServices services; private RestCatalogServer server; @Before public void setUp() throws Exception { delegate = new RecordingThriftIface(); - delegate.allDatabases = List.of("sales", "marketing"); + // "default" and "catalog2__default" back catalog2's clean view (translated + // to/from "default"); "sales"/"marketing" back the existing catalog1 cases. + delegate.allDatabases = List.of("sales", "marketing", "default", "catalog2__default"); delegate.databases.put("sales", RecordingThriftIface.database("sales")); delegate.databases.put("marketing", RecordingThriftIface.database("marketing")); + delegate.databases.put("default", RecordingThriftIface.database("default")); + delegate.databases.put("catalog2__default", RecordingThriftIface.database("catalog2__default")); delegate.tablesByDatabase.put("sales", List.of("orders")); delegate.tables.put("sales.orders", RecordingThriftIface.table("sales", "orders")); + delegate.tablesByDatabase.put("catalog2__default", List.of("events")); + delegate.tables.put("catalog2__default.events", RecordingThriftIface.table("catalog2__default", "events")); ProxyConfig config = buildConfig(); - service = new IcebergRestService(CATALOG_NAME, delegate.iface, null); - server = RestCatalogServer.open(config, service); + services = IcebergRestServices.open(config, delegate.iface); + server = RestCatalogServer.open(config, services); Assert.assertNotNull("server must start", server); } @@ -47,8 +54,8 @@ public void tearDown() throws Exception { if (server != null) { server.close(); } - if (service != null) { - service.close(); + if (services != null) { + services.close(); } } @@ -111,6 +118,33 @@ public void loadingNonIcebergTableReturnsErrorResponse() throws Exception { response.statusCode() >= 400 && response.statusCode() < 500); } + @Test + public void configWithWarehouseSelectsCatalogPrefix() throws Exception { + HttpResponse response = get("/v1/config?warehouse=catalog2"); + Assert.assertEquals(200, response.statusCode()); + Assert.assertTrue(response.body(), response.body().contains("\"prefix\":\"catalog2\"")); + } + + @Test + public void configWithUnknownWarehouseReturns400() throws Exception { + Assert.assertEquals(400, get("/v1/config?warehouse=nope").statusCode()); + } + + @Test + public void secondPrefixShowsCleanView() throws Exception { + HttpResponse response = get("/v1/catalog2/namespaces"); + Assert.assertEquals(200, response.statusCode()); + Assert.assertTrue(response.body(), response.body().contains("[\"default\"]")); + Assert.assertFalse(response.body(), response.body().contains("catalog2__default")); + } + + @Test + public void defaultPrefixKeepsFederatedView() throws Exception { + HttpResponse response = get("/v1/catalog1/namespaces"); + Assert.assertEquals(200, response.statusCode()); + Assert.assertTrue(response.body(), response.body().contains("[\"catalog2__default\"]")); + } + private HttpResponse get(String path) throws Exception { HttpClient client = HttpClient.newBuilder().connectTimeout(HTTP_TIMEOUT).build(); HttpRequest request = HttpRequest.newBuilder() @@ -124,18 +158,29 @@ private HttpResponse get(String path) throws Exception { private static ProxyConfig buildConfig() { return ProxyConfig.builder() .server(new ServerConfig("hms-proxy-test", "127.0.0.1", 9083, 1, 4)) - .catalogDbSeparator(".") + .catalogDbSeparator("__") .defaultCatalog(CATALOG_NAME) - .catalogs(Map.of(CATALOG_NAME, new CatalogConfig( - CATALOG_NAME, - null, - null, - false, - CatalogAccessMode.READ_WRITE, - List.of(), - null, - null, - Map.of("hive.metastore.uris", "thrift://hms-test:9083")))) + .catalogs(Map.of( + CATALOG_NAME, new CatalogConfig( + CATALOG_NAME, + null, + null, + false, + CatalogAccessMode.READ_WRITE, + List.of(), + null, + null, + Map.of("hive.metastore.uris", "thrift://hms-test:9083")), + CATALOG2_NAME, new CatalogConfig( + CATALOG2_NAME, + null, + null, + false, + CatalogAccessMode.READ_WRITE, + List.of(), + null, + null, + Map.of("hive.metastore.uris", "thrift://hms-test:9084")))) .backend(new BackendConfig(Map.of())) .restCatalog(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)) .syntheticReadLockStore(SyntheticReadLockStoreConfig.inMemory()) From 425126ad96a41f3997a3c90f7edffab6714f8cc8 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 17:17:48 +0300 Subject: [PATCH 11/50] Document multi-catalog Iceberg REST prefixes --- CHANGELOG.md | 9 +++++++++ CHANGELOG.ru.md | 10 ++++++++++ README.md | 37 ++++++++++++++++++++++++++++++++----- README.ru.md | 38 +++++++++++++++++++++++++++++++++----- 4 files changed, 84 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d85bb90..9b1e565 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,15 @@ For a Russian version, see [CHANGELOG.ru.md](CHANGELOG.ru.md). `--scenario all` when `HMS_SMOKE_REST_URL` is unset. The local stand enables the listener on its plain profile (host port 19183) and registers a minimal Iceberg table for the load check. +- The Iceberg REST frontend now exposes every configured catalog as its own + prefix, `/v1//...`, instead of only `routing.default-catalog`. + `GET /v1/config?warehouse=` returns `overrides.prefix=` + for warehouse discovery; an unknown warehouse is a 400 + (`BadRequestException`), and an unknown prefix is still a 404 + (`NoSuchCatalogException`). The default catalog's prefix keeps the phase-1 + federated view (its own databases plus every other catalog's databases + under `` names) for compatibility; every other + prefix is a clean, per-catalog view where those federated names never leak. ### Fixed diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 73884d5..12e9136 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -35,6 +35,16 @@ English version: [CHANGELOG.md](CHANGELOG.md). пропускается, если `HMS_SMOKE_REST_URL` не задан. Локальный стенд включает listener в plain-профиле (host-порт 19183) и регистрирует минимальную Iceberg-таблицу для проверки load. +- Iceberg REST frontend теперь отдаёт каждый настроенный каталог под своим + prefix, `/v1//...`, а не только под `routing.default-catalog`. + `GET /v1/config?warehouse=` возвращает `overrides.prefix=` + для warehouse discovery; неизвестный warehouse — это 400 + (`BadRequestException`), а неизвестный prefix по-прежнему 404 + (`NoSuchCatalogException`). Prefix дефолтного каталога сохраняет + federated-представление из phase 1 (его собственные базы плюс базы всех + остальных каталогов под именами ``) для + совместимости; любой другой prefix — чистое, per-catalog представление, в + которое эти federated-имена не просачиваются. ### Исправлено diff --git a/README.md b/README.md index 765522f..3c62bba 100644 --- a/README.md +++ b/README.md @@ -981,8 +981,7 @@ Catalog spec, backed by the same routing/federation pipeline as the Thrift HMS front door. Status: **experimental, read-only**. Iceberg clients (PyIceberg, Spark `iceberg-rest`, Trino `iceberg-rest`) can discover and load Iceberg tables stored in HMS via the standard `metadata_location` table parameter; -writes, commits, view operations, and multi-catalog routing are NOT supported -in this iteration. +writes, commits, and view operations are NOT supported in this iteration. Enable it via: @@ -1005,9 +1004,22 @@ rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab | `GET /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | supported (Iceberg tables only) | | `POST`, `DELETE`, view operations, commits | unsupported | -`{prefix}` must equal the proxy's `routing.default-catalog`. Multi-catalog REST -(prefix-per-catalog with namespace prefix rewriting on both request and -response paths) is a planned follow-up. +`{prefix}` is any catalog listed in `catalogs=`: every configured catalog is +exposed as its own REST prefix, `/v1//...`. `GET /v1/config` supports +warehouse discovery: pass `?warehouse=` and the response's +`overrides.prefix` names that catalog, so a client can bind itself to the +right prefix without hardcoding it (see the client examples below). Without +`warehouse`, `/v1/config` advertises `routing.default-catalog`, matching +phase-1 behavior; an unknown `warehouse` value returns HTTP 400 +(`BadRequestException`). + +The default catalog's prefix keeps the phase-1 federated view: its own +databases plus every other catalog's databases under their +`` names (see [HiveServer2](#hiveserver2) above). +Every other prefix is a clean view: only that catalog's own databases, under +their internal names — the federated `` names never +leak into a non-default prefix. An unknown prefix still returns 404 +(`NoSuchCatalogException`). ### SPNEGO setup @@ -1037,6 +1049,21 @@ spark.sql.catalog.my_catalog.catalog-impl=org.apache.iceberg.rest.RESTCatalog spark.sql.catalog.my_catalog.uri=http://hms-proxy:9183 ``` +To target a non-default catalog, pass `warehouse=`; the client sends +it to `GET /v1/config` during discovery and binds itself to that catalog's +prefix for every following request: + +```python +catalog = RestCatalog("sales-catalog", **{ + "uri": "http://hms-proxy:9183", + "warehouse": "sales", +}) +``` + +```properties +spark.sql.catalog.sales_catalog.warehouse=sales +``` + ### Caveats - Iceberg REST is **Iceberg-spec only** by design. Non-Iceberg Hive tables diff --git a/README.ru.md b/README.ru.md index 15969be..2d7e459 100644 --- a/README.ru.md +++ b/README.ru.md @@ -978,8 +978,8 @@ Iceberg REST Catalog, использующий тот же routing/federation pi Thrift HMS front door. Статус: **экспериментально, read-only**. Iceberg-клиенты (PyIceberg, Spark `iceberg-rest`, Trino `iceberg-rest`) могут discover и load Iceberg-таблицы, хранящиеся в HMS через стандартный параметр -`metadata_location`. Writes, commits, view-операции и multi-catalog routing в -этой итерации НЕ поддерживаются. +`metadata_location`. Writes, commits и view-операции в этой итерации НЕ +поддерживаются. Включается так: @@ -1002,9 +1002,22 @@ rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab | `GET /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | поддержан (только Iceberg-таблицы) | | `POST`, `DELETE`, view-операции, commits | не поддержан | -`{prefix}` должен совпадать с `routing.default-catalog` прокси. Multi-catalog -REST (prefix-per-catalog с rewriting namespace prefix на request и response -путях) — запланированный follow-up. +`{prefix}` — любой каталог, перечисленный в `catalogs=`: каждый настроенный +каталог получает собственный REST prefix, `/v1//...`. `GET +/v1/config` поддерживает warehouse discovery: передайте `?warehouse=`, +и в ответе `overrides.prefix` назовёт этот каталог, так что клиент может +привязаться к нужному prefix, не зашивая его в конфигурацию (см. примеры +клиентов ниже). Без `warehouse` `/v1/config` объявляет `routing.default-catalog` +— как и в phase 1; неизвестное значение `warehouse` возвращает HTTP 400 +(`BadRequestException`). + +Prefix дефолтного каталога сохраняет federated-представление из phase 1: его +собственные базы плюс базы всех остальных каталогов под именами +`` (см. [HiveServer2](#hiveserver2) выше). Любой +другой prefix — чистое представление: только собственные базы этого +каталога, под их внутренними именами — federated-имена +`` в non-default prefix не просачиваются. На +неизвестный prefix по-прежнему отвечает 404 (`NoSuchCatalogException`). ### Настройка SPNEGO @@ -1034,6 +1047,21 @@ spark.sql.catalog.my_catalog.catalog-impl=org.apache.iceberg.rest.RESTCatalog spark.sql.catalog.my_catalog.uri=http://hms-proxy:9183 ``` +Чтобы обратиться к non-default каталогу, передайте `warehouse=`: +клиент отправит его в `GET /v1/config` при discovery и привяжется к prefix +этого каталога для всех последующих запросов: + +```python +catalog = RestCatalog("sales-catalog", **{ + "uri": "http://hms-proxy:9183", + "warehouse": "sales", +}) +``` + +```properties +spark.sql.catalog.sales_catalog.warehouse=sales +``` + ### Особенности и ограничения - Iceberg REST по дизайну работает **только с Iceberg-таблицами**. Native From 8161f7b537faffc9e5fa6df9b64cf41ef734742b Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 17:29:32 +0300 Subject: [PATCH 12/50] Cover multi-catalog REST prefixes in the stand smoke Add an optional second-prefix block to run_rest_smoke, driven by HMS_SMOKE_REST_SECOND_PREFIX (+ HMS_SMOKE_REST_SECOND_ICEBERG_TABLE): warehouse discovery advertises the chosen prefix, an unknown warehouse gets a clean 400, the clean view under the second prefix lists the namespace without leaking catalog__ external names, and the second-prefix table load returns metadata-location. Wire the stand's simple.env to the apache catalog and its second Iceberg table (smoke_iceberg_tbl_ap, registered on the second HDFS cluster namenode-b) and document the registration in both stand README locales. Extend TEST-MATRIX (EN+RU) section G with rows G8-G11 and note the same-day revalidation. --- ...real-installation-smoke.simple.env.example | 6 ++++ scripts/run-real-installation-smoke.sh | 32 +++++++++++++++++++ smoke-stand/README.md | 28 ++++++++++++++++ smoke-stand/README.ru.md | 28 ++++++++++++++++ smoke-stand/TEST-MATRIX.md | 7 ++++ smoke-stand/TEST-MATRIX.ru.md | 7 ++++ smoke-stand/env/simple.env | 5 +++ 7 files changed, 113 insertions(+) diff --git a/scripts/hms-real-installation-smoke.simple.env.example b/scripts/hms-real-installation-smoke.simple.env.example index 33d1e44..a1e4887 100644 --- a/scripts/hms-real-installation-smoke.simple.env.example +++ b/scripts/hms-real-installation-smoke.simple.env.example @@ -103,3 +103,9 @@ HMS_SMOKE_PARTITION_LOCK_TRANSACTIONAL=false # HMS_SMOKE_REST_NAMESPACE=default # HMS_SMOKE_REST_ICEBERG_TABLE=smoke_iceberg_tbl # HMS_SMOKE_REST_NON_ICEBERG_TABLE=smoke_read_tbl +# A non-default catalog prefix. When set, the REST smoke also checks warehouse discovery +# (GET /v1/config?warehouse=) advertises this prefix, an unknown warehouse gets HTTP 400, +# and the clean view under this prefix lists the namespace without leaking any catalog__ external names. +# HMS_SMOKE_REST_SECOND_PREFIX=apache +# An Iceberg table under the second prefix, loaded the same way as HMS_SMOKE_REST_ICEBERG_TABLE. +# HMS_SMOKE_REST_SECOND_ICEBERG_TABLE=smoke_iceberg_tbl_ap diff --git a/scripts/run-real-installation-smoke.sh b/scripts/run-real-installation-smoke.sh index a4cc363..5f6b17d 100755 --- a/scripts/run-real-installation-smoke.sh +++ b/scripts/run-real-installation-smoke.sh @@ -75,6 +75,10 @@ Optional Iceberg REST env vars: HMS_SMOKE_REST_NAMESPACE default: default HMS_SMOKE_REST_ICEBERG_TABLE Iceberg table to list and load; skipped when unset HMS_SMOKE_REST_NON_ICEBERG_TABLE plain Hive table that must NOT appear in the listing + HMS_SMOKE_REST_SECOND_PREFIX non-default catalog prefix; enables warehouse discovery + and clean-view checks under it; skipped when unset + HMS_SMOKE_REST_SECOND_ICEBERG_TABLE Iceberg table under the second prefix to load; skipped + when unset EOF cat <<'EOF' @@ -967,6 +971,34 @@ run_rest_smoke() { code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/no_such_table_smoke" "${body}")" [[ "${code}" =~ ^2 ]] && fail "REST write route unexpectedly succeeded with HTTP ${code}: $(cat "${body}")" + # Optional second prefix: proves warehouse discovery and the clean view work for a + # non-default catalog too, not only for the one /v1/config already advertised. + local second_prefix="${HMS_SMOKE_REST_SECOND_PREFIX:-}" + if [[ -n "${second_prefix}" ]]; then + code="$(rest_request GET "/v1/config?warehouse=${second_prefix}" "${body}")" + [[ "${code}" == "200" ]] || fail "config?warehouse=${second_prefix} returned HTTP ${code}: $(cat "${body}")" + grep -q "\"prefix\"[[:space:]]*:[[:space:]]*\"${second_prefix}\"" "${body}" \ + || fail "warehouse discovery did not advertise prefix '${second_prefix}': $(cat "${body}")" + + code="$(rest_request GET "/v1/config?warehouse=no_such_warehouse_smoke" "${body}")" + [[ "${code}" == "400" ]] || fail "unknown warehouse expected HTTP 400, got ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${second_prefix}/namespaces" "${body}")" + [[ "${code}" == "200" ]] || fail "GET /v1/${second_prefix}/namespaces returned HTTP ${code}: $(cat "${body}")" + grep -q "\[\"${namespace}\"\]" "${body}" \ + || fail "namespace '${namespace}' missing under prefix '${second_prefix}': $(cat "${body}")" + if grep -q "${second_prefix}__" "${body}"; then + fail "external names leaked into the clean view of '${second_prefix}': $(cat "${body}")" + fi + + if [[ -n "${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE:-}" ]]; then + code="$(rest_request GET "/v1/${second_prefix}/namespaces/${namespace}/tables/${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE}" "${body}")" + [[ "${code}" == "200" ]] || fail "REST load under '${second_prefix}' returned HTTP ${code}: $(cat "${body}")" + grep -q '"metadata-location"' "${body}" \ + || fail "second-prefix load carries no metadata-location: $(cat "${body}")" + fi + fi + log "Iceberg REST smoke passed (prefix '${prefix}', namespace '${namespace}')" } diff --git a/smoke-stand/README.md b/smoke-stand/README.md index f8a56ec..5f5df1b 100644 --- a/smoke-stand/README.md +++ b/smoke-stand/README.md @@ -216,6 +216,34 @@ so a passing load proves the whole chain: REST route → HiveCatalog → the pro layer → HMS → HDFS. Plain Hive tables of the same database (`smoke_read_hdp`, `smoke_txn_tbl`) must stay invisible through REST — the smoke asserts that too. +### A second table, on the second catalog + +`HMS_SMOKE_REST_SECOND_PREFIX` (see `smoke-stand/env/simple.env`) points the REST smoke at the +`apache` catalog too, which proves warehouse discovery and the clean view also work for a +non-default prefix. It needs a second Iceberg table, registered the same way but on the +`apache` catalog's own cluster (`namenode-b`): + +```bash +# 1. Put a minimal Iceberg table metadata file onto the apache catalog's cluster +docker cp stand-namenode-b:/tmp/00000-smoke.metadata.json +docker exec stand-namenode-b bash -c \ + 'hdfs dfs -mkdir -p /warehouse/apache/smoke_iceberg_tbl_ap/metadata && + hdfs dfs -put -f /tmp/00000-smoke.metadata.json /warehouse/apache/smoke_iceberg_tbl_ap/metadata/' + +# 2. Register the table in the apache catalog +docker exec stand-hs2 bash -c "java -cp '/opt/hs2/conf:/opt/hs2/lib/*' org.apache.hive.beeline.BeeLine \ + -u 'jdbc:hive2://localhost:10000/default' -n hive --silent=true \ + -e \"create external table if not exists apache__default.smoke_iceberg_tbl_ap (id int, ds string) + stored as parquet + location 'hdfs://namenode-b:8020/warehouse/apache/smoke_iceberg_tbl_ap' + tblproperties ( + 'table_type'='ICEBERG', + 'metadata_location'='hdfs://namenode-b:8020/warehouse/apache/smoke_iceberg_tbl_ap/metadata/00000-smoke.metadata.json');\"" +``` + +The `metadata.json` is a copy of the first table's, with `location` pointed at the +`smoke_iceberg_tbl_ap` path above and a fresh `table-uuid`. + The Kerberos profile leaves the REST listener off: SPNEGO needs a GSS-enabled curl inside the network, and the handshake itself is already covered end-to-end by `SpnegoIntegrationTest` on hadoop-minikdc. diff --git a/smoke-stand/README.ru.md b/smoke-stand/README.ru.md index 565ed85..049740e 100644 --- a/smoke-stand/README.ru.md +++ b/smoke-stand/README.ru.md @@ -215,6 +215,34 @@ docker exec stand-hs2 bash -c "java -cp '/opt/hs2/conf:/opt/hs2/lib/*' org.apach прокси → HMS → HDFS. Обычные Hive-таблицы той же базы (`smoke_read_hdp`, `smoke_txn_tbl`) через REST видны быть не должны — smoke проверяет и это. +### Вторая таблица, на втором каталоге + +`HMS_SMOKE_REST_SECOND_PREFIX` (см. `smoke-stand/env/simple.env`) направляет REST-smoke ещё и +на каталог `apache` — это доказывает, что warehouse discovery и чистое представление работают +и для non-default prefix. Ему нужна вторая Iceberg-таблица, зарегистрированная так же, но на +собственном кластере каталога `apache` (`namenode-b`): + +```bash +# 1. Положить минимальный файл table metadata Iceberg на кластер каталога apache +docker cp stand-namenode-b:/tmp/00000-smoke.metadata.json +docker exec stand-namenode-b bash -c \ + 'hdfs dfs -mkdir -p /warehouse/apache/smoke_iceberg_tbl_ap/metadata && + hdfs dfs -put -f /tmp/00000-smoke.metadata.json /warehouse/apache/smoke_iceberg_tbl_ap/metadata/' + +# 2. Зарегистрировать таблицу в каталоге apache +docker exec stand-hs2 bash -c "java -cp '/opt/hs2/conf:/opt/hs2/lib/*' org.apache.hive.beeline.BeeLine \ + -u 'jdbc:hive2://localhost:10000/default' -n hive --silent=true \ + -e \"create external table if not exists apache__default.smoke_iceberg_tbl_ap (id int, ds string) + stored as parquet + location 'hdfs://namenode-b:8020/warehouse/apache/smoke_iceberg_tbl_ap' + tblproperties ( + 'table_type'='ICEBERG', + 'metadata_location'='hdfs://namenode-b:8020/warehouse/apache/smoke_iceberg_tbl_ap/metadata/00000-smoke.metadata.json');\"" +``` + +`metadata.json` — копия файла первой таблицы, с `location`, указывающим на путь +`smoke_iceberg_tbl_ap` выше, и новым `table-uuid`. + Kerberos-профиль оставляет REST listener выключенным: SPNEGO требует curl с GSS внутри сети, а сам handshake уже покрыт end-to-end тестом `SpnegoIntegrationTest` на hadoop-minikdc. diff --git a/smoke-stand/TEST-MATRIX.md b/smoke-stand/TEST-MATRIX.md index cff3a24..39e47b9 100644 --- a/smoke-stand/TEST-MATRIX.md +++ b/smoke-stand/TEST-MATRIX.md @@ -94,6 +94,10 @@ table is the hand-registered `smoke_iceberg_tbl` (see the stand README). | G5 | Unknown prefix → clean 404 `NoSuchCatalogException` | ✅ | n/a | | G6 | Unknown table → clean 404 | ✅ | n/a | | G7 | Write route (`DELETE` table) refused, non-2xx | ✅ | n/a | +| G8 | `GET /v1/config?warehouse=apache` advertises `prefix=apache` | ✅ | n/a | +| G9 | Unknown warehouse (`GET /v1/config?warehouse=no_such_warehouse_smoke`) → clean 400 | ✅ | n/a | +| G10 | Clean namespace view under the `apache` prefix lists `default` with no `apache__`-prefixed external names | ✅ | n/a | +| G11 | Table load under the `apache` prefix (`smoke_iceberg_tbl_ap`, second HDFS cluster) returns `metadata-location` | ✅ | n/a | ## F. Not covered, and why @@ -124,6 +128,9 @@ executed is claimed; a row not listed was not repeated and its ✅ stands on the Later the same day the branch's Iceberg REST listener was enabled on the plain profile and section G was run for the first time (`--scenario rest`, and again as the REST step of a full green `--scenario all`). + The same day, after the `apache` catalog's second Iceberg table (`smoke_iceberg_tbl_ap`) was + registered on its own cluster (`namenode-b`), the new multi-catalog REST rows (G8-G11) were + run too, in the same `--scenario rest` and `--scenario all` passes. ## Two caveats on faithfulness diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index 5baaa70..3909ac9 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -94,6 +94,10 @@ HDP-клиент не может пользоваться Apache-listener — Th | G5 | Неизвестный prefix → чистый 404 `NoSuchCatalogException` | ✅ | n/a | | G6 | Неизвестная таблица → чистый 404 | ✅ | n/a | | G7 | Write-роут (`DELETE` таблицы) отклонён, не-2xx | ✅ | n/a | +| G8 | `GET /v1/config?warehouse=apache` объявляет `prefix=apache` | ✅ | n/a | +| G9 | Неизвестный warehouse (`GET /v1/config?warehouse=no_such_warehouse_smoke`) → чистый 400 | ✅ | n/a | +| G10 | Чистое представление namespace под prefix `apache` показывает `default` без утечки внешних имён с префиксом `apache__` | ✅ | n/a | +| G11 | Load таблицы под prefix `apache` (`smoke_iceberg_tbl_ap`, второй HDFS-кластер) возвращает `metadata-location` | ✅ | n/a | ## F. Что не покрыто и почему @@ -125,6 +129,9 @@ HDP-клиент не может пользоваться Apache-listener — Th Позже в тот же день на plain-профиле был включён Iceberg REST listener ветки и впервые прогнан раздел G (`--scenario rest`, затем ещё раз как REST-шаг полностью зелёного `--scenario all`). + В тот же день, после регистрации второй Iceberg-таблицы каталога `apache` + (`smoke_iceberg_tbl_ap`) на её собственном кластере (`namenode-b`), были прогнаны и новые + multi-catalog REST-строки (G8-G11) — в тех же прогонах `--scenario rest` и `--scenario all`. ## Две оговорки честности diff --git a/smoke-stand/env/simple.env b/smoke-stand/env/simple.env index d25ec28..d9d9a1d 100644 --- a/smoke-stand/env/simple.env +++ b/smoke-stand/env/simple.env @@ -90,3 +90,8 @@ HMS_SMOKE_REST_PREFIX=hdp HMS_SMOKE_REST_NAMESPACE=default HMS_SMOKE_REST_ICEBERG_TABLE=smoke_iceberg_tbl HMS_SMOKE_REST_NON_ICEBERG_TABLE=smoke_read_hdp + +# Second catalog (apache), lives on the second HDFS cluster (namenode-b). Proves warehouse +# discovery and the clean view work for a non-default prefix too. +HMS_SMOKE_REST_SECOND_PREFIX=apache +HMS_SMOKE_REST_SECOND_ICEBERG_TABLE=smoke_iceberg_tbl_ap From f4d133b475ef1b3e6f5829446a664a5a32b1dd3c Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 17:39:06 +0300 Subject: [PATCH 13/50] Apply final-review fixes for Iceberg REST phase 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - IcebergRestService: rewrite class Javadoc to describe the actual per-prefix service model instead of the stale MVP/single-catalog note. - IcebergRestServices#close: attempt close() on every service instead of stopping at the first failure; collect the first IOException, add later ones as suppressed, and rethrow. - TEST-MATRIX.ru.md: keep "prefix" in Latin script in the G10 row instead of the Cyrillic "префиксом". - AGENTS.md: document the restcatalog module in the architecture notes. --- AGENTS.md | 1 + smoke-stand/TEST-MATRIX.ru.md | 2 +- .../hmsproxy/restcatalog/IcebergRestService.java | 8 +++++--- .../hmsproxy/restcatalog/IcebergRestServices.java | 14 +++++++++++++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6e77c14..e09a570 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -84,6 +84,7 @@ scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple. - Знание о служебном плейсхолдере Hive (`_dummy_database`/`_dummy_table`, константы `SemanticAnalyzer.DUMMY_DATABASE`/`DUMMY_TABLE`, которые Hive шлёт в `LockRequest` для `INSERT ... VALUES`) живёт только в `routing/HivePlaceholderNamespace`. Плейсхолдер не выбирает каталог, не считается вторым namespace lock-запроса и не переписывается интернализацией в реальную backend-базу. Не добавляй локальных сравнений с `_dummy_database` в других классах. - Для RPC, у которых Hive IDL не объявляет исключений (`add_write_notification_log`, `open_txns`, `show_locks` и другие: у их `_result` есть только поле `success`), libthrift 0.9.3 подменяет любое серверное исключение на `TApplicationException("Internal error processing ")`. Текст отказа таких методов виден только в логе - не рассчитывай, что его получит клиент, и не проверяй его в smoke-скриптах. Настоящий HMS теряет свои тексты ошибок так же. - Классификация backend Thrift-ошибок живёт только в `thriftbridge/ThriftFailureClassifier`: «метода нет» - это `TApplicationException` с типом `UNKNOWN_METHOD` (или отсутствие метода в загруженном runtime), transport failure и protocol desync - отдельные категории. Не пиши локальные `instanceof TApplicationException` для решений про fallback, downgrade или переоткрытие соединения. +- `restcatalog` - Iceberg REST front door поверх routing-слоя: per-catalog сервисы работают через in-process прокси `IMetaStoreClient`, доступ только на чтение, для не-default каталогов включён name translation, чтобы клиент видел внутренние имена баз этого каталога. ## Парсинг конфигурации diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index 3909ac9..81ab7ba 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -96,7 +96,7 @@ HDP-клиент не может пользоваться Apache-listener — Th | G7 | Write-роут (`DELETE` таблицы) отклонён, не-2xx | ✅ | n/a | | G8 | `GET /v1/config?warehouse=apache` объявляет `prefix=apache` | ✅ | n/a | | G9 | Неизвестный warehouse (`GET /v1/config?warehouse=no_such_warehouse_smoke`) → чистый 400 | ✅ | n/a | -| G10 | Чистое представление namespace под prefix `apache` показывает `default` без утечки внешних имён с префиксом `apache__` | ✅ | n/a | +| G10 | Чистое представление namespace под prefix `apache` показывает `default` без утечки внешних имён вида `apache__*` | ✅ | n/a | | G11 | Load таблицы под prefix `apache` (`smoke_iceberg_tbl_ap`, второй HDFS-кластер) возвращает `metadata-location` | ✅ | n/a | ## F. Что не покрыто и почему diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java index 46a2cbf..1c68888 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java @@ -16,9 +16,11 @@ /** * Bridges Iceberg REST calls to the proxy's ThriftHiveMetastore.Iface via a - * single shared RoutingHiveCatalog. The MVP supports the proxy's default catalog - * only; multi-catalog support requires namespace prefix rewriting on both - * request and response paths (planned for a follow-up step). + * RoutingHiveCatalog. Each instance serves a single catalog prefix. The + * default catalog's service exposes the federated view as-is, with no name + * translation; every other catalog's service is given a CatalogNameTranslation + * so REST clients see that catalog's internal database names instead of the + * federated ones. */ public final class IcebergRestService implements AutoCloseable { private static final String UNUSED_URI = "thrift://hms-proxy-loopback:0"; diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java index daaa0c6..f8026ae 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java @@ -49,8 +49,20 @@ public String defaultPrefix() { @Override public void close() throws IOException { + IOException first = null; for (IcebergRestService service : byPrefix.values()) { - service.close(); + try { + service.close(); + } catch (IOException e) { + if (first == null) { + first = e; + } else { + first.addSuppressed(e); + } + } + } + if (first != null) { + throw first; } } } From fcd72c89c42494fe1d7c72d9f10883218bd8037b Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 18:40:24 +0300 Subject: [PATCH 14/50] Mention the service registry in the IcebergRestService Javadoc --- .../hmsproxy/restcatalog/IcebergRestService.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java index 1c68888..f896b91 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java @@ -16,11 +16,11 @@ /** * Bridges Iceberg REST calls to the proxy's ThriftHiveMetastore.Iface via a - * RoutingHiveCatalog. Each instance serves a single catalog prefix. The - * default catalog's service exposes the federated view as-is, with no name - * translation; every other catalog's service is given a CatalogNameTranslation - * so REST clients see that catalog's internal database names instead of the - * federated ones. + * RoutingHiveCatalog. Each instance serves a single catalog prefix and is + * looked up through the IcebergRestServices registry. The default catalog's + * service exposes the federated view as-is, with no name translation; every + * other catalog's service is given a CatalogNameTranslation so REST clients + * see that catalog's internal database names instead of the federated ones. */ public final class IcebergRestService implements AutoCloseable { private static final String UNUSED_URI = "thrift://hms-proxy-loopback:0"; From d938f88d63b4f6994d94f2c07c76eab09cbd5be1 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 18:46:46 +0300 Subject: [PATCH 15/50] Assert both sides of the multi-catalog REST view in the smoke The second-prefix block now checks the federation side too: the name stays visible under the default prefix, and the second catalog's Iceberg table is listed and loaded through that federated name. The clean-view side gains the cross-catalog negatives - a default-catalog table and the external namespace name both 404 under the second prefix - and the second catalog's plain Hive table must stay out of its listing (HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE). The catalog-db separator is no longer hardwired into the leak grep (HMS_SMOKE_REST_SEPARATOR, default __). Recorded as rows G12-G16 in TEST-MATRIX, both locales. --- ...real-installation-smoke.simple.env.example | 10 +++- scripts/run-real-installation-smoke.sh | 52 +++++++++++++++++-- smoke-stand/TEST-MATRIX.md | 9 +++- smoke-stand/TEST-MATRIX.ru.md | 8 +++ smoke-stand/env/simple.env | 1 + 5 files changed, 74 insertions(+), 6 deletions(-) diff --git a/scripts/hms-real-installation-smoke.simple.env.example b/scripts/hms-real-installation-smoke.simple.env.example index a1e4887..110734a 100644 --- a/scripts/hms-real-installation-smoke.simple.env.example +++ b/scripts/hms-real-installation-smoke.simple.env.example @@ -105,7 +105,13 @@ HMS_SMOKE_PARTITION_LOCK_TRANSACTIONAL=false # HMS_SMOKE_REST_NON_ICEBERG_TABLE=smoke_read_tbl # A non-default catalog prefix. When set, the REST smoke also checks warehouse discovery # (GET /v1/config?warehouse=) advertises this prefix, an unknown warehouse gets HTTP 400, -# and the clean view under this prefix lists the namespace without leaking any catalog__ external names. +# the clean view under this prefix lists the namespace without leaking external names, the +# federated name stays visible (and loadable) under the default prefix, +# and cross-catalog names produce clean 404s on the wrong side. # HMS_SMOKE_REST_SECOND_PREFIX=apache -# An Iceberg table under the second prefix, loaded the same way as HMS_SMOKE_REST_ICEBERG_TABLE. +# An Iceberg table under the second prefix, loaded both directly and through the federated name. # HMS_SMOKE_REST_SECOND_ICEBERG_TABLE=smoke_iceberg_tbl_ap +# A plain Hive table of the second catalog that must stay invisible in its REST listing. +# HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE=smoke_read_tbl2 +# Catalog-db separator of the proxy, used to build federated names. Default: __ +# HMS_SMOKE_REST_SEPARATOR=__ diff --git a/scripts/run-real-installation-smoke.sh b/scripts/run-real-installation-smoke.sh index 5f6b17d..d783923 100755 --- a/scripts/run-real-installation-smoke.sh +++ b/scripts/run-real-installation-smoke.sh @@ -77,8 +77,12 @@ Optional Iceberg REST env vars: HMS_SMOKE_REST_NON_ICEBERG_TABLE plain Hive table that must NOT appear in the listing HMS_SMOKE_REST_SECOND_PREFIX non-default catalog prefix; enables warehouse discovery and clean-view checks under it; skipped when unset - HMS_SMOKE_REST_SECOND_ICEBERG_TABLE Iceberg table under the second prefix to load; skipped - when unset + HMS_SMOKE_REST_SECOND_ICEBERG_TABLE Iceberg table under the second prefix to load (also + listed and loaded through the federated name under the + default prefix); skipped when unset + HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE plain Hive table of the second catalog that must NOT + appear in its REST listing + HMS_SMOKE_REST_SEPARATOR catalog-db separator of the proxy; default: __ EOF cat <<'EOF' @@ -974,7 +978,13 @@ run_rest_smoke() { # Optional second prefix: proves warehouse discovery and the clean view work for a # non-default catalog too, not only for the one /v1/config already advertised. local second_prefix="${HMS_SMOKE_REST_SECOND_PREFIX:-}" + local separator="${HMS_SMOKE_REST_SEPARATOR:-__}" if [[ -n "${second_prefix}" ]]; then + # The second catalog's databases appear twice on purpose: under the default + # prefix as federated names, and under their own + # prefix as bare internal names. Both sides are asserted here. + local fed_ns="${second_prefix}${separator}${namespace}" + code="$(rest_request GET "/v1/config?warehouse=${second_prefix}" "${body}")" [[ "${code}" == "200" ]] || fail "config?warehouse=${second_prefix} returned HTTP ${code}: $(cat "${body}")" grep -q "\"prefix\"[[:space:]]*:[[:space:]]*\"${second_prefix}\"" "${body}" \ @@ -987,15 +997,51 @@ run_rest_smoke() { [[ "${code}" == "200" ]] || fail "GET /v1/${second_prefix}/namespaces returned HTTP ${code}: $(cat "${body}")" grep -q "\[\"${namespace}\"\]" "${body}" \ || fail "namespace '${namespace}' missing under prefix '${second_prefix}': $(cat "${body}")" - if grep -q "${second_prefix}__" "${body}"; then + if grep -q "${second_prefix}${separator}" "${body}"; then fail "external names leaked into the clean view of '${second_prefix}': $(cat "${body}")" fi + code="$(rest_request GET "/v1/${prefix}/namespaces" "${body}")" + [[ "${code}" == "200" ]] || fail "GET /v1/${prefix}/namespaces returned HTTP ${code}: $(cat "${body}")" + grep -q "\[\"${fed_ns}\"\]" "${body}" \ + || fail "federated namespace '${fed_ns}' missing under the default prefix '${prefix}': $(cat "${body}")" + + code="$(rest_request GET "/v1/${second_prefix}/namespaces/${fed_ns}" "${body}")" + [[ "${code}" == "404" ]] \ + || fail "external namespace name '${fed_ns}' under prefix '${second_prefix}' expected HTTP 404, got ${code}: $(cat "${body}")" + + if [[ -n "${iceberg_table}" ]]; then + code="$(rest_request GET "/v1/${second_prefix}/namespaces/${namespace}/tables/${iceberg_table}" "${body}")" + [[ "${code}" == "404" ]] \ + || fail "default-catalog table '${iceberg_table}' under prefix '${second_prefix}' expected HTTP 404, got ${code}: $(cat "${body}")" + fi + if [[ -n "${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE:-}" ]]; then + code="$(rest_request GET "/v1/${second_prefix}/namespaces/${namespace}/tables" "${body}")" + [[ "${code}" == "200" ]] || fail "GET tables under '${second_prefix}' returned HTTP ${code}: $(cat "${body}")" + grep -q "\"name\"[[:space:]]*:[[:space:]]*\"${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE}\"" "${body}" \ + || fail "Iceberg table '${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE}' missing from the '${second_prefix}' listing: $(cat "${body}")" + if [[ -n "${HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE:-}" ]]; then + if grep -q "\"name\"[[:space:]]*:[[:space:]]*\"${HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE}\"" "${body}"; then + fail "non-Iceberg table '${HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE}' leaked into the '${second_prefix}' listing: $(cat "${body}")" + fi + fi + code="$(rest_request GET "/v1/${second_prefix}/namespaces/${namespace}/tables/${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE}" "${body}")" [[ "${code}" == "200" ]] || fail "REST load under '${second_prefix}' returned HTTP ${code}: $(cat "${body}")" grep -q '"metadata-location"' "${body}" \ || fail "second-prefix load carries no metadata-location: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${fed_ns}/tables" "${body}")" + [[ "${code}" == "200" ]] || fail "GET tables of '${fed_ns}' under '${prefix}' returned HTTP ${code}: $(cat "${body}")" + grep -q "\"name\"[[:space:]]*:[[:space:]]*\"${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE}\"" "${body}" \ + || fail "Iceberg table '${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE}' missing from the federated '${fed_ns}' listing: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${fed_ns}/tables/${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE}" "${body}")" + [[ "${code}" == "200" ]] \ + || fail "federated load of '${fed_ns}.${HMS_SMOKE_REST_SECOND_ICEBERG_TABLE}' returned HTTP ${code}: $(cat "${body}")" + grep -q '"metadata-location"' "${body}" \ + || fail "federated load carries no metadata-location: $(cat "${body}")" fi fi diff --git a/smoke-stand/TEST-MATRIX.md b/smoke-stand/TEST-MATRIX.md index 39e47b9..57a478e 100644 --- a/smoke-stand/TEST-MATRIX.md +++ b/smoke-stand/TEST-MATRIX.md @@ -98,6 +98,11 @@ table is the hand-registered `smoke_iceberg_tbl` (see the stand README). | G9 | Unknown warehouse (`GET /v1/config?warehouse=no_such_warehouse_smoke`) → clean 400 | ✅ | n/a | | G10 | Clean namespace view under the `apache` prefix lists `default` with no `apache__`-prefixed external names | ✅ | n/a | | G11 | Table load under the `apache` prefix (`smoke_iceberg_tbl_ap`, second HDFS cluster) returns `metadata-location` | ✅ | n/a | +| G12 | Federated namespace `apache__default` stays visible under the default prefix | ✅ | n/a | +| G13 | Listing and load of `smoke_iceberg_tbl_ap` through the federated `apache__default` name under the default prefix | ✅ | n/a | +| G14 | A default-catalog table under the `apache` prefix → clean 404 | ✅ | n/a | +| G15 | The external name `apache__default` used as a namespace under the `apache` prefix → clean 404 | ✅ | n/a | +| G16 | The second catalog's plain Hive table (`smoke_read_ap`) stays invisible in the `apache` listing | ✅ | n/a | ## F. Not covered, and why @@ -130,7 +135,9 @@ executed is claimed; a row not listed was not repeated and its ✅ stands on the full green `--scenario all`). The same day, after the `apache` catalog's second Iceberg table (`smoke_iceberg_tbl_ap`) was registered on its own cluster (`namenode-b`), the new multi-catalog REST rows (G8-G11) were - run too, in the same `--scenario rest` and `--scenario all` passes. + run too, in the same `--scenario rest` and `--scenario all` passes. A follow-up run the same + day added and passed the federation/isolation rows G12-G16: the federated name under the + default prefix (listing and load included) and clean 404s for every cross-catalog shape. ## Two caveats on faithfulness diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index 81ab7ba..88a9177 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -98,6 +98,11 @@ HDP-клиент не может пользоваться Apache-listener — Th | G9 | Неизвестный warehouse (`GET /v1/config?warehouse=no_such_warehouse_smoke`) → чистый 400 | ✅ | n/a | | G10 | Чистое представление namespace под prefix `apache` показывает `default` без утечки внешних имён вида `apache__*` | ✅ | n/a | | G11 | Load таблицы под prefix `apache` (`smoke_iceberg_tbl_ap`, второй HDFS-кластер) возвращает `metadata-location` | ✅ | n/a | +| G12 | Federated namespace `apache__default` остаётся виден под default-prefix | ✅ | n/a | +| G13 | Листинг и load `smoke_iceberg_tbl_ap` через federated-имя `apache__default` под default-prefix | ✅ | n/a | +| G14 | Таблица default-каталога под prefix `apache` → чистый 404 | ✅ | n/a | +| G15 | Внешнее имя `apache__default`, использованное как namespace под prefix `apache` → чистый 404 | ✅ | n/a | +| G16 | Обычная Hive-таблица второго каталога (`smoke_read_ap`) не видна в листинге под prefix `apache` | ✅ | n/a | ## F. Что не покрыто и почему @@ -132,6 +137,9 @@ HDP-клиент не может пользоваться Apache-listener — Th В тот же день, после регистрации второй Iceberg-таблицы каталога `apache` (`smoke_iceberg_tbl_ap`) на её собственном кластере (`namenode-b`), были прогнаны и новые multi-catalog REST-строки (G8-G11) — в тех же прогонах `--scenario rest` и `--scenario all`. + Дополнительный прогон в тот же день добавил и прошёл строки G12-G16 про федерацию и изоляцию: + federated-имя под default-prefix (включая листинг и load) и чистые 404 на каждую + кросс-каталожную форму. ## Две оговорки честности diff --git a/smoke-stand/env/simple.env b/smoke-stand/env/simple.env index d9d9a1d..d5f41b7 100644 --- a/smoke-stand/env/simple.env +++ b/smoke-stand/env/simple.env @@ -95,3 +95,4 @@ HMS_SMOKE_REST_NON_ICEBERG_TABLE=smoke_read_hdp # discovery and the clean view work for a non-default prefix too. HMS_SMOKE_REST_SECOND_PREFIX=apache HMS_SMOKE_REST_SECOND_ICEBERG_TABLE=smoke_iceberg_tbl_ap +HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE=smoke_read_ap From 0b47130f147d6a8257238cdf2d43bbcfcb9329c5 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 20:16:28 +0300 Subject: [PATCH 16/50] Add REST listener series to the Prometheus metrics --- ...07-27-iceberg-rest-phase3-observability.md | 92 +++++++++++++++++++ ...ceberg-rest-phase3-observability-design.md | 68 ++++++++++++++ .../observability/PrometheusMetrics.java | 27 +++++- .../observability/PrometheusMetricsTest.java | 17 ++++ 4 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/plans/2026-07-27-iceberg-rest-phase3-observability.md create mode 100644 docs/superpowers/specs/2026-07-27-iceberg-rest-phase3-observability-design.md diff --git a/docs/superpowers/plans/2026-07-27-iceberg-rest-phase3-observability.md b/docs/superpowers/plans/2026-07-27-iceberg-rest-phase3-observability.md new file mode 100644 index 0000000..92c8e8a --- /dev/null +++ b/docs/superpowers/plans/2026-07-27-iceberg-rest-phase3-observability.md @@ -0,0 +1,92 @@ +# Iceberg REST Phase 3: Observability Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** REST traffic visible in the proxy's Prometheus metrics per `docs/superpowers/specs/2026-07-27-iceberg-rest-phase3-observability-design.md`. + +**Architecture:** Three new series in the hand-rolled `PrometheusMetrics`; `IcebergHttpHandler` records prefix/route/status/duration around `doHandle`; the listener info gauge is set on successful bind. No audit changes, no response-behavior changes. + +**Tech Stack:** Java 17, JUnit 4. `JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19`, offline Maven (`mvn -o`). No new dependencies. + +## Global Constraints + +- Java 17; 2-space indent; explicit imports; no new dependencies. +- Metric label rules from the spec verbatim: `route` = lower-case adapter `Route` enum name or pseudo-routes `unknown_prefix` / `unknown_route` / `bad_request`; `prefix` = catalog prefix or `unknown`; `status` = numeric HTTP status written. Never a raw URL in a label. +- Metrics recording must never fail or alter a response; record after the response is written. +- Bilingual docs: every EN doc change lands with its RU counterpart in the same commit. +- English commit messages, no attribution footers. Commits pre-approved; do NOT push. + +--- + +### Task 1: PrometheusMetrics REST series + +**Files:** +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetrics.java` +- Test: `src/test/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetricsTest.java` (add cases; if the class does not exist, check for the existing metrics test class with `grep -rn 'class PrometheusMetrics' src/test` and extend that) + +**Interfaces:** +- Produces: `void recordRestRequest(String prefix, String route, int status, double durationSeconds)`; `void setRestListenerInfo(String bindHost, int port)`. + +- [ ] **Step 1: Write failing tests** — after `recordRestRequest("hdp", "load_table", 200, 0.05)` the `render()` output contains `hms_proxy_rest_requests_total{prefix="hdp",route="load_table",status="200"} 1` and a `hms_proxy_rest_request_duration_seconds` histogram series for those labels; after `setRestListenerInfo("0.0.0.0", 9183)` render contains `hms_proxy_rest_listener_info{bind_host="0.0.0.0",port="9183"} 1`. Follow the existing test class's assertion style (substring checks on `render()`). +- [ ] **Step 2: Run** `mvn -o -Dtest= test` — expect compile failure. +- [ ] **Step 3: Implement** — declare the three series next to the existing ones, same builder/registration style, duration histogram reusing the same bucket array as `hms_proxy_request_duration_seconds`; `recordRestRequest` increments the counter with `String.valueOf(status)` and observes the histogram; `setRestListenerInfo` sets the gauge to 1. +- [ ] **Step 4: Run to green.** +- [ ] **Step 5: Commit** (include `docs/superpowers/specs/...phase3...md` and this plan file): `git commit -m "Add REST listener series to the Prometheus metrics"`. + +--- + +### Task 2: Handler instrumentation and wiring + +**Files:** +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java` +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java` (`open(ProxyConfig, IcebergRestServices, PrometheusMetrics)`) +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java` (pass `observability.metrics()`) +- Test: `src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java` (+ update every `RestCatalogServer.open` call site: `grep -rn 'RestCatalogServer.open' src/`) + +**Interfaces:** +- Consumes: `recordRestRequest` / `setRestListenerInfo` from Task 1. +- Produces: instrumented handler; `RestCatalogServer.open` requires non-null metrics. + +- [ ] **Step 1: Add failing integration tests** — after `get("/v1/catalog1/namespaces")` (200), `get("/v1/nope/namespaces")` (404) and `get("/v1/config?warehouse=nope")` (400), the shared `PrometheusMetrics` instance's `render()` contains: a `route="list_namespaces",status="200"` series with `prefix="catalog1"`; a `route="unknown_prefix",prefix="unknown",status="404"` series; a `route="bad_request",prefix="unknown",status="400"` series; and `hms_proxy_rest_listener_info`. +- [ ] **Step 2: Implement** — handler constructor takes `PrometheusMetrics`; `handle()` wraps `doHandle` with `System.nanoTime()`; the handler records in `finally` using fields captured during dispatch: resolved prefix (or `unknown`), normalized route (lower-case `Route` enum name via the parsed route; `config` for the discovery path; pseudo-routes for refusals), and the status passed to the write helpers (track it where the response is written). `RestCatalogServer.open` requires metrics (`Objects.requireNonNull`) and calls `setRestListenerInfo(bindHost, boundPort)` after `server.start()`. +- [ ] **Step 3: Run the restcatalog suite** (`mvn -o -Dtest='io/github/mmalykhin/hmsproxy/restcatalog/*' test`), then the FULL suite (`mvn -o test`) — 0 failures, 0 skipped. +- [ ] **Step 4: Commit**: `git commit -m "Record REST request metrics in the Iceberg HTTP handler"`. + +--- + +### Task 3: Smoke step, stand run, docs + +**Files:** +- Modify: `scripts/run-real-installation-smoke.sh` — at the end of `run_rest_smoke`, before the final log line: + +```bash + local metrics_url="${HMS_SMOKE_REST_METRICS_URL:-}" + if [[ -n "${metrics_url}" ]]; then + code="$(rest_request GET "" "${body}")" # placeholder removed below; use curl directly: + curl -sS -o "${body}" "${metrics_url}" || fail "cannot fetch metrics from ${metrics_url}" + grep -q 'hms_proxy_rest_requests_total{' "${body}" \ + || fail "metrics endpoint carries no hms_proxy_rest_requests_total series" + grep -q 'hms_proxy_rest_listener_info{' "${body}" \ + || fail "metrics endpoint carries no hms_proxy_rest_listener_info series" + fi +``` + + (Use the curl call only — do not call `rest_request` for the metrics URL; drop the placeholder line. Document `HMS_SMOKE_REST_METRICS_URL` in usage next to the other REST vars.) +- Modify: `smoke-stand/env/simple.env` — `HMS_SMOKE_REST_METRICS_URL=http://localhost:19090/metrics`. +- Modify: `scripts/hms-real-installation-smoke.simple.env.example` — commented key + one-line comment. +- Modify: `README.md` + `README.ru.md` — extend the metrics table with the three new series; one sentence in the Iceberg REST section that the listener is covered by metrics. Same content both locales. +- Modify: `CHANGELOG.md` + `CHANGELOG.ru.md` — bullet under 2026-07-27 Added. +- Modify after the run: `smoke-stand/TEST-MATRIX.md` + `.ru.md` — row G17 (REST metrics visible on the management endpoint) + a revalidation-log sentence. + +- [ ] **Step 1: Implement runner/env/docs; `bash -n` passes.** +- [ ] **Step 2: Rebuild + restage + restart the stand proxy** (`mvn -o -DskipTests package`, `cd smoke-stand && ./prepare.sh`, `docker compose up -d --build`), wait healthy. +- [ ] **Step 3: Run** `--scenario rest` then `--scenario all` — both `completed successfully`. +- [ ] **Step 4: Update TEST-MATRIX both locales; commit everything**: `git commit -m "Cover REST metrics in the stand smoke"`. + +--- + +## Self-Review + +- Spec coverage: three series (T1), label rules + wiring + non-null metrics (T2), never-fail recording (T2 finally-after-write), smoke + docs (T3). No audit/readiness changes anywhere — matches out-of-scope. +- Placeholders: the T3 snippet's stray `rest_request` line is explicitly instructed to be dropped; no TBDs. +- Type consistency: `recordRestRequest(String,String,int,double)` and `setRestListenerInfo(String,int)` used identically in T1/T2. diff --git a/docs/superpowers/specs/2026-07-27-iceberg-rest-phase3-observability-design.md b/docs/superpowers/specs/2026-07-27-iceberg-rest-phase3-observability-design.md new file mode 100644 index 0000000..a91bfbe --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-iceberg-rest-phase3-observability-design.md @@ -0,0 +1,68 @@ +# Iceberg REST frontend, phase 3: observability — design + +Date: 2026-07-27 +Branch: `feature/iceberg-rest-fe-phase1` +Status: approved + +## Goal + +Make REST traffic visible in the proxy's existing observability stack. Metrics only: +per-RPC audit events already flow through `RoutingMetaStoreProxy` with the right +`authenticatedUser`, and no separate HTTP audit log is added. + +## Metrics + +All in the existing hand-rolled `PrometheusMetrics` (same style, same label-cardinality +cap, no new dependencies): + +- `hms_proxy_rest_requests_total{prefix, route, status}` — counter, one increment per + HTTP request handled by the Iceberg REST listener. +- `hms_proxy_rest_request_duration_seconds{prefix, route}` — histogram, same buckets as + `hms_proxy_request_duration_seconds`. +- `hms_proxy_rest_listener_info{bind_host, port}` — info gauge set to 1 when the + listener starts (mirrors `hms_proxy_synthetic_read_lock_store_info`). + +Label rules: + +- `route` is never the raw URL. It is the adapter's parsed `Route` enum name in lower + case (`config`, `list_namespaces`, `load_table`, ...), or one of the pseudo-routes + `unknown_prefix`, `unknown_route`, `bad_request` for requests refused before dispatch. +- `prefix` is the catalog prefix, or `unknown` when the request failed before a + configured catalog was resolved. +- `status` is the numeric HTTP status actually written. + +Cardinality: prefixes are bounded by the catalog list plus `unknown`; routes by the +enum plus three pseudo-values; status by HTTP codes — all inside the existing series cap. + +## Wiring + +`ProxyObservability.metrics()` is already available in `HmsProxyApplication`. It is +passed through `RestCatalogServer.open(config, services, metrics)` into the +`IcebergHttpHandler` constructor. `doHandle` is wrapped with a timer; the record happens +in `finally` with the status that was actually written (the handler tracks the last +written status). The info gauge is set in `RestCatalogServer.open` after a successful +bind. Metrics are required (non-null): tests construct a real `PrometheusMetrics` and +may assert on its `render()` output — no null/no-op path exists. + +## Error handling + +No behavioral change to responses. Metric recording must never fail a request: recording +happens after the response bytes are written. + +## Testing + +- Unit: route normalization (enum name mapping, pseudo-routes) and label values. +- Integration (`IcebergRestEndpointIntegrationTest`): after hitting 200/400/404 paths, + `PrometheusMetrics.render()` contains the expected series. +- Smoke: `--scenario rest` gains a metrics step — after the REST checks, fetch the + management `/metrics` endpoint (`HMS_SMOKE_REST_METRICS_URL`, stand: + `http://localhost:19090/metrics`) and grep `hms_proxy_rest_requests_total` and + `hms_proxy_rest_listener_info`; skipped when the variable is unset. +- Docs: README/CHANGELOG both locales (metrics table extended), stand TEST-MATRIX row + G17 after the stand run. + +## Out of scope + +- Separate HTTP-level audit events. +- Readiness/health changes. +- Any change to REST response behavior. diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetrics.java b/src/main/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetrics.java index 3f1083e..8c9b0ad 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetrics.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetrics.java @@ -127,6 +127,19 @@ public final class PrometheusMetrics { "hms_proxy_synthetic_read_lock_store_info", "Configured synthetic read-lock store mode for this proxy instance", List.of("store_mode")); + private final Counter restRequestsTotal = new Counter( + "hms_proxy_rest_requests_total", + "Total Iceberg REST requests by catalog prefix, route, and terminal HTTP status", + List.of("prefix", "route", "status")); + private final Histogram restRequestDurationSeconds = new Histogram( + "hms_proxy_rest_request_duration_seconds", + "Iceberg REST request duration in seconds", + List.of("prefix", "route"), + REQUEST_DURATION_BUCKETS); + private final Gauge restListenerInfo = new Gauge( + "hms_proxy_rest_listener_info", + "Configured Iceberg REST listener bind host and port for this proxy instance", + List.of("bind_host", "port")); public void recordRequest(String method, String catalog, String backend, String status, double durationSeconds) { requestsTotal.inc(labels("method", method, "catalog", catalog, "backend", backend, "status", status)); @@ -276,6 +289,15 @@ public void setSyntheticReadLockStoreMode(String storeMode) { syntheticReadLockStoreInfo.set(labels("store_mode", storeMode), 1.0); } + public void recordRestRequest(String prefix, String route, int status, double durationSeconds) { + restRequestsTotal.inc(labels("prefix", prefix, "route", route, "status", String.valueOf(status))); + restRequestDurationSeconds.observe(labels("prefix", prefix, "route", route), durationSeconds); + } + + public void setRestListenerInfo(String bindHost, int port) { + restListenerInfo.set(labels("bind_host", bindHost, "port", String.valueOf(port)), 1.0); + } + // Declared last so every metric field above is already initialized; render order is the // exposition order of /metrics. private final List exposedMetrics = List.of( @@ -299,7 +321,10 @@ public void setSyntheticReadLockStoreMode(String storeMode) { syntheticReadLockStoreFailuresTotal, syntheticReadLockHandoffsTotal, syntheticReadLocksActive, - syntheticReadLockStoreInfo); + syntheticReadLockStoreInfo, + restRequestsTotal, + restRequestDurationSeconds, + restListenerInfo); public String render() { int estimatedSize = 0; diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetricsTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetricsTest.java index ff6e871..3b93ee0 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetricsTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/observability/PrometheusMetricsTest.java @@ -47,6 +47,23 @@ public void rendersConfiguredCountersAndHistogramSamples() { Assert.assertTrue(rendered.contains("hms_proxy_synthetic_read_lock_store_info{store_mode=\"zookeeper\"} 1.0")); } + @Test + public void rendersRestListenerSeriesAfterRecordingRestRequestsAndListenerInfo() { + PrometheusMetrics metrics = new PrometheusMetrics(); + + metrics.recordRestRequest("hdp", "load_table", 200, 0.05); + metrics.setRestListenerInfo("0.0.0.0", 9183); + + String rendered = metrics.render(); + + Assert.assertTrue(rendered.contains( + "hms_proxy_rest_requests_total{prefix=\"hdp\",route=\"load_table\",status=\"200\"} 1")); + Assert.assertTrue(rendered.contains( + "hms_proxy_rest_request_duration_seconds_count{prefix=\"hdp\",route=\"load_table\"} 1")); + Assert.assertTrue(rendered.contains( + "hms_proxy_rest_listener_info{bind_host=\"0.0.0.0\",port=\"9183\"} 1")); + } + @Test public void unknownExceptionTypesCollapseIntoOtherLabel() { PrometheusMetrics metrics = new PrometheusMetrics(); From eec20f1ae70b5cc733188a62ab74489518f70235 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 20:25:05 +0300 Subject: [PATCH 17/50] Record REST request metrics in the Iceberg HTTP handler --- .../hmsproxy/app/HmsProxyApplication.java | 2 +- .../restcatalog/IcebergHttpHandler.java | 82 ++++++++++++++----- .../restcatalog/RestCatalogServer.java | 13 +-- .../IcebergRestEndpointIntegrationTest.java | 21 ++++- .../restcatalog/RestCatalogServerTest.java | 11 +-- .../restcatalog/SpnegoIntegrationTest.java | 7 +- 6 files changed, 100 insertions(+), 36 deletions(-) diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java index 816c857..1ace0b9 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java @@ -59,7 +59,7 @@ public static void main(String[] args) throws Exception { AdditionalFrontendThriftServers.open(config, proxy, frontDoorSecurity); IcebergRestServices restServices = config.restCatalog().enabled() ? IcebergRestServices.open(config, proxy) : null; - RestCatalogServer restServer = RestCatalogServer.open(config, restServices)) { + RestCatalogServer restServer = RestCatalogServer.open(config, restServices, observability.metrics())) { MetastoreThriftServer server = new MetastoreThriftServer(config, proxy, frontDoorSecurity); installShutdownHook(server, teardownComplete, config.server().shutdownTimeoutSeconds()); LOG.info("Starting HMS proxy '{}' on {}:{}", config.server().name(), diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java index 6678fc9..8dc8890 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java @@ -3,6 +3,7 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpPrincipal; +import io.github.mmalykhin.hmsproxy.observability.PrometheusMetrics; import io.github.mmalykhin.hmsproxy.security.ClientRequestContext; import java.io.IOException; import java.io.InputStream; @@ -11,7 +12,9 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.LinkedHashMap; +import java.util.Locale; import java.util.Map; +import java.util.Objects; import org.apache.iceberg.exceptions.RESTException; import org.apache.iceberg.rest.RESTCatalogAdapter.HTTPMethod; import org.apache.iceberg.rest.RESTCatalogAdapter.Route; @@ -34,11 +37,30 @@ final class IcebergHttpHandler implements HttpHandler { private static final String V1_PREFIX = "/v1/"; private static final String CONFIG_SEGMENT = "config"; private static final String WAREHOUSE_PARAM = "warehouse"; + private static final String UNKNOWN_PREFIX_LABEL = "unknown"; + private static final String ROUTE_CONFIG = "config"; + private static final String ROUTE_UNKNOWN_PREFIX = "unknown_prefix"; + private static final String ROUTE_UNKNOWN_ROUTE = "unknown_route"; + private static final String ROUTE_BAD_REQUEST = "bad_request"; private final IcebergRestServices services; + private final PrometheusMetrics metrics; - IcebergHttpHandler(IcebergRestServices services) { + IcebergHttpHandler(IcebergRestServices services, PrometheusMetrics metrics) { this.services = services; + this.metrics = Objects.requireNonNull(metrics, "metrics"); + } + + /** + * Per-request mutable state threaded through {@link #doHandle} and the write helpers so the + * resolved prefix/route/status can be recorded once in {@link #handle}'s finally block. The + * handler instance itself is shared across the HTTP executor's threads, so this state must + * never live in handler fields. + */ + private static final class RequestOutcome { + private String prefix = UNKNOWN_PREFIX_LABEL; + private String route = ROUTE_UNKNOWN_ROUTE; + private int status; } @Override @@ -50,26 +72,32 @@ public void handle(HttpExchange exchange) throws IOException { String remoteUser = principal != null ? principal.getUsername() : null; String previousAddress = ClientRequestContext.setRemoteAddress(remoteAddress); String previousUser = ClientRequestContext.setRemoteUser(remoteUser); + RequestOutcome outcome = new RequestOutcome(); + long startNanos = System.nanoTime(); try { - doHandle(exchange); + doHandle(exchange, outcome); } finally { + double durationSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0; + metrics.recordRestRequest(outcome.prefix, outcome.route, outcome.status, durationSeconds); ClientRequestContext.restoreRemoteAddress(previousAddress); ClientRequestContext.restoreRemoteUser(previousUser); } } - private void doHandle(HttpExchange exchange) throws IOException { + private void doHandle(HttpExchange exchange, RequestOutcome outcome) throws IOException { try { String rawPath = exchange.getRequestURI().getPath(); if (!rawPath.startsWith(V1_PREFIX) && !rawPath.equals("/v1")) { - writeError(exchange, 404, "NotImplementedException", "Path not handled by Iceberg REST endpoint"); + outcome.route = ROUTE_UNKNOWN_ROUTE; + writeError(exchange, outcome, 404, "NotImplementedException", "Path not handled by Iceberg REST endpoint"); return; } HTTPMethod method; try { method = HTTPMethod.valueOf(exchange.getRequestMethod().toUpperCase()); } catch (IllegalArgumentException e) { - writeError(exchange, 405, "BadRequestException", + outcome.route = ROUTE_BAD_REQUEST; + writeError(exchange, outcome, 405, "BadRequestException", "Method not allowed: " + exchange.getRequestMethod()); return; } @@ -81,25 +109,29 @@ private void doHandle(HttpExchange exchange) throws IOException { String remainder = slash < 0 ? "" : trimmed.substring(slash + 1); if (CONFIG_SEGMENT.equals(firstSegment) && remainder.isEmpty()) { - handleConfig(exchange, queryParams); + handleConfig(exchange, queryParams, outcome); return; } IcebergRestService service = services.serviceFor(firstSegment); if (service == null) { - writeError(exchange, 404, "NoSuchCatalogException", + outcome.route = ROUTE_UNKNOWN_PREFIX; + writeError(exchange, outcome, 404, "NoSuchCatalogException", "Unknown catalog prefix in URL: " + rawPath); return; } + outcome.prefix = service.catalogName(); String relativePath = remainder.isEmpty() ? "v1" : "v1/" + remainder; Pair> routeAndVars = Route.from(method, relativePath); if (routeAndVars == null) { - writeError(exchange, 404, "NotImplementedException", + outcome.route = ROUTE_UNKNOWN_ROUTE; + writeError(exchange, outcome, 404, "NotImplementedException", "Route not supported: " + method + " " + relativePath); return; } Route route = routeAndVars.first(); + outcome.route = route.name().toLowerCase(Locale.ROOT); Object body = readBody(exchange, route); Class responseType = route.responseClass(); @@ -114,41 +146,46 @@ private void doHandle(HttpExchange exchange) throws IOException { // RESTCatalogAdapter always rethrows after invoking the error handler, // so an error response is already captured when we get here. if (capturedError[0] != null) { - writeErrorResponse(exchange, capturedError[0]); + writeErrorResponse(exchange, outcome, capturedError[0]); } else { - writeError(exchange, 500, e.getClass().getSimpleName(), e.getMessage()); + writeError(exchange, outcome, 500, e.getClass().getSimpleName(), e.getMessage()); } return; } if (capturedError[0] != null) { - writeErrorResponse(exchange, capturedError[0]); + writeErrorResponse(exchange, outcome, capturedError[0]); return; } if (responseType == null || response == null) { + outcome.status = 204; exchange.sendResponseHeaders(204, -1); exchange.getResponseBody().close(); return; } - writeJson(exchange, 200, IcebergRestMapper.mapper().writeValueAsString(response)); + writeJson(exchange, outcome, 200, IcebergRestMapper.mapper().writeValueAsString(response)); } catch (Exception e) { LOG.warn("Unhandled error serving {} {}", exchange.getRequestMethod(), exchange.getRequestURI(), e); - writeError(exchange, 500, e.getClass().getSimpleName(), + writeError(exchange, outcome, 500, e.getClass().getSimpleName(), e.getMessage() == null ? "internal error" : e.getMessage()); } } - private void handleConfig(HttpExchange exchange, Map queryParams) throws IOException { + private void handleConfig(HttpExchange exchange, Map queryParams, RequestOutcome outcome) + throws IOException { String warehouse = queryParams.get(WAREHOUSE_PARAM); IcebergRestService service = services.byWarehouse(warehouse); if (service == null) { - writeError(exchange, 400, "BadRequestException", "Unknown warehouse: " + warehouse); + outcome.route = ROUTE_BAD_REQUEST; + writeError(exchange, outcome, 400, "BadRequestException", "Unknown warehouse: " + warehouse); return; } + outcome.prefix = service.catalogName(); + outcome.route = ROUTE_CONFIG; ConfigResponse cfg = service.loadConfig(); - writeJson(exchange, 200, IcebergRestMapper.mapper().writeValueAsString(cfg)); + writeJson(exchange, outcome, 200, IcebergRestMapper.mapper().writeValueAsString(cfg)); } private T dispatchInternal( @@ -196,7 +233,9 @@ private static Map parseQueryString(String rawQuery) { return result; } - private static void writeJson(HttpExchange exchange, int status, String body) throws IOException { + private static void writeJson(HttpExchange exchange, RequestOutcome outcome, int status, String body) + throws IOException { + outcome.status = status; byte[] bytes = body.getBytes(StandardCharsets.UTF_8); exchange.getResponseHeaders().set("Content-Type", JSON_CONTENT_TYPE); exchange.sendResponseHeaders(status, bytes.length); @@ -205,18 +244,21 @@ private static void writeJson(HttpExchange exchange, int status, String body) th } } - private static void writeError(HttpExchange exchange, int status, String type, String message) throws IOException { + private static void writeError( + HttpExchange exchange, RequestOutcome outcome, int status, String type, String message) throws IOException { ErrorResponse error = ErrorResponse.builder() .responseCode(status) .withType(type) .withMessage(message == null ? "" : message) .withStackTrace(Arrays.asList()) .build(); - writeErrorResponse(exchange, error); + writeErrorResponse(exchange, outcome, error); } - private static void writeErrorResponse(HttpExchange exchange, ErrorResponse error) throws IOException { + private static void writeErrorResponse(HttpExchange exchange, RequestOutcome outcome, ErrorResponse error) + throws IOException { int status = error.code() > 0 ? error.code() : 500; + outcome.status = status; String body = IcebergRestMapper.mapper().writeValueAsString(error); byte[] bytes = body.getBytes(StandardCharsets.UTF_8); exchange.getResponseHeaders().set("Content-Type", JSON_CONTENT_TYPE); diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java index ca2a04e..55cddc8 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServer.java @@ -6,11 +6,13 @@ import com.sun.net.httpserver.HttpServer; import io.github.mmalykhin.hmsproxy.config.ProxyConfig; import io.github.mmalykhin.hmsproxy.config.restcatalog.RestCatalogConfig; +import io.github.mmalykhin.hmsproxy.observability.PrometheusMetrics; import java.io.IOException; import java.io.OutputStream; import java.net.BindException; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; @@ -34,11 +36,9 @@ private RestCatalogServer(HttpServer server, ThreadPoolExecutor executor) { this.executor = executor; } - public static RestCatalogServer open(ProxyConfig config) throws IOException { - return open(config, null); - } - - public static RestCatalogServer open(ProxyConfig config, IcebergRestServices services) throws IOException { + public static RestCatalogServer open( + ProxyConfig config, IcebergRestServices services, PrometheusMetrics metrics) throws IOException { + Objects.requireNonNull(metrics, "metrics"); RestCatalogConfig restConfig = config.restCatalog(); if (!restConfig.enabled()) { return null; @@ -69,7 +69,7 @@ public static RestCatalogServer open(ProxyConfig config, IcebergRestServices ser HttpContext v1Context; if (services != null) { - v1Context = server.createContext("/v1/", new IcebergHttpHandler(services)); + v1Context = server.createContext("/v1/", new IcebergHttpHandler(services, metrics)); } else { v1Context = server.createContext("/v1/config", new ConfigHandler()); } @@ -88,6 +88,7 @@ public static RestCatalogServer open(ProxyConfig config, IcebergRestServices ser new ThreadPoolExecutor.CallerRunsPolicy()); server.setExecutor(executor); server.start(); + metrics.setRestListenerInfo(restConfig.bindHost(), server.getAddress().getPort()); LOG.info("Iceberg REST catalog listener started on {}:{} (threads: {}..{})", restConfig.bindHost(), restConfig.port(), restConfig.minWorkerThreads(), restConfig.maxWorkerThreads()); diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java index 47af1de..b073634 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java @@ -7,6 +7,7 @@ import io.github.mmalykhin.hmsproxy.config.routing.BackendConfig; import io.github.mmalykhin.hmsproxy.config.server.ServerConfig; import io.github.mmalykhin.hmsproxy.config.syntheticlock.SyntheticReadLockStoreConfig; +import io.github.mmalykhin.hmsproxy.observability.PrometheusMetrics; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -27,6 +28,7 @@ public class IcebergRestEndpointIntegrationTest { private RecordingThriftIface delegate; private IcebergRestServices services; private RestCatalogServer server; + private PrometheusMetrics metrics; @Before public void setUp() throws Exception { @@ -45,7 +47,8 @@ public void setUp() throws Exception { ProxyConfig config = buildConfig(); services = IcebergRestServices.open(config, delegate.iface); - server = RestCatalogServer.open(config, services); + metrics = new PrometheusMetrics(); + server = RestCatalogServer.open(config, services, metrics); Assert.assertNotNull("server must start", server); } @@ -145,6 +148,22 @@ public void defaultPrefixKeepsFederatedView() throws Exception { Assert.assertTrue(response.body(), response.body().contains("[\"catalog2__default\"]")); } + @Test + public void recordsRestRequestMetricsAndListenerInfo() throws Exception { + Assert.assertEquals(200, get("/v1/" + CATALOG_NAME + "/namespaces").statusCode()); + Assert.assertEquals(404, get("/v1/nope/namespaces").statusCode()); + Assert.assertEquals(400, get("/v1/config?warehouse=nope").statusCode()); + + String rendered = metrics.render(); + Assert.assertTrue("rendered: " + rendered, + rendered.contains("prefix=\"" + CATALOG_NAME + "\",route=\"list_namespaces\",status=\"200\"")); + Assert.assertTrue("rendered: " + rendered, + rendered.contains("prefix=\"unknown\",route=\"unknown_prefix\",status=\"404\"")); + Assert.assertTrue("rendered: " + rendered, + rendered.contains("prefix=\"unknown\",route=\"bad_request\",status=\"400\"")); + Assert.assertTrue("rendered: " + rendered, rendered.contains("hms_proxy_rest_listener_info")); + } + private HttpResponse get(String path) throws Exception { HttpClient client = HttpClient.newBuilder().connectTimeout(HTTP_TIMEOUT).build(); HttpRequest request = HttpRequest.newBuilder() diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java index a7adcc4..0ed85c1 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RestCatalogServerTest.java @@ -7,6 +7,7 @@ import io.github.mmalykhin.hmsproxy.config.routing.BackendConfig; import io.github.mmalykhin.hmsproxy.config.server.ServerConfig; import io.github.mmalykhin.hmsproxy.config.syntheticlock.SyntheticReadLockStoreConfig; +import io.github.mmalykhin.hmsproxy.observability.PrometheusMetrics; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -23,14 +24,14 @@ public class RestCatalogServerTest { @Test public void returnsNullWhenDisabled() throws Exception { ProxyConfig config = buildConfig(new RestCatalogConfig(false, "127.0.0.1", 0, 1, 4, null, null)); - RestCatalogServer server = RestCatalogServer.open(config); + RestCatalogServer server = RestCatalogServer.open(config, null, new PrometheusMetrics()); Assert.assertNull(server); } @Test public void servesEmptyIcebergConfigOnGet() throws Exception { ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)); - try (RestCatalogServer server = RestCatalogServer.open(config)) { + try (RestCatalogServer server = RestCatalogServer.open(config, null, new PrometheusMetrics())) { Assert.assertNotNull(server); HttpResponse response = request(server, "/v1/config", "GET"); Assert.assertEquals(200, response.statusCode()); @@ -43,7 +44,7 @@ public void servesEmptyIcebergConfigOnGet() throws Exception { @Test public void rejectsNonReadMethodsOnConfig() throws Exception { ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)); - try (RestCatalogServer server = RestCatalogServer.open(config)) { + try (RestCatalogServer server = RestCatalogServer.open(config, null, new PrometheusMetrics())) { HttpResponse response = request(server, "/v1/config", "POST"); Assert.assertEquals(405, response.statusCode()); Assert.assertEquals("GET, HEAD", response.headers().firstValue("Allow").orElse("")); @@ -53,7 +54,7 @@ public void rejectsNonReadMethodsOnConfig() throws Exception { @Test public void respondsWithNotFoundForUnknownPath() throws Exception { ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)); - try (RestCatalogServer server = RestCatalogServer.open(config)) { + try (RestCatalogServer server = RestCatalogServer.open(config, null, new PrometheusMetrics())) { HttpResponse response = request(server, "/v1/namespaces", "GET"); Assert.assertEquals(404, response.statusCode()); Assert.assertTrue(response.body().contains("\"code\":404")); @@ -63,7 +64,7 @@ public void respondsWithNotFoundForUnknownPath() throws Exception { @Test public void boundPortReflectsActualListener() throws Exception { ProxyConfig config = buildConfig(new RestCatalogConfig(true, "127.0.0.1", 0, 1, 4, null, null)); - try (RestCatalogServer server = RestCatalogServer.open(config)) { + try (RestCatalogServer server = RestCatalogServer.open(config, null, new PrometheusMetrics())) { Assert.assertTrue("port must be allocated", server.boundPort() > 0); } } diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoIntegrationTest.java index e890699..d6d444e 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoIntegrationTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/SpnegoIntegrationTest.java @@ -7,6 +7,7 @@ import io.github.mmalykhin.hmsproxy.config.routing.BackendConfig; import io.github.mmalykhin.hmsproxy.config.server.ServerConfig; import io.github.mmalykhin.hmsproxy.config.syntheticlock.SyntheticReadLockStoreConfig; +import io.github.mmalykhin.hmsproxy.observability.PrometheusMetrics; import java.io.File; import java.net.URI; import java.net.http.HttpClient; @@ -92,7 +93,7 @@ public static void stopKdc() { @Test public void unauthenticatedRequestGetsNegotiateChallenge() throws Exception { - try (RestCatalogServer server = RestCatalogServer.open(buildProxyConfig(), null)) { + try (RestCatalogServer server = RestCatalogServer.open(buildProxyConfig(), null, new PrometheusMetrics())) { HttpResponse response = HttpClient.newBuilder().connectTimeout(HTTP_TIMEOUT).build() .send(HttpRequest.newBuilder() .uri(URI.create("http://127.0.0.1:" + server.boundPort() + "/v1/config")) @@ -108,7 +109,7 @@ public void unauthenticatedRequestGetsNegotiateChallenge() throws Exception { @Test public void invalidNegotiateTokenIsRejected() throws Exception { - try (RestCatalogServer server = RestCatalogServer.open(buildProxyConfig(), null)) { + try (RestCatalogServer server = RestCatalogServer.open(buildProxyConfig(), null, new PrometheusMetrics())) { HttpResponse response = HttpClient.newBuilder().connectTimeout(HTTP_TIMEOUT).build() .send(HttpRequest.newBuilder() .uri(URI.create("http://127.0.0.1:" + server.boundPort() + "/v1/config")) @@ -124,7 +125,7 @@ public void invalidNegotiateTokenIsRejected() throws Exception { @Test public void authenticatedRequestSucceeds() throws Exception { - try (RestCatalogServer server = RestCatalogServer.open(buildProxyConfig(), null)) { + try (RestCatalogServer server = RestCatalogServer.open(buildProxyConfig(), null, new PrometheusMetrics())) { UserGroupInformation clientUgi = UserGroupInformation .loginUserFromKeytabAndReturnUGI(clientPrincipal, clientKeytab.getAbsolutePath()); String token = clientUgi.doAs((PrivilegedExceptionAction) () -> { From becda09c9dafeee9ded0819dd7a3cc5a9491e743 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 20:36:44 +0300 Subject: [PATCH 18/50] Cover REST metrics in the stand smoke The REST smoke now optionally fetches the management /metrics endpoint (HMS_SMOKE_REST_METRICS_URL) and checks it carries the hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series, proving the Iceberg REST listener's own Prometheus metrics actually show up where an operator would look for them. Documented in both READMEs (new table rows plus a one-line note in the Iceberg REST section), the changelog, and the stand's test matrix (row G17 plus a revalidation-log note). --- CHANGELOG.md | 6 ++++++ CHANGELOG.ru.md | 6 ++++++ README.md | 10 ++++++++++ README.ru.md | 10 ++++++++++ .../hms-real-installation-smoke.simple.env.example | 3 +++ scripts/run-real-installation-smoke.sh | 12 ++++++++++++ smoke-stand/TEST-MATRIX.md | 5 +++++ smoke-stand/TEST-MATRIX.ru.md | 5 +++++ smoke-stand/env/simple.env | 4 ++++ 9 files changed, 61 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b1e565..c461282 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,12 @@ For a Russian version, see [CHANGELOG.ru.md](CHANGELOG.ru.md). federated view (its own databases plus every other catalog's databases under `` names) for compatibility; every other prefix is a clean, per-catalog view where those federated names never leak. +- The Iceberg REST frontend is now covered by Prometheus metrics: + `hms_proxy_rest_requests_total{prefix,route,status}`, + `hms_proxy_rest_request_duration_seconds{prefix,route}`, and + `hms_proxy_rest_listener_info{bind_host,port}`. `--scenario rest` in the + smoke runners checks the management `/metrics` endpoint carries the first + and third series when `HMS_SMOKE_REST_METRICS_URL` is set. ### Fixed diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 12e9136..9bd20bf 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -45,6 +45,12 @@ English version: [CHANGELOG.md](CHANGELOG.md). остальных каталогов под именами ``) для совместимости; любой другой prefix — чистое, per-catalog представление, в которое эти federated-имена не просачиваются. +- Iceberg REST frontend теперь покрыт Prometheus-метриками: + `hms_proxy_rest_requests_total{prefix,route,status}`, + `hms_proxy_rest_request_duration_seconds{prefix,route}` и + `hms_proxy_rest_listener_info{bind_host,port}`. `--scenario rest` в + smoke-раннерах проверяет, что management-endpoint `/metrics` несёт первую и + третью серии, если задан `HMS_SMOKE_REST_METRICS_URL`. ### Исправлено diff --git a/README.md b/README.md index 3c62bba..f9a772d 100644 --- a/README.md +++ b/README.md @@ -442,6 +442,9 @@ Current Prometheus metrics: - `hms_proxy_backend_session_acquire_timeouts_total{catalog,operation}` - `hms_proxy_adaptive_timeout_reconnect_total{catalog}` - `hms_proxy_adaptive_timeout_reconnect_skipped_total{catalog,reason}` +- `hms_proxy_rest_requests_total{prefix,route,status}` +- `hms_proxy_rest_request_duration_seconds{prefix,route}` +- `hms_proxy_rest_listener_info{bind_host,port}` Example Prometheus scrape config: @@ -472,6 +475,9 @@ Metric semantics: - `hms_proxy_backend_session_acquire_timeouts_total` counts fail-fast events when the shared backend metastore session pool runs out of permits within the catalog's `latencyBudgetMs` (or 30s default); `operation=borrow` covers regular RPC dispatch, `operation=reconnect` covers admin reconnect attempts that could not quiesce the pool - `hms_proxy_adaptive_timeout_reconnect_total` counts how often the adaptive socket timeout reconnected the shared backend client (and forced impersonation-cache eviction); use it to spot reconnect storms under volatile latency - `hms_proxy_adaptive_timeout_reconnect_skipped_total` counts adaptive-timeout adjustments suppressed by the throttles (`reason=hysteresis` for sub-threshold deltas, `reason=cooldown` for events too close to a previous reconnect) +- `hms_proxy_rest_requests_total` counts Iceberg REST HTTP requests by catalog prefix, route, and terminal HTTP status +- `hms_proxy_rest_request_duration_seconds` measures Iceberg REST request duration grouped by catalog prefix and route +- `hms_proxy_rest_listener_info` is a constant-info gauge that exposes the configured bind host and port of the Iceberg REST listener Despite the historical `synthetic_read_lock` metric names, the shim now also serves eligible non-transactional `NO_TXN` DDL locks and non-transactional write locks on non-default catalogs. @@ -993,6 +999,10 @@ rest-catalog.kerberos.principal=HTTP/_HOST@EXAMPLE.COM rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab ``` +Requests to this listener are covered by the Prometheus metrics described in +[Prometheus metrics](#prometheus-metrics): `hms_proxy_rest_requests_total`, +`hms_proxy_rest_request_duration_seconds`, and `hms_proxy_rest_listener_info`. + ### Supported endpoints | Endpoint | Status | diff --git a/README.ru.md b/README.ru.md index 2d7e459..cc9146e 100644 --- a/README.ru.md +++ b/README.ru.md @@ -444,6 +444,9 @@ state, а `probeAgeMs` показывает, насколько устарели - `hms_proxy_backend_session_acquire_timeouts_total{catalog,operation}` - `hms_proxy_adaptive_timeout_reconnect_total{catalog}` - `hms_proxy_adaptive_timeout_reconnect_skipped_total{catalog,reason}` +- `hms_proxy_rest_requests_total{prefix,route,status}` +- `hms_proxy_rest_request_duration_seconds{prefix,route}` +- `hms_proxy_rest_listener_info{bind_host,port}` Пример Prometheus scrape config: @@ -474,6 +477,9 @@ scrape_configs: - `hms_proxy_backend_session_acquire_timeouts_total` считает fail-fast события, когда пул shared backend metastore session исчерпан и permit не освобождается за `latencyBudgetMs` каталога (или 30s по умолчанию); `operation=borrow` для обычной диспетчеризации RPC, `operation=reconnect` для админских реконнектов, которым не удалось quiesce пул - `hms_proxy_adaptive_timeout_reconnect_total` считает, сколько раз adaptive socket timeout приводил к reconnect shared backend client (с принудительным сбросом impersonation-кэша); полезен для отслеживания reconnect storm при нестабильной latency - `hms_proxy_adaptive_timeout_reconnect_skipped_total` считает adaptive-timeout правки, подавленные троттлингом (`reason=hysteresis` для дельт ниже порога, `reason=cooldown` для срабатываний раньше cooldown окна после предыдущего reconnect) +- `hms_proxy_rest_requests_total` считает HTTP-запросы Iceberg REST с группировкой по catalog prefix, route и terminal HTTP-статусу +- `hms_proxy_rest_request_duration_seconds` измеряет длительность запросов Iceberg REST с группировкой по catalog prefix и route +- `hms_proxy_rest_listener_info` это constant-info gauge, который показывает настроенные bind host и port Iceberg REST listener'а Несмотря на исторические имена метрик `synthetic_read_lock`, этот shim теперь также обслуживает допустимые non-transactional `NO_TXN` DDL lock и non-transactional write lock на non-default @@ -991,6 +997,10 @@ rest-catalog.kerberos.principal=HTTP/_HOST@EXAMPLE.COM rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab ``` +Запросы к этому listener'у покрыты Prometheus-метриками из раздела +[Prometheus-метрики](#prometheus-метрики): `hms_proxy_rest_requests_total`, +`hms_proxy_rest_request_duration_seconds` и `hms_proxy_rest_listener_info`. + ### Поддерживаемые endpoint'ы | Endpoint | Статус | diff --git a/scripts/hms-real-installation-smoke.simple.env.example b/scripts/hms-real-installation-smoke.simple.env.example index 110734a..d52560a 100644 --- a/scripts/hms-real-installation-smoke.simple.env.example +++ b/scripts/hms-real-installation-smoke.simple.env.example @@ -115,3 +115,6 @@ HMS_SMOKE_PARTITION_LOCK_TRANSACTIONAL=false # HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE=smoke_read_tbl2 # Catalog-db separator of the proxy, used to build federated names. Default: __ # HMS_SMOKE_REST_SEPARATOR=__ +# Management /metrics endpoint. When set, the REST smoke also checks it carries the +# hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series. +# HMS_SMOKE_REST_METRICS_URL=http://proxy-host:9090/metrics diff --git a/scripts/run-real-installation-smoke.sh b/scripts/run-real-installation-smoke.sh index d783923..f7399f2 100755 --- a/scripts/run-real-installation-smoke.sh +++ b/scripts/run-real-installation-smoke.sh @@ -83,6 +83,9 @@ Optional Iceberg REST env vars: HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE plain Hive table of the second catalog that must NOT appear in its REST listing HMS_SMOKE_REST_SEPARATOR catalog-db separator of the proxy; default: __ + HMS_SMOKE_REST_METRICS_URL management /metrics endpoint; when set, the REST smoke + checks it carries hms_proxy_rest_requests_total and + hms_proxy_rest_listener_info series EOF cat <<'EOF' @@ -1045,6 +1048,15 @@ run_rest_smoke() { fi fi + local metrics_url="${HMS_SMOKE_REST_METRICS_URL:-}" + if [[ -n "${metrics_url}" ]]; then + curl -sS -o "${body}" "${metrics_url}" || fail "cannot fetch metrics from ${metrics_url}" + grep -q 'hms_proxy_rest_requests_total{' "${body}" \ + || fail "metrics endpoint carries no hms_proxy_rest_requests_total series" + grep -q 'hms_proxy_rest_listener_info{' "${body}" \ + || fail "metrics endpoint carries no hms_proxy_rest_listener_info series" + fi + log "Iceberg REST smoke passed (prefix '${prefix}', namespace '${namespace}')" } diff --git a/smoke-stand/TEST-MATRIX.md b/smoke-stand/TEST-MATRIX.md index 57a478e..d426da5 100644 --- a/smoke-stand/TEST-MATRIX.md +++ b/smoke-stand/TEST-MATRIX.md @@ -103,6 +103,7 @@ table is the hand-registered `smoke_iceberg_tbl` (see the stand README). | G14 | A default-catalog table under the `apache` prefix → clean 404 | ✅ | n/a | | G15 | The external name `apache__default` used as a namespace under the `apache` prefix → clean 404 | ✅ | n/a | | G16 | The second catalog's plain Hive table (`smoke_read_ap`) stays invisible in the `apache` listing | ✅ | n/a | +| G17 | REST metrics (`requests_total`, `listener_info`) visible on the management `/metrics` endpoint | ✅ | n/a | ## F. Not covered, and why @@ -138,6 +139,10 @@ executed is claimed; a row not listed was not repeated and its ✅ stands on the run too, in the same `--scenario rest` and `--scenario all` passes. A follow-up run the same day added and passed the federation/isolation rows G12-G16: the federated name under the default prefix (listing and load included) and clean 404s for every cross-catalog shape. + Later still, jar `1.0.20-eec20f1a` added row G17: with `HMS_SMOKE_REST_METRICS_URL` set to the + stand's management endpoint, both `--scenario rest` and `--scenario all` fetched it with curl + and confirmed the `hms_proxy_rest_requests_total` and `hms_proxy_rest_listener_info` series were + present and populated after the REST checks ran. ## Two caveats on faithfulness diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index 88a9177..ea57012 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -103,6 +103,7 @@ HDP-клиент не может пользоваться Apache-listener — Th | G14 | Таблица default-каталога под prefix `apache` → чистый 404 | ✅ | n/a | | G15 | Внешнее имя `apache__default`, использованное как namespace под prefix `apache` → чистый 404 | ✅ | n/a | | G16 | Обычная Hive-таблица второго каталога (`smoke_read_ap`) не видна в листинге под prefix `apache` | ✅ | n/a | +| G17 | REST-метрики (`requests_total`, `listener_info`) видны на management-endpoint `/metrics` | ✅ | n/a | ## F. Что не покрыто и почему @@ -140,6 +141,10 @@ HDP-клиент не может пользоваться Apache-listener — Th Дополнительный прогон в тот же день добавил и прошёл строки G12-G16 про федерацию и изоляцию: federated-имя под default-prefix (включая листинг и load) и чистые 404 на каждую кросс-каталожную форму. + Ещё позже jar `1.0.20-eec20f1a` добавил строку G17: с `HMS_SMOKE_REST_METRICS_URL`, указывающим + на management-endpoint стенда, оба прогона — `--scenario rest` и `--scenario all` — забрали его + curl'ом и подтвердили, что серии `hms_proxy_rest_requests_total` и `hms_proxy_rest_listener_info` + присутствуют и заполнены после того, как отработали REST-проверки. ## Две оговорки честности diff --git a/smoke-stand/env/simple.env b/smoke-stand/env/simple.env index d5f41b7..24d0ab6 100644 --- a/smoke-stand/env/simple.env +++ b/smoke-stand/env/simple.env @@ -96,3 +96,7 @@ HMS_SMOKE_REST_NON_ICEBERG_TABLE=smoke_read_hdp HMS_SMOKE_REST_SECOND_PREFIX=apache HMS_SMOKE_REST_SECOND_ICEBERG_TABLE=smoke_iceberg_tbl_ap HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE=smoke_read_ap + +# Management metrics endpoint, reachable from the host. When set, the REST smoke also checks it +# carries the hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series. +HMS_SMOKE_REST_METRICS_URL=http://localhost:19090/metrics From eca202f3421332e6eb387d4ff3ca929a75da4517 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 20:47:18 +0300 Subject: [PATCH 19/50] Add an Iceberg REST row to the Grafana dashboard Rate/error-ratio/latency stats, a listener-up stat driven by hms_proxy_rest_listener_info, latency quantiles, and breakdowns by HTTP status, catalog prefix and route (pseudo-routes mark refusals). Panels are cloned from the dashboard's existing reference panels; series names were checked against a live /metrics scrape, including the histogram _bucket series the quantile panels rely on. --- CHANGELOG.md | 4 +- CHANGELOG.ru.md | 4 +- README.md | 4 +- README.ru.md | 4 +- monitoring/grafana/hms-proxy-dashboard.json | 2905 ++++++++++++++++--- 5 files changed, 2503 insertions(+), 418 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c461282..aee9153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,9 @@ For a Russian version, see [CHANGELOG.ru.md](CHANGELOG.ru.md). `hms_proxy_rest_request_duration_seconds{prefix,route}`, and `hms_proxy_rest_listener_info{bind_host,port}`. `--scenario rest` in the smoke runners checks the management `/metrics` endpoint carries the first - and third series when `HMS_SMOKE_REST_METRICS_URL` is set. + and third series when `HMS_SMOKE_REST_METRICS_URL` is set. The bundled + Grafana dashboard gains an Iceberg REST row: rate/error-ratio/latency + stats, quantiles, and breakdowns by HTTP status, catalog prefix and route. ### Fixed diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 9bd20bf..5355efb 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -50,7 +50,9 @@ English version: [CHANGELOG.md](CHANGELOG.md). `hms_proxy_rest_request_duration_seconds{prefix,route}` и `hms_proxy_rest_listener_info{bind_host,port}`. `--scenario rest` в smoke-раннерах проверяет, что management-endpoint `/metrics` несёт первую и - третью серии, если задан `HMS_SMOKE_REST_METRICS_URL`. + третью серии, если задан `HMS_SMOKE_REST_METRICS_URL`. В комплектный + Grafana dashboard добавлен ряд Iceberg REST: stat'ы rate/error ratio/latency, + квантили и разбивки по HTTP-статусу, catalog prefix и route. ### Исправлено diff --git a/README.md b/README.md index f9a772d..99c7cb5 100644 --- a/README.md +++ b/README.md @@ -515,7 +515,9 @@ computed for output nobody reads. A ready-to-import Grafana dashboard is included in `monitoring/grafana/hms-proxy-dashboard.json`. It covers request rate, latency, backend failures, -fallbacks, default-catalog routing, and ambiguous routing events. +fallbacks, default-catalog routing, and ambiguous routing events, plus an Iceberg REST row: +request rate, error ratio and latency quantiles of the REST listener, breakdowns by HTTP status, +catalog prefix and route, and a listener-up stat. ### Selective federation exposure diff --git a/README.ru.md b/README.ru.md index cc9146e..68d3e64 100644 --- a/README.ru.md +++ b/README.ru.md @@ -518,7 +518,9 @@ Proxy также пишет один structured audit log на каждый за Готовый Grafana dashboard лежит в `monitoring/grafana/hms-proxy-dashboard.json`. В нём уже есть панели по request rate, latency, -backend failures, fallbacks, default-catalog routing и ambiguous routing. +backend failures, fallbacks, default-catalog routing и ambiguous routing, а также ряд Iceberg +REST: request rate, error ratio и квантили латентности REST-listener'а, разбивки по HTTP-статусу, +catalog prefix и route, и stat «listener up». ### Selective federation exposure diff --git a/monitoring/grafana/hms-proxy-dashboard.json b/monitoring/grafana/hms-proxy-dashboard.json index 7ccc228..e56ab0e 100644 --- a/monitoring/grafana/hms-proxy-dashboard.json +++ b/monitoring/grafana/hms-proxy-dashboard.json @@ -24,39 +24,69 @@ "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1000, "panels": [], "title": "Requests & Latency (hms_proxy_requests_total, hms_proxy_request_duration_seconds)", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "sum(rate(hms_proxy_requests_total)) — total request rate served by the proxy.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 100 }, - { "color": "red", "value": 1000 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, "id": 1, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -73,32 +103,57 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Share of requests with status=error.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.01 }, - { "color": "red", "value": 0.05 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } ] }, "unit": "percentunit" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, "id": 2, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -115,32 +170,57 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Share of requests with status=fallback.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.01 }, - { "color": "red", "value": 0.05 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } ] }, "unit": "percentunit" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, "id": 3, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -157,32 +237,57 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "P95 of hms_proxy_request_duration_seconds across all matched series.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.25 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.25 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, "id": 4, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -199,11 +304,16 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Request rate broken down by terminal status (ok/fallback/degraded/error/throttled).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -214,27 +324,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, "id": 5, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -250,11 +387,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "P50 / P95 / P99 latency derived from hms_proxy_request_duration_seconds_bucket.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -265,27 +407,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, "id": 6, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -315,11 +484,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Top 10 RPC methods by request rate.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -330,27 +504,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, "id": 7, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -366,11 +567,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Request rate broken down by routed backend instance.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -381,27 +587,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, "id": 8, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -416,42 +649,71 @@ "title": "Requests by Backend / Catalog", "type": "timeseries" }, - { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, "id": 1001, "panels": [], "title": "Backend Operations (hms_proxy_backend_failures_total, hms_proxy_backend_fallback_total, hms_proxy_backend_session_acquire_timeouts_total)", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "sum(rate(hms_proxy_backend_failures_total)) — backend invocation exceptions per second.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.1 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 8, "x": 0, "y": 22 }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 22 + }, "id": 9, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -468,32 +730,57 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "sum(rate(hms_proxy_backend_fallback_total)) — compatibility fallbacks served per second.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.1 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 8, "x": 8, "y": 22 }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 22 + }, "id": 10, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -510,32 +797,57 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Number of distinct exception types currently producing backend failures.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 1 }, - { "color": "red", "value": 3 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 3 + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 8, "x": 16, "y": 22 }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 22 + }, "id": 11, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -552,11 +864,16 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Backend failure rate split by backend.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -567,27 +884,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, "id": 12, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -603,11 +947,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Backend failure rate split by exception class.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -618,27 +967,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, "id": 13, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -654,11 +1030,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Compatibility fallback rate by RPC method.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -669,27 +1050,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 34 + }, "id": 14, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -705,11 +1113,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Compatibility fallback rate split by API conversion (from_api → to_api).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -720,27 +1133,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, "id": 15, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -756,32 +1196,57 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "sum(rate(hms_proxy_backend_session_acquire_timeouts_total)) — backend metastore session acquisitions that timed out waiting for a free pool permit (fail-fast). Non-zero rate indicates pool exhaustion or a stuck backend.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.01 }, - { "color": "red", "value": 0.1 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 8, "x": 0, "y": 42 }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 42 + }, "id": 16, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -798,11 +1263,16 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Backend session acquire timeouts split by catalog and operation (borrow vs reconnect). Persistent rate per catalog usually means the shared session pool is undersized or backend calls are slower than the configured latency budget.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -813,27 +1283,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 16, "x": 8, "y": 42 }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 42 + }, "id": 17, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -848,42 +1345,71 @@ "title": "Backend Session Acquire Timeouts by Catalog/Operation", "type": "timeseries" }, - { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, "id": 1006, "panels": [], "title": "Adaptive Timeout (hms_proxy_adaptive_timeout_reconnect_total, hms_proxy_adaptive_timeout_reconnect_skipped_total)", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Rate of backend client reconnects triggered by adaptive socket timeout adjustments. A persistent rate suggests latency volatility — consider raising routing.adaptive-timeout.reconnect-cooldown-ms or alpha.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.05 }, - { "color": "red", "value": 0.2 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.05 + }, + { + "color": "red", + "value": 0.2 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 8, "x": 0, "y": 51 }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 51 + }, "id": 42, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -900,11 +1426,16 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Adaptive timeout reconnects per catalog. Each event tears down the shared backend session and evicts impersonation clients (Kerberos re-login).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -915,27 +1446,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 16, "x": 8, "y": 51 }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 51 + }, "id": 43, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -951,11 +1509,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Reconnect attempts suppressed by hysteresis (delta below threshold) or cooldown (too soon after a previous reconnect). Healthy throttling keeps reconnects bounded under volatile latency.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -966,27 +1529,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 59 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 59 + }, "id": 44, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -1001,42 +1591,71 @@ "title": "Adaptive Timeout Reconnects Suppressed (hysteresis / cooldown)", "type": "timeseries" }, - { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 67 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, "id": 1002, "panels": [], "title": "Routing (hms_proxy_routing_ambiguous_total, hms_proxy_default_catalog_routed_total)", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Requests routed to the default catalog because no explicit catalog namespace was provided.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 1 }, - { "color": "red", "value": 100 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 100 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 12, "x": 0, "y": 51 }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 51 + }, "id": 16, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -1053,32 +1672,57 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Requests safely failed because deterministic routing detected conflicting namespaces. Should stay at zero.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.001 }, - { "color": "red", "value": 0.01 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + }, + { + "color": "red", + "value": 0.01 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 12, "x": 12, "y": 51 }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 51 + }, "id": 17, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -1095,11 +1739,16 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Default-catalog routing rate per RPC method.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1110,27 +1759,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 55 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 55 + }, "id": 18, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -1145,42 +1821,71 @@ "title": "Default-Catalog Routes by Method", "type": "timeseries" }, - { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 63 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, "id": 1003, "panels": [], "title": "Rate Limiting (hms_proxy_rate_limited_total)", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Requests rejected by overload protection per second.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.1 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 8, "x": 0, "y": 64 }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 64 + }, "id": 19, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -1197,32 +1902,57 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Distinct limiting dimensions currently throttling traffic (principal, source, source_cidr, method_family, catalog, rpc_class).", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 1 }, - { "color": "red", "value": 3 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 3 + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 8, "x": 8, "y": 64 }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 64 + }, "id": 20, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -1239,32 +1969,57 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Distinct catalogs currently experiencing rejected requests.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 1 }, - { "color": "red", "value": 3 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 3 + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 8, "x": 16, "y": 64 }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 64 + }, "id": 21, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -1281,11 +2036,16 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Throttled rate split by limiting dimension.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1296,27 +2056,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 68 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 68 + }, "id": 22, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -1332,11 +2119,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Throttled rate split by RPC method family (read/write/lock/etc.).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1347,27 +2139,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 68 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 68 + }, "id": 23, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -1383,11 +2202,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Throttled rate split by limiter scope.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1398,27 +2222,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 76 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 76 + }, "id": 24, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -1434,11 +2285,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Throttled rate split by catalog.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1449,27 +2305,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 76 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 76 + }, "id": 25, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -1485,34 +2368,62 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Top throttled label combinations across all dimensions.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "custom": { "align": "auto", - "cellOptions": { "type": "auto" }, + "cellOptions": { + "type": "auto" + }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.1 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 84 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 84 + }, "id": 26, "options": { "cellHeight": "sm", - "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, "showHeader": true }, "pluginVersion": "11.1.0", @@ -1533,48 +2444,76 @@ { "id": "organize", "options": { - "excludeByName": { "Time": true, "__name__": true }, + "excludeByName": { + "Time": true, + "__name__": true + }, "indexByName": {}, - "renameByName": { "Value": "rps" } + "renameByName": { + "Value": "rps" + } } } ], "type": "table" }, - { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 92 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 92 + }, "id": 1004, "panels": [], "title": "Metadata Filtering (hms_proxy_filtered_objects_total)", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Metadata objects hidden by selective federation filters per second.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 12, "x": 0, "y": 93 }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 93 + }, "id": 27, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -1591,30 +2530,49 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Distinct object types currently being filtered (databases, tables, etc.).", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 12, "x": 12, "y": 93 }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 93 + }, "id": 28, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -1631,11 +2589,16 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Filtered objects rate split by object_type.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1646,27 +2609,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 97 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 97 + }, "id": 29, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -1682,11 +2672,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Filtered objects rate split by catalog.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1697,27 +2692,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 97 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 97 + }, "id": 30, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -1733,11 +2755,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Filtered objects rate split by RPC method (top 10).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1748,27 +2775,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 105 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 105 + }, "id": 31, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -1784,32 +2838,54 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Top filtered (method, catalog, object_type) tuples.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "custom": { "align": "auto", - "cellOptions": { "type": "auto" }, + "cellOptions": { + "type": "auto" + }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 113 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 113 + }, "id": 32, "options": { "cellHeight": "sm", - "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, "showHeader": true }, "pluginVersion": "11.1.0", @@ -1830,50 +2906,84 @@ { "id": "organize", "options": { - "excludeByName": { "Time": true, "__name__": true }, + "excludeByName": { + "Time": true, + "__name__": true + }, "indexByName": {}, - "renameByName": { "Value": "ops/s" } + "renameByName": { + "Value": "ops/s" + } } } ], "type": "table" }, - { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 121 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 121 + }, "id": 1005, "panels": [], "title": "Synthetic Read Locks (hms_proxy_synthetic_read_lock_* + hms_proxy_synthetic_read_locks_active)", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "hms_proxy_synthetic_read_locks_active — current number of active synthetic read locks.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 100 }, - { "color": "red", "value": 1000 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 122 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 122 + }, "id": 33, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -1890,30 +3000,49 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "hms_proxy_synthetic_read_lock_store_info — configured lock store mode (in_memory or zookeeper). Series name shows the active mode.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, "unit": "none" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 122 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 122 + }, "id": 34, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "value_and_name" }, "pluginVersion": "11.1.0", @@ -1930,30 +3059,49 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Synthetic read-lock acquire rate (operation=acquire, result=acquired).", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 122 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 122 + }, "id": 35, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -1970,32 +3118,57 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Synthetic read-lock store failures per second.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.001 }, - { "color": "red", "value": 0.05 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + }, + { + "color": "red", + "value": 0.05 + } ] }, "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 122 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 122 + }, "id": 36, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -2012,11 +3185,16 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Synthetic read-lock event rate split by operation.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -2027,27 +3205,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 126 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 126 + }, "id": 37, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -2063,11 +3268,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Synthetic read-lock event rate split by result.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -2078,27 +3288,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 126 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 126 + }, "id": 38, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -2114,11 +3351,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Synthetic read-lock handoff rate (lock served by a different proxy instance than the original owner) split by operation.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -2129,27 +3371,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 134 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 134 + }, "id": 39, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -2165,11 +3434,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Synthetic read-lock store failure rate split by exception type.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -2180,27 +3454,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 134 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 134 + }, "id": 40, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -2216,32 +3517,54 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "Top (operation, result, store_mode, catalog) lock-event combinations.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "custom": { "align": "auto", - "cellOptions": { "type": "auto" }, + "cellOptions": { + "type": "auto" + }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 142 }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 142 + }, "id": 41, "options": { "cellHeight": "sm", - "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, "showHeader": true }, "pluginVersion": "11.1.0", @@ -2262,9 +3585,14 @@ { "id": "organize", "options": { - "excludeByName": { "Time": true, "__name__": true }, + "excludeByName": { + "Time": true, + "__name__": true + }, "indexByName": {}, - "renameByName": { "Value": "ops/s" } + "renameByName": { + "Value": "ops/s" + } } } ], @@ -2272,18 +3600,28 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 150 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 150 + }, "id": 1010, "panels": [], "title": "Impersonation Pool (hms_proxy_impersonation_pool_users, hms_proxy_impersonation_pool_sessions, hms_proxy_impersonation_session_acquire_timeouts_total, hms_proxy_impersonation_session_evictions_total)", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "sum by (catalog) (hms_proxy_impersonation_pool_users) — distinct users currently holding a per-user impersonation session pool. Compare against catalog..impersonation-max-clients.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -2294,27 +3632,54 @@ "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 151 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 151 + }, "id": 60, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -2330,11 +3695,16 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "sum by (catalog, state) (hms_proxy_impersonation_pool_sessions) — backend Thrift sessions held across all per-user impersonation pools, split into active (currently borrowed) and idle (returned, kept warm). Active staying close to (users × catalog..impersonation-pool-max-size) means pools are saturated and callers may queue.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -2345,27 +3715,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 151 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 151 + }, "id": 61, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -2381,32 +3778,57 @@ "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "sum(rate(hms_proxy_impersonation_session_acquire_timeouts_total)) — borrow attempts on a per-user impersonation pool that timed out. Persistent rate means the per-user pool is undersized for the user's concurrency or backend calls exceed the latency budget.", "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.01 }, - { "color": "red", "value": 0.1 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 8, "x": 0, "y": 159 }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 159 + }, "id": 62, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, "pluginVersion": "11.1.0", @@ -2423,11 +3845,16 @@ "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "description": "sum by (catalog, reason) (rate(hms_proxy_impersonation_session_evictions_total)) — per-user impersonation backend sessions discarded grouped by reason: idle (TTL), transport_failure (retry-once after Thrift error), user_evicted (whole user pool dropped on adaptive-timeout reconnect or LRU capacity), user_capacity (per-catalog impersonation-max-clients eviction).", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -2438,27 +3865,54 @@ "drawStyle": "line", "fillOpacity": 12, "gradientMode": "opacity", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 2, "pointSize": 4, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "never", "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 16, "x": 8, "y": 159 }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 159 + }, "id": 63, "options": { - "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "pluginVersion": "11.1.0", "targets": [ @@ -2472,6 +3926,629 @@ ], "title": "Impersonation Session Evictions by Reason", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 167 + }, + "id": 1011, + "panels": [], + "title": "Iceberg REST (hms_proxy_rest_requests_total, hms_proxy_rest_request_duration_seconds, hms_proxy_rest_listener_info)", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "sum(rate(hms_proxy_rest_requests_total)) — HTTP request rate on the Iceberg REST listener.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 168 + }, + "id": 200, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(hms_proxy_rest_requests_total[$__rate_interval]))", + "legendFormat": "rps", + "range": true, + "refId": "A" + } + ], + "title": "REST Request Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Share of REST requests answered with a 4xx/5xx status.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 168 + }, + "id": 201, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum(rate(hms_proxy_rest_requests_total{status=~\"4..|5..\"}[$__rate_interval])) / sum(rate(hms_proxy_rest_requests_total[$__rate_interval]))", + "legendFormat": "errors", + "range": true, + "refId": "A" + } + ], + "title": "REST Error Ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "p95 of hms_proxy_rest_request_duration_seconds.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.25 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 168 + }, + "id": 202, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(hms_proxy_rest_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "range": true, + "refId": "A" + } + ], + "title": "REST P95 Latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "hms_proxy_rest_listener_info — 1 while the Iceberg REST listener is bound.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 168 + }, + "id": 203, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum(hms_proxy_rest_listener_info)", + "legendFormat": "up", + "range": true, + "refId": "A" + } + ], + "title": "REST Listener Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "REST request rate broken down by terminal HTTP status.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 172 + }, + "id": 204, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (status) (rate(hms_proxy_rest_requests_total[$__rate_interval]))", + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + } + ], + "title": "REST Requests by Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Latency quantiles of the Iceberg REST listener.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 172 + }, + "id": 205, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(hms_proxy_rest_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "range": true, + "refId": "A" + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(hms_proxy_rest_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "range": true, + "refId": "B" + }, + { + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(hms_proxy_rest_request_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "range": true, + "refId": "C" + } + ], + "title": "REST Latency p50 / p95 / p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "REST request rate broken down by catalog prefix (unknown = refused before catalog resolution).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 180 + }, + "id": 206, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "sum by (prefix) (rate(hms_proxy_rest_requests_total[$__rate_interval]))", + "legendFormat": "{{prefix}}", + "range": true, + "refId": "A" + } + ], + "title": "REST Requests by Prefix", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Most active REST routes; pseudo-routes unknown_prefix/unknown_route/bad_request mark refusals.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 12, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 180 + }, + "id": 207, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "editorMode": "code", + "expr": "topk(10, sum by (route) (rate(hms_proxy_rest_requests_total[$__rate_interval])))", + "legendFormat": "{{route}}", + "range": true, + "refId": "A" + } + ], + "title": "REST Requests by Route (top 10)", + "type": "timeseries" } ], "refresh": "30s", @@ -2625,4 +4702,4 @@ "uid": "hms-proxy-all-metrics", "version": 1, "weekStart": "" -} +} \ No newline at end of file From 613b7a1e720d4b483a08ae7602b89d88ef4dc231 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Mon, 27 Jul 2026 21:27:03 +0300 Subject: [PATCH 20/50] Move the Iceberg REST front door to Iceberg 1.9.2 Bump iceberg.version to 1.9.2, pin Jackson core/databind to 2.18.3 to match what Iceberg 1.9.2 is compiled against (Hive 3.1.3 otherwise drags in databind 2.12), and exclude the newly-transitive slf4j-api from the three Iceberg dependencies so the tree keeps a single provider. Re-vendor RESTCatalogAdapter from the matching apache-iceberg-1.9.2 tag. Route stays nested and gets the same public-promotion patch as before; HTTPMethod moved out into HTTPRequest.HTTPMethod, and the adapter's public entry point is now handleRequest(...) instead of the removed execute(...) overload. Migrate IcebergRestService.dispatch and IcebergHttpHandler's route dispatch to the new signature: the handler now merges query params and route path vars itself, calls dispatch(route, vars, body, type) directly, and maps thrown RuntimeExceptions to error responses via RESTCatalogAdapter.configureResponseFromException instead of the old error-handler callback. Request outcome/metrics bookkeeping is unchanged. Include the phase 4a design spec and plan alongside the code change. --- ...2026-07-27-iceberg-rest-phase4a-upgrade.md | 282 +++++++++++++++++ ...-27-iceberg-rest-phase4a-upgrade-design.md | 161 ++++++++++ pom.xml | 29 +- .../restcatalog/IcebergHttpHandler.java | 39 +-- .../restcatalog/IcebergRestService.java | 14 +- .../iceberg/rest/RESTCatalogAdapter.java | 290 ++++++++++-------- 6 files changed, 650 insertions(+), 165 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-27-iceberg-rest-phase4a-upgrade.md create mode 100644 docs/superpowers/specs/2026-07-27-iceberg-rest-phase4a-upgrade-design.md diff --git a/docs/superpowers/plans/2026-07-27-iceberg-rest-phase4a-upgrade.md b/docs/superpowers/plans/2026-07-27-iceberg-rest-phase4a-upgrade.md new file mode 100644 index 0000000..e4ccc03 --- /dev/null +++ b/docs/superpowers/plans/2026-07-27-iceberg-rest-phase4a-upgrade.md @@ -0,0 +1,282 @@ +# Iceberg REST Phase 4a: Upgrade to Iceberg 1.9.2 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Move the REST front door from Iceberg `1.5.2` to `1.9.2` per `docs/superpowers/specs/2026-07-27-iceberg-rest-phase4a-upgrade-design.md`, keeping every existing behavior intact. + +**Architecture:** Bump the version, pin Jackson to the version Iceberg expects, re-vendor `RESTCatalogAdapter` from `1.9.2`, and migrate our dispatch from the removed `execute(...)` overload to the public `handleRequest(...)` plus exception-based error mapping. No new REST features — exists routes and `endpoints` advertising are phase 4b. + +**Tech Stack:** Java 17 (`JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19`), Maven, JUnit 4, Iceberg 1.9.2, the local docker-compose smoke stand. + +## Global Constraints + +- Java 17; 2-space indent; explicit imports; no new dependencies beyond the Iceberg version move and the Jackson pin. +- Target version `1.9.2`. Fallback if the stand shows unfixable Hive breakage: `1.8.1`; if that also fails, abandon the upgrade and report back. Record the outcome in the changelog. +- Jackson: pin `jackson-core` and `jackson-databind` to `2.18.3` in `dependencyManagement`. +- `org.slf4j:slf4j-api` must resolve to exactly one version, the project's `1.7.36`; no `log4j:log4j` anywhere in the tree. +- Behavior must be unchanged except the deltas the spec allows: view routes start returning real data, and `/v1/config` may carry new optional fields. +- Bilingual docs: every EN doc change lands with its RU counterpart in the same commit. +- English commit messages, no attribution footers. Commits are pre-approved; do NOT push. +- Offline Maven (`mvn -o`) is the norm, but the first build after the version bump needs the network to fetch `1.9.2` — use plain `mvn` for that one fetch, then `mvn -o` afterwards. + +--- + +### Task 1: Version bump, Jackson pin, re-vendored adapter, dispatch migration + +This is one task on purpose: the tree does not compile between the version bump and the dispatch migration, so there is no intermediate state a reviewer could accept. + +**Files:** +- Modify: `pom.xml` (`iceberg.version`, `dependencyManagement`, slf4j exclusions on the three Iceberg dependencies) +- Modify: `src/main/java/org/apache/iceberg/rest/RESTCatalogAdapter.java` (replace wholesale with the `1.9.2` source) +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java` (dispatch signature) +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java` (imports, route dispatch, error mapping) +- Possibly modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java` (only if a read path needs a method the proxy lacks) + +**Interfaces:** +- Produces: `IcebergRestService.dispatch(Route route, Map vars, Object body, Class responseType)` returning `T`, throwing catalog exceptions instead of reporting through a callback. + +- [ ] **Step 1: Fetch the 1.9.2 adapter source** + +The vendored file must come from the matching upstream tag, not be hand-edited: + +```bash +curl -sS -o /tmp/RESTCatalogAdapter-1.9.2.java \ + https://raw.githubusercontent.com/apache/iceberg/apache-iceberg-1.9.2/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java +head -30 /tmp/RESTCatalogAdapter-1.9.2.java +``` + +Expected: the Apache licence header followed by `package org.apache.iceberg.rest;`. If the tag path 404s, list the tags with `curl -sS https://api.github.com/repos/apache/iceberg/tags | grep -o 'apache-iceberg-1\.9\.[0-9]*' | head` and use the exact `1.9.2` tag name. + +- [ ] **Step 2: Install the file, preserving our vendoring header** + +Copy the fetched source over `src/main/java/org/apache/iceberg/rest/RESTCatalogAdapter.java`, then re-add the two-comment header that currently sits between the licence and the `package` line, with the version updated: + +```java +// Vendored verbatim from apache-iceberg-1.9.2 (core/src/test/java/org/apache/iceberg/rest). +// In 1.9.x this class lives in test sources of iceberg-core; the copy here lets the +// proxy use it without depending on the tests classifier. When bumping the Iceberg +// version, diff this file against the matching upstream tag. +``` + +- [ ] **Step 3: Update pom.xml** + +Set the version property: + +```xml +1.9.2 +``` + +Add to the existing `` block (which currently pins only curator): + +```xml + + + com.fasterxml.jackson.core + jackson-core + 2.18.3 + + + com.fasterxml.jackson.core + jackson-databind + 2.18.3 + +``` + +Add this exclusion inside the `` of **each** of the three Iceberg dependencies (`iceberg-core`, `iceberg-bundled-guava`, `iceberg-hive-metastore`); `iceberg-bundled-guava` has no `` element today, so add one: + +```xml + + org.slf4j + slf4j-api + +``` + +- [ ] **Step 4: Migrate the dispatch layer** + +In `IcebergRestService`, replace the `dispatch(...)` method with: + +```java + public T dispatch( + RESTCatalogAdapter.Route route, + Map vars, + Object body, + Class responseType) { + return adapter.handleRequest(route, vars, body, responseType); + } +``` + +Adjust its imports: drop `HTTPMethod`, `ErrorResponse` and `Consumer` if they become unused; keep `RESTResponse`. + +In `IcebergHttpHandler`: +- change the import `org.apache.iceberg.rest.RESTCatalogAdapter.HTTPMethod` to `org.apache.iceberg.rest.HTTPRequest.HTTPMethod` (the `Route` import stays as it is); +- replace the `capturedError` callback scheme around the dispatch call with a catch that maps the exception: + +```java + try { + response = service.dispatch(route, routeAndVars.second(), body, effectiveResponseType); + } catch (RuntimeException e) { + ErrorResponse.Builder builder = ErrorResponse.builder(); + RESTCatalogAdapter.configureResponseFromException(e, builder); + writeErrorResponse(exchange, outcome, builder.build()); + return; + } +``` + +Keep the existing `writeErrorResponse` helper and the per-request `RequestOutcome` bookkeeping exactly as they are — metrics recording must keep working unchanged. Import `org.apache.iceberg.rest.RESTCatalogAdapter` for the static helper. + +- [ ] **Step 5: Build (online, one time) and fix what the compiler reports** + +```bash +export JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 +mvn -DskipTests compile +``` + +Expected: `BUILD SUCCESS`. Compilation errors here are the migration surface — resolve them inside the files listed above; do not widen the change. + +- [ ] **Step 6: Run the restcatalog suite** + +```bash +export JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 +mvn -o -Dtest='io/github/mmalykhin/hmsproxy/restcatalog/*' test +``` + +Expected: all green. A failure naming `UnsupportedOperationException: HMS proxy REST gateway does not support IMetaStoreClient.` means `1.9.2` read paths call a method the proxy lacks: add that branch to `RoutingMetaStoreClient`'s switch, applying the same name translation the neighbouring branches use, and re-run. + +- [ ] **Step 7: Run the full suite** + +```bash +mvn -o test +``` + +Expected: `BUILD SUCCESS`, 0 failures, 0 skipped. + +- [ ] **Step 8: Verify the dependency tree** + +```bash +mvn -o dependency:tree > /tmp/tree-4a.txt 2>&1 +grep -E 'slf4j-api|log4j:log4j|jackson-core:jar|jackson-databind:jar' /tmp/tree-4a.txt +``` + +Expected: exactly one `org.slf4j:slf4j-api:jar:1.7.36`, no `log4j:log4j`, and `jackson-core` and `jackson-databind` both at `2.18.3`. + +- [ ] **Step 9: Build the fat jar and check the shade report** + +```bash +mvn -o -DskipTests package 2>&1 | grep -E 'overlapping classes|BUILD' +``` + +Expected: `BUILD SUCCESS` and no new `overlapping classes` warning that names an `iceberg-` jar (resource overlaps such as LICENSE/NOTICE are pre-existing and fine). + +- [ ] **Step 10: Commit** (include the spec and this plan, both currently untracked) + +```bash +git add pom.xml src/main docs/superpowers +git commit -m "Move the Iceberg REST front door to Iceberg 1.9.2" +``` + +--- + +### Task 2: Stand validation + +**Files:** +- No source changes expected. If the stand exposes a defect, fix it in the files from Task 1 and note it in the report. + +- [ ] **Step 1: Rebuild, restage and restart the stand** + +The stand containers are running with the previous jar. + +```bash +export JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 +mvn -o -DskipTests package +cd smoke-stand && ./prepare.sh && docker compose up -d --build +``` + +Then poll until healthy (up to ~8 minutes), e.g. `docker compose ps` in a loop with `sleep 20`. Never run `docker compose down` — the stand's HDFS data and registered Iceberg tables live in its volumes. + +- [ ] **Step 2: Run the REST scenario** + +```bash +JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 \ + scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple.env --scenario rest +``` + +Expected: `scenario 'rest' completed successfully`. + +- [ ] **Step 3: Run the full scenario** + +```bash +JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 \ + scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple.env --scenario all +``` + +Expected: `scenario 'all' completed successfully`. + +- [ ] **Step 4: Run the SQL layer — the Jackson-regression detector** + +This is the acceptance criterion for "Hive still works" after the Jackson pin. It runs from inside the HDP HiveServer2 container: + +```bash +docker cp scripts/run-real-installation-smoke.sh stand-hs2-hdp:/opt/hs2-hdp/run-smoke.sh +docker exec stand-hs2-hdp /opt/hs2-hdp/run-smoke.sh --env-file /opt/hs2-hdp/sql.env --scenario sql +``` + +Expected: `scenario 'sql' completed successfully`. If `stand-hs2-hdp` is not running, start it with `cd smoke-stand && docker compose --profile hdp up -d` and wait for health (it runs emulated and is slow to start). If `/opt/hs2-hdp/sql.env` is missing, recreate it from `smoke-stand/env/simple.env` plus the SQL keys documented in that file's trailing comment block, and copy in the fat jar as `/opt/hs2-hdp/hms-proxy.jar` with `HMS_SMOKE_FAT_JAR` pointing at it. + +A failure here that names Jackson (`NoSuchMethodError`, `ClassNotFoundException` under `com.fasterxml.jackson`) is the fallback trigger: report it and stop rather than improvising. + +- [ ] **Step 5: Verify client-visible behavior held** + +```bash +curl -s http://localhost:19183/v1/hdp/namespaces/default/tables/smoke_iceberg_tbl \ + | python3 -c "import sys,json; m=json.load(sys.stdin)['metadata']; print('format-version:', m['format-version']); print('fields:', len(m))" +curl -s http://localhost:19183/v1/hdp/namespaces | head -c 200; echo +curl -s http://localhost:19183/v1/apache/namespaces | head -c 200; echo +curl -s 'http://localhost:19183/v1/config?warehouse=apache'; echo +curl -s http://localhost:19090/metrics | grep -c 'hms_proxy_rest_requests_total{' +``` + +Expected: `format-version: 2` (the upgrade must not promote the table); the federated view under `hdp` still lists `default` and `apache__default`; the clean view under `apache` still lists only `default`; discovery still advertises `prefix":"apache"`; the metrics grep returns a non-zero count. + +- [ ] **Step 6: Record the view-route delta** + +```bash +curl -s -w '\n[%{http_code}]\n' http://localhost:19183/v1/hdp/namespaces/default/views +``` + +Before the upgrade this answered `204` with an empty body. Note the new status and body verbatim in the task report — it is the spec's expected delta and feeds the changelog wording in Task 3. + +- [ ] **Step 7: Commit only if something needed fixing** + +If Steps 1-6 required no source change, there is nothing to commit; say so in the report. + +--- + +### Task 3: Documentation + +**Files:** +- Modify: `README.md`, `README.ru.md` — the Iceberg REST section's note that pins `RoutingHiveCatalog` to Iceberg `1.5.2` (find it with `grep -n '1\.5\.2' README.md`) must name `1.9.2`, and the sentence about running `RoutingHiveCatalogTest` after a bump stays. +- Modify: `CHANGELOG.md`, `CHANGELOG.ru.md` — a bullet under `## 2026-07-27` / `### Changed` (create the subsection if the date section has none). +- Modify: `smoke-stand/TEST-MATRIX.md`, `smoke-stand/TEST-MATRIX.ru.md` — one sentence appended to the `2026-07-27` revalidation-log entry recording that sections A-D and G were re-run on the upgraded jar. + +- [ ] **Step 1: Write the changelog bullet** + +It must state: the REST front door moved from Iceberg `1.5.2` to `1.9.2`; Jackson is pinned to `2.18.3` because Iceberg expects `2.18` while Hive `3.1.3` brings `2.12`; view routes now return real data instead of an empty `204` because `HiveCatalog` became a `ViewCatalog`; the dispatch layer moved to `handleRequest` with exception-based error mapping; client compatibility is unaffected because the REST endpoint is a wire protocol and a v2 table still loads as v2. Mirror it in Russian, keeping terms like `prefix`, `route`, `front door` in Latin script. + +- [ ] **Step 2: Update the READMEs and TEST-MATRIX files, both locales.** + +- [ ] **Step 3: Commit** + +```bash +git add README.md README.ru.md CHANGELOG.md CHANGELOG.ru.md smoke-stand/TEST-MATRIX.md smoke-stand/TEST-MATRIX.ru.md +git commit -m "Document the Iceberg 1.9.2 upgrade" +``` + +--- + +## Self-Review + +- **Spec coverage:** version bump + Jackson pin + slf4j exclusion (T1 S3); re-vendored adapter (T1 S1-S2); adapter API migration incl. `configureResponseFromException` (T1 S4); `RoutingMetaStoreClient` gap handling (T1 S6); dependency-tree and shade acceptance (T1 S8-S9); stand + SQL-layer Jackson detector (T2 S2-S4); v2-format client-compatibility assertion (T2 S5); view-route delta recorded (T2 S6); docs both locales incl. the README `1.5.2` pin (T3). Fallback trigger is stated in the Global Constraints and at T2 S4. No gaps. +- **Placeholder scan:** none — every step carries the exact command or code, including the conditional `RoutingMetaStoreClient` branch instructions. +- **Type consistency:** `dispatch(Route, Map, Object, Class)` is declared in T1's Interfaces block and used with exactly that shape in T1 S4; `RESTCatalogAdapter.Route` stays the nested type throughout; `configureResponseFromException(Exception, ErrorResponse.Builder)` matches the verified upstream signature. diff --git a/docs/superpowers/specs/2026-07-27-iceberg-rest-phase4a-upgrade-design.md b/docs/superpowers/specs/2026-07-27-iceberg-rest-phase4a-upgrade-design.md new file mode 100644 index 0000000..85a1675 --- /dev/null +++ b/docs/superpowers/specs/2026-07-27-iceberg-rest-phase4a-upgrade-design.md @@ -0,0 +1,161 @@ +# Iceberg REST frontend, phase 4a: Iceberg upgrade — design + +Date: 2026-07-27 +Branch: `feature/iceberg-rest-fe-phase1` +Status: approved + +## Goal + +Move the REST front door off Iceberg `1.5.2` onto `1.9.2`. This sub-project is the +dependency and API-migration work only: everything that worked before must still work. +The read-path features the upgrade unlocks (exists routes, `endpoints` advertising) are +phase 4b. + +Phase 4 was split in two because the upgrade carries dependency risk that deserves its +own acceptance gate: if it proves unworkable, it can be abandoned without losing 4b. + +## Why 1.9.2 + +Verified against the published artifacts before choosing, across every candidate: + +| version | exists routes | native views | `Route` | adapter entry point | jackson | +| --- | --- | --- | --- | --- | --- | +| 1.5.2 (today) | no | no | nested | `execute(...)`, public | 2.14 / 2.12 | +| 1.8.1 | yes | yes | nested | `handleRequest(route, vars, body, type)` | ~2.18 | +| **1.9.2** | **yes** | **yes** | **nested** | **`handleRequest(route, vars, body, type)`** | **2.18.3** | +| 1.10.2 | yes | yes | nested | same | 2.21.2 | +| 1.11.0 | yes | yes | top-level | `handleRequest` over `HTTPRequest` | 2.21.3 | + +Exists routes (`NAMESPACE_EXISTS`, `TABLE_EXISTS`, `VIEW_EXISTS`) arrive in `1.8.x` and +are what phase 4b needs; `CatalogHandlers` in the main jar carries the matching +`namespaceExists` / `tableExists` / `viewExists`. Everything above `1.9.2` adds only +scan-planning routes this project does not want, while `1.10.2` and `1.11.0` cost a +wider Jackson jump and `1.11.0` additionally restructures the adapter into three +vendored files. `1.9.2` is therefore the newest release that buys every feature at the +mildest dependency and API cost. + +Also verified for `1.9.2`: + +- `HiveCatalog` extends `BaseMetastoreViewCatalog`, so view read paths become real + instead of the current empty `204`. +- The private `clients` field survives with the same type, so the reflection inject in + `RoutingHiveCatalog` still applies. +- Bytecode major version 55 (Java 11) — no new runtime requirement. +- No slf4j 2.x fluent-API call sites, so Iceberg runs on the project's slf4j `1.7.36`. +- `iceberg-hive-metastore` is published at `1.9.2`, and its pom declares no Hive + dependency (upstream keeps Hive `compileOnly`), so Hive `3.1.3` keeps providing + `IMetaStoreClient`. + +## The Jackson problem + +Measured on the real tree by temporarily setting `iceberg.version=1.9.2` and running +`mvn dependency:tree`: + +| | jackson-core | jackson-databind | +| --- | --- | --- | +| today (1.5.2) | 2.14.2 | 2.12.0 (Hive) | +| with 1.9.2 | 2.18.3 (Iceberg) | 2.12.0 (Hive) | + +Iceberg wins `jackson-core`, Hive wins `jackson-databind`, and the split widens from two +minor versions to six. Iceberg `1.9.2` is compiled against databind `2.18`; served `2.12` +it would break in `TableMetadataParser` — the exact path that reads `metadata.json`. + +Resolution: pin `jackson-core` and `jackson-databind` to `2.18.3` through +`dependencyManagement`. This hands the newer Jackson to the Hive/Hadoop paths too, which +AGENTS.md flags as fragile — so "Hive still works" becomes an acceptance criterion +measured on the stand, not an assumption. + +The rest of the measured tree is undramatic: `slf4j-api` stays at the project's `1.7.36`, +`avro` stays at Hive's `1.7.4`, `caffeine` stays at `2.9.3`, and `httpclient5` moves +`5.3.1` → `5.4.3`. + +**Fallback.** If the stand shows Hive breakage that Jackson alignment cannot fix, step +down to `1.8.1` — the only lower rung that still carries exists routes and native views. +If that fails too, abandon the upgrade and take phase 4b on `1.5.2`, losing native views +and implementing exists checks by hand. Record the outcome in the changelog. + +## Adapter API migration + +The upgrade is not import-only: the `execute(HTTPMethod, path, queryParams, body, +responseType, headers, errorHandler)` overload that `IcebergRestService.dispatch` +currently calls is gone in every candidate version. `1.9.2` replaces it with the public +`handleRequest(Route route, Map vars, Object body, Class responseType)`. + +Three consequences, all inside `IcebergRestService` and `IcebergHttpHandler`: + +- The route and its path variables are already parsed by the handler + (`Route.from(...)` returns `Pair>`), so they are passed + straight into `handleRequest`. +- `HTTPMethod` moved out of the adapter: the import becomes + `org.apache.iceberg.rest.HTTPRequest.HTTPMethod`. `Route` itself stays nested in + `RESTCatalogAdapter`, so that import is unchanged and only one file is vendored. +- `handleRequest` takes no error-handler callback; catalog exceptions propagate instead. + The handler maps them with the adapter's public + `RESTCatalogAdapter.configureResponseFromException(Exception, ErrorResponse.Builder)`, + which fills in both the message and the response code, replacing today's + captured-callback scheme. + +## Scope + +- `iceberg.version` `1.5.2` → `1.9.2`; Jackson pinned to `2.18.3` via + `dependencyManagement`; an explicit `org.slf4j:slf4j-api` exclusion on the Iceberg + dependencies so the transitive `2.0.17` can never win. Existing exclusions stay. +- Re-vendor `RESTCatalogAdapter` from `1.9.2` test sources — still one file, since + `Route` remains nested. The vendoring header comment names the new version. +- Migrate `IcebergRestService.dispatch` and `IcebergHttpHandler` to `handleRequest` and + exception-based error mapping, as described above. +- Extend `RoutingMetaStoreClient` if `1.9.2` read paths call `IMetaStoreClient` methods + the proxy's dynamic proxy does not implement. The proxy throws + `UnsupportedOperationException` with the method signature, so any gap surfaces as a + clear test failure; new branches must apply the same name translation as the existing + ones. + +Out of scope: enabling exists routes, advertising `endpoints`, view smoke coverage, +pagination (`CatalogHandlers` still returns whole lists, and HMS does too). + +## Expected behavior deltas + +Strict neutrality is not achievable and is not claimed: + +- View routes (`GET .../views`, `GET .../views/{name}`) start returning real data instead + of the current empty `204`, because `HiveCatalog` is a `ViewCatalog` from `1.7` on. +- Exists routes become reachable in the adapter. Wiring them into the handler is 4b; + until then they behave as they do today. +- `/v1/config` may carry the new optional `endpoints` / `idempotency-key-lifetime` + fields. Populating `endpoints` is 4b. + +Everything else — namespace and table listings, table load, multi-catalog prefixes, name +translation, metrics, SPNEGO — must be unchanged. + +## Client compatibility + +The REST front door is a wire protocol, so a client's own Iceberg version is independent +of ours; unlike the Thrift front door, one endpoint serves every client version. Two +properties make the upgrade safe for existing clients, and both are asserted rather than +assumed: + +- **The proxy re-serializes table metadata rather than streaming the file.** Verified on + the stand: the raw `metadata.json` in HDFS carries 18 top-level fields while the served + response carries 21 (`refs`, `statistics`, `partition-statistics` are added by the + parser). The emitted field set therefore follows our library version. Older clients + tolerate this because Iceberg's JSON parsers read fields by name and ignore unknown + ones. +- **Table format version comes from the data, not from our library.** A `format-version: 2` + table stays v2 after the upgrade; the parser does not silently promote tables. What + limits an old client is the format version of the tables in HMS, not the proxy. + +Acceptance therefore includes: after the upgrade a v2 table still loads with +`"format-version": 2` and keeps the fields the v2 spec requires. + +## Testing + +- Full `mvn -o test` green. +- `mvn -o dependency:tree`: exactly one `org.slf4j:slf4j-api` (the project's `1.7.36`), + no `log4j:log4j`, and matching `jackson-core` / `jackson-databind` versions. +- Fat jar builds with no new overlapping *classes* in the shade report. +- Stand, plain profile: `--scenario rest` and `--scenario all` green, plus the **SQL + layer** through both HiveServer2 instances — that pass is the Jackson-regression + detector for the Hive paths. +- A stand assertion that the served v2 table still reports `"format-version": 2`. +- Docs: README and CHANGELOG in both locales; the README note pinning `RoutingHiveCatalog` + to Iceberg `1.5.2` must name `1.9.2` instead. diff --git a/pom.xml b/pom.xml index 6907dc8..c09b8d0 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ 2.12.0 3.1.3 2.6.5 - 1.5.2 + 1.9.2 4.13.2 + + com.fasterxml.jackson.core + jackson-core + 2.18.3 + + + com.fasterxml.jackson.core + jackson-databind + 2.18.3 + @@ -177,6 +190,10 @@ org.slf4j slf4j-log4j12 + + org.slf4j + slf4j-api + + + org.apache.hadoop + hadoop-hdfs + + + + + org.apache.hadoop + hadoop-hdfs + 2.6.0 + + + jdk.tools + jdk.tools + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java index 5a48fd5..351db21 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java @@ -193,11 +193,15 @@ private void doHandle(HttpExchange exchange, RequestOutcome outcome) throws IOEx return; } writeJson(exchange, outcome, 200, IcebergRestMapper.mapper().writeValueAsString(response)); - } catch (Exception e) { + } catch (Throwable t) { + // Catches Throwable, not just Exception: a backend classpath mismatch (e.g. a + // NoSuchMethodError from a stale transitive Hadoop jar) is an Error, and letting + // it escape here would leave the JDK HTTP server abandoning the exchange with no + // response at all - the client hangs instead of getting a 5xx. LOG.warn("Unhandled error serving {} {}", - exchange.getRequestMethod(), exchange.getRequestURI(), e); - writeError(exchange, outcome, 500, e.getClass().getSimpleName(), - e.getMessage() == null ? "internal error" : e.getMessage()); + exchange.getRequestMethod(), exchange.getRequestURI(), t); + writeError(exchange, outcome, 500, t.getClass().getSimpleName(), + t.getMessage() == null ? "internal error" : t.getMessage()); } } diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java index c9856a1..e5997ce 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java @@ -148,6 +148,21 @@ public void loadMissingNamespaceReturns404() throws Exception { Assert.assertEquals("body: " + response.body(), 404, response.statusCode()); } + @Test + public void unhandledErrorFromDispatchYieldsResponseInsteadOfHang() throws Exception { + // Regression test for IcebergHttpHandler.doHandle's catch-all: it used to catch only + // Exception, so an Error thrown below the REST adapter (like the real + // NoSuchMethodError from a classpath mismatch this phase hit) would escape doHandle and + // handle() entirely, leaving the JDK HTTP server to abandon the exchange with no + // response - the client would hang until its own timeout instead of seeing a 5xx. If + // that regressed, this get() call would throw (timeout or connection reset) rather than + // returning a response, so this test fails loudly instead of asserting a wrong status. + HttpResponse response = + get("/v1/" + CATALOG_NAME + "/namespaces/" + RecordingThriftIface.THROWS_ERROR_PROBE_DB); + Assert.assertEquals("body: " + response.body(), 500, response.statusCode()); + Assert.assertTrue(response.body(), response.body().contains("\"type\":\"Error\"")); + } + @Test public void listTablesReturnsTableNames() throws Exception { HttpResponse response = get("/v1/" + CATALOG_NAME + "/namespaces/sales/tables"); diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java index 1a00de4..4416828 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java @@ -23,6 +23,13 @@ final class RecordingThriftIface { static final long LOCK_ID = 42L; + // Magic database name that makes get_database throw an Error instead of a checked + // Thrift exception. Proxy.newProxyInstance() passes Error/RuntimeException through the + // invocation handler unwrapped (only checked exceptions get boxed in + // UndeclaredThrowableException), so this reaches IcebergHttpHandler's catch-all exactly + // like the real NoSuchMethodError this proxy phase needed to stop from hanging the client. + static final String THROWS_ERROR_PROBE_DB = "throws_error_probe"; + final List calls = new ArrayList<>(); final Map databases = new HashMap<>(); final Map tables = new HashMap<>(); @@ -67,6 +74,9 @@ private Object handle(Object proxy, Method method, Object[] args) throws Throwab case "get_database": { String db = (String) args[0]; calls.add("get_database:" + db); + if (THROWS_ERROR_PROBE_DB.equals(db)) { + throw new Error("simulated classpath-mismatch failure (e.g. NoSuchMethodError)"); + } Database value = databases.get(db); if (value == null) { throw new NoSuchObjectException("no database " + db); From c4685ef7b30f5d5fcf5700552a75cd8cad11df62 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 12:21:47 +0300 Subject: [PATCH 40/50] Cover Iceberg REST table writes in the stand smoke Add HMS_SMOKE_REST_WRITE_TABLE-guarded coverage to run_rest_smoke: create, load (asserting metadata-location) and drop a table on the default catalog, then two negatives - a direct create under a non-default prefix and a create under that prefix's federated namespace name reached through the default prefix - both asserting 403. Also fix the pre-existing config assertions, which still expected a read-only listing: the default catalog's GET /v1/config and GET /v1/{prefix}/config now must advertise the five table-write routes, and a new check on the non-default prefix's own config proves the write/read asymmetry is actually observable via discovery. Document, in both locales, that table writes are default-catalog only and why (every other catalog is served by the synthetic lock shim, which grants locks without conflict checking), that the gate is enforced on the resolved catalog so a federated name cannot bypass it, and that GET /v1/config advertises the asymmetry. Record the two fixes phase 5a needed to make writes work at all: the hadoop-hdfs/hadoop-common version alignment (writes are the first code path opening an HDFS output stream from the proxy's own JVM) and the IcebergHttpHandler catch-all widened from Exception to Throwable, so a JVM Error can no longer hang a client's connection forever. Verified on the local stand: --scenario rest and --scenario all both completed successfully, and the SQL layer (both HiveServer2 instances) passed as the regression check for the Hadoop dependency change, since table writes and Hive's own ACID commits share the same lock path. --- AGENTS.md | 4 +- CHANGELOG.md | 61 +++++++++++ CHANGELOG.ru.md | 60 +++++++++++ README.md | 85 ++++++++++++--- README.ru.md | 75 +++++++++++-- ...real-installation-smoke.simple.env.example | 10 ++ scripts/run-real-installation-smoke.sh | 102 ++++++++++++++---- smoke-stand/TEST-MATRIX.md | 28 ++++- smoke-stand/TEST-MATRIX.ru.md | 25 ++++- smoke-stand/env/simple.env | 7 ++ 10 files changed, 407 insertions(+), 50 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e09a570..8301edf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -84,7 +84,7 @@ scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple. - Знание о служебном плейсхолдере Hive (`_dummy_database`/`_dummy_table`, константы `SemanticAnalyzer.DUMMY_DATABASE`/`DUMMY_TABLE`, которые Hive шлёт в `LockRequest` для `INSERT ... VALUES`) живёт только в `routing/HivePlaceholderNamespace`. Плейсхолдер не выбирает каталог, не считается вторым namespace lock-запроса и не переписывается интернализацией в реальную backend-базу. Не добавляй локальных сравнений с `_dummy_database` в других классах. - Для RPC, у которых Hive IDL не объявляет исключений (`add_write_notification_log`, `open_txns`, `show_locks` и другие: у их `_result` есть только поле `success`), libthrift 0.9.3 подменяет любое серверное исключение на `TApplicationException("Internal error processing ")`. Текст отказа таких методов виден только в логе - не рассчитывай, что его получит клиент, и не проверяй его в smoke-скриптах. Настоящий HMS теряет свои тексты ошибок так же. - Классификация backend Thrift-ошибок живёт только в `thriftbridge/ThriftFailureClassifier`: «метода нет» - это `TApplicationException` с типом `UNKNOWN_METHOD` (или отсутствие метода в загруженном runtime), transport failure и protocol desync - отдельные категории. Не пиши локальные `instanceof TApplicationException` для решений про fallback, downgrade или переоткрытие соединения. -- `restcatalog` - Iceberg REST front door поверх routing-слоя: per-catalog сервисы работают через in-process прокси `IMetaStoreClient`, доступ только на чтение, для не-default каталогов включён name translation, чтобы клиент видел внутренние имена баз этого каталога. +- `restcatalog` - Iceberg REST front door поверх routing-слоя: per-catalog сервисы работают через in-process прокси `IMetaStoreClient`; для не-default каталогов включён name translation, чтобы клиент видел внутренние имена баз этого каталога. Write-запросы к таблицам (create/commit/drop/rename/register) поддержаны, но только когда namespace резолвится в `routing.default-catalog` - только его таблицы подкреплены реальным HMS-локом, остальные каталоги обслуживает synthetic lock shim без проверки конфликтов. `WriteRouteGate` проверяет резолвленный каталог, а не prefix запроса, поэтому federated-имя под default-prefix отказывается так же, как прямой запрос к non-default prefix; `GET /v1/config` объявляет эту асимметрию в `endpoints`. Не добавляй локальных проверок "разрешён ли write" в других классах - весь gate живёт в `WriteRouteGate`. ## Парсинг конфигурации @@ -99,7 +99,7 @@ scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple. - Держи изменения узко сфокусированными. Пути routing, security, compatibility и class-loading чувствительны для production. - Предпочитай deterministic routing и явные safe failures вместо догадок, когда catalog ownership или namespace context неоднозначны. - Сохраняй совместимость с Java 17. Не добавляй требования к более новым версиям языка или runtime. -- Не добавляй новые зависимости без реальной необходимости; Hive/Hadoop dependency convergence хрупкий. +- Не добавляй новые зависимости без реальной необходимости; Hive/Hadoop dependency convergence хрупкий. `hadoop-hdfs` и `hadoop-common` обязаны быть одной версии: Maven их не сравнивает (разные artifact ID), а рассинхрон ломает `DFSOutputStream` (`NoSuchMethodError: FSOutputSummer.`) при любой записи в HDFS изнутри JVM прокси - `pom.xml` явно исключает транзитивный `hadoop-hdfs` из `orc-core` и держит `hadoop-hdfs:2.6.0` напрямую, чтобы совпасть с `hadoop-common`. Проверяй `mvn -o dependency:tree | grep -iE "hadoop-hdfs|hadoop-common"` при трогании этих зависимостей. - Логирование: slf4j-api и binding `slf4j-reload4j` держи на одной версии `${slf4j.version}`. `org.apache.log4j` должен приходить только из reload4j, поэтому `log4j:log4j` исключён во всех зависимостях, которые его тянут (обе Hive-зависимости, `hadoop-mapreduce-client-core`, `curator-test`). Добавляя зависимость, проверь `mvn -o dependency:tree | grep log4j`: второй провайдер этого пакета даст в fat jar классы, выбранные shade-плагином произвольно. - Комментарии оставляй короткими и только там, где они объясняют неочевидное compatibility, security, routing или class-loading поведение. - Не форматируй несвязанные файлы и не меняй generated docs, если твоя задача не требует их обновления. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cae732..9e9229d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,67 @@ tagged release, `v1.0.0`, was cut on 2026-04-29. For a Russian version, see [CHANGELOG.ru.md](CHANGELOG.ru.md). +## 2026-07-28 + +### Added + +- The Iceberg REST front door now supports table writes - create, commit + (update), drop, rename, register - when the request's namespace resolves + to `routing.default-catalog`. `RoutingMetaStoreClient` now implements + `createTable`, `dropTable`, `alter_table_with_environmentContext` and the + commit-lock RPCs (`lock`, `checkLock`, `unlock`, `heartbeat`, `showLocks`) + instead of throwing `UnsupportedOperationException` for all of them; every + other `IMetaStoreClient` method it does not need for this stays + unsupported. +- Every Iceberg REST write route is refused with `403` + (`ForbiddenException`) whenever its namespace resolves to any catalog + other than the default one: only the default catalog's tables are backed + by a real HMS lock, and every other catalog is served by the synthetic + lock shim, which grants an `EXCLUSIVE` lock unconditionally with no + conflict checking - a commit routed there would race a concurrent writer + into a silently lost update. The new `WriteRouteGate` checks the + **resolved** catalog, not the request's own URL prefix, so a federated + `` name reached through the default prefix is + refused exactly like a direct request against the non-default prefix; the + gate covers every write route `RESTCatalogAdapter` exposes (table and view + CRUD, namespace CRUD, rename, multi-table transaction commit), not only + the five table-write routes this phase actually implements. +- `GET /v1/config` and `GET /v1/{prefix}/config` now advertise the write/read + asymmetry between catalogs: the default catalog's `endpoints` field + carries the five table-write routes on top of the nine read routes from + the previous phase; every other catalog's carries only the nine read + routes. A spec-compliant client can discover the restriction instead of + learning about it from a failed request. +- `--scenario rest` in the smoke runners drives the table write round trip - + create (asserting `200`), load (asserting `metadata-location` comes back), + drop (asserting a `2xx`) - and the two negative cases: a direct create + under a non-default prefix, and a create under that prefix's federated + namespace name reached through the default prefix, both asserting `403`. + Guarded by the new `HMS_SMOKE_REST_WRITE_TABLE`; skipped when unset. The + runner also now asserts the config write/read asymmetry above, for both + the default catalog and a configured second catalog. + +### Fixed + +- Every HDFS write from inside the proxy's own JVM failed with + `NoSuchMethodError: FSOutputSummer.`, deep inside + `DFSOutputStream` - table writes are the first proxy code path to open an + HDFS output stream itself; reads use a different, unaffected class path. + `orc-core` (pulled in transitively by `hive-standalone-metastore`) was + dragging a stale `hadoop-hdfs:2.2.0` alongside `hadoop-common:2.6.0` + elsewhere in the tree, and Maven's mediation never compared them (they are + different artifact IDs). `pom.xml` now excludes that transitive + `hadoop-hdfs` and depends on `hadoop-hdfs:2.6.0` directly, to match + `hadoop-common`. +- The Iceberg REST request dispatcher (`IcebergHttpHandler`) only caught + `Exception`, so a `NoSuchMethodError` (or any other `java.lang.Error`) + escaping request handling unwound past both catch blocks with no response + ever sent - the JDK HTTP server logged the stack trace to stderr and + abandoned the exchange, leaving the client's connection hanging + indefinitely with no timeout on the server side. The catch-all is now + `Throwable`, so such failures map to the usual error response like any + other failure instead of hanging the caller. + ## 2026-07-27 ### Added diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 1a2e93b..178a3c9 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -6,6 +6,66 @@ English version: [CHANGELOG.md](CHANGELOG.md). +## 2026-07-28 + +### Добавлено + +- Iceberg REST front door теперь поддерживает write-запросы к таблицам — + create, commit (update), drop, rename, register — когда namespace запроса + резолвится в `routing.default-catalog`. `RoutingMetaStoreClient` теперь + реализует `createTable`, `dropTable`, `alter_table_with_environmentContext` + и commit-lock RPC (`lock`, `checkLock`, `unlock`, `heartbeat`, + `showLocks`) вместо того, чтобы кидать `UnsupportedOperationException` на + все они; любой другой метод `IMetaStoreClient`, который для этого не + нужен, по-прежнему не поддержан. +- Любой write-роут Iceberg REST отказывается с `403` (`ForbiddenException`), + если его namespace резолвится в любой каталог, кроме дефолтного: только + таблицы дефолтного каталога подкреплены реальным HMS-локом, а любой + другой каталог обслуживается синтетическим lock shim, который выдаёт + `EXCLUSIVE`-лок безусловно, без проверки конфликтов — commit, направленный + туда, гонялся бы наперегонки с конкурентным writer'ом и молча терял + апдейт. Новый `WriteRouteGate` проверяет **резолвленный** каталог, а не + prefix из URL запроса, так что federated-имя ``, + достигнутое через дефолтный prefix, отказывается точно так же, как прямой + запрос к non-default prefix; gate покрывает каждый write-роут, который + выставляет `RESTCatalogAdapter` (table и view CRUD, namespace CRUD, + rename, multi-table transaction commit), а не только пять table-write + роутов, которые эта фаза реально реализует. +- `GET /v1/config` и `GET /v1/{prefix}/config` теперь объявляют + write/read-асимметрию между каталогами: в `endpoints` дефолтного каталога + дополнительно к девяти read-роутам предыдущей фазы перечислены пять + table-write роутов; у любого другого каталога — только девять read-роутов. + Спецификация-совместимый клиент может обнаружить это ограничение через + discovery, а не из проваленного запроса. +- `--scenario rest` в smoke-раннерах гоняет write round trip таблицы — + create (проверка `200`), load (проверка, что вернулся `metadata-location`), + drop (проверка `2xx`) — и два негативных случая: прямой create под + non-default prefix и create под federated-именем этого prefix, достигнутым + через дефолтный prefix, — оба с ожиданием `403`. Настраивается новой + `HMS_SMOKE_REST_WRITE_TABLE`; пропускается, если не задана. Раннер также + теперь проверяет write/read-асимметрию в config, описанную выше, — и для + дефолтного каталога, и для настроенного второго каталога. + +### Исправлено + +- Любая запись в HDFS изнутри JVM прокси падала с `NoSuchMethodError: + FSOutputSummer.` глубоко внутри `DFSOutputStream` — write таблицы + оказался первым путём в прокси, который сам открывает output stream в + HDFS; чтение идёт по другому, незатронутому classpath. `orc-core` + (приходит транзитивно через `hive-standalone-metastore`) тянул устаревший + `hadoop-hdfs:2.2.0` рядом с `hadoop-common:2.6.0` в другом месте дерева, и + мавеновская медиация никогда их не сравнивала (это разные artifact ID). + `pom.xml` теперь исключает этот транзитивный `hadoop-hdfs` и напрямую + зависит от `hadoop-hdfs:2.6.0`, чтобы совпасть с `hadoop-common`. +- Диспетчер запросов Iceberg REST (`IcebergHttpHandler`) ловил только + `Exception`, поэтому `NoSuchMethodError` (или любой другой + `java.lang.Error`), ускользнувший из обработки запроса, улетал мимо обоих + catch-блоков без единого ответа — JDK HTTP server логировал stack trace в + stderr и бросал exchange, оставляя соединение клиента висеть бесконечно, + без тайм-аута даже на стороне сервера. Catch-all теперь ловит `Throwable`, + так что такие сбои маппятся в обычный error-ответ как любой другой сбой, + вместо того чтобы вешать вызывающую сторону. + ## 2026-07-27 ### Добавлено diff --git a/README.md b/README.md index 7a81cb5..4f54519 100644 --- a/README.md +++ b/README.md @@ -986,13 +986,36 @@ ambiguously, so `__` is usually the safer choice. The proxy can also run a parallel HTTP listener that speaks the Iceberg REST Catalog spec, backed by the same routing/federation pipeline as the Thrift HMS -front door. Status: **experimental, read-only**. Iceberg clients (PyIceberg, -Spark `iceberg-rest`, Trino `iceberg-rest`) can discover and load Iceberg -tables stored in HMS via the standard `metadata_location` table parameter. +front door. Status: **experimental**; table writes (create, commit, drop, +rename, register) are supported, but **only when the target namespace +resolves to `routing.default-catalog`**. Iceberg clients (PyIceberg, Spark +`iceberg-rest`, Trino `iceberg-rest`) can discover and load Iceberg tables +stored in HMS via the standard `metadata_location` table parameter. View routes now return real data instead of an empty `204`, since -`HiveCatalog` is a `ViewCatalog` as of Iceberg `1.7`; writes, commits, and -view mutations (create/drop/rename) are still NOT supported in this -iteration. +`HiveCatalog` is a `ViewCatalog` as of Iceberg `1.7`; view mutations +(create/drop/rename), namespace mutations and multi-table transaction +commits are still NOT supported in this iteration, on any catalog. + +**Why writes are default-catalog only:** only the default catalog's tables +are backed by a real HMS lock (see [ZooKeeper storage for synthetic read +locks](#zookeeper-storage-for-synthetic-read-locks)); every other +catalog is served by the synthetic lock shim, which grants an `EXCLUSIVE` +lock unconditionally with no conflict checking. A commit routed there would +believe it owns the table while silently racing - and possibly losing to - a +concurrent writer, corrupting `metadata.json` without ever reporting a +conflict. `WriteRouteGate` enforces this on the **resolved** catalog, not on +the request's own URL prefix: the default catalog's own prefix also exposes +every other catalog's databases as federated `` +names (see [Supported endpoints](#supported-endpoints) below), so a create +under `/v1/{default-prefix}/namespaces/apache__default/tables` is refused +exactly like a direct create under `/v1/apache/namespaces/default/tables` - +both resolve to the `apache` catalog and get the same `403` +(`ForbiddenException`) with a message naming the resolved catalog. `GET +/v1/config` and `GET /v1/{prefix}/config` advertise this asymmetry +directly: the default catalog's `endpoints` list carries the five write +routes below, every other catalog's carries only the nine read routes, so a +spec-compliant client discovers the restriction instead of learning about it +from a failed request. Enable it via: @@ -1023,7 +1046,12 @@ Requests to this listener are covered by the Prometheus metrics described in | `HEAD /v1/{prefix}/namespaces/{ns}` | supported (204 if exists, 404 if not) | | `HEAD /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | supported (204 if exists, 404 if not) | | `HEAD /v1/{prefix}/namespaces/{ns}/views/{view}` | supported (204 if exists, 404 if not) | -| `POST`, `DELETE`, commits (including view mutations) | unsupported | +| `POST /v1/{prefix}/namespaces/{ns}/tables` | supported for the default catalog only (create); `403` elsewhere | +| `POST /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | supported for the default catalog only (commit/update); `403` elsewhere | +| `DELETE /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | supported for the default catalog only (drop); `403` elsewhere | +| `POST /v1/{prefix}/tables/rename` | supported for the default catalog only (rename); `403` elsewhere | +| `POST /v1/{prefix}/namespaces/{ns}/register` | supported for the default catalog only (register); `403` elsewhere | +| Namespace/view mutations, multi-table transaction commits | unsupported on every catalog | `{prefix}` is any catalog listed in `catalogs=`: every configured catalog is exposed as its own REST prefix, `/v1//...`. `GET /v1/config` supports @@ -1032,14 +1060,23 @@ warehouse discovery: pass `?warehouse=` and the response's right prefix without hardcoding it (see the client examples below). Without `warehouse`, `/v1/config` advertises `routing.default-catalog`, matching phase-1 behavior; an unknown `warehouse` value returns HTTP 400 -(`BadRequestException`). The response's `endpoints` field lists exactly the -nine read routes above (list/load namespace + namespace-exists, list/load -table + table-exists, list/load view + view-exists), so a modern client knows -not to attempt a write; older clients that don't look at the field are -unaffected. `GET /v1/{prefix}/config` answers the same way from the proxy's -own handler — `overrides.prefix` names the catalog from the path segment -instead of the `warehouse` query param — and an unknown prefix there is still -a 404. +(`BadRequestException`). The response's `endpoints` field lists the nine +read routes above for every catalog (list/load namespace + namespace-exists, +list/load table + table-exists, list/load view + view-exists); the default +catalog's `endpoints` additionally carry the five table-write routes above, +so a modern client can discover the write/read asymmetry between catalogs +instead of learning about it from a failed request. `GET /v1/{prefix}/config` +answers the same way from the proxy's own handler — `overrides.prefix` names +the catalog from the path segment instead of the `warehouse` query param — +and an unknown prefix there is still a 404. + +A refused write answers `403` (`ForbiddenException`) with a message naming +the resolved catalog, for example: `Writes are only supported in the +default catalog 'hdp'; namespace 'apache__default' belongs to catalog +'apache', which is served by the synthetic lock shim and provides no writer +isolation.` This is enforced on the namespace the request **resolves** to, +not on the URL prefix it arrived under - see [Why writes are default-catalog +only](#iceberg-rest-catalog-frontend) above. Error responses carry the mapped HTTP status, `type` and `message` but never a server stack trace, since this listener can be reached without @@ -1048,7 +1085,11 @@ authentication. A request body that fails to parse answers 400 every route that takes a body. A `HEAD` response never writes a body, per RFC 9110 — including on an error status — so an exists-check against a missing namespace, table or view returns a plain 404 with no body, not a 404 with a -JSON payload. +JSON payload. The request dispatcher catches `Throwable`, not just +`Exception`: any `Error` that escapes handling (for example a dependency- +version `NoSuchMethodError` surfacing deep inside a write) is mapped to the +usual error response instead of unwinding past the handler and leaving the +client's connection to hang forever with no response at all. The default catalog's prefix keeps the phase-1 federated view: its own databases plus every other catalog's databases under their @@ -1110,6 +1151,18 @@ spark.sql.catalog.sales_catalog.warehouse=sales - `RoutingHiveCatalog` uses reflection on Iceberg's private `HiveCatalog.clients` field, pinned to Iceberg `1.9.2`. Bumping the Iceberg version requires running `RoutingHiveCatalogTest` to confirm the inject still works. +- A table write opens an HDFS output stream (`metadata.json`) from inside the + proxy's own JVM - the first code path in the proxy to do so; reads use a + different, unaffected class path. This requires `hadoop-hdfs` and + `hadoop-common` to be the same version in the dependency tree. Maven's + mediation never compared them (they are different artifact IDs), so + `orc-core` (pulled in transitively by `hive-standalone-metastore`) was + dragging a stale `hadoop-hdfs:2.2.0` alongside `hadoop-common:2.6.0` + elsewhere in the tree, and every write failed with `NoSuchMethodError: + FSOutputSummer.` deep inside `DFSOutputStream`. `pom.xml` now + excludes that transitive `hadoop-hdfs` and depends on `hadoop-hdfs:2.6.0` + directly, to match `hadoop-common`. Keep the two aligned if you ever + override either version. ## Security diff --git a/README.ru.md b/README.ru.md index 6ed0b2f..1eac4e7 100644 --- a/README.ru.md +++ b/README.ru.md @@ -983,13 +983,37 @@ catalog.catalog2.conf.hms.proxy.external-table-drop-purge.allowed-prefixes=hdfs: Proxy дополнительно умеет поднять параллельный HTTP listener со спецификацией Iceberg REST Catalog, использующий тот же routing/federation pipeline что и -Thrift HMS front door. Статус: **экспериментально, read-only**. Iceberg-клиенты +Thrift HMS front door. Статус: **экспериментально**; write-запросы к таблицам +(create, commit, drop, rename, register) поддержаны, но **только когда +целевой namespace резолвится в `routing.default-catalog`**. Iceberg-клиенты (PyIceberg, Spark `iceberg-rest`, Trino `iceberg-rest`) могут discover и load Iceberg-таблицы, хранящиеся в HMS через стандартный параметр `metadata_location`. View routes теперь возвращают реальные данные вместо пустого `204`, так как `HiveCatalog` стал `ViewCatalog`, начиная с Iceberg -`1.7`; writes, commits и view-мутации (create/drop/rename) в этой итерации -по-прежнему НЕ поддерживаются. +`1.7`; view-мутации (create/drop/rename), namespace-мутации и multi-table +transaction commit по-прежнему НЕ поддерживаются в этой итерации, ни для +одного каталога. + +**Почему writes работают только в default-каталоге:** реальным HMS-локом +подкреплены только таблицы дефолтного каталога (см. [ZooKeeper storage для +synthetic read locks](#zookeeper-storage-для-synthetic-read-locks)); +любой другой каталог обслуживается синтетическим lock shim, который выдаёт +`EXCLUSIVE`-лок безусловно, без проверки конфликтов. Commit, направленный +туда, решил бы, что владеет таблицей, молча гоняясь наперегонки с — и, +возможно, проигрывая — конкурентным writer'ом, что портит `metadata.json` без +единого сообщения о конфликте. `WriteRouteGate` проверяет это на +**резолвленном** каталоге, а не на prefix из URL запроса: prefix дефолтного +каталога тоже выставляет базы всех остальных каталогов под federated-именами +`` (см. [Поддерживаемые endpoint'ы](#поддерживаемые-endpointы) +ниже), так что create по +`/v1/{default-prefix}/namespaces/apache__default/tables` отказывается точно +так же, как прямой create по `/v1/apache/namespaces/default/tables` — оба +резолвятся в каталог `apache` и получают один и тот же `403` +(`ForbiddenException`). `GET /v1/config` и `GET /v1/{prefix}/config` +объявляют эту асимметрию напрямую: в `endpoints` дефолтного каталога +перечислены пять write-роутов из таблицы ниже, у любого другого каталога — +только девять read-роутов, так что спецификация-совместимый клиент узнаёт об +ограничении из discovery, а не из проваленного запроса. Включается так: @@ -1020,7 +1044,12 @@ rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab | `HEAD /v1/{prefix}/namespaces/{ns}` | поддержан (204, если существует, 404 — если нет) | | `HEAD /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | поддержан (204, если существует, 404 — если нет) | | `HEAD /v1/{prefix}/namespaces/{ns}/views/{view}` | поддержан (204, если существует, 404 — если нет) | -| `POST`, `DELETE`, commits (включая view-мутации) | не поддержан | +| `POST /v1/{prefix}/namespaces/{ns}/tables` | поддержан только для дефолтного каталога (create); иначе `403` | +| `POST /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | поддержан только для дефолтного каталога (commit/update); иначе `403` | +| `DELETE /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | поддержан только для дефолтного каталога (drop); иначе `403` | +| `POST /v1/{prefix}/tables/rename` | поддержан только для дефолтного каталога (rename); иначе `403` | +| `POST /v1/{prefix}/namespaces/{ns}/register` | поддержан только для дефолтного каталога (register); иначе `403` | +| Namespace/view-мутации, multi-table transaction commit | не поддержаны ни для одного каталога | `{prefix}` — любой каталог, перечисленный в `catalogs=`: каждый настроенный каталог получает собственный REST prefix, `/v1//...`. `GET @@ -1029,15 +1058,25 @@ rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab привязаться к нужному prefix, не зашивая его в конфигурацию (см. примеры клиентов ниже). Без `warehouse` `/v1/config` объявляет `routing.default-catalog` — как и в phase 1; неизвестное значение `warehouse` возвращает HTTP 400 -(`BadRequestException`). Поле `endpoints` в ответе перечисляет ровно девять -read-роутов из таблицы выше (list/load namespace + namespace-exists, list/load -table + table-exists, list/load view + view-exists), чтобы современный клиент -знал, что писать сюда не стоит; старые клиенты, которые это поле не смотрят, -ничего не теряют. `GET /v1/{prefix}/config` отвечает так же, но из +(`BadRequestException`). Поле `endpoints` в ответе перечисляет девять +read-роутов из таблицы выше для любого каталога (list/load namespace + +namespace-exists, list/load table + table-exists, list/load view + +view-exists); у дефолтного каталога `endpoints` дополнительно несёт пять +write-роутов таблиц из таблицы выше, так что современный клиент может +обнаружить write/read-асимметрию между каталогами через discovery, а не из +провалившегося запроса. `GET /v1/{prefix}/config` отвечает так же, но из собственного handler'а прокси — `overrides.prefix` называет каталог из сегмента пути, а не из query-параметра `warehouse`, — и неизвестный prefix здесь по-прежнему даёт 404. +Отказ в write отвечает `403` (`ForbiddenException`) с сообщением, называющим +резолвленный каталог, например: `Writes are only supported in the default +catalog 'hdp'; namespace 'apache__default' belongs to catalog 'apache', +which is served by the synthetic lock shim and provides no writer +isolation.` Это проверяется на namespace, в который запрос **резолвится**, а +не на prefix из URL — см. [Почему writes работают только в +default-каталоге](#iceberg-rest-catalog-frontend) выше. + Error-ответы несут смапленный HTTP-статус, `type` и `message`, но никогда — server stack trace, потому что этот listener может быть доступен без аутентификации. Тело запроса, которое не удаётся распарсить, отвечает 400 @@ -1045,7 +1084,11 @@ server stack trace, потому что этот listener может быть д принимающего тело. `HEAD`-ответ никогда не пишет тело, как того требует RFC 9110, — включая error-статусы, — так что exists-check на отсутствующий namespace, таблицу или view возвращает обычный 404 без тела, а не 404 с -JSON-телом. +JSON-телом. Диспетчер запросов ловит `Throwable`, а не только `Exception`: +любой `Error`, который ускользнул бы из обработки (например, +`NoSuchMethodError` от несовпадения версий зависимостей, всплывший глубоко +внутри write), теперь маппится в обычный error-ответ, а не улетает мимо +handler'а, оставляя соединение клиента висеть без ответа навсегда. Prefix дефолтного каталога сохраняет federated-представление из phase 1: его собственные базы плюс базы всех остальных каталогов под именами @@ -1108,6 +1151,18 @@ spark.sql.catalog.sales_catalog.warehouse=sales `HiveCatalog.clients`, привязанное к Iceberg `1.9.2`. При апгрейде Iceberg обязательно прогнать `RoutingHiveCatalogTest`, чтобы убедиться, что inject ещё работает. +- Write таблицы открывает output stream в HDFS (`metadata.json`) прямо из + JVM прокси — это первый путь в прокси, который так делает; чтение идёт по + другому, незатронутому classpath. Это требует, чтобы `hadoop-hdfs` и + `hadoop-common` были одной версии в дереве зависимостей. Мавеновская + медиация никогда их не сравнивала (это разные artifact ID), поэтому + `orc-core` (приходит транзитивно через `hive-standalone-metastore`) тянул + устаревший `hadoop-hdfs:2.2.0` рядом с `hadoop-common:2.6.0` в другом месте + дерева, и каждый write падал с `NoSuchMethodError: + FSOutputSummer.` глубоко внутри `DFSOutputStream`. `pom.xml` теперь + исключает этот транзитивный `hadoop-hdfs` и напрямую зависит от + `hadoop-hdfs:2.6.0`, чтобы совпасть с `hadoop-common`. Держите обе версии + согласованными, если переопределяете любую из них. ## Безопасность diff --git a/scripts/hms-real-installation-smoke.simple.env.example b/scripts/hms-real-installation-smoke.simple.env.example index d52560a..9882def 100644 --- a/scripts/hms-real-installation-smoke.simple.env.example +++ b/scripts/hms-real-installation-smoke.simple.env.example @@ -118,3 +118,13 @@ HMS_SMOKE_PARTITION_LOCK_TRANSACTIONAL=false # Management /metrics endpoint. When set, the REST smoke also checks it carries the # hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series. # HMS_SMOKE_REST_METRICS_URL=http://proxy-host:9090/metrics +# Table write round trip: create, load (asserting metadata-location comes back) and drop a table +# through the REST write routes. If left unset, the write step is skipped. Writes only work when +# the request resolves to the default catalog - every other catalog is served by the synthetic +# lock shim, which grants locks without conflict checking, so writes there are refused with 403 +# rather than silently racing concurrent writers. With HMS_SMOKE_REST_SECOND_PREFIX also set, the +# step additionally asserts 403 for a direct create under that prefix and for a create under its +# federated namespace name reached through the default prefix - proving +# the gate is enforced on the resolved catalog, not the request's own prefix. Use a name distinct +# from every other table already on the target so a rerun cannot collide with a leftover. +# HMS_SMOKE_REST_WRITE_TABLE=smoke_rest_written diff --git a/scripts/run-real-installation-smoke.sh b/scripts/run-real-installation-smoke.sh index a07b28e..7c1dd74 100755 --- a/scripts/run-real-installation-smoke.sh +++ b/scripts/run-real-installation-smoke.sh @@ -86,6 +86,16 @@ Optional Iceberg REST env vars: HMS_SMOKE_REST_METRICS_URL management /metrics endpoint; when set, the REST smoke checks it carries hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series + HMS_SMOKE_REST_WRITE_TABLE table name to create/load/drop through the REST write + routes; skipped when unset. Writes only work when the + advertised default-prefix catalog resolves to the real + backend (non-default catalogs are served by the synthetic + lock shim and refuse writes with 403). Requires + HMS_SMOKE_REST_SECOND_PREFIX to also exercise the two + negative checks (a direct create under that prefix and a + create under its federated name under the default prefix, + both expected 403); use a name distinct from every other + table already on the stand so a rerun cannot collide EOF cat <<'EOF' @@ -974,9 +984,11 @@ run_rest_smoke() { [[ "${code}" == "404" ]] \ || fail "unknown REST table expected HTTP 404, got ${code}: $(cat "${body}")" - # Writes are out of scope for this phase; the route must refuse, not half-apply. + # Dropping a table that was never created must not silently succeed with a 2xx - true for + # every catalog, independent of whether that catalog's writes are gated (see the write round + # trip and its negatives further below, guarded by HMS_SMOKE_REST_WRITE_TABLE). code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/no_such_table_smoke" "${body}")" - [[ "${code}" =~ ^2 ]] && fail "REST write route unexpectedly succeeded with HTTP ${code}: $(cat "${body}")" + [[ "${code}" =~ ^2 ]] && fail "dropping a non-existent table unexpectedly succeeded with HTTP ${code}: $(cat "${body}")" # Error responses keep the mapped status, type and message but must not leak the server # stack trace: this listener may be unauthenticated, so a trace would expose internal @@ -997,32 +1009,29 @@ run_rest_smoke() { log "skipping REST unparseable-body check because HMS_SMOKE_REST_ICEBERG_TABLE is not set" fi - # GET /v1/config must advertise the read routes this phase actually serves and none of the - # write routes it refuses: at least one concrete read entry (the namespaces listing route), - # no POST namespaces-create entry and no DELETE entry at all. Checked against both the + # GET /v1/config resolves to the default catalog (no warehouse override), which since phase 5a + # is also this proxy's only write-capable catalog: it must advertise the namespaces read route + # AND its write routes (table create, commit, drop, rename, register). Checked against both the # unprefixed /v1/config and the prefixed /v1/{prefix}/config, which clients pin to via the - # "prefix" override and use identically for discovery. + # "prefix" override and use identically for discovery. The matching non-default-catalog check, + # proving the write routes stay absent there, lives below in the optional second-prefix block. code="$(rest_request GET "/v1/config" "${body}")" [[ "${code}" == "200" ]] || fail "GET /v1/config returned HTTP ${code}: $(cat "${body}")" grep -qF '"GET /v1/{prefix}/namespaces"' "${body}" \ || fail "config does not advertise the namespaces read route: $(cat "${body}")" - if grep -qF 'POST /v1/{prefix}/namespaces"' "${body}"; then - fail "config unexpectedly advertises a namespaces write route: $(cat "${body}")" - fi - if grep -qF 'DELETE ' "${body}"; then - fail "config unexpectedly advertises a write (DELETE) route: $(cat "${body}")" - fi + grep -qF '"POST /v1/{prefix}/namespaces/{namespace}/tables"' "${body}" \ + || fail "default-catalog config does not advertise the table-create write route: $(cat "${body}")" + grep -qF '"DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}"' "${body}" \ + || fail "default-catalog config does not advertise the table-drop write route: $(cat "${body}")" code="$(rest_request GET "/v1/${prefix}/config" "${body}")" [[ "${code}" == "200" ]] || fail "GET /v1/${prefix}/config returned HTTP ${code}: $(cat "${body}")" grep -qF '"GET /v1/{prefix}/namespaces"' "${body}" \ || fail "prefixed config does not advertise the namespaces read route: $(cat "${body}")" - if grep -qF 'POST /v1/{prefix}/namespaces"' "${body}"; then - fail "prefixed config unexpectedly advertises a namespaces write route: $(cat "${body}")" - fi - if grep -qF 'DELETE ' "${body}"; then - fail "prefixed config unexpectedly advertises a write (DELETE) route: $(cat "${body}")" - fi + grep -qF '"POST /v1/{prefix}/namespaces/{namespace}/tables"' "${body}" \ + || fail "prefixed default-catalog config does not advertise the table-create write route: $(cat "${body}")" + grep -qF '"DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}"' "${body}" \ + || fail "prefixed default-catalog config does not advertise the table-drop write route: $(cat "${body}")" # Optional second prefix: proves warehouse discovery and the clean view work for a # non-default catalog too, not only for the one /v1/config already advertised. @@ -1042,6 +1051,20 @@ run_rest_smoke() { code="$(rest_request GET "/v1/config?warehouse=no_such_warehouse_smoke" "${body}")" [[ "${code}" == "400" ]] || fail "unknown warehouse expected HTTP 400, got ${code}: $(cat "${body}")" + # Discovery must advertise the write asymmetry: the non-default catalog's own config carries + # the namespaces read route but none of the write routes the default catalog's config asserted + # above, since only the default catalog's writes reach a real backend lock. + code="$(rest_request GET "/v1/${second_prefix}/config" "${body}")" + [[ "${code}" == "200" ]] || fail "GET /v1/${second_prefix}/config returned HTTP ${code}: $(cat "${body}")" + grep -qF '"GET /v1/{prefix}/namespaces"' "${body}" \ + || fail "non-default-catalog config does not advertise the namespaces read route: $(cat "${body}")" + if grep -qF 'POST /v1/{prefix}/namespaces/{namespace}/tables"' "${body}"; then + fail "non-default-catalog config unexpectedly advertises the table-create write route: $(cat "${body}")" + fi + if grep -qF 'DELETE ' "${body}"; then + fail "non-default-catalog config unexpectedly advertises a write (DELETE) route: $(cat "${body}")" + fi + code="$(rest_request GET "/v1/${second_prefix}/namespaces" "${body}")" [[ "${code}" == "200" ]] || fail "GET /v1/${second_prefix}/namespaces returned HTTP ${code}: $(cat "${body}")" grep -q "\[\"${namespace}\"\]" "${body}" \ @@ -1094,6 +1117,49 @@ run_rest_smoke() { fi fi + # Table writes: supported only where the request resolves to the default catalog, because + # only that catalog's commit path reaches the real backend's Hive lock; every other catalog is + # served by the synthetic lock shim, which grants locks without conflict checking, so a commit + # there would race concurrent writers into silently lost updates. Guarded by + # HMS_SMOKE_REST_WRITE_TABLE so the check stays off unless a stand is known to support it. The + # create/load/drop round trip proves the default catalog actually commits; the two negatives + # prove the gate is enforced on the *resolved* catalog, not just the request's own prefix - a + # federated name under the default prefix is refused exactly like a direct request against the + # non-default prefix. + local write_table="${HMS_SMOKE_REST_WRITE_TABLE:-}" + if [[ -n "${write_table}" ]]; then + local create_body='{"name":"'"${write_table}"'","schema":{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"}]}}' + + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data "${create_body}" "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${namespace}/tables")" + [[ "${code}" == "200" ]] || fail "REST create of '${write_table}' returned HTTP ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${namespace}/tables/${write_table}" "${body}")" + [[ "${code}" == "200" ]] || fail "REST load of freshly-created '${write_table}' returned HTTP ${code}: $(cat "${body}")" + grep -q '"metadata-location"' "${body}" \ + || fail "REST load of '${write_table}' carries no metadata-location: $(cat "${body}")" + + code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/${write_table}" "${body}")" + [[ "${code}" =~ ^2 ]] || fail "REST drop of '${write_table}' returned HTTP ${code}: $(cat "${body}")" + + if [[ -n "${second_prefix}" ]]; then + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data "${create_body}" "${HMS_SMOKE_REST_URL}/v1/${second_prefix}/namespaces/${namespace}/tables")" + [[ "${code}" == "403" ]] \ + || fail "REST create under non-default prefix '${second_prefix}' expected HTTP 403, got ${code}: $(cat "${body}")" + + local write_fed_ns="${second_prefix}${separator}${namespace}" + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data "${create_body}" "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${write_fed_ns}/tables")" + [[ "${code}" == "403" ]] \ + || fail "REST create under federated namespace '${write_fed_ns}' expected HTTP 403, got ${code}: $(cat "${body}")" + else + log "skipping REST write-gate negative checks because HMS_SMOKE_REST_SECOND_PREFIX is not set" + fi + else + log "skipping REST write round trip because HMS_SMOKE_REST_WRITE_TABLE is not set" + fi + local metrics_url="${HMS_SMOKE_REST_METRICS_URL:-}" if [[ -n "${metrics_url}" ]]; then curl -sS -o "${body}" "${metrics_url}" || fail "cannot fetch metrics from ${metrics_url}" diff --git a/smoke-stand/TEST-MATRIX.md b/smoke-stand/TEST-MATRIX.md index ea4628e..d0d4b41 100644 --- a/smoke-stand/TEST-MATRIX.md +++ b/smoke-stand/TEST-MATRIX.md @@ -93,7 +93,7 @@ table is the hand-registered `smoke_iceberg_tbl` (see the stand README). | G4 | Table load returns `metadata-location` and full metadata read from HDFS by the proxy itself | ✅ | n/a | | G5 | Unknown prefix → clean 404 `NoSuchCatalogException` | ✅ | n/a | | G6 | Unknown table → clean 404 | ✅ | n/a | -| G7 | Write route (`DELETE` table) refused, non-2xx | ✅ | n/a | +| G7 | `DELETE` of a non-existent table answers a clean 404, not a silent 2xx | ✅ | n/a | | G8 | `GET /v1/config?warehouse=apache` advertises `prefix=apache` | ✅ | n/a | | G9 | Unknown warehouse (`GET /v1/config?warehouse=no_such_warehouse_smoke`) → clean 400 | ✅ | n/a | | G10 | Clean namespace view under the `apache` prefix lists `default` with no `apache__`-prefixed external names | ✅ | n/a | @@ -107,7 +107,11 @@ table is the hand-registered `smoke_iceberg_tbl` (see the stand README). | G18 | `HEAD` on namespaces/tables answers `204` when present and `404` when absent, including under the non-default `apache` prefix and for a plain Hive table (`smoke_read_hdp`) | ✅ | n/a | | G19 | Error response for a missing namespace carries the mapped `404`, `type` and `message` but no `"stack":[...]` server trace | ✅ | n/a | | G20 | An unparseable `POST .../metrics` body answers `400` (`BadRequestException`), not a `500` | ✅ | n/a | -| G21 | `GET /v1/config` and `GET /v1/{prefix}/config` both advertise the `GET /v1/{prefix}/namespaces` read route and carry no write route (no `POST /v1/{prefix}/namespaces`, no `DELETE` entry) | ✅ | n/a | +| G21 | `GET /v1/config` and `GET /v1/{prefix}/config` (both resolving to the default catalog) advertise the table-create and table-drop write routes, on top of the namespaces read route | ✅ | n/a | +| G22 | `GET /v1/{second-prefix}/config` (non-default catalog) advertises the namespaces read route and carries no write route - proves discovery advertises the write/read asymmetry, not only the default side | ✅ | n/a | +| G23 | Table write round trip on the default catalog: `POST` create (`200`), `GET` load (`metadata-location` present), `DELETE` drop (`2xx`) | ✅ | n/a | +| G24 | Direct `POST` create under the non-default `apache` prefix refused with `403` (`ForbiddenException`) | ✅ | n/a | +| G25 | `POST` create under the federated `apache__default` namespace, reached through the default prefix, refused with `403` - proves the write gate is enforced on the *resolved* catalog, not the request's own prefix | ✅ | n/a | ## F. Not covered, and why @@ -163,6 +167,26 @@ executed is claimed; a row not listed was not repeated and its ✅ stands on the the rebuilt jar; the strengthened assertion was proven to discriminate by temporarily pointing it at a route name the server does not serve and confirming the runner failed with "config does not advertise the namespaces read route" before restoring it. + Later still, jar `1.0.41-931b78d4` (phase 5a: table writes for the default + catalog, the write gate, and asymmetric endpoint advertising; a Hadoop + `hadoop-hdfs`/`hadoop-common` version-alignment fix and the widened + `Throwable` catch-all in `IcebergHttpHandler` landed on top of it) added + rows G22-G25 and updated G7, G21. `--scenario rest` and `--scenario all` + both re-ran green: `GET /v1/config` and `GET /v1/{prefix}/config` (default + catalog) were confirmed to carry the table-create and table-drop write + routes; `GET /v1/apache/config` was confirmed to carry neither. A table + created through `POST /v1/hdp/namespaces/default/tables` loaded back with a + `metadata-location` and dropped with `204`; a direct create under + `/v1/apache/namespaces/default/tables` and a create under + `/v1/hdp/namespaces/apache__default/tables` both answered `403`. The SQL + layer (sections B and C, both HiveServer2 instances) was re-run as the + regression check for the Hadoop dependency change, since table writes and + Hive's own ACID commits now share the same lock path; it passed, with + `stand-hs2-hdp` restarted first (its HiveServer2 session had gone stale + after the stand rebuild - a fresh session opened cleanly against the same, + otherwise-untouched HDFS state) and `HMS_SMOKE_SQL_HDP_SESSION_INIT=set + hive.execution.engine=mr;` supplied for the Hortonworks pass, as documented + in `smoke-stand/env/simple.env`. ## Two caveats on faithfulness diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index 1181a42..47d65e0 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -93,7 +93,7 @@ HDP-клиент не может пользоваться Apache-listener — Th | G4 | Load таблицы возвращает `metadata-location` и полные метаданные, прочитанные из HDFS самим прокси | ✅ | n/a | | G5 | Неизвестный prefix → чистый 404 `NoSuchCatalogException` | ✅ | n/a | | G6 | Неизвестная таблица → чистый 404 | ✅ | n/a | -| G7 | Write-роут (`DELETE` таблицы) отклонён, не-2xx | ✅ | n/a | +| G7 | `DELETE` несуществующей таблицы отвечает чистым 404, а не тихим 2xx | ✅ | n/a | | G8 | `GET /v1/config?warehouse=apache` объявляет `prefix=apache` | ✅ | n/a | | G9 | Неизвестный warehouse (`GET /v1/config?warehouse=no_such_warehouse_smoke`) → чистый 400 | ✅ | n/a | | G10 | Чистое представление namespace под prefix `apache` показывает `default` без утечки внешних имён вида `apache__*` | ✅ | n/a | @@ -107,7 +107,11 @@ HDP-клиент не может пользоваться Apache-listener — Th | G18 | `HEAD` на namespace/таблицу отвечает `204`, если объект существует, и `404`, если нет — в том числе под не-default prefix `apache` и для обычной Hive-таблицы (`smoke_read_hdp`) | ✅ | n/a | | G19 | Error-ответ на отсутствующий namespace несёт смапленные `404`, `type` и `message`, но без `"stack":[...]` server trace | ✅ | n/a | | G20 | Нераспарсиваемое тело `POST .../metrics` отвечает `400` (`BadRequestException`), а не `500` | ✅ | n/a | -| G21 | `GET /v1/config` и `GET /v1/{prefix}/config` оба объявляют read-роут `GET /v1/{prefix}/namespaces` и не несут ни одного write-роута (ни `POST /v1/{prefix}/namespaces`, ни записи `DELETE`) | ✅ | n/a | +| G21 | `GET /v1/config` и `GET /v1/{prefix}/config` (оба резолвятся в default-каталог) объявляют write-роуты create и drop таблицы поверх read-роута namespaces | ✅ | n/a | +| G22 | `GET /v1/{second-prefix}/config` (non-default каталог) объявляет read-роут namespaces и не несёт ни одного write-роута — доказывает, что discovery объявляет write/read-асимметрию, а не только default-сторону | ✅ | n/a | +| G23 | Write round trip таблицы на default-каталоге: `POST` create (`200`), `GET` load (`metadata-location` присутствует), `DELETE` drop (`2xx`) | ✅ | n/a | +| G24 | Прямой `POST` create под non-default prefix `apache` отклонён с `403` (`ForbiddenException`) | ✅ | n/a | +| G25 | `POST` create под federated-namespace `apache__default`, достигнутым через default-prefix, отклонён с `403` — доказывает, что write gate проверяется на *резолвленном* каталоге, а не на prefix запроса | ✅ | n/a | ## F. Что не покрыто и почему @@ -165,6 +169,23 @@ HDP-клиент не может пользоваться Apache-listener — Th пересобранном jar'е; то, что укреплённая проверка действительно различает случаи, подтверждено временной подменой ожидаемого имени роута на несуществующее — раннер упал с сообщением "config does not advertise the namespaces read route", после чего подмена была отменена. + Ещё позже jar `1.0.41-931b78d4` (phase 5a: write-запросы к таблицам для default-каталога, + write gate и асимметричное объявление endpoint'ов; поверх легли фикс выравнивания версий + `hadoop-hdfs`/`hadoop-common` и расширенный catch-all `Throwable` в `IcebergHttpHandler`) + добавил строки G22-G25 и обновил G7, G21. `--scenario rest` и `--scenario all` оба + перепрогнаны зелёными: подтверждено, что `GET /v1/config` и `GET /v1/{prefix}/config` + (default-каталог) несут write-роуты create и drop таблицы; подтверждено, что + `GET /v1/apache/config` не несёт ни одного. Таблица, созданная через + `POST /v1/hdp/namespaces/default/tables`, загрузилась обратно с `metadata-location` и + удалилась `204`; прямой create под `/v1/apache/namespaces/default/tables` и create под + `/v1/hdp/namespaces/apache__default/tables` оба ответили `403`. SQL-слой (разделы B и C, + оба HiveServer2) перепрогнан как регрессионная проверка на изменение Hadoop-зависимостей, + раз write-запросы к таблицам и собственные ACID-коммиты Hive теперь идут по одному и тому же + lock-пути; прошёл, при этом `stand-hs2-hdp` пришлось сначала перезапустить (его сессия + HiveServer2 протухла после пересборки стенда — свежая сессия открылась штатно против того же, + иначе не тронутого состояния HDFS), а для прохода через Hortonworks понадобился + `HMS_SMOKE_SQL_HDP_SESSION_INIT=set hive.execution.engine=mr;`, как документировано в + `smoke-stand/env/simple.env`. ## Две оговорки честности diff --git a/smoke-stand/env/simple.env b/smoke-stand/env/simple.env index 24d0ab6..49a3b6f 100644 --- a/smoke-stand/env/simple.env +++ b/smoke-stand/env/simple.env @@ -97,6 +97,13 @@ HMS_SMOKE_REST_SECOND_PREFIX=apache HMS_SMOKE_REST_SECOND_ICEBERG_TABLE=smoke_iceberg_tbl_ap HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE=smoke_read_ap +# Table write round trip (create/load/drop) plus the two negatives (direct create under the +# non-default prefix, and create under the federated name of that prefix's namespace under the +# default prefix) - both must answer 403, since only the default catalog's commit path reaches a +# real backend lock. The name is distinct from every other table already on the stand so a rerun +# cannot collide with a leftover from a previous pass. +HMS_SMOKE_REST_WRITE_TABLE=smoke_rest_written + # Management metrics endpoint, reachable from the host. When set, the REST smoke also checks it # carries the hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series. HMS_SMOKE_REST_METRICS_URL=http://localhost:19090/metrics From 902edf027a3f79eb0cb650f531e9260c7181b3d8 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 12:40:22 +0300 Subject: [PATCH 41/50] Exclude Xerces from hadoop-hdfs and add write-route drift guard hadoop-hdfs:2.6.0 pulls in xerces:xercesImpl/xml-apis for its offline image/edits viewer only, never for the DFSClient runtime path this proxy uses. xercesImpl ships META-INF/services/javax.xml.parsers.* entries, which in the shaded fat jar would make a 2007-vintage parser with known CVEs the JVM-wide JAXP provider for every DocumentBuilderFactory/SAXParserFactory lookup in the process. Exclude both from the hadoop-hdfs dependency block and document the convention in AGENTS.md. Also add an exhaustiveness test tying WriteRouteGate.WRITE_ROUTES to every RESTCatalogAdapter.Route constant, so a route added on the next vendored adapter upgrade fails the test until explicitly classified as a write or a deliberate non-write - closing the exact drift that once let COMMIT_TRANSACTION and the view/namespace writes through unguarded. --- AGENTS.md | 1 + pom.xml | 15 +++++++++ .../hmsproxy/restcatalog/WriteRouteGate.java | 9 +++++ .../restcatalog/WriteRouteGateTest.java | 33 +++++++++++++++++++ 4 files changed, 58 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 8301edf..ec64e19 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -101,6 +101,7 @@ scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple. - Сохраняй совместимость с Java 17. Не добавляй требования к более новым версиям языка или runtime. - Не добавляй новые зависимости без реальной необходимости; Hive/Hadoop dependency convergence хрупкий. `hadoop-hdfs` и `hadoop-common` обязаны быть одной версии: Maven их не сравнивает (разные artifact ID), а рассинхрон ломает `DFSOutputStream` (`NoSuchMethodError: FSOutputSummer.`) при любой записи в HDFS изнутри JVM прокси - `pom.xml` явно исключает транзитивный `hadoop-hdfs` из `orc-core` и держит `hadoop-hdfs:2.6.0` напрямую, чтобы совпасть с `hadoop-common`. Проверяй `mvn -o dependency:tree | grep -iE "hadoop-hdfs|hadoop-common"` при трогании этих зависимостей. - Логирование: slf4j-api и binding `slf4j-reload4j` держи на одной версии `${slf4j.version}`. `org.apache.log4j` должен приходить только из reload4j, поэтому `log4j:log4j` исключён во всех зависимостях, которые его тянут (обе Hive-зависимости, `hadoop-mapreduce-client-core`, `curator-test`). Добавляя зависимость, проверь `mvn -o dependency:tree | grep log4j`: второй провайдер этого пакета даст в fat jar классы, выбранные shade-плагином произвольно. +- `hadoop-hdfs` тянет `xerces:xercesImpl`/`xml-apis` (нужны только offline-вьюеру image/edits, не рантайм-пути `DFSClient`, который использует этот прокси) - оба исключены в блоке `hadoop-hdfs` в `pom.xml`. `xercesImpl` несёт `META-INF/services/javax.xml.parsers.*`, что в shaded fat jar делает 2007-летний парсер (с известными CVE) JVM-wide JAXP-провайдером для каждого `DocumentBuilderFactory`/`SAXParserFactory` в процессе, включая Hadoop `Configuration` и парсинг hive-site - и тихо вытесняет парсер JDK. Добавляя зависимость, проверь `mvn -o dependency:tree | grep -iE "xerces|xml-apis"`. - Комментарии оставляй короткими и только там, где они объясняют неочевидное compatibility, security, routing или class-loading поведение. - Не форматируй несвязанные файлы и не меняй generated docs, если твоя задача не требует их обновления. diff --git a/pom.xml b/pom.xml index 80f4fd7..eddf8e8 100644 --- a/pom.xml +++ b/pom.xml @@ -127,6 +127,21 @@ log4j log4j + + + xerces + xercesImpl + + + xml-apis + xml-apis + diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGate.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGate.java index d224a9b..edb0fc7 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGate.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGate.java @@ -42,6 +42,15 @@ final class WriteRouteGate { Route.COMMIT_TRANSACTION); private static final String NAMESPACE_VAR = "namespace"; + /** + * Test-only accessor so {@code WriteRouteGateTest} can assert every {@link Route} constant is + * explicitly classified as a write or a deliberate non-write - never widen this beyond + * package-private. + */ + static Set writeRoutesForTesting() { + return WRITE_ROUTES; + } + private final String defaultCatalogName; private final Function catalogForNamespace; diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGateTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGateTest.java index 9ba8e81..e5483ab 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGateTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGateTest.java @@ -1,7 +1,9 @@ package io.github.mmalykhin.hmsproxy.restcatalog; +import java.util.EnumSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.UpdateRequirement; import org.apache.iceberg.catalog.Namespace; @@ -33,6 +35,18 @@ public class WriteRouteGateTest { private static final WriteRouteGate GATE = new WriteRouteGate(DEFAULT_CATALOG, WriteRouteGateTest::catalogForNamespace); + // Every Route constant that is deliberately NOT a write, spelled out one by one so that adding + // a new constant to RESTCatalogAdapter.Route (e.g. on the next vendored-adapter upgrade) fails + // everyRouteIsClassifiedAsWriteOrDeliberateNonWrite below until someone classifies it - this is + // the exact gap that once let COMMIT_TRANSACTION and the view/namespace writes through + // unguarded (see git history around "Close write-gate hole"). + private static final Set DELIBERATE_NON_WRITE_ROUTES = EnumSet.of( + Route.TOKENS, Route.SEPARATE_AUTH_TOKENS_URI, Route.CONFIG, + Route.LIST_NAMESPACES, Route.NAMESPACE_EXISTS, Route.LOAD_NAMESPACE, + Route.LIST_TABLES, Route.TABLE_EXISTS, Route.LOAD_TABLE, + Route.REPORT_METRICS, + Route.LIST_VIEWS, Route.VIEW_EXISTS, Route.LOAD_VIEW); + private static String catalogForNamespace(String externalDbName) { if (externalDbName != null && externalDbName.startsWith(OTHER_CATALOG + SEPARATOR)) { return OTHER_CATALOG; @@ -40,6 +54,25 @@ private static String catalogForNamespace(String externalDbName) { return DEFAULT_CATALOG; } + @Test + public void everyRouteIsClassifiedAsWriteOrDeliberateNonWrite() { + Set writeRoutes = WriteRouteGate.writeRoutesForTesting(); + for (Route route : Route.values()) { + boolean isWrite = writeRoutes.contains(route); + boolean isDeliberateNonWrite = DELIBERATE_NON_WRITE_ROUTES.contains(route); + Assert.assertFalse( + "Route." + route.name() + " is classified as both a write route (in " + + "WriteRouteGate.WRITE_ROUTES) and a deliberate non-write route (in this test's " + + "DELIBERATE_NON_WRITE_ROUTES); fix whichever set is wrong.", + isWrite && isDeliberateNonWrite); + Assert.assertTrue( + "Route." + route.name() + " is not classified anywhere: classify the new route as a " + + "write (add it to WriteRouteGate.WRITE_ROUTES) or as a deliberate non-write (add " + + "it to this test's DELIBERATE_NON_WRITE_ROUTES).", + isWrite || isDeliberateNonWrite); + } + } + @Test public void readRoutesAreAlwaysAllowed() { Map vars = Map.of("namespace", OTHER_CATALOG + SEPARATOR + "default"); From 80aedd3201cff616bb08a826f63940f8be7698c2 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 12:57:33 +0300 Subject: [PATCH 42/50] Cover REST commit, rename and gate negatives beyond CREATE_TABLE in smoke The REST write smoke previously only drove create/load/drop plus two CREATE_TABLE negatives. Extend the HMS_SMOKE_REST_WRITE_TABLE-guarded block to add a real commit against the just-created table (asserting the returned metadata-location changed, proving HiveTableOperations.commit actually wrote a new metadata file instead of silently no-oping), a rename round trip, and three more write-route gate negatives - COMMIT_TRANSACTION on a federated table, CREATE_NAMESPACE with a federated name, and rename with a federated destination namespace. COMMIT_TRANSACTION was a critical bypass found during phase 5a and had so far only been covered by unit tests. The block now drops both the write table's original and renamed names defensively before creating, so a rerun on a dirty stand cannot half-fail. Document the new coverage in TEST-MATRIX.md/.ru.md (rows G26-G30) and the 2026-07-28 revalidation-log entry. --- scripts/run-real-installation-smoke.sh | 107 +++++++++++++++++++++---- smoke-stand/TEST-MATRIX.md | 19 +++++ smoke-stand/TEST-MATRIX.ru.md | 20 +++++ 3 files changed, 129 insertions(+), 17 deletions(-) diff --git a/scripts/run-real-installation-smoke.sh b/scripts/run-real-installation-smoke.sh index 7c1dd74..2e75a6b 100755 --- a/scripts/run-real-installation-smoke.sh +++ b/scripts/run-real-installation-smoke.sh @@ -86,16 +86,19 @@ Optional Iceberg REST env vars: HMS_SMOKE_REST_METRICS_URL management /metrics endpoint; when set, the REST smoke checks it carries hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series - HMS_SMOKE_REST_WRITE_TABLE table name to create/load/drop through the REST write - routes; skipped when unset. Writes only work when the - advertised default-prefix catalog resolves to the real - backend (non-default catalogs are served by the synthetic - lock shim and refuse writes with 403). Requires - HMS_SMOKE_REST_SECOND_PREFIX to also exercise the two - negative checks (a direct create under that prefix and a - create under its federated name under the default prefix, - both expected 403); use a name distinct from every other - table already on the stand so a rerun cannot collide + HMS_SMOKE_REST_WRITE_TABLE table name to create, commit, rename and drop through + the REST write routes; skipped when unset. Writes only + work when the advertised default-prefix catalog resolves + to the real backend (non-default catalogs are served by + the synthetic lock shim and refuse writes with 403). The + block drops both this name and "_renamed" + defensively before creating, so a rerun on a dirty stand + cannot half-fail. Requires HMS_SMOKE_REST_SECOND_PREFIX to + also exercise the gate negatives: CREATE_TABLE under that + prefix and under its federated name, COMMIT_TRANSACTION + naming a federated table, CREATE_NAMESPACE with a + federated name, and RENAME with a federated destination - + all expected 403 EOF cat <<'EOF' @@ -1121,26 +1124,61 @@ run_rest_smoke() { # only that catalog's commit path reaches the real backend's Hive lock; every other catalog is # served by the synthetic lock shim, which grants locks without conflict checking, so a commit # there would race concurrent writers into silently lost updates. Guarded by - # HMS_SMOKE_REST_WRITE_TABLE so the check stays off unless a stand is known to support it. The - # create/load/drop round trip proves the default catalog actually commits; the two negatives - # prove the gate is enforced on the *resolved* catalog, not just the request's own prefix - a - # federated name under the default prefix is refused exactly like a direct request against the - # non-default prefix. + # HMS_SMOKE_REST_WRITE_TABLE so the check stays off unless a stand is known to support it. + # + # The round trip below exercises create, a REAL commit against the just-created table, and a + # rename - not just create/load/drop - because the commit is the lock-taking path the whole + # phase-5a safety argument is about, and rename is a second write route entirely. The gate + # negatives cover write routes beyond CREATE_TABLE too: COMMIT_TRANSACTION was a critical + # bypass found during this phase and, unlike CREATE_TABLE, is otherwise only pinned down by + # unit tests. All gate negatives prove enforcement on the *resolved* catalog, not just the + # request's own prefix - a federated name under the default prefix is refused exactly like a + # direct request against the non-default prefix. local write_table="${HMS_SMOKE_REST_WRITE_TABLE:-}" if [[ -n "${write_table}" ]]; then + local renamed_write_table="${write_table}_renamed" + # A rerun against a dirty stand must not half-fail on a leftover from a previous run left + # under either name this block can leave the table under; neither drop result is asserted. + local discard="" + discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/${write_table}" "${body}")" + discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/${renamed_write_table}" "${body}")" + local create_body='{"name":"'"${write_table}"'","schema":{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"}]}}' code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ --data "${create_body}" "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${namespace}/tables")" [[ "${code}" == "200" ]] || fail "REST create of '${write_table}' returned HTTP ${code}: $(cat "${body}")" + local create_metadata_location="" + create_metadata_location="$(grep -o '"metadata-location"[[:space:]]*:[[:space:]]*"[^"]*"' "${body}" | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/')" + [[ -n "${create_metadata_location}" ]] \ + || fail "REST create of '${write_table}' carries no metadata-location: $(cat "${body}")" + + local table_uuid="" + table_uuid="$(grep -o '"table-uuid"[[:space:]]*:[[:space:]]*"[^"]*"' "${body}" | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/')" + [[ -n "${table_uuid}" ]] \ + || fail "REST create of '${write_table}' carries no metadata.table-uuid: $(cat "${body}")" + code="$(rest_request GET "/v1/${prefix}/namespaces/${namespace}/tables/${write_table}" "${body}")" [[ "${code}" == "200" ]] || fail "REST load of freshly-created '${write_table}' returned HTTP ${code}: $(cat "${body}")" grep -q '"metadata-location"' "${body}" \ || fail "REST load of '${write_table}' carries no metadata-location: $(cat "${body}")" - code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/${write_table}" "${body}")" - [[ "${code}" =~ ^2 ]] || fail "REST drop of '${write_table}' returned HTTP ${code}: $(cat "${body}")" + # Real commit against the existing table: the phase-5a safety argument rests on this path + # taking a real Hive lock via HiveTableOperations.commit. A commit that silently no-ops + # would still answer 200 but hand back the SAME metadata-location, so the check that matters + # is the new location differing from create's, not merely the status code. + local commit_body='{"requirements":[{"type":"assert-table-uuid","uuid":"'"${table_uuid}"'"}],"updates":[{"action":"set-properties","updates":{"smoke":"committed"}}]}' + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data "${commit_body}" "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${namespace}/tables/${write_table}")" + [[ "${code}" == "200" ]] || fail "REST commit of '${write_table}' returned HTTP ${code}: $(cat "${body}")" + + local commit_metadata_location="" + commit_metadata_location="$(grep -o '"metadata-location"[[:space:]]*:[[:space:]]*"[^"]*"' "${body}" | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/')" + [[ -n "${commit_metadata_location}" ]] \ + || fail "REST commit of '${write_table}' carries no metadata-location: $(cat "${body}")" + [[ "${commit_metadata_location}" != "${create_metadata_location}" ]] \ + || fail "REST commit of '${write_table}' did not write a new metadata file: metadata-location is still '${commit_metadata_location}'" if [[ -n "${second_prefix}" ]]; then code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ @@ -1153,9 +1191,44 @@ run_rest_smoke() { --data "${create_body}" "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${write_fed_ns}/tables")" [[ "${code}" == "403" ]] \ || fail "REST create under federated namespace '${write_fed_ns}' expected HTTP 403, got ${code}: $(cat "${body}")" + + # COMMIT_TRANSACTION naming a federated table: the bypass this phase actually found, and + # otherwise only covered by unit tests. + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"table-changes":[{"identifier":{"namespace":["'"${write_fed_ns}"'"],"name":"t"},"requirements":[],"updates":[]}]}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/transactions/commit")" + [[ "${code}" == "403" ]] \ + || fail "REST COMMIT_TRANSACTION naming federated table in '${write_fed_ns}' expected HTTP 403, got ${code}: $(cat "${body}")" + + # CREATE_NAMESPACE with a federated name. + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"namespace":["'"${second_prefix}${separator}zzz_smoke"'"]}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces")" + [[ "${code}" == "403" ]] \ + || fail "REST CREATE_NAMESPACE with federated name '${second_prefix}${separator}zzz_smoke' expected HTTP 403, got ${code}: $(cat "${body}")" + + # RENAME with a federated destination: proves the destination-side check, not just the + # source. Must run while the source table still exists under its current name. + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"source":{"namespace":["'"${namespace}"'"],"name":"'"${write_table}"'"},"destination":{"namespace":["'"${write_fed_ns}"'"],"name":"'"${write_table}"'"}}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/tables/rename")" + [[ "${code}" == "403" ]] \ + || fail "REST rename with federated destination namespace '${write_fed_ns}' expected HTTP 403, got ${code}: $(cat "${body}")" else log "skipping REST write-gate negative checks because HMS_SMOKE_REST_SECOND_PREFIX is not set" fi + + # Rename round trip: the real table must survive under its new name, not just answer 204. + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"source":{"namespace":["'"${namespace}"'"],"name":"'"${write_table}"'"},"destination":{"namespace":["'"${namespace}"'"],"name":"'"${renamed_write_table}"'"}}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/tables/rename")" + [[ "${code}" == "204" ]] || fail "REST rename of '${write_table}' returned HTTP ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${namespace}/tables/${renamed_write_table}" "${body}")" + [[ "${code}" == "200" ]] || fail "REST load of renamed '${renamed_write_table}' returned HTTP ${code}: $(cat "${body}")" + + code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/${renamed_write_table}" "${body}")" + [[ "${code}" =~ ^2 ]] || fail "REST drop of '${renamed_write_table}' returned HTTP ${code}: $(cat "${body}")" else log "skipping REST write round trip because HMS_SMOKE_REST_WRITE_TABLE is not set" fi diff --git a/smoke-stand/TEST-MATRIX.md b/smoke-stand/TEST-MATRIX.md index d0d4b41..2de77f1 100644 --- a/smoke-stand/TEST-MATRIX.md +++ b/smoke-stand/TEST-MATRIX.md @@ -112,6 +112,11 @@ table is the hand-registered `smoke_iceberg_tbl` (see the stand README). | G23 | Table write round trip on the default catalog: `POST` create (`200`), `GET` load (`metadata-location` present), `DELETE` drop (`2xx`) | ✅ | n/a | | G24 | Direct `POST` create under the non-default `apache` prefix refused with `403` (`ForbiddenException`) | ✅ | n/a | | G25 | `POST` create under the federated `apache__default` namespace, reached through the default prefix, refused with `403` - proves the write gate is enforced on the *resolved* catalog, not the request's own prefix | ✅ | n/a | +| G26 | Real `POST` commit against the just-created table (`assert-table-uuid` requirement + `set-properties` update) answers `200` and the returned `metadata-location` differs from create's - proof a new metadata file was actually written through `HiveTableOperations.commit`, not a silent no-op | ✅ | n/a | +| G27 | `POST /v1/{prefix}/tables/rename` answers `204`, and `GET` on the new name answers `200` | ✅ | n/a | +| G28 | `POST /v1/{prefix}/transactions/commit` naming a table in the federated `apache__default` namespace refused with `403` | ✅ | n/a | +| G29 | `POST /v1/{prefix}/namespaces` with a federated name (`apache__zzz_smoke`) refused with `403` | ✅ | n/a | +| G30 | `POST /v1/{prefix}/tables/rename` with a federated *destination* namespace (source table still under its current name) refused with `403` - proves the destination side of the gate, not just the source | ✅ | n/a | ## F. Not covered, and why @@ -188,6 +193,20 @@ executed is claimed; a row not listed was not repeated and its ✅ stands on the hive.execution.engine=mr;` supplied for the Hortonworks pass, as documented in `smoke-stand/env/simple.env`. +- **2026-07-28**, jar `1.0.43-c4685ef7` (unchanged on the stand; only the smoke script grew new + checks against it). Added rows G26-G30: the write round trip now includes a REAL commit against + the just-created table and a rename round trip, not just create/load/drop, and the gate + negatives now cover COMMIT_TRANSACTION, CREATE_NAMESPACE and rename-with-federated-destination + on top of the existing CREATE_TABLE pair - COMMIT_TRANSACTION in particular was a critical + bypass found during phase 5a and had until now only been pinned down by unit tests. `--scenario + rest` and `--scenario all` both re-ran green: the create response's `metadata-location` (ending + `00000-...`) differed from the commit response's (`00001-...`), the renamed table loaded back + with `200`, and all three new negatives answered `403`. The G26 assertion was proven to + discriminate by temporarily requiring the commit's `metadata-location` to equal create's + (i.e. asserting a no-op commit); the runner failed with "did not write a new metadata file", + confirming the check would catch a silently no-opped commit; the assertion was restored and + both scenarios re-ran green. + ## Two caveats on faithfulness - The Kerberos profile is complete end to end — client → HiveServer2 → proxy → metastores → HDFS, diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index 47d65e0..fa244d9 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -112,6 +112,11 @@ HDP-клиент не может пользоваться Apache-listener — Th | G23 | Write round trip таблицы на default-каталоге: `POST` create (`200`), `GET` load (`metadata-location` присутствует), `DELETE` drop (`2xx`) | ✅ | n/a | | G24 | Прямой `POST` create под non-default prefix `apache` отклонён с `403` (`ForbiddenException`) | ✅ | n/a | | G25 | `POST` create под federated-namespace `apache__default`, достигнутым через default-prefix, отклонён с `403` — доказывает, что write gate проверяется на *резолвленном* каталоге, а не на prefix запроса | ✅ | n/a | +| G26 | Настоящий `POST` commit против только что созданной таблицы (requirement `assert-table-uuid` + update `set-properties`) отвечает `200`, и возвращённый `metadata-location` отличается от того, что дал create — доказательство, что новый metadata-файл действительно записан через `HiveTableOperations.commit`, а не тихий no-op | ✅ | n/a | +| G27 | `POST /v1/{prefix}/tables/rename` отвечает `204`, а `GET` по новому имени отвечает `200` | ✅ | n/a | +| G28 | `POST /v1/{prefix}/transactions/commit`, называющий таблицу в federated-namespace `apache__default`, отклонён с `403` | ✅ | n/a | +| G29 | `POST /v1/{prefix}/namespaces` с federated-именем (`apache__zzz_smoke`) отклонён с `403` | ✅ | n/a | +| G30 | `POST /v1/{prefix}/tables/rename` с federated destination-namespace (source-таблица ещё под текущим именем) отклонён с `403` — доказывает проверку именно destination-стороны gate, а не только source | ✅ | n/a | ## F. Что не покрыто и почему @@ -187,6 +192,21 @@ HDP-клиент не может пользоваться Apache-listener — Th `HMS_SMOKE_SQL_HDP_SESSION_INIT=set hive.execution.engine=mr;`, как документировано в `smoke-stand/env/simple.env`. +- **2026-07-28**, jar `1.0.43-c4685ef7` (на стенде не менялся; новые проверки добавлены только + в smoke-скрипт). Добавлены строки G26-G30: write round trip теперь включает НАСТОЯЩИЙ commit + против только что созданной таблицы и rename round trip, а не только create/load/drop, а + негативы gate теперь покрывают COMMIT_TRANSACTION, CREATE_NAMESPACE и rename с federated + destination — поверх уже существующей пары CREATE_TABLE. COMMIT_TRANSACTION в частности был + критическим обходом, найденным в ходе этой фазы, и до сих пор был закрыт только unit-тестами. + `--scenario rest` и `--scenario all` оба перепрогнаны зелёными: `metadata-location` из ответа + create (оканчивающийся на `00000-...`) отличался от `metadata-location` из ответа commit + (`00001-...`), переименованная таблица загрузилась обратно с `200`, все три новых негатива + ответили `403`. Проверка G26 доказала свою различающую способность: она была временно изменена + так, чтобы требовать равенства `metadata-location` commit'а и create (то есть утверждать + no-op commit); раннер упал с сообщением "did not write a new metadata file", подтвердив, что + проверка ловит тихо не сработавший commit; проверка была восстановлена, оба сценария + перепрогнаны зелёными. + ## Две оговорки честности - Kerberos-профиль полный сквозняком — клиент → HiveServer2 → прокси → метасторы → HDFS, ни один From 74ce37f4b70b66fc490ec4b230624881dcaa3da0 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 13:43:59 +0300 Subject: [PATCH 43/50] Turn on Iceberg REST under Kerberos and fix its HDFS write path The stand's Kerberos profile has kept the Iceberg REST listener off since phase 1 because SPNEGO needed a GSS-capable curl inside the network; the proxy container's curl is GSS-capable now, so there is no reason left to keep the listener untested there while phase 5a's REST writes and the hadoop-hdfs 2.6.0 bump both land on paths the Kerberos profile also uses. Give the proxy an HTTP/proxy@REALM principal in its existing keytab, enable rest-catalog.* with SPNEGO in the Kerberos properties, and fix two real bugs the first real write under Kerberos surfaced: IcebergRestService was building its own Kerberos-blind Configuration instead of reusing the catalog's HiveConf (breaks with "Failed to specify server's Kerberos principal name"), and the per-catalog conf was missing dfs.data.transfer.protection, which the datanodes require for SASL data transfer once an actual block write - not just a NameNode RPC - is involved. Update TEST-MATRIX(.ru).md and README(.ru).md to reflect the checks that were actually run against the Kerberos profile. --- AGENTS.md | 2 +- smoke-stand/README.md | 16 +++++- smoke-stand/README.ru.md | 16 +++++- smoke-stand/TEST-MATRIX.md | 57 ++++++++++++++++--- smoke-stand/TEST-MATRIX.ru.md | 57 ++++++++++++++++--- smoke-stand/kdc/entrypoint.sh | 3 + .../proxy/hms-proxy-kerberos.properties | 15 +++++ .../hmsproxy/app/HmsProxyApplication.java | 6 +- .../restcatalog/IcebergRestService.java | 11 +++- .../restcatalog/IcebergRestServices.java | 18 +++++- 10 files changed, 174 insertions(+), 27 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ec64e19..1c762d4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -84,7 +84,7 @@ scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple. - Знание о служебном плейсхолдере Hive (`_dummy_database`/`_dummy_table`, константы `SemanticAnalyzer.DUMMY_DATABASE`/`DUMMY_TABLE`, которые Hive шлёт в `LockRequest` для `INSERT ... VALUES`) живёт только в `routing/HivePlaceholderNamespace`. Плейсхолдер не выбирает каталог, не считается вторым namespace lock-запроса и не переписывается интернализацией в реальную backend-базу. Не добавляй локальных сравнений с `_dummy_database` в других классах. - Для RPC, у которых Hive IDL не объявляет исключений (`add_write_notification_log`, `open_txns`, `show_locks` и другие: у их `_result` есть только поле `success`), libthrift 0.9.3 подменяет любое серверное исключение на `TApplicationException("Internal error processing ")`. Текст отказа таких методов виден только в логе - не рассчитывай, что его получит клиент, и не проверяй его в smoke-скриптах. Настоящий HMS теряет свои тексты ошибок так же. - Классификация backend Thrift-ошибок живёт только в `thriftbridge/ThriftFailureClassifier`: «метода нет» - это `TApplicationException` с типом `UNKNOWN_METHOD` (или отсутствие метода в загруженном runtime), transport failure и protocol desync - отдельные категории. Не пиши локальные `instanceof TApplicationException` для решений про fallback, downgrade или переоткрытие соединения. -- `restcatalog` - Iceberg REST front door поверх routing-слоя: per-catalog сервисы работают через in-process прокси `IMetaStoreClient`; для не-default каталогов включён name translation, чтобы клиент видел внутренние имена баз этого каталога. Write-запросы к таблицам (create/commit/drop/rename/register) поддержаны, но только когда namespace резолвится в `routing.default-catalog` - только его таблицы подкреплены реальным HMS-локом, остальные каталоги обслуживает synthetic lock shim без проверки конфликтов. `WriteRouteGate` проверяет резолвленный каталог, а не prefix запроса, поэтому federated-имя под default-prefix отказывается так же, как прямой запрос к non-default prefix; `GET /v1/config` объявляет эту асимметрию в `endpoints`. Не добавляй локальных проверок "разрешён ли write" в других классах - весь gate живёт в `WriteRouteGate`. +- `restcatalog` - Iceberg REST front door поверх routing-слоя: per-catalog сервисы работают через in-process прокси `IMetaStoreClient`; для не-default каталогов включён name translation, чтобы клиент видел внутренние имена баз этого каталога. Write-запросы к таблицам (create/commit/drop/rename/register) поддержаны, но только когда namespace резолвится в `routing.default-catalog` - только его таблицы подкреплены реальным HMS-локом, остальные каталоги обслуживает synthetic lock shim без проверки конфликтов. `WriteRouteGate` проверяет резолвленный каталог, а не prefix запроса, поэтому federated-имя под default-prefix отказывается так же, как прямой запрос к non-default prefix; `GET /v1/config` объявляет эту асимметрию в `endpoints`. Не добавляй локальных проверок "разрешён ли write" в других классах - весь gate живёт в `WriteRouteGate`. Каждый `IcebergRestService` получает per-catalog Hadoop `Configuration` через `IcebergRestServices.open(..., hadoopConfForCatalog)`, которая в проде резолвится в `router.requireBackend(catalog).hiveConf()` - тот же объект, что уже несёт `fs.defaultFS`/Kerberos-настройки catalog..conf.* для Thrift-пути. Не заводи для REST-пути отдельную голую `new Configuration()`: под Kerberos это провалит запись в HDFS ("Failed to specify server's Kerberos principal name"), потому что namenode-принципал у каждого каталога свой и известен только через его собственный conf. ## Парсинг конфигурации diff --git a/smoke-stand/README.md b/smoke-stand/README.md index 5f5df1b..03cba66 100644 --- a/smoke-stand/README.md +++ b/smoke-stand/README.md @@ -244,9 +244,19 @@ docker exec stand-hs2 bash -c "java -cp '/opt/hs2/conf:/opt/hs2/lib/*' org.apach The `metadata.json` is a copy of the first table's, with `location` pointed at the `smoke_iceberg_tbl_ap` path above and a fresh `table-uuid`. -The Kerberos profile leaves the REST listener off: SPNEGO needs a GSS-enabled curl inside -the network, and the handshake itself is already covered end-to-end by -`SpnegoIntegrationTest` on hadoop-minikdc. +The Kerberos profile runs the REST listener too, on the same port (19183) as the plain profile. +It answers SPNEGO: the KDC issues an `HTTP/proxy@SMOKE.LOCAL` principal into the same keytab the +Thrift front door uses, and `hms-proxy-kerberos.properties` points `rest-catalog.kerberos.*` at +it. The handshake itself is also covered end-to-end by `SpnegoIntegrationTest` on hadoop-minikdc; +the stand additionally exercises it with `curl --negotiate` run *inside* `stand-proxy` (the KDC +and the `proxy` hostname only resolve in-network): + +```bash +docker exec stand-proxy kinit -kt /keytabs/smoke-user.keytab smoke-user@SMOKE.LOCAL +docker exec stand-proxy curl -sS --negotiate -u : http://proxy:9183/v1/config +``` + +See `TEST-MATRIX.md` section G for exactly which checks have been run against this profile. ## Hortonworks HiveServer2 (`--profile hdp`) diff --git a/smoke-stand/README.ru.md b/smoke-stand/README.ru.md index 049740e..8781906 100644 --- a/smoke-stand/README.ru.md +++ b/smoke-stand/README.ru.md @@ -243,9 +243,19 @@ docker exec stand-hs2 bash -c "java -cp '/opt/hs2/conf:/opt/hs2/lib/*' org.apach `metadata.json` — копия файла первой таблицы, с `location`, указывающим на путь `smoke_iceberg_tbl_ap` выше, и новым `table-uuid`. -Kerberos-профиль оставляет REST listener выключенным: SPNEGO требует curl с GSS внутри -сети, а сам handshake уже покрыт end-to-end тестом `SpnegoIntegrationTest` на -hadoop-minikdc. +Kerberos-профиль тоже гоняет REST listener, на том же порту (19183), что и plain-профиль. +Он отвечает на SPNEGO: KDC выдаёт принципал `HTTP/proxy@SMOKE.LOCAL` в тот же keytab, которым +пользуется Thrift front door, а `hms-proxy-kerberos.properties` указывает `rest-catalog.kerberos.*` +на него. Сам handshake по-прежнему покрыт end-to-end тестом `SpnegoIntegrationTest` на +hadoop-minikdc; дополнительно стенд проверяет его curl'ом с `--negotiate` *изнутри* +`stand-proxy` (KDC и hostname `proxy` резолвятся только внутри сети): + +```bash +docker exec stand-proxy kinit -kt /keytabs/smoke-user.keytab smoke-user@SMOKE.LOCAL +docker exec stand-proxy curl -sS --negotiate -u : http://proxy:9183/v1/config +``` + +Какие именно проверки прогонялись на этом профиле — раздел G в `TEST-MATRIX.ru.md`. ## Hortonworks HiveServer2 (`--profile hdp`) diff --git a/smoke-stand/TEST-MATRIX.md b/smoke-stand/TEST-MATRIX.md index 2de77f1..80f8211 100644 --- a/smoke-stand/TEST-MATRIX.md +++ b/smoke-stand/TEST-MATRIX.md @@ -81,13 +81,20 @@ only path that covers the Hortonworks front door with a real client. ## G. Iceberg REST catalog front door (host port 19183) -Plain profile only; the Kerberos profile keeps the listener off (SPNEGO is covered in-JVM by -`SpnegoIntegrationTest`). Driven by `--scenario rest` with curl from the host; the loaded -table is the hand-registered `smoke_iceberg_tbl` (see the stand README). +Driven by `--scenario rest` with curl from the host (plain) or curl `--negotiate` from inside +`stand-proxy` (kerberos - the KDC and the `proxy` hostname only resolve in-network, and the +container's curl is GSS-capable). The loaded table is the hand-registered `smoke_iceberg_tbl` +(see the stand README). The Kerberos profile carried the listener disabled through phase 5a +because SPNEGO needed a GSS-capable curl inside the network; once that stopped being true the +listener was turned on there too (`rest-catalog.kerberos.principal=HTTP/proxy@SMOKE.LOCAL`, +same keytab as the Thrift front door) and the write round trip, the write gate and an +unauthenticated-request check were run against it - see the 2026-07-28 kerberos entry below. +The read-only rows (G2-G22, G27-G30) have not yet been re-run against the Kerberos profile and +stay `n/a` until they are. | # | Check | plain | kerberos | | --- | --- | --- | --- | -| G1 | `GET /v1/config` advertises `prefix=hdp` (the default catalog) | ✅ | n/a | +| G1 | `GET /v1/config` advertises `prefix=hdp` (the default catalog) | ✅ | ✅ | | G2 | Namespace list and load (`default`) | ✅ | n/a | | G3 | Table listing shows the Iceberg table and hides plain Hive tables of the same database | ✅ | n/a | | G4 | Table load returns `metadata-location` and full metadata read from HDFS by the proxy itself | ✅ | n/a | @@ -109,14 +116,15 @@ table is the hand-registered `smoke_iceberg_tbl` (see the stand README). | G20 | An unparseable `POST .../metrics` body answers `400` (`BadRequestException`), not a `500` | ✅ | n/a | | G21 | `GET /v1/config` and `GET /v1/{prefix}/config` (both resolving to the default catalog) advertise the table-create and table-drop write routes, on top of the namespaces read route | ✅ | n/a | | G22 | `GET /v1/{second-prefix}/config` (non-default catalog) advertises the namespaces read route and carries no write route - proves discovery advertises the write/read asymmetry, not only the default side | ✅ | n/a | -| G23 | Table write round trip on the default catalog: `POST` create (`200`), `GET` load (`metadata-location` present), `DELETE` drop (`2xx`) | ✅ | n/a | -| G24 | Direct `POST` create under the non-default `apache` prefix refused with `403` (`ForbiddenException`) | ✅ | n/a | -| G25 | `POST` create under the federated `apache__default` namespace, reached through the default prefix, refused with `403` - proves the write gate is enforced on the *resolved* catalog, not the request's own prefix | ✅ | n/a | -| G26 | Real `POST` commit against the just-created table (`assert-table-uuid` requirement + `set-properties` update) answers `200` and the returned `metadata-location` differs from create's - proof a new metadata file was actually written through `HiveTableOperations.commit`, not a silent no-op | ✅ | n/a | +| G23 | Table write round trip on the default catalog: `POST` create (`200`), `GET` load (`metadata-location` present), `DELETE` drop (`2xx`) | ✅ | ✅ | +| G24 | Direct `POST` create under the non-default `apache` prefix refused with `403` (`ForbiddenException`) | ✅ | ✅ | +| G25 | `POST` create under the federated `apache__default` namespace, reached through the default prefix, refused with `403` - proves the write gate is enforced on the *resolved* catalog, not the request's own prefix | ✅ | ✅ | +| G26 | Real `POST` commit against the just-created table (`assert-table-uuid` requirement + `set-properties` update) answers `200` and the returned `metadata-location` differs from create's - proof a new metadata file was actually written through `HiveTableOperations.commit`, not a silent no-op | ✅ | ✅ | | G27 | `POST /v1/{prefix}/tables/rename` answers `204`, and `GET` on the new name answers `200` | ✅ | n/a | | G28 | `POST /v1/{prefix}/transactions/commit` naming a table in the federated `apache__default` namespace refused with `403` | ✅ | n/a | | G29 | `POST /v1/{prefix}/namespaces` with a federated name (`apache__zzz_smoke`) refused with `403` | ✅ | n/a | | G30 | `POST /v1/{prefix}/tables/rename` with a federated *destination* namespace (source table still under its current name) refused with `403` - proves the destination side of the gate, not just the source | ✅ | n/a | +| G31 | A request without `--negotiate` is rejected `401` with a `WWW-Authenticate: Negotiate` challenge and an empty body | n/a | ✅ | ## F. Not covered, and why @@ -207,6 +215,39 @@ executed is claimed; a row not listed was not repeated and its ✅ stands on the confirming the check would catch a silently no-opped commit; the assertion was restored and both scenarios re-ran green. +- **2026-07-28** (second entry), the Iceberg REST listener was turned on for the first time in + the Kerberos profile: the KDC gained an `HTTP/proxy@SMOKE.LOCAL` principal in the same keytab + the Thrift front door already uses, and `hms-proxy-kerberos.properties` gained a + `rest-catalog.*` block pointing at it, on the same port 19183 the plain profile uses. Bringing + the stand up this way surfaced a real bug, not just a missing config row: `IcebergRestService` + built its own bare `Configuration` instead of reusing the catalog's Kerberos-aware `HiveConf`, + so every REST write failed with "Failed to specify server's Kerberos principal name" once the + NameNode RPC was reached; fixed by threading `CatalogBackend.hiveConf()` through + `IcebergRestServices.open(...)`. A second, stand-only gap followed once the NameNode RPC + itself worked: the per-catalog Hadoop conf was missing `dfs.data.transfer.protection`, so a + create's actual block write to the datanode reset the connection ("could only be written to 0 + of the 1 minReplication nodes") even though a plain NameNode-only RPC (the existing purge-path + delete) had never needed it; added `catalog.hdp.conf.dfs.data.transfer.protection=authentication` + and the same key for `catalog.apache` to `hms-proxy-kerberos.properties`, matching what + `hdfs/hadoop-kerberos*.env` already requires of the datanodes. With both fixed, first + `docker exec stand-proxy /opt/hms-proxy/scripts/run-real-installation-smoke-kerberos.sh + --scenario all` was re-run to confirm the Hadoop dependency bump the REST feature travelled in + on (`hadoop-hdfs` 2.2.0 -> 2.6.0) had not regressed the existing kerberized Thrift/lock paths - + it completed with `scenario 'all' completed successfully` (the notification-negative check's + `TApplicationException` is the documented libthrift 0.9.3 behavior for exception-less RPCs, not + a failure). Then, from inside `stand-proxy` after `kinit -kt smoke-user.keytab`, curl + `--negotiate` drove rows G1, G23-G26 and the new G31 (below): an unauthenticated request got a + clean `401`/`WWW-Authenticate: Negotiate`; `GET /v1/config` advertised `prefix=hdp` with the + write routes; a table was created (`200`), loaded back (`200`), committed for real (`200`, + `metadata-location` moved from a `00000-...` file to a `00001-...` one), refused with `403` + both directly under the `apache` prefix and via the federated `apache__default` namespace + under the default prefix, and dropped (`204`). `docker logs stand-proxy` traced the create's + and commit's `lock`/`unlock` to `catalog=hdp, backend=hdp` with small sequential lock IDs (387, + 388 - the real backend's scheme, not the synthetic shim's), and + `logs/hms-proxy-audit.log` carried `"authenticatedUser":"smoke-user@SMOKE.LOCAL"` on every one + of those entries. The remaining Kerberos-column read-only rows (G2-G22, G27-G30) were not + re-run and stay `n/a`. + ## Two caveats on faithfulness - The Kerberos profile is complete end to end — client → HiveServer2 → proxy → metastores → HDFS, diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index fa244d9..de155b2 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -81,13 +81,19 @@ HDP-клиент не может пользоваться Apache-listener — Th ## G. Iceberg REST catalog front door (host-порт 19183) -Только plain-профиль; Kerberos-профиль держит listener выключенным (SPNEGO покрыт in-JVM -тестом `SpnegoIntegrationTest`). Гоняется через `--scenario rest` curl'ом с хоста; -загружаемая таблица — зарегистрированная вручную `smoke_iceberg_tbl` (см. README стенда). +Гоняется через `--scenario rest` curl'ом с хоста (plain) либо curl'ом с `--negotiate` изнутри +`stand-proxy` (kerberos — KDC и hostname `proxy` резолвятся только внутри сети, а curl в +контейнере собран с GSS). Загружаемая таблица — зарегистрированная вручную `smoke_iceberg_tbl` +(см. README стенда). Kerberos-профиль всю фазу 5a держал listener выключенным, потому что SPNEGO +требовал GSS-способный curl внутри сети; как только это перестало быть верным, listener включили +и там тоже (`rest-catalog.kerberos.principal=HTTP/proxy@SMOKE.LOCAL`, тот же keytab, что и у +Thrift front door), и против него прогнали write round trip, write gate и проверку +неаутентифицированного запроса — см. вторую запись за 2026-07-28 в журнале ниже. Read-only строки +(G2-G22, G27-G30) на Kerberos-профиле пока не перепрогонялись и остаются `n/a`. | # | Проверка | plain | kerberos | | --- | --- | --- | --- | -| G1 | `GET /v1/config` объявляет `prefix=hdp` (default-каталог) | ✅ | n/a | +| G1 | `GET /v1/config` объявляет `prefix=hdp` (default-каталог) | ✅ | ✅ | | G2 | Листинг и load namespace (`default`) | ✅ | n/a | | G3 | Листинг таблиц показывает Iceberg-таблицу и прячет обычные Hive-таблицы той же базы | ✅ | n/a | | G4 | Load таблицы возвращает `metadata-location` и полные метаданные, прочитанные из HDFS самим прокси | ✅ | n/a | @@ -109,14 +115,15 @@ HDP-клиент не может пользоваться Apache-listener — Th | G20 | Нераспарсиваемое тело `POST .../metrics` отвечает `400` (`BadRequestException`), а не `500` | ✅ | n/a | | G21 | `GET /v1/config` и `GET /v1/{prefix}/config` (оба резолвятся в default-каталог) объявляют write-роуты create и drop таблицы поверх read-роута namespaces | ✅ | n/a | | G22 | `GET /v1/{second-prefix}/config` (non-default каталог) объявляет read-роут namespaces и не несёт ни одного write-роута — доказывает, что discovery объявляет write/read-асимметрию, а не только default-сторону | ✅ | n/a | -| G23 | Write round trip таблицы на default-каталоге: `POST` create (`200`), `GET` load (`metadata-location` присутствует), `DELETE` drop (`2xx`) | ✅ | n/a | -| G24 | Прямой `POST` create под non-default prefix `apache` отклонён с `403` (`ForbiddenException`) | ✅ | n/a | -| G25 | `POST` create под federated-namespace `apache__default`, достигнутым через default-prefix, отклонён с `403` — доказывает, что write gate проверяется на *резолвленном* каталоге, а не на prefix запроса | ✅ | n/a | -| G26 | Настоящий `POST` commit против только что созданной таблицы (requirement `assert-table-uuid` + update `set-properties`) отвечает `200`, и возвращённый `metadata-location` отличается от того, что дал create — доказательство, что новый metadata-файл действительно записан через `HiveTableOperations.commit`, а не тихий no-op | ✅ | n/a | +| G23 | Write round trip таблицы на default-каталоге: `POST` create (`200`), `GET` load (`metadata-location` присутствует), `DELETE` drop (`2xx`) | ✅ | ✅ | +| G24 | Прямой `POST` create под non-default prefix `apache` отклонён с `403` (`ForbiddenException`) | ✅ | ✅ | +| G25 | `POST` create под federated-namespace `apache__default`, достигнутым через default-prefix, отклонён с `403` — доказывает, что write gate проверяется на *резолвленном* каталоге, а не на prefix запроса | ✅ | ✅ | +| G26 | Настоящий `POST` commit против только что созданной таблицы (requirement `assert-table-uuid` + update `set-properties`) отвечает `200`, и возвращённый `metadata-location` отличается от того, что дал create — доказательство, что новый metadata-файл действительно записан через `HiveTableOperations.commit`, а не тихий no-op | ✅ | ✅ | | G27 | `POST /v1/{prefix}/tables/rename` отвечает `204`, а `GET` по новому имени отвечает `200` | ✅ | n/a | | G28 | `POST /v1/{prefix}/transactions/commit`, называющий таблицу в federated-namespace `apache__default`, отклонён с `403` | ✅ | n/a | | G29 | `POST /v1/{prefix}/namespaces` с federated-именем (`apache__zzz_smoke`) отклонён с `403` | ✅ | n/a | | G30 | `POST /v1/{prefix}/tables/rename` с federated destination-namespace (source-таблица ещё под текущим именем) отклонён с `403` — доказывает проверку именно destination-стороны gate, а не только source | ✅ | n/a | +| G31 | Запрос без `--negotiate` отклоняется `401` с вызовом `WWW-Authenticate: Negotiate` и пустым телом | n/a | ✅ | ## F. Что не покрыто и почему @@ -207,6 +214,40 @@ HDP-клиент не может пользоваться Apache-listener — Th проверка ловит тихо не сработавший commit; проверка была восстановлена, оба сценария перепрогнаны зелёными. +- **2026-07-28** (вторая запись), Iceberg REST listener впервые включён на Kerberos-профиле: + у KDC появился принципал `HTTP/proxy@SMOKE.LOCAL` в том же keytab, которым уже пользуется + Thrift front door, а `hms-proxy-kerberos.properties` получил блок `rest-catalog.*`, + указывающий на него, на том же порту 19183, что и plain-профиль. Подъём стенда в таком виде + вскрыл настоящий баг, а не просто отсутствующую строку конфига: `IcebergRestService` строил + собственную голую `Configuration` вместо того, чтобы переиспользовать Kerberos-осведомлённый + `HiveConf` каталога, поэтому любой REST-write падал с "Failed to specify server's Kerberos + principal name" сразу после того, как RPC до NameNode доходил; починено протягиванием + `CatalogBackend.hiveConf()` через `IcebergRestServices.open(...)`. Следом обнаружился второй, + специфичный только для стенда пробел — уже после того, как сам RPC к NameNode заработал: + в per-catalog Hadoop-конфиге не хватало `dfs.data.transfer.protection`, так что настоящая + запись блока на datanode при create рвала соединение ("could only be written to 0 of the 1 + minReplication nodes"), хотя чисто NameNode-овый RPC (существующий delete в purge-пути) в + этом ключе никогда не нуждался; добавлены `catalog.hdp.conf.dfs.data.transfer.protection=authentication` + и та же настройка для `catalog.apache` в `hms-proxy-kerberos.properties`, в соответствии с тем, + что `hdfs/hadoop-kerberos*.env` уже требует от datanode'ов. После обоих исправлений сначала + перепрогнан `docker exec stand-proxy /opt/hms-proxy/scripts/run-real-installation-smoke-kerberos.sh + --scenario all`, чтобы подтвердить, что апгрейд Hadoop-зависимости, вместе с которым приехала + REST-фича (`hadoop-hdfs` 2.2.0 -> 2.6.0), не сломал уже существующие керберизованные + Thrift/lock-пути — прогон завершился `scenario 'all' completed successfully` + (`TApplicationException` у негативной notification-проверки — задокументированное поведение + libthrift 0.9.3 для RPC без объявленных исключений, а не провал). Затем, изнутри `stand-proxy` + после `kinit -kt smoke-user.keytab`, curl с `--negotiate` прогнал строки G1, G23-G26 и новую + G31 (ниже): неаутентифицированный запрос получил чистый `401`/`WWW-Authenticate: Negotiate`; + `GET /v1/config` объявил `prefix=hdp` вместе с write-роутами; таблица создана (`200`), + загружена обратно (`200`), закоммичена по-настоящему (`200`, `metadata-location` сместился с + файла `00000-...` на `00001-...`), отклонена с `403` и напрямую под prefix `apache`, и через + federated-namespace `apache__default` под default-prefix, и удалена (`204`). + `docker logs stand-proxy` показал, что `lock`/`unlock` create и commit прошли через + `catalog=hdp, backend=hdp` с небольшими последовательными lock ID (387, 388 — схема настоящего + бэкенда, а не synthetic-shim'а), а `logs/hms-proxy-audit.log` нёс + `"authenticatedUser":"smoke-user@SMOKE.LOCAL"` в каждой из этих записей. Остальные read-only + строки Kerberos-колонки (G2-G22, G27-G30) не перепрогонялись и остаются `n/a`. + ## Две оговорки честности - Kerberos-профиль полный сквозняком — клиент → HiveServer2 → прокси → метасторы → HDFS, ни один diff --git a/smoke-stand/kdc/entrypoint.sh b/smoke-stand/kdc/entrypoint.sh index 6d1bfe6..7942a1d 100644 --- a/smoke-stand/kdc/entrypoint.sh +++ b/smoke-stand/kdc/entrypoint.sh @@ -20,6 +20,9 @@ principals=( "hive/hms-apache@${REALM}:hms-apache.keytab" "hive/hms-hdp@${REALM}:hms-hdp.keytab" "hive/proxy@${REALM}:proxy.keytab" + # SPNEGO for the proxy's Iceberg REST listener; same keytab as the Thrift service principal + # since both run in the same container under the same hostname. + "HTTP/proxy@${REALM}:proxy.keytab" "hive/hs2@${REALM}:hs2.keytab" "hive/hs2-hdp@${REALM}:hs2-hdp.keytab" "hdfs/namenode@${REALM}:namenode.keytab" diff --git a/smoke-stand/proxy/hms-proxy-kerberos.properties b/smoke-stand/proxy/hms-proxy-kerberos.properties index 9087248..3f8edfd 100644 --- a/smoke-stand/proxy/hms-proxy-kerberos.properties +++ b/smoke-stand/proxy/hms-proxy-kerberos.properties @@ -20,6 +20,14 @@ additional-frontends.hdp.port=9084 additional-frontends.hdp.frontend-profile=HORTONWORKS_3_1_0_3_1_0_78 additional-frontends.hdp.standalone-metastore-jar=/opt/hms-proxy/hive-metastore/hive-standalone-metastore-3.1.0.3.1.0.0-78.jar +# Iceberg REST catalog front door (read-only, experimental) over SPNEGO. The KDC container's +# curl is GSS-capable (mit-krb5), so the round trip is exercised with --negotiate the same way +# the plain profile exercises it unauthenticated. +rest-catalog.enabled=true +rest-catalog.port=9183 +rest-catalog.kerberos.principal=HTTP/proxy@SMOKE.LOCAL +rest-catalog.kerberos.keytab=/keytabs/proxy.keytab + routing.default-catalog=hdp routing.catalog-db-separator=__ @@ -55,6 +63,13 @@ catalog.hdp.conf.dfs.namenode.kerberos.principal=hdfs/namenode@SMOKE.LOCAL catalog.apache.conf.dfs.namenode.kerberos.principal=hdfs/namenode-b@SMOKE.LOCAL catalog.hdp.conf.dfs.namenode.kerberos.principal.pattern=* catalog.apache.conf.dfs.namenode.kerberos.principal.pattern=* +# Both HDFS clusters run their datanodes with SASL data-transfer protection instead of +# privileged ports (see hdfs/hadoop-kerberos*.env); a client that skips this key can still +# delete (NameNode-only RPC, which is all the purge path above needed) but a real block write - +# the REST create/commit path below - gets "Connection reset" from the datanode and the write +# fails with 0 of 1 replicas. +catalog.hdp.conf.dfs.data.transfer.protection=authentication +catalog.apache.conf.dfs.data.transfer.protection=authentication catalog.hdp.conf.hadoop.security.authentication=kerberos catalog.apache.conf.hadoop.security.authentication=kerberos diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java index 375d728..39b727e 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/app/HmsProxyApplication.java @@ -73,8 +73,12 @@ public static void main(String[] args) throws Exception { }; try (AdditionalFrontendThriftServers extras = AdditionalFrontendThriftServers.open(config, proxy, frontDoorSecurity); + // Reuses each catalog's own HiveConf (fs.defaultFS, Kerberos namenode principal, ...) + // instead of building REST a second, independent Configuration: see + // IcebergRestServices.open javadoc for why that would break HDFS writes under Kerberos. IcebergRestServices restServices = config.restCatalog().enabled() - ? IcebergRestServices.open(config, proxy, catalogForExternalDb) + ? IcebergRestServices.open(config, proxy, catalogForExternalDb, + catalog -> router.requireBackend(catalog).hiveConf()) : null; RestCatalogServer restServer = RestCatalogServer.open(config, restServices, observability.metrics())) { MetastoreThriftServer server = new MetastoreThriftServer(config, proxy, frontDoorSecurity); diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java index 81dd772..29549a1 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java @@ -75,13 +75,20 @@ public IcebergRestService( ThriftHiveMetastore.Iface delegate, CatalogNameTranslation translationOrNull, String defaultCatalogName, - Function catalogForExternalDb) { + Function catalogForExternalDb, + Configuration hadoopConf) { this.catalogName = Objects.requireNonNull(catalogName, "catalogName"); Objects.requireNonNull(delegate, "delegate"); Objects.requireNonNull(defaultCatalogName, "defaultCatalogName"); Objects.requireNonNull(catalogForExternalDb, "catalogForExternalDb"); + Objects.requireNonNull(hadoopConf, "hadoopConf"); IMetaStoreClient client = RoutingMetaStoreClient.create(delegate, translationOrNull); - this.catalog = new RoutingHiveCatalog(client, new Configuration()); + // Must be the same Configuration CatalogBackend.hiveConf() builds for this catalog's Thrift + // path (fs.defaultFS, dfs.namenode.kerberos.principal, hadoop.security.authentication, ...). + // A bare `new Configuration()` here writes to HDFS fine unauthenticated, but under Kerberos + // HadoopFileIO fails with "Failed to specify server's Kerberos principal name" because the + // namenode's service principal is only known through the catalog's own conf. + this.catalog = new RoutingHiveCatalog(client, hadoopConf); this.catalog.initialize(catalogName, Map.of(CatalogProperties.URI, UNUSED_URI)); this.adapter = new RESTCatalogAdapter(catalog); // A name-translated (non-default) service only ever exposes its own databases, so its own diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java index 791f282..886516d 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestServices.java @@ -5,6 +5,7 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.function.Function; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; /** @@ -32,13 +33,28 @@ private IcebergRestServices(Map byPrefix, String def */ public static IcebergRestServices open( ProxyConfig config, ThriftHiveMetastore.Iface delegate, Function catalogForExternalDb) { + return open(config, delegate, catalogForExternalDb, catalog -> new Configuration()); + } + + /** + * @param hadoopConfForCatalog supplies the Hadoop {@link Configuration} each catalog's REST + * service uses to reach its warehouse filesystem. Production wiring passes {@code + * catalog -> router.requireBackend(catalog).hiveConf()} - the same Configuration the + * Thrift path already built for that catalog - so REST writes see the same + * fs.defaultFS/Kerberos settings instead of a second, independently-built (and + * Kerberos-blind) Configuration. + */ + public static IcebergRestServices open( + ProxyConfig config, ThriftHiveMetastore.Iface delegate, Function catalogForExternalDb, + Function hadoopConfForCatalog) { Map services = new LinkedHashMap<>(); for (String catalog : config.catalogNames()) { CatalogNameTranslation translation = catalog.equals(config.defaultCatalog()) ? null : new CatalogNameTranslation(catalog, config.catalogDbSeparator()); services.put(catalog, - new IcebergRestService(catalog, delegate, translation, config.defaultCatalog(), catalogForExternalDb)); + new IcebergRestService(catalog, delegate, translation, config.defaultCatalog(), catalogForExternalDb, + hadoopConfForCatalog.apply(catalog))); } return new IcebergRestServices(services, config.defaultCatalog()); } From 154a04eb711426d841b689085ca01af26c444907 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 14:07:01 +0300 Subject: [PATCH 44/50] Let the REST metastore client create, alter and drop namespaces RoutingMetaStoreClient now answers IMetaStoreClient.createDatabase, dropDatabase(String,boolean,boolean,boolean) and alterDatabase instead of throwing UnsupportedOperationException, delegating to the Thrift create_database/drop_database/alter_database RPCs and translating names through the existing CatalogNameTranslation. The Database payload passed to create/alter is translated on a copy via the Thrift copy constructor, never by mutating the caller's object. RecordingThriftIface now records both the argument and the Database payload's own name for alter_database, so a missing payload translation fails the test (verified: removing it breaks alterDatabaseTranslatesBothArgumentAndPayload with the expected mismatch, restoring it goes back to green). --- ...7-28-iceberg-rest-phase5b-write-surface.md | 267 ++++++++++++++++++ ...eberg-rest-phase5b-write-surface-design.md | 89 ++++++ .../restcatalog/RoutingMetaStoreClient.java | 39 ++- .../restcatalog/RecordingThriftIface.java | 19 ++ .../RoutingMetaStoreClientTest.java | 27 ++ 5 files changed, 437 insertions(+), 4 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-28-iceberg-rest-phase5b-write-surface.md create mode 100644 docs/superpowers/specs/2026-07-28-iceberg-rest-phase5b-write-surface-design.md diff --git a/docs/superpowers/plans/2026-07-28-iceberg-rest-phase5b-write-surface.md b/docs/superpowers/plans/2026-07-28-iceberg-rest-phase5b-write-surface.md new file mode 100644 index 0000000..d5d6639 --- /dev/null +++ b/docs/superpowers/plans/2026-07-28-iceberg-rest-phase5b-write-surface.md @@ -0,0 +1,267 @@ +# Iceberg REST Phase 5b: Completing the Write Surface Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Implement namespace DDL and make the view writes and transaction commits phase 5a inadvertently enabled official — advertised, tested, documented — per `docs/superpowers/specs/2026-07-28-iceberg-rest-phase5b-write-surface-design.md`. + +**Architecture:** Three client methods (`createDatabase`, `dropDatabase`, `alterDatabase`) are the only missing implementation; everything else already works because 5a's table plumbing is what `HiveViewOperations` and the transaction commit path use. The rest of the phase is advertising, coverage and documentation. + +**Tech Stack:** Java 17 (`JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19`), Maven offline (`mvn -o`), JUnit 4, Iceberg 1.9.2, the docker-compose smoke stand. + +## Global Constraints + +- Java 17; 2-space indent; explicit imports; no new dependencies. +- The default-catalog write restriction is unchanged and must not be relaxed: `WriteRouteGate` keys on the catalog the namespace resolves to, never on the URL prefix, and already covers all thirteen write routes. No task may narrow it. +- Advertise only what is actually served and permitted; non-default catalogs keep advertising reads only. +- Phase-3 per-request metrics bookkeeping (`RequestOutcome`, the `finally` recording) must keep working. +- Bilingual docs: every EN doc change lands with its RU counterpart in the same commit. +- English commit messages, no attribution footers. Commits pre-approved; do NOT push. +- Never run `docker compose down` on the stand — HDFS data and registered tables live in volumes. The stand is currently in the **Kerberos** profile; compose calls for it need `--env-file .env.kerberos --profile kerberos`. + +--- + +### Task 1: Namespace DDL in RoutingMetaStoreClient + +**Files:** +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java` +- Test: `src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java` +- Test fixture: `src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java` + +**Interfaces:** +- Produces: the client answers `createDatabase`, `dropDatabase`, `alterDatabase` instead of throwing `UnsupportedOperationException`. + +- [ ] **Step 1: Learn the exact signatures** + +```bash +JAVAP=~/Library/Java/JavaVirtualMachines/liberica-17.0.19/bin/javap +mkdir -p /tmp/imsc && unzip -o -q hive-metastore/hive-standalone-metastore-3.1.3.jar \ + 'org/apache/hadoop/hive/metastore/IMetaStoreClient.class' -d /tmp/imsc +$JAVAP -cp /tmp/imsc org.apache.hadoop.hive.metastore.IMetaStoreClient \ + | grep -E 'createDatabase|dropDatabase|alterDatabase' +``` + +Record the output in the report and implement only the overloads Iceberg calls — confirm which by disassembling `HiveCatalog`: + +```bash +mkdir -p /tmp/hto && unzip -o -q ~/.m2/repository/org/apache/iceberg/iceberg-hive-metastore/1.9.2/iceberg-hive-metastore-1.9.2.jar -d /tmp/hto +$JAVAP -c -p -cp /tmp/hto org.apache.iceberg.hive.HiveCatalog | grep -E 'createDatabase|dropDatabase|alterDatabase' +``` + +Leave every other overload throwing, as the existing branches do. + +- [ ] **Step 2: Write the failing tests** + +Add to `RoutingMetaStoreClientTest`, in the file's existing style (build a `RecordingThriftIface`, wrap it, call, assert on `recording.calls` and on the returned value). Cover the untranslated client and a `new CatalogNameTranslation("apache", "__")` one. The `Database` argument carries its own name, so — exactly as the phase-5a `alter_table` test learned the hard way — the fixture must record the **Database's own name**, not only the string argument, or a missing translation would not fail the test: + +```java + @Test + public void createDatabaseTranslatesName() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + client.createDatabase(RecordingThriftIface.database("sales")); + assertEquals(List.of("create_database:apache__sales"), recording.calls); + } + + @Test + public void dropDatabaseTranslatesName() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + client.dropDatabase("sales", false, true, false); + assertEquals(List.of("drop_database:apache__sales"), recording.calls); + } + + @Test + public void alterDatabaseTranslatesBothArgumentAndPayload() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + client.alterDatabase("sales", RecordingThriftIface.database("sales")); + assertEquals(List.of("alter_database:apache__sales:apache__sales"), recording.calls); + } +``` + +Adjust the `dropDatabase` overload and the recorded strings to whatever Step 1 showed; the third test's two-part recording (argument name and payload name) is the point — keep that shape. + +- [ ] **Step 3: Run to verify red** + +```bash +export JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 +mvn -o -Dtest=RoutingMetaStoreClientTest test +``` + +Expected: `UnsupportedOperationException: HMS proxy REST gateway does not support IMetaStoreClient.`. + +- [ ] **Step 4: Implement** + +Add the branches to the switch in `RoutingInvocationHandler.invoke`, translating with the existing `db(...)` helper. For `createDatabase` and `alterDatabase` translate the `Database` payload's own name on a **copy** (thrift copy constructor, as `rewriteDatabase` already does), never by mutating the caller's object. Delegate to `delegate.create_database(...)`, `delegate.drop_database(...)`, `delegate.alter_database(...)` per the Thrift interface. + +- [ ] **Step 5: Prove the tests discriminate** + +Temporarily remove the payload translation from `alterDatabase`, run `alterDatabaseTranslatesBothArgumentAndPayload`, confirm it FAILS, restore, confirm green. Record both runs — phase 5a shipped a test that could not fail because the fixture ignored the payload, and this step exists so that cannot recur. + +- [ ] **Step 6: Run green, then the package** + +```bash +mvn -o -Dtest=RoutingMetaStoreClientTest test +mvn -o -Dtest='io/github/mmalykhin/hmsproxy/restcatalog/*' test +``` + +- [ ] **Step 7: Commit** (include the phase-5b spec and this plan, both untracked) + +```bash +git add src/main src/test docs/superpowers +git commit -m "Let the REST metastore client create, alter and drop namespaces" +``` + +--- + +### Task 2: Advertise the full served write surface + +**Files:** +- Modify: `src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java` (`WRITE_ENDPOINTS`) +- Test: `src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java` + +**Interfaces:** +- Consumes: namespace DDL from Task 1. +- Produces: the default catalog's `endpoints` lists table writes, view writes, namespace DDL and the transaction commit; other catalogs list reads only. + +- [ ] **Step 1: Confirm which `Endpoint` constants exist and how they render** + +Iceberg 1.9.2's `org.apache.iceberg.rest.Endpoint` is the source of truth. List its constants and print the ones you intend to add so the assertions match the real serialized form: + +```bash +JAVAP=~/Library/Java/JavaVirtualMachines/liberica-17.0.19/bin/javap +mkdir -p /tmp/ep && unzip -o -q ~/.m2/repository/org/apache/iceberg/iceberg-core/1.9.2/iceberg-core-1.9.2.jar \ + 'org/apache/iceberg/rest/Endpoint.class' -d /tmp/ep +$JAVAP -constants -cp /tmp/ep org.apache.iceberg.rest.Endpoint | grep -E 'V1_(CREATE|UPDATE|DELETE|REPLACE|RENAME)_(VIEW|NAMESPACE)|V1_COMMIT_TRANSACTION|V1_UPDATE_NAMESPACE' +``` + +Report which constants exist. If one you expected is absent, do not invent it — say so and leave that route unadvertised, noting the mismatch. + +- [ ] **Step 2: Write the failing tests** + +```java + @Test + public void defaultCatalogAdvertisesViewAndNamespaceWrites() throws Exception { + String body = get("/v1/catalog1/config").body(); + assertTrue(body, body.contains("POST /v1/{prefix}/namespaces/{namespace}/views")); + assertTrue(body, body.contains("POST /v1/{prefix}/namespaces")); + assertTrue(body, body.contains("POST /v1/{prefix}/transactions/commit")); + } + + @Test + public void nonDefaultCatalogAdvertisesNoWritesAtAll() throws Exception { + String body = get("/v1/catalog2/config").body(); + assertFalse(body, body.contains("POST /v1/{prefix}/namespaces/{namespace}/views")); + assertFalse(body, body.contains("POST /v1/{prefix}/transactions/commit")); + assertTrue(body, body.contains("GET /v1/{prefix}/namespaces")); + } +``` + +Replace each expected string with the real rendering Step 1 printed if it differs. + +- [ ] **Step 3: Run red, implement, run green** + +Extend `WRITE_ENDPOINTS` with the constants Step 1 confirmed. Keep the read list and the non-default behaviour untouched. + +```bash +mvn -o -Dtest=IcebergRestEndpointIntegrationTest test +mvn -o -Dtest='io/github/mmalykhin/hmsproxy/restcatalog/*' test +``` + +- [ ] **Step 4: Full suite** + +```bash +mvn -o test +``` + +Expected: 0 failures, 0 skipped. + +- [ ] **Step 5: Commit** + +```bash +git add src/main src/test +git commit -m "Advertise the view, namespace and transaction writes the front door serves" +``` + +--- + +### Task 3: Smoke coverage, stand runs, documentation + +**Files:** +- Modify: `scripts/run-real-installation-smoke.sh` — extend the `HMS_SMOKE_REST_WRITE_TABLE`-guarded block +- Modify: `smoke-stand/env/simple.env`, `scripts/hms-real-installation-smoke.simple.env.example` if new variables are needed +- Modify: `README.md`, `README.ru.md`, `CHANGELOG.md`, `CHANGELOG.ru.md` +- Modify after the runs: `smoke-stand/TEST-MATRIX.md`, `smoke-stand/TEST-MATRIX.ru.md` + +- [ ] **Step 1: Add the smoke checks** + +Three round trips, each asserting the effect rather than only a status. I verified all three shapes against the live stand, so these are known-good: + +*Namespace DDL:* `POST /v1/${prefix}/namespaces` with `{"namespace":["smoke_rest_ns"]}` → 200; `GET /v1/${prefix}/namespaces/smoke_rest_ns` → 200; `POST /v1/${prefix}/namespaces/smoke_rest_ns/properties` with `{"removals":[],"updates":{"smoke":"yes"}}` → 200; `GET` it again and assert the property is present in the body; `DELETE /v1/${prefix}/namespaces/smoke_rest_ns` → 204; `GET` again → 404. + +*View writes:* `POST /v1/${prefix}/namespaces/${namespace}/views` with +`{"name":"smoke_rest_view","schema":{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"}]},"view-version":{"version-id":1,"timestamp-ms":1753700000000,"schema-id":0,"summary":{"operation":"create"},"default-namespace":["${namespace}"],"representations":[{"type":"sql","sql":"select 1","dialect":"hive"}]},"properties":{}}` +→ 200 and the body carries a `metadata-location`; `GET .../views` lists it; `DELETE .../views/smoke_rest_view` → 204. + +*Transaction commit:* create a table, capture its `table-uuid` and `metadata-location`, then `POST /v1/${prefix}/transactions/commit` with +`{"table-changes":[{"identifier":{"namespace":["${namespace}"],"name":"
"},"requirements":[{"type":"assert-table-uuid","uuid":""}],"updates":[{"action":"set-properties","updates":{"txn":"yes"}}]}]}` +→ 204, then GET the table and assert its `metadata-location` DIFFERS from the captured one. That difference is the proof the transaction actually committed; a no-op must fail the smoke. + +Use the runner's existing `rest_request`/`fail` idioms and its grep/sed JSON extraction (no jq). Drop leftovers defensively at the start so a rerun on a dirty stand cannot half-fail. + +- [ ] **Step 2: `bash -n scripts/run-real-installation-smoke.sh` passes.** + +- [ ] **Step 3: Rebuild, restage and run on the plain profile** + +The stand is currently in the Kerberos profile. Rebuild and restage the jar, then switch it to the plain profile for these runs: + +```bash +export JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 +mvn -o -DskipTests package +cd smoke-stand && ./prepare.sh && docker compose up -d --build +``` + +Poll until healthy, then: + +```bash +JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 \ + scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple.env --scenario rest +JAVA_HOME=~/Library/Java/JavaVirtualMachines/liberica-17.0.19 \ + scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple.env --scenario all +``` + +Both must end `completed successfully`. + +- [ ] **Step 4: Prove one new assertion can fail** + +Break the transaction-commit assertion (demand the `metadata-location` be unchanged), rerun, confirm the runner reports the failure, restore, rerun green. Record both runs. + +- [ ] **Step 5: Re-run the Kerberos profile for the write paths** + +Phase 5a's Kerberos pass is what exposed the missing `HiveConf` wiring, so the new write paths get the same treatment: + +```bash +cd smoke-stand && docker compose --env-file .env.kerberos --profile kerberos up -d --build +``` + +Poll until healthy, then from inside `stand-proxy` (`kinit -kt /keytabs/smoke-user.keytab smoke-user@SMOKE.LOCAL`, then `curl --negotiate -u :` against `http://proxy:9183`), drive the namespace round trip and the view round trip and report each response. If either fails under Kerberos but passed on the plain profile, that is a real finding — report it rather than working around it. + +- [ ] **Step 6: Docs and matrix, both locales; commit everything** + +The documentation must say plainly that view writes and transaction commits were already reachable after phase 5a and that this phase makes them official — advertised, covered and documented — rather than implying they are new. State that namespace DDL is genuinely new. Repeat the default-catalog restriction and that it applies to every one of these routes. + +```bash +git add scripts smoke-stand README.md README.ru.md CHANGELOG.md CHANGELOG.ru.md +git commit -m "Cover the completed Iceberg REST write surface in the stand smoke" +``` + +--- + +## Self-Review + +- **Spec coverage:** namespace DDL client methods (T1); advertising the full served write surface (T2); smoke for all three round trips plus the discriminating transaction assertion (T3 Steps 1, 4); Kerberos re-run for the new write paths (T3 Step 5); bilingual docs stating honestly what was already reachable (T3 Step 6); the `CREATE_NAMESPACE` federated-name gate test already exists from 5a and is explicitly left in place (Global Constraints forbid narrowing the gate). No gaps. +- **Placeholder scan:** none — every request body is spelled out and was probed against the live stand; the one conditional (a missing `Endpoint` constant) names exactly what to do and to report. +- **Type consistency:** `createDatabase`/`dropDatabase`/`alterDatabase` are declared in T1's Interfaces and consumed by T2's advertising; `WRITE_ENDPOINTS` is the existing field name from phase 5a Task 3; `RoutingMetaStoreClient.create(delegate, translation)` is the existing two-arg factory. diff --git a/docs/superpowers/specs/2026-07-28-iceberg-rest-phase5b-write-surface-design.md b/docs/superpowers/specs/2026-07-28-iceberg-rest-phase5b-write-surface-design.md new file mode 100644 index 0000000..88a5861 --- /dev/null +++ b/docs/superpowers/specs/2026-07-28-iceberg-rest-phase5b-write-surface-design.md @@ -0,0 +1,89 @@ +# Iceberg REST frontend, phase 5b: completing the write surface — design + +Date: 2026-07-28 +Branch: `feature/iceberg-rest-fe-phase1` +Status: approved + +## Goal + +Finish the write surface of the Iceberg REST front door: implement namespace DDL, and make +the view writes and multi-table transaction commits that phase 5a inadvertently enabled +official — advertised, tested and documented. The default-catalog restriction and its gate +are unchanged. + +## What phase 5a actually shipped + +The roadmap assumed views, namespaces and transactions were all unimplemented. Probing the +running stand under Kerberos shows otherwise: + +| Route group | State after 5a | +| --- | --- | +| Namespace DDL | **Not implemented** — `HiveCatalog` needs `createDatabase`, `dropDatabase`, `alterDatabase`, none of which `RoutingMetaStoreClient` answers | +| View writes | **Working** — `POST .../views` returned 200 with a real `metadata-location`, the view listed, `DELETE` returned 204 | +| Multi-table transactions | **Working** — `POST .../transactions/commit` returned 204 and advanced the table's metadata from `00000-…` to `00001-…`, proving a real commit | + +The cause is the same mechanism that handed phase 4a its exists routes for free: the handler +dispatches on the whole `Route` enum with no allowlist, and the client plumbing 5a added for +tables — `createTable`, `MetastoreUtil.alterTable`, the lock family — is exactly what +`HiveViewOperations` and the transaction commit path use. `HiveViewOperations` implements +`HiveOperationsBase` and takes the same `HiveLock`. + +## The problem this creates + +`/v1/config` advertises nine reads and five table writes. View writes and transaction +commits work and are **not** advertised, so a spec-compliant client reading `endpoints` +concludes they are unsupported and will not use them. That is the same discovery lie phase 4b +set out to remove, pointing the other way — under-advertising rather than over-advertising. + +Worse, working functionality has no unit tests, no smoke coverage and no documentation. It +holds together only because nothing has disturbed it. + +## Scope + +1. **Namespace DDL.** Add `createDatabase`, `dropDatabase` and `alterDatabase` to + `RoutingMetaStoreClient`, each applying the same name translation as the existing + branches. This is the only genuinely missing implementation. +2. **Legitimise what works.** Advertise the view-write and transaction endpoints — and the + namespace-DDL ones once they work — in the default catalog's `endpoints` list, alongside + the table writes. Non-default catalogs keep advertising reads only. +3. **Cover it.** Unit tests for the new client branches and for the widened endpoint list; + smoke coverage for a view write round trip, a namespace DDL round trip, and a transaction + commit, each asserting the effect rather than only the status code. +4. **Document it**, both locales, including the honest note that views and transactions were + already reachable before this phase made them official. + +## Safety + +Unchanged and non-negotiable: writes are permitted only where the target namespace resolves +to `routing.default-catalog`, enforced by `WriteRouteGate` on the resolved catalog rather +than the URL prefix. The gate already covers all thirteen write routes, including the ones +this phase legitimises, and phase 5a added the drift-guard test that fails when a new route +appears unclassified. Nothing here relaxes that. + +Namespace DDL raises one question the table work did not: `CREATE_NAMESPACE` names a +namespace that does not exist yet, so "the catalog it resolves to" is decided purely by the +name's own prefix. A request to create `apache__foo` under the default prefix must be refused +exactly as a write into an existing federated namespace is — the gate already does this, and +the phase adds a test pinning it. + +## Error handling + +No new status codes. Namespace DDL failures surface through the same +`configureResponseFromException` mapping as every other route, with the stack stripped. + +## Testing + +- Unit: the three new client branches with translation, mirroring the existing write-method + tests; the widened endpoint list per catalog. +- Smoke (`--scenario rest`, in the existing write-guarded block): a namespace create → + properties update → drop round trip; a view create → list → drop round trip; a transaction + commit asserting the table's `metadata-location` advances. Plus the gate negative for + creating a federated namespace, which already exists and stays. +- Stand: plain profile for the round trips; the Kerberos profile is re-run for the write + paths, since phase 5a's Kerberos pass is what exposed the missing `HiveConf` wiring. +- Docs: README and CHANGELOG both locales; TEST-MATRIX rows after the run. + +## Out of scope + +Relaxing the default-catalog restriction; giving non-default catalogs real writer isolation; +scan planning; OAuth. diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java index 7ab5c56..7ae08cd 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java @@ -22,10 +22,11 @@ /** * Bridges Iceberg's HiveCatalog (which requires IMetaStoreClient) to the proxy's - * ThriftHiveMetastore.Iface. Read paths, table writes (create/drop/alter) and the - * commit-lock RPCs Iceberg's write path needs are implemented; everything else throws - * UnsupportedOperationException. This class only wires the client through: it does not - * restrict writes to the default catalog, that gate is a separate concern. + * ThriftHiveMetastore.Iface. Read paths, table writes (create/drop/alter), namespace DDL + * (create/drop/alter database) and the commit-lock RPCs Iceberg's write path needs are + * implemented; everything else throws UnsupportedOperationException. This class only wires + * the client through: it does not restrict writes to the default catalog, that gate is a + * separate concern. */ public final class RoutingMetaStoreClient { private RoutingMetaStoreClient() { @@ -94,6 +95,16 @@ private Table translateDbName(Table table) { return copy; } + /** Translates a caller-supplied Database's own name to the external name, on a copy. */ + private Database translateDatabaseName(Database database) { + if (translation == null || database == null) { + return database; + } + Database copy = new Database(database); + copy.setName(db(database.getName())); + return copy; + } + @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { String name = method.getName(); @@ -172,6 +183,26 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl return null; } break; + case "createDatabase": + if (paramTypes.length == 1) { + delegate.create_database(translateDatabaseName((Database) args[0])); + return null; + } + break; + case "dropDatabase": + if (paramTypes.length == 4) { + delegate.drop_database( + db((String) args[0]), (Boolean) args[1], (Boolean) args[3]); + return null; + } + break; + case "alterDatabase": + if (paramTypes.length == 2) { + delegate.alter_database( + db((String) args[0]), translateDatabaseName((Database) args[1])); + return null; + } + break; case "dropTable": if (paramTypes.length == 4 && paramTypes[0] == String.class && paramTypes[1] == String.class diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java index 4416828..f887177 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java @@ -126,6 +126,25 @@ private Object handle(Object proxy, Method method, Object[] args) throws Throwab } return result; } + case "create_database": { + Database database = (Database) args[0]; + calls.add("create_database:" + database.getName()); + databases.put(database.getName(), database); + return null; + } + case "drop_database": { + String db = (String) args[0]; + calls.add("drop_database:" + db); + databases.remove(db); + return null; + } + case "alter_database": { + String db = (String) args[0]; + Database database = (Database) args[1]; + calls.add("alter_database:" + db + ":" + database.getName()); + databases.put(db, database); + return null; + } case "create_table": { Table table = (Table) args[0]; calls.add("create_table:" + table.getDbName() + "." + table.getTableName()); diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java index 923480e..2c05d98 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java @@ -156,6 +156,33 @@ public void alterTableTranslatesDatabaseName() throws Exception { List.of("alter_table:apache__default.t1:table=apache__default"), recording.calls); } + @Test + public void createDatabaseTranslatesName() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + client.createDatabase(RecordingThriftIface.database("sales")); + Assert.assertEquals(List.of("create_database:apache__sales"), recording.calls); + } + + @Test + public void dropDatabaseTranslatesName() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + client.dropDatabase("sales", false, true, false); + Assert.assertEquals(List.of("drop_database:apache__sales"), recording.calls); + } + + @Test + public void alterDatabaseTranslatesBothArgumentAndPayload() throws Exception { + RecordingThriftIface recording = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create( + recording.iface, new CatalogNameTranslation("apache", "__")); + client.alterDatabase("sales", RecordingThriftIface.database("sales")); + Assert.assertEquals(List.of("alter_database:apache__sales:apache__sales"), recording.calls); + } + @Test public void lockAndUnlockReachTheDelegate() throws Exception { RecordingThriftIface recording = new RecordingThriftIface(); From 87676b4ff2ffb80ac4b0b735abf5d70d5d4b78f3 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 14:15:21 +0300 Subject: [PATCH 45/50] Fix dropDatabase overload dispatch and honor ignoreUnknownDb The dropDatabase case matched on 4-arg overloads by length alone, so IMetaStoreClient's (String,String,boolean,boolean) default overload (catName, dbName, ...) also matched and cast the dbName String to Boolean, throwing ClassCastException instead of the usual UnsupportedOperationException. Narrow the match to the exact (String,boolean,boolean,boolean) signature, mirroring dropTable's explicit paramTypes checks. Also mirror dropTable's ignoreUnknownTab handling for ignoreUnknownDb: catch NoSuchObjectException from the delegate call and rethrow only when the flag is false, instead of silently discarding it. Strengthen RecordingThriftIface's drop_database case to record the deleteData/cascade booleans (and throw NoSuchObjectException for an unknown database) so argument-order regressions and the ignoreUnknownDb behavior are covered by tests. --- .../restcatalog/RoutingMetaStoreClient.java | 16 ++++-- .../restcatalog/RecordingThriftIface.java | 7 ++- .../RoutingMetaStoreClientTest.java | 51 +++++++++++++++++-- 3 files changed, 67 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java index 7ae08cd..5e4941e 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClient.java @@ -190,9 +190,19 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl } break; case "dropDatabase": - if (paramTypes.length == 4) { - delegate.drop_database( - db((String) args[0]), (Boolean) args[1], (Boolean) args[3]); + if (paramTypes.length == 4 + && paramTypes[0] == String.class && paramTypes[1] == boolean.class + && paramTypes[2] == boolean.class && paramTypes[3] == boolean.class) { + boolean deleteData = (Boolean) args[1]; + boolean ignoreUnknownDb = (Boolean) args[2]; + boolean cascade = (Boolean) args[3]; + try { + delegate.drop_database(db((String) args[0]), deleteData, cascade); + } catch (NoSuchObjectException e) { + if (!ignoreUnknownDb) { + throw e; + } + } return null; } break; diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java index f887177..094dc79 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RecordingThriftIface.java @@ -134,7 +134,12 @@ private Object handle(Object proxy, Method method, Object[] args) throws Throwab } case "drop_database": { String db = (String) args[0]; - calls.add("drop_database:" + db); + boolean deleteData = (Boolean) args[1]; + boolean cascade = (Boolean) args[2]; + calls.add("drop_database:" + db + ":" + deleteData + ":" + cascade); + if (!databases.containsKey(db)) { + throw new NoSuchObjectException("no database " + db); + } databases.remove(db); return null; } diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java index 2c05d98..1d93af2 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/RoutingMetaStoreClientTest.java @@ -5,6 +5,7 @@ import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.LockRequest; import org.apache.hadoop.hive.metastore.api.LockResponse; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.ShowLocksRequest; import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; import org.apache.hadoop.hive.metastore.api.Table; @@ -166,12 +167,56 @@ public void createDatabaseTranslatesName() throws Exception { } @Test - public void dropDatabaseTranslatesName() throws Exception { + public void dropDatabaseTranslatesNameAndForwardsFlagsInOrder() throws Exception { RecordingThriftIface recording = new RecordingThriftIface(); + recording.databases.put("apache__sales", RecordingThriftIface.database("apache__sales")); IMetaStoreClient client = RoutingMetaStoreClient.create( recording.iface, new CatalogNameTranslation("apache", "__")); - client.dropDatabase("sales", false, true, false); - Assert.assertEquals(List.of("drop_database:apache__sales"), recording.calls); + client.dropDatabase("sales", true, false, false); + Assert.assertEquals( + List.of("drop_database:apache__sales:true:false"), recording.calls); + } + + @Test + public void dropDatabaseWithCatalogNameOverloadThrowsUnsupportedOperation() { + RecordingThriftIface delegate = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + try { + client.dropDatabase("hive", "sales", false, true); + Assert.fail("expected UnsupportedOperationException"); + } catch (UnsupportedOperationException expected) { + Assert.assertTrue(expected.getMessage(), + expected.getMessage().contains("dropDatabase")); + } catch (Exception other) { + Assert.fail("expected UnsupportedOperationException, got " + other.getClass().getName()); + } + } + + @Test + public void dropDatabaseIgnoresMissingDatabaseWhenFlagSet() throws Exception { + RecordingThriftIface delegate = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + client.dropDatabase("missing", false, true, false); + + Assert.assertEquals( + List.of("drop_database:missing:false:false"), delegate.calls); + } + + @Test + public void dropDatabasePropagatesMissingDatabaseWhenFlagUnset() { + RecordingThriftIface delegate = new RecordingThriftIface(); + IMetaStoreClient client = RoutingMetaStoreClient.create(delegate.iface); + + try { + client.dropDatabase("missing", false, false, false); + Assert.fail("expected NoSuchObjectException"); + } catch (NoSuchObjectException expected) { + // expected: ignoreUnknownDb was false, so the exception must propagate. + } catch (Exception other) { + Assert.fail("expected NoSuchObjectException, got " + other.getClass().getName()); + } } @Test From 2b778592ff698620432f4ca842fa835ddbd645c0 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 14:22:18 +0300 Subject: [PATCH 46/50] Advertise the view, namespace and transaction writes the front door serves The front door's client plumbing already dispatches view writes, namespace DDL and multi-table transaction commits through the same generic RESTCatalogAdapter/RoutingHiveCatalog path table writes use, and WriteRouteGate already gates all thirteen write routes for the default catalog - but GET /v1/config only advertised the five table-write routes, so a spec-compliant client would conclude the rest were unsupported. Extend WRITE_ENDPOINTS with the eight confirmed Iceberg 1.9.2 Endpoint constants for view CRUD+rename, namespace CRUD and transaction commit. Other catalogs are unaffected: they still advertise reads only, matching the synthetic lock shim that backs their writes with no real conflict checking. --- AGENTS.md | 2 +- .../restcatalog/IcebergRestService.java | 29 ++++++++---- .../IcebergRestEndpointIntegrationTest.java | 44 +++++++++++++++---- 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1c762d4..4995ff5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -84,7 +84,7 @@ scripts/run-real-installation-smoke-simple.sh --env-file smoke-stand/env/simple. - Знание о служебном плейсхолдере Hive (`_dummy_database`/`_dummy_table`, константы `SemanticAnalyzer.DUMMY_DATABASE`/`DUMMY_TABLE`, которые Hive шлёт в `LockRequest` для `INSERT ... VALUES`) живёт только в `routing/HivePlaceholderNamespace`. Плейсхолдер не выбирает каталог, не считается вторым namespace lock-запроса и не переписывается интернализацией в реальную backend-базу. Не добавляй локальных сравнений с `_dummy_database` в других классах. - Для RPC, у которых Hive IDL не объявляет исключений (`add_write_notification_log`, `open_txns`, `show_locks` и другие: у их `_result` есть только поле `success`), libthrift 0.9.3 подменяет любое серверное исключение на `TApplicationException("Internal error processing ")`. Текст отказа таких методов виден только в логе - не рассчитывай, что его получит клиент, и не проверяй его в smoke-скриптах. Настоящий HMS теряет свои тексты ошибок так же. - Классификация backend Thrift-ошибок живёт только в `thriftbridge/ThriftFailureClassifier`: «метода нет» - это `TApplicationException` с типом `UNKNOWN_METHOD` (или отсутствие метода в загруженном runtime), transport failure и protocol desync - отдельные категории. Не пиши локальные `instanceof TApplicationException` для решений про fallback, downgrade или переоткрытие соединения. -- `restcatalog` - Iceberg REST front door поверх routing-слоя: per-catalog сервисы работают через in-process прокси `IMetaStoreClient`; для не-default каталогов включён name translation, чтобы клиент видел внутренние имена баз этого каталога. Write-запросы к таблицам (create/commit/drop/rename/register) поддержаны, но только когда namespace резолвится в `routing.default-catalog` - только его таблицы подкреплены реальным HMS-локом, остальные каталоги обслуживает synthetic lock shim без проверки конфликтов. `WriteRouteGate` проверяет резолвленный каталог, а не prefix запроса, поэтому federated-имя под default-prefix отказывается так же, как прямой запрос к non-default prefix; `GET /v1/config` объявляет эту асимметрию в `endpoints`. Не добавляй локальных проверок "разрешён ли write" в других классах - весь gate живёт в `WriteRouteGate`. Каждый `IcebergRestService` получает per-catalog Hadoop `Configuration` через `IcebergRestServices.open(..., hadoopConfForCatalog)`, которая в проде резолвится в `router.requireBackend(catalog).hiveConf()` - тот же объект, что уже несёт `fs.defaultFS`/Kerberos-настройки catalog..conf.* для Thrift-пути. Не заводи для REST-пути отдельную голую `new Configuration()`: под Kerberos это провалит запись в HDFS ("Failed to specify server's Kerberos principal name"), потому что namenode-принципал у каждого каталога свой и известен только через его собственный conf. +- `restcatalog` - Iceberg REST front door поверх routing-слоя: per-catalog сервисы работают через in-process прокси `IMetaStoreClient`; для не-default каталогов включён name translation, чтобы клиент видел внутренние имена баз этого каталога. Write-запросы к таблицам, view (create/update/drop/rename), namespace DDL (create/update/drop) и multi-table transaction commit поддержаны, но только когда namespace резолвится в `routing.default-catalog` - только его таблицы подкреплены реальным HMS-локом, остальные каталоги обслуживает synthetic lock shim без проверки конфликтов. `WriteRouteGate` проверяет резолвленный каталог, а не prefix запроса, поэтому federated-имя под default-prefix отказывается так же, как прямой запрос к non-default prefix; `GET /v1/config` объявляет всю эту асимметрию в `endpoints` - default-каталог видит все тринадцать write-маршрутов, что WriteRouteGate проверяет, остальные каталоги видят только чтение. Не добавляй локальных проверок "разрешён ли write" в других классах - весь gate живёт в `WriteRouteGate`. Каждый `IcebergRestService` получает per-catalog Hadoop `Configuration` через `IcebergRestServices.open(..., hadoopConfForCatalog)`, которая в проде резолвится в `router.requireBackend(catalog).hiveConf()` - тот же объект, что уже несёт `fs.defaultFS`/Kerberos-настройки catalog..conf.* для Thrift-пути. Не заводи для REST-пути отдельную голую `new Configuration()`: под Kerberos это провалит запись в HDFS ("Failed to specify server's Kerberos principal name"), потому что namenode-принципал у каждого каталога свой и известен только через его собственный conf. ## Парсинг конфигурации diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java index 29549a1..5b26a20 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java @@ -23,8 +23,9 @@ * other catalog's service is given a CatalogNameTranslation so REST clients * see that catalog's internal database names instead of the federated ones. * Only the default catalog's service advertises (and, via {@link WriteRouteGate}, - * actually allows) table writes - every other catalog is discovery-only, matching - * the synthetic lock shim that backs its writes with no real conflict checking. + * actually allows) writes - table, view, namespace and transaction-commit alike - + * every other catalog is discovery-only, matching the synthetic lock shim that + * backs its writes with no real conflict checking. */ public final class IcebergRestService implements AutoCloseable { private static final String UNUSED_URI = "thrift://hms-proxy-loopback:0"; @@ -48,18 +49,30 @@ public final class IcebergRestService implements AutoCloseable { Endpoint.V1_VIEW_EXISTS); /** - * Table write routes phase 5a actually implements: only the default catalog's tables are - * backed by a real HMS lock, so only the default catalog's service advertises these. {@link - * WriteRouteGate} additionally gates view, namespace and transaction-commit writes for safety, - * but this phase does not implement any of those as working features, so they must stay off - * this list - advertising them would promise a capability the proxy does not deliver. + * Write routes actually served for the default catalog: only its tables, views and namespaces + * are backed by a real HMS lock, so only its service advertises these. Every entry here has a + * counterpart in {@link WriteRouteGate}'s WRITE_ROUTES - the gate covers all thirteen write + * routes, but a route belongs on this list only once it is a genuinely working feature, not + * merely gated for safety; advertising a gated-but-unimplemented route would promise a + * capability the proxy does not deliver. View writes (HiveViewOperations), namespace DDL and + * the multi-table transaction commit all dispatch through the same generic + * RESTCatalogAdapter/RoutingHiveCatalog path the table routes use, so they are genuine features + * as of phase 5b, not gate-only placeholders. */ private static final List WRITE_ENDPOINTS = List.of( Endpoint.V1_CREATE_TABLE, Endpoint.V1_UPDATE_TABLE, Endpoint.V1_DELETE_TABLE, Endpoint.V1_RENAME_TABLE, - Endpoint.V1_REGISTER_TABLE); + Endpoint.V1_REGISTER_TABLE, + Endpoint.V1_CREATE_VIEW, + Endpoint.V1_UPDATE_VIEW, + Endpoint.V1_DELETE_VIEW, + Endpoint.V1_RENAME_VIEW, + Endpoint.V1_CREATE_NAMESPACE, + Endpoint.V1_UPDATE_NAMESPACE, + Endpoint.V1_DELETE_NAMESPACE, + Endpoint.V1_COMMIT_TRANSACTION); private static final List DEFAULT_CATALOG_ENDPOINTS = Stream.concat(READ_ENDPOINTS.stream(), WRITE_ENDPOINTS.stream()).toList(); diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java index e5997ce..d4c9164 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java @@ -355,9 +355,10 @@ public void commitTransactionMixingFederatedTableIsRefused() throws Exception { @Test public void configAdvertisesOnlyServedEndpoints() throws Exception { // /v1/config with no warehouse resolves to the default catalog (catalog1 in this fixture), - // which now serves table writes (phase 5a). Other write kinds (view, namespace, - // transaction-commit) are gated by WriteRouteGate for safety but are not implemented - // features of this phase, so they must never be advertised, even for the default catalog. + // which serves table writes (phase 5a) plus view writes, namespace DDL and the + // transaction-commit (phase 5b) - all thirteen routes WriteRouteGate gates are now genuinely + // working features, dispatched through the same generic RESTCatalogAdapter/RoutingHiveCatalog + // path the table routes use, so all of them belong on this list. HttpResponse response = get("/v1/config"); Assert.assertEquals(200, response.statusCode()); String body = response.body(); @@ -367,15 +368,42 @@ public void configAdvertisesOnlyServedEndpoints() throws Exception { body.contains("POST /v1/{prefix}/namespaces/{namespace}/tables")); Assert.assertTrue("default catalog must advertise table deletes: " + body, body.contains("DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}")); - Assert.assertFalse( - "view writes are gated but not implemented in this phase, must not be advertised: " + body, + Assert.assertTrue( + "default catalog must advertise view writes: " + body, body.contains("POST /v1/{prefix}/namespaces/{namespace}/views")); - Assert.assertFalse( - "namespace/transaction writes are gated but not implemented in this phase, " - + "must not be advertised: " + body, + Assert.assertTrue( + "default catalog must advertise view renames: " + body, + body.contains("POST /v1/{prefix}/views/rename")); + Assert.assertTrue( + "default catalog must advertise namespace creation: " + body, + body.contains("POST /v1/{prefix}/namespaces")); + Assert.assertTrue( + "default catalog must advertise namespace property updates: " + body, + body.contains("POST /v1/{prefix}/namespaces/{namespace}/properties")); + Assert.assertTrue( + "default catalog must advertise namespace deletes: " + body, + body.contains("DELETE /v1/{prefix}/namespaces/{namespace}")); + Assert.assertTrue( + "default catalog must advertise the transaction commit: " + body, body.contains("POST /v1/{prefix}/transactions/commit")); } + @Test + public void defaultCatalogAdvertisesViewAndNamespaceWrites() throws Exception { + String body = get("/v1/" + CATALOG_NAME + "/config").body(); + Assert.assertTrue(body, body.contains("POST /v1/{prefix}/namespaces/{namespace}/views")); + Assert.assertTrue(body, body.contains("POST /v1/{prefix}/namespaces")); + Assert.assertTrue(body, body.contains("POST /v1/{prefix}/transactions/commit")); + } + + @Test + public void nonDefaultCatalogAdvertisesNoWritesAtAll() throws Exception { + String body = get("/v1/" + CATALOG2_NAME + "/config").body(); + Assert.assertFalse(body, body.contains("POST /v1/{prefix}/namespaces/{namespace}/views")); + Assert.assertFalse(body, body.contains("POST /v1/{prefix}/transactions/commit")); + Assert.assertTrue(body, body.contains("GET /v1/{prefix}/namespaces")); + } + @Test public void defaultCatalogConfigAdvertisesWrites() throws Exception { String body = get("/v1/" + CATALOG_NAME + "/config").body(); From a3d5356a2118c149b0f5455c9319d506f3b7a431 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 14:51:37 +0300 Subject: [PATCH 47/50] Cover the completed Iceberg REST write surface in the stand smoke Extend the HMS_SMOKE_REST_WRITE_TABLE-guarded block with namespace DDL, view write and multi-table transaction-commit round trips, each asserting the effect (property actually set, namespace actually gone, view actually listed, metadata-location actually changed) rather than trusting the status code alone. Prove the new transaction assertion discriminates by temporarily demanding a no-op commit, confirming the runner fails, then restoring it. Rebuilt and restaged the jar and validated on the stand: namespace DDL had never before reached a real metastore (the pre-phase jar answered 406 for createDatabase), and now passes end to end on both the plain and Kerberos profiles, the latter confirmed by genuine create_database/alter_database/ drop_database audit entries. Correct stale "read-only" claims in the smoke-stand README (both locales) and proxy properties comments left over from before phase 5a's write gate and the Kerberos REST turn-on. Update README/CHANGELOG (both locales) to state that view writes and transaction commit were already reachable after phase 5a and are now officially advertised and covered, while namespace DDL is genuinely new this phase. Update TEST-MATRIX (both locales) with the new rows and a revalidation-log entry for this run. --- CHANGELOG.md | 22 +++ CHANGELOG.ru.md | 23 +++ README.md | 56 ++++--- README.ru.md | 56 ++++--- ...real-installation-smoke.simple.env.example | 25 ++-- scripts/run-real-installation-smoke.sh | 138 ++++++++++++++++-- smoke-stand/README.md | 12 +- smoke-stand/README.ru.md | 13 +- smoke-stand/TEST-MATRIX.md | 38 +++++ smoke-stand/TEST-MATRIX.ru.md | 37 +++++ smoke-stand/env/simple.env | 9 +- .../proxy/hms-proxy-kerberos.properties | 3 +- smoke-stand/proxy/hms-proxy.properties | 6 +- 13 files changed, 359 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e9229d..990b8aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,28 @@ For a Russian version, see [CHANGELOG.ru.md](CHANGELOG.ru.md). Guarded by the new `HMS_SMOKE_REST_WRITE_TABLE`; skipped when unset. The runner also now asserts the config write/read asymmetry above, for both the default catalog and a configured second catalog. +- `RoutingMetaStoreClient` now implements `createDatabase`, + `dropDatabase(String, boolean, boolean, boolean)` and `alterDatabase` + instead of throwing `UnsupportedOperationException` - genuinely new: until + now every namespace-DDL Iceberg REST route answered unsupported regardless + of catalog. Names are translated through the existing + `CatalogNameTranslation`, and the `Database` payload passed to + create/alter is translated on a copy, never by mutating the caller's + object. +- `GET /v1/config` and `GET /v1/{prefix}/config` now advertise the full + served write surface: view CRUD/rename and namespace CRUD were already + reachable through the same generic dispatch path table writes use, and + `WriteRouteGate` already gated all thirteen write routes - only discovery + and smoke lagged behind. The default catalog's `endpoints` now carry all + thirteen write routes (table, view and namespace DDL, transaction + commit); every other catalog still advertises the nine read routes only. +- `--scenario rest` now also drives a namespace DDL round trip + (create/load/update-property/drop), a view round trip + (create/list/drop, asserting a real `metadata-location`) and a + multi-table transaction-commit round trip via + `POST /v1/{prefix}/transactions/commit`, asserting the table's + `metadata-location` actually changed rather than trusting the `204` + alone. All three are guarded by the existing `HMS_SMOKE_REST_WRITE_TABLE`. ### Fixed diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 178a3c9..6766e54 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -45,6 +45,29 @@ English version: [CHANGELOG.md](CHANGELOG.md). `HMS_SMOKE_REST_WRITE_TABLE`; пропускается, если не задана. Раннер также теперь проверяет write/read-асимметрию в config, описанную выше, — и для дефолтного каталога, и для настроенного второго каталога. +- `RoutingMetaStoreClient` теперь реализует `createDatabase`, + `dropDatabase(String, boolean, boolean, boolean)` и `alterDatabase` вместо + того, чтобы кидать `UnsupportedOperationException` — по-настоящему новое: + до сих пор любой namespace-DDL роут Iceberg REST отвечал unsupported + независимо от каталога. Имена транслируются через существующий + `CatalogNameTranslation`, а payload `Database`, передаваемый в + create/alter, транслируется на копии, а не мутацией объекта вызывающего. +- `GET /v1/config` и `GET /v1/{prefix}/config` теперь объявляют весь + обслуживаемый write-роут целиком: view CRUD/rename и namespace CRUD уже + были достижимы через тот же общий dispatch-путь, которым пользуется write + таблиц, и `WriteRouteGate` уже гейтил все тринадцать write-роутов — отставали + только discovery и smoke. В `endpoints` дефолтного каталога теперь + перечислены все тринадцать write-роутов (write таблиц, view и namespace + DDL, transaction commit); у любого другого каталога по-прежнему только + девять read-роутов. +- `--scenario rest` теперь также гоняет namespace DDL round trip + (create/load/update-property/drop), view round trip (create/list/drop, + с проверкой реального `metadata-location`) и multi-table + transaction-commit round trip через + `POST /v1/{prefix}/transactions/commit`, проверяя, что + `metadata-location` таблицы реально изменился, а не доверяя одному + только `204`. Все три настраиваются существующей + `HMS_SMOKE_REST_WRITE_TABLE`. ### Исправлено diff --git a/README.md b/README.md index 4f54519..b053e45 100644 --- a/README.md +++ b/README.md @@ -986,15 +986,24 @@ ambiguously, so `__` is usually the safer choice. The proxy can also run a parallel HTTP listener that speaks the Iceberg REST Catalog spec, backed by the same routing/federation pipeline as the Thrift HMS -front door. Status: **experimental**; table writes (create, commit, drop, -rename, register) are supported, but **only when the target namespace -resolves to `routing.default-catalog`**. Iceberg clients (PyIceberg, Spark -`iceberg-rest`, Trino `iceberg-rest`) can discover and load Iceberg tables -stored in HMS via the standard `metadata_location` table parameter. -View routes now return real data instead of an empty `204`, since -`HiveCatalog` is a `ViewCatalog` as of Iceberg `1.7`; view mutations -(create/drop/rename), namespace mutations and multi-table transaction -commits are still NOT supported in this iteration, on any catalog. +front door. Status: **experimental**; the full write surface `RESTCatalogAdapter` +exposes - table writes (create, commit, drop, rename, register), view writes +(create, commit, drop, rename) and namespace DDL (create, update properties, +drop), plus multi-table transaction commit - is supported, but **only when the +target namespace resolves to `routing.default-catalog`**. Iceberg clients +(PyIceberg, Spark `iceberg-rest`, Trino `iceberg-rest`) can discover and load +Iceberg tables stored in HMS via the standard `metadata_location` table +parameter. + +Table writes and the default-catalog-only gate landed first; view writes and +multi-table transaction commit were already reachable at that point too - the +REST dispatch path for them shares the same generic `RoutingHiveCatalog`/ +`RoutingMetaStoreClient` plumbing table writes use, and `WriteRouteGate` +already classified all thirteen routes as writes - they were simply not yet +advertised in `GET /v1/config` or covered by smoke. Namespace DDL is genuinely +new: `RoutingMetaStoreClient` did not implement `createDatabase`, +`alterDatabase` or `dropDatabase` until now, so `POST /v1/{prefix}/namespaces` +and friends answered `UnsupportedOperationException` regardless of catalog. **Why writes are default-catalog only:** only the default catalog's tables are backed by a real HMS lock (see [ZooKeeper storage for synthetic read @@ -1010,12 +1019,13 @@ names (see [Supported endpoints](#supported-endpoints) below), so a create under `/v1/{default-prefix}/namespaces/apache__default/tables` is refused exactly like a direct create under `/v1/apache/namespaces/default/tables` - both resolve to the `apache` catalog and get the same `403` -(`ForbiddenException`) with a message naming the resolved catalog. `GET -/v1/config` and `GET /v1/{prefix}/config` advertise this asymmetry -directly: the default catalog's `endpoints` list carries the five write -routes below, every other catalog's carries only the nine read routes, so a -spec-compliant client discovers the restriction instead of learning about it -from a failed request. +(`ForbiddenException`) with a message naming the resolved catalog. This +applies uniformly to every one of the thirteen write routes below - table, +view and namespace DDL, and transaction commit alike. `GET /v1/config` and +`GET /v1/{prefix}/config` advertise this asymmetry directly: the default +catalog's `endpoints` list carries all thirteen write routes below, every +other catalog's carries only the nine read routes, so a spec-compliant client +discovers the restriction instead of learning about it from a failed request. Enable it via: @@ -1051,7 +1061,14 @@ Requests to this listener are covered by the Prometheus metrics described in | `DELETE /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | supported for the default catalog only (drop); `403` elsewhere | | `POST /v1/{prefix}/tables/rename` | supported for the default catalog only (rename); `403` elsewhere | | `POST /v1/{prefix}/namespaces/{ns}/register` | supported for the default catalog only (register); `403` elsewhere | -| Namespace/view mutations, multi-table transaction commits | unsupported on every catalog | +| `POST /v1/{prefix}/namespaces/{ns}/views` | supported for the default catalog only (create); `403` elsewhere | +| `POST /v1/{prefix}/namespaces/{ns}/views/{view}` | supported for the default catalog only (commit/update); `403` elsewhere | +| `DELETE /v1/{prefix}/namespaces/{ns}/views/{view}` | supported for the default catalog only (drop); `403` elsewhere | +| `POST /v1/{prefix}/views/rename` | supported for the default catalog only (rename); `403` elsewhere | +| `POST /v1/{prefix}/namespaces` | supported for the default catalog only (create); `403` elsewhere | +| `POST /v1/{prefix}/namespaces/{ns}/properties` | supported for the default catalog only (update properties); `403` elsewhere | +| `DELETE /v1/{prefix}/namespaces/{ns}` | supported for the default catalog only (drop); `403` elsewhere | +| `POST /v1/{prefix}/transactions/commit` | supported for the default catalog only (multi-table commit); `403` elsewhere | `{prefix}` is any catalog listed in `catalogs=`: every configured catalog is exposed as its own REST prefix, `/v1//...`. `GET /v1/config` supports @@ -1063,9 +1080,10 @@ phase-1 behavior; an unknown `warehouse` value returns HTTP 400 (`BadRequestException`). The response's `endpoints` field lists the nine read routes above for every catalog (list/load namespace + namespace-exists, list/load table + table-exists, list/load view + view-exists); the default -catalog's `endpoints` additionally carry the five table-write routes above, -so a modern client can discover the write/read asymmetry between catalogs -instead of learning about it from a failed request. `GET /v1/{prefix}/config` +catalog's `endpoints` additionally carry all thirteen write routes above +(table, view and namespace DDL, transaction commit), so a modern client can +discover the write/read asymmetry between catalogs instead of learning about +it from a failed request. `GET /v1/{prefix}/config` answers the same way from the proxy's own handler — `overrides.prefix` names the catalog from the path segment instead of the `warehouse` query param — and an unknown prefix there is still a 404. diff --git a/README.ru.md b/README.ru.md index 1eac4e7..c5b0251 100644 --- a/README.ru.md +++ b/README.ru.md @@ -983,16 +983,24 @@ catalog.catalog2.conf.hms.proxy.external-table-drop-purge.allowed-prefixes=hdfs: Proxy дополнительно умеет поднять параллельный HTTP listener со спецификацией Iceberg REST Catalog, использующий тот же routing/federation pipeline что и -Thrift HMS front door. Статус: **экспериментально**; write-запросы к таблицам -(create, commit, drop, rename, register) поддержаны, но **только когда -целевой namespace резолвится в `routing.default-catalog`**. Iceberg-клиенты -(PyIceberg, Spark `iceberg-rest`, Trino `iceberg-rest`) могут discover и load -Iceberg-таблицы, хранящиеся в HMS через стандартный параметр -`metadata_location`. View routes теперь возвращают реальные данные вместо -пустого `204`, так как `HiveCatalog` стал `ViewCatalog`, начиная с Iceberg -`1.7`; view-мутации (create/drop/rename), namespace-мутации и multi-table -transaction commit по-прежнему НЕ поддерживаются в этой итерации, ни для -одного каталога. +Thrift HMS front door. Статус: **экспериментально**; весь write-роут, который +выставляет `RESTCatalogAdapter`, — write таблиц (create, commit, drop, +rename, register), write view (create, commit, drop, rename) и namespace DDL +(create, update properties, drop), а также multi-table transaction commit — +поддержан, но **только когда целевой namespace резолвится в +`routing.default-catalog`**. Iceberg-клиенты (PyIceberg, Spark `iceberg-rest`, +Trino `iceberg-rest`) могут discover и load Iceberg-таблицы, хранящиеся в HMS +через стандартный параметр `metadata_location`. + +Write таблиц и gate «только default-каталог» появились первыми; write view и +multi-table transaction commit к этому моменту уже были достижимы — их +REST-путь идёт через тот же общий `RoutingHiveCatalog`/`RoutingMetaStoreClient`, +которым пользуется write таблиц, и `WriteRouteGate` уже классифицировал все +тринадцать роутов как write — их просто ещё не объявляли в `GET /v1/config` и +не покрывали smoke. Namespace DDL — по-настоящему новое: `RoutingMetaStoreClient` +не реализовывал `createDatabase`, `alterDatabase` и `dropDatabase` до сих пор, +поэтому `POST /v1/{prefix}/namespaces` и соседние роуты отвечали +`UnsupportedOperationException` независимо от каталога. **Почему writes работают только в default-каталоге:** реальным HMS-локом подкреплены только таблицы дефолтного каталога (см. [ZooKeeper storage для @@ -1009,11 +1017,13 @@ synthetic read locks](#zookeeper-storage-для-synthetic-read-locks)); `/v1/{default-prefix}/namespaces/apache__default/tables` отказывается точно так же, как прямой create по `/v1/apache/namespaces/default/tables` — оба резолвятся в каталог `apache` и получают один и тот же `403` -(`ForbiddenException`). `GET /v1/config` и `GET /v1/{prefix}/config` -объявляют эту асимметрию напрямую: в `endpoints` дефолтного каталога -перечислены пять write-роутов из таблицы ниже, у любого другого каталога — -только девять read-роутов, так что спецификация-совместимый клиент узнаёт об -ограничении из discovery, а не из проваленного запроса. +(`ForbiddenException`). Это касается всех тринадцати write-роутов из таблицы +ниже одинаково — write таблиц, view и namespace DDL, transaction commit. +`GET /v1/config` и `GET /v1/{prefix}/config` объявляют эту асимметрию +напрямую: в `endpoints` дефолтного каталога перечислены все тринадцать +write-роутов из таблицы ниже, у любого другого каталога — только девять +read-роутов, так что спецификация-совместимый клиент узнаёт об ограничении +из discovery, а не из проваленного запроса. Включается так: @@ -1049,7 +1059,14 @@ rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab | `DELETE /v1/{prefix}/namespaces/{ns}/tables/{tbl}` | поддержан только для дефолтного каталога (drop); иначе `403` | | `POST /v1/{prefix}/tables/rename` | поддержан только для дефолтного каталога (rename); иначе `403` | | `POST /v1/{prefix}/namespaces/{ns}/register` | поддержан только для дефолтного каталога (register); иначе `403` | -| Namespace/view-мутации, multi-table transaction commit | не поддержаны ни для одного каталога | +| `POST /v1/{prefix}/namespaces/{ns}/views` | поддержан только для дефолтного каталога (create); иначе `403` | +| `POST /v1/{prefix}/namespaces/{ns}/views/{view}` | поддержан только для дефолтного каталога (commit/update); иначе `403` | +| `DELETE /v1/{prefix}/namespaces/{ns}/views/{view}` | поддержан только для дефолтного каталога (drop); иначе `403` | +| `POST /v1/{prefix}/views/rename` | поддержан только для дефолтного каталога (rename); иначе `403` | +| `POST /v1/{prefix}/namespaces` | поддержан только для дефолтного каталога (create); иначе `403` | +| `POST /v1/{prefix}/namespaces/{ns}/properties` | поддержан только для дефолтного каталога (update properties); иначе `403` | +| `DELETE /v1/{prefix}/namespaces/{ns}` | поддержан только для дефолтного каталога (drop); иначе `403` | +| `POST /v1/{prefix}/transactions/commit` | поддержан только для дефолтного каталога (multi-table commit); иначе `403` | `{prefix}` — любой каталог, перечисленный в `catalogs=`: каждый настроенный каталог получает собственный REST prefix, `/v1//...`. `GET @@ -1061,9 +1078,10 @@ rest-catalog.kerberos.keytab=/etc/security/keytabs/spnego.service.keytab (`BadRequestException`). Поле `endpoints` в ответе перечисляет девять read-роутов из таблицы выше для любого каталога (list/load namespace + namespace-exists, list/load table + table-exists, list/load view + -view-exists); у дефолтного каталога `endpoints` дополнительно несёт пять -write-роутов таблиц из таблицы выше, так что современный клиент может -обнаружить write/read-асимметрию между каталогами через discovery, а не из +view-exists); у дефолтного каталога `endpoints` дополнительно несёт все +тринадцать write-роутов из таблицы выше (write таблиц, view и namespace DDL, +transaction commit), так что современный клиент может обнаружить +write/read-асимметрию между каталогами через discovery, а не из провалившегося запроса. `GET /v1/{prefix}/config` отвечает так же, но из собственного handler'а прокси — `overrides.prefix` называет каталог из сегмента пути, а не из query-параметра `warehouse`, — и неизвестный prefix diff --git a/scripts/hms-real-installation-smoke.simple.env.example b/scripts/hms-real-installation-smoke.simple.env.example index 9882def..180cdb1 100644 --- a/scripts/hms-real-installation-smoke.simple.env.example +++ b/scripts/hms-real-installation-smoke.simple.env.example @@ -118,13 +118,20 @@ HMS_SMOKE_PARTITION_LOCK_TRANSACTIONAL=false # Management /metrics endpoint. When set, the REST smoke also checks it carries the # hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series. # HMS_SMOKE_REST_METRICS_URL=http://proxy-host:9090/metrics -# Table write round trip: create, load (asserting metadata-location comes back) and drop a table -# through the REST write routes. If left unset, the write step is skipped. Writes only work when -# the request resolves to the default catalog - every other catalog is served by the synthetic -# lock shim, which grants locks without conflict checking, so writes there are refused with 403 -# rather than silently racing concurrent writers. With HMS_SMOKE_REST_SECOND_PREFIX also set, the -# step additionally asserts 403 for a direct create under that prefix and for a create under its -# federated namespace name reached through the default prefix - proving -# the gate is enforced on the resolved catalog, not the request's own prefix. Use a name distinct -# from every other table already on the target so a rerun cannot collide with a leftover. +# Table write round trip: create, commit, rename and drop a table through the REST write routes. +# If left unset, the write step (and the namespace, view and transaction round trips below) is +# skipped. Writes only work when the request resolves to the default catalog - every other catalog +# is served by the synthetic lock shim, which grants locks without conflict checking, so writes +# there are refused with 403 rather than silently racing concurrent writers. With +# HMS_SMOKE_REST_SECOND_PREFIX also set, the step additionally asserts 403 for a direct create +# under that prefix, a create under its federated namespace name reached +# through the default prefix, a COMMIT_TRANSACTION naming a federated table, a CREATE_NAMESPACE +# with a federated name, and a rename with a federated destination - proving the gate is enforced +# on the resolved catalog, not the request's own prefix. Use a name distinct from every other table +# already on the target so a rerun cannot collide with a leftover. Also gates a namespace DDL round +# trip (create/load/update-property/drop of namespace "smoke_rest_ns"), a view round trip +# (create/list/drop of view "smoke_rest_view" in HMS_SMOKE_REST_NAMESPACE) and a multi-table +# transaction commit round trip (create "_txn", commit through +# POST /v1/{prefix}/transactions/commit, assert its metadata-location changed, drop) - the same +# default-catalog restriction applies to every one of these routes. # HMS_SMOKE_REST_WRITE_TABLE=smoke_rest_written diff --git a/scripts/run-real-installation-smoke.sh b/scripts/run-real-installation-smoke.sh index 2e75a6b..af0d420 100755 --- a/scripts/run-real-installation-smoke.sh +++ b/scripts/run-real-installation-smoke.sh @@ -87,18 +87,26 @@ Optional Iceberg REST env vars: checks it carries hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series HMS_SMOKE_REST_WRITE_TABLE table name to create, commit, rename and drop through - the REST write routes; skipped when unset. Writes only - work when the advertised default-prefix catalog resolves - to the real backend (non-default catalogs are served by - the synthetic lock shim and refuse writes with 403). The - block drops both this name and "_renamed" - defensively before creating, so a rerun on a dirty stand - cannot half-fail. Requires HMS_SMOKE_REST_SECOND_PREFIX to - also exercise the gate negatives: CREATE_TABLE under that - prefix and under its federated name, COMMIT_TRANSACTION - naming a federated table, CREATE_NAMESPACE with a - federated name, and RENAME with a federated destination - - all expected 403 + the REST write routes; skipped when unset. Also gates a + namespace DDL round trip (create/load/update-property/ + drop of namespace "smoke_rest_ns"), a view round trip + (create/list/drop of view "smoke_rest_view" in + HMS_SMOKE_REST_NAMESPACE) and a multi-table transaction + commit round trip (create "_txn", commit through + POST /v1/{prefix}/transactions/commit, assert its + metadata-location changed, drop) - the same default- + catalog restriction applies to every one of these routes. + Writes only work when the advertised default-prefix + catalog resolves to the real backend (non-default + catalogs are served by the synthetic lock shim and refuse + writes with 403). The block drops leftovers under every + name it can create defensively before creating, so a + rerun on a dirty stand cannot half-fail. Requires + HMS_SMOKE_REST_SECOND_PREFIX to also exercise the gate + negatives: CREATE_TABLE under that prefix and under its + federated name, COMMIT_TRANSACTION naming a federated + table, CREATE_NAMESPACE with a federated name, and RENAME + with a federated destination - all expected 403 EOF cat <<'EOF' @@ -904,9 +912,11 @@ EOF fi } -# Iceberg REST catalog smoke. The endpoint is read-only by design, so the positive checks -# are discovery and loads, and the negative checks pin down that an unknown prefix, an -# unknown table and a write route all fail cleanly instead of half-working. +# Iceberg REST catalog smoke. Discovery and loads are always checked; when HMS_SMOKE_REST_WRITE_TABLE +# is set, the default catalog's full served write surface (table, view and namespace DDL plus +# multi-table transaction commit) is exercised as round trips, not just status codes, and the +# negative checks pin down that an unknown prefix, an unknown table and a write route on any other +# catalog all fail cleanly instead of half-working. rest_is_configured() { [[ -n "${HMS_SMOKE_REST_URL:-}" ]] } @@ -1229,6 +1239,104 @@ run_rest_smoke() { code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/${renamed_write_table}" "${body}")" [[ "${code}" =~ ^2 ]] || fail "REST drop of '${renamed_write_table}' returned HTTP ${code}: $(cat "${body}")" + + # Namespace DDL round trip: create, load, update a property and drop - genuinely new since + # RoutingMetaStoreClient only gained createDatabase/alterDatabase/dropDatabase this phase. + # Same default-catalog-only restriction as table writes (WriteRouteGate covers CREATE_NAMESPACE + # by namespace, not by the request's own prefix). Dropped defensively first so a rerun on a + # dirty stand cannot half-fail. + local ns_smoke="smoke_rest_ns" + discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${ns_smoke}" "${body}")" + + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"namespace":["'"${ns_smoke}"'"]}' "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces")" + [[ "${code}" == "200" ]] || fail "REST namespace create of '${ns_smoke}' returned HTTP ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${ns_smoke}" "${body}")" + [[ "${code}" == "200" ]] || fail "REST namespace load of '${ns_smoke}' returned HTTP ${code}: $(cat "${body}")" + + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"removals":[],"updates":{"smoke":"yes"}}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${ns_smoke}/properties")" + [[ "${code}" == "200" ]] || fail "REST namespace property update of '${ns_smoke}' returned HTTP ${code}: $(cat "${body}")" + + # The effect that matters: the property must actually be there, not just a 200 status - a + # no-op update route would still answer 200. + code="$(rest_request GET "/v1/${prefix}/namespaces/${ns_smoke}" "${body}")" + [[ "${code}" == "200" ]] || fail "REST namespace reload of '${ns_smoke}' returned HTTP ${code}: $(cat "${body}")" + grep -q '"smoke"[[:space:]]*:[[:space:]]*"yes"' "${body}" \ + || fail "REST namespace property update of '${ns_smoke}' did not stick: $(cat "${body}")" + + code="$(rest_request DELETE "/v1/${prefix}/namespaces/${ns_smoke}" "${body}")" + [[ "${code}" == "204" ]] || fail "REST namespace drop of '${ns_smoke}' returned HTTP ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${ns_smoke}" "${body}")" + [[ "${code}" == "404" ]] \ + || fail "REST namespace load of dropped '${ns_smoke}' expected HTTP 404, got ${code}: $(cat "${body}")" + + # View write round trip: create (asserting a real metadata-location), list, and drop. View + # writes were already reachable after phase 5a's commit path; this smoke makes that officially + # covered rather than merely advertised, under the same default-catalog-only restriction. + local view_smoke="smoke_rest_view" + discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/views/${view_smoke}" "${body}")" + + local view_create_body='{"name":"'"${view_smoke}"'","schema":{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"}]},"view-version":{"version-id":1,"timestamp-ms":1753700000000,"schema-id":0,"summary":{"operation":"create"},"default-namespace":["'"${namespace}"'"],"representations":[{"type":"sql","sql":"select 1","dialect":"hive"}]},"properties":{}}' + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data "${view_create_body}" "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${namespace}/views")" + [[ "${code}" == "200" ]] || fail "REST view create of '${view_smoke}' returned HTTP ${code}: $(cat "${body}")" + grep -q '"metadata-location"' "${body}" \ + || fail "REST view create of '${view_smoke}' carries no metadata-location: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${namespace}/views" "${body}")" + [[ "${code}" == "200" ]] || fail "REST view listing of '${namespace}' returned HTTP ${code}: $(cat "${body}")" + grep -q "\"name\"[[:space:]]*:[[:space:]]*\"${view_smoke}\"" "${body}" \ + || fail "view '${view_smoke}' missing from the REST view listing: $(cat "${body}")" + + code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/views/${view_smoke}" "${body}")" + [[ "${code}" == "204" ]] || fail "REST view drop of '${view_smoke}' returned HTTP ${code}: $(cat "${body}")" + + # Transaction commit round trip: a fresh table committed through + # POST /v1/{prefix}/transactions/commit rather than the per-table commit route already + # exercised above - the multi-table atomic-commit path is what phase 5a's write gate had a + # critical bypass in (COMMIT_TRANSACTION naming a federated table), so it gets its own + # positive proof here too. A no-op commit would still answer 204, so the check that matters is + # the metadata-location actually changing, not the status code. + local txn_table="${write_table}_txn" + discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/${txn_table}" "${body}")" + + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"name":"'"${txn_table}"'","schema":{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"}]}}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${namespace}/tables")" + [[ "${code}" == "200" ]] || fail "REST create of '${txn_table}' returned HTTP ${code}: $(cat "${body}")" + + local txn_create_metadata_location="" + txn_create_metadata_location="$(grep -o '"metadata-location"[[:space:]]*:[[:space:]]*"[^"]*"' "${body}" | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/')" + [[ -n "${txn_create_metadata_location}" ]] \ + || fail "REST create of '${txn_table}' carries no metadata-location: $(cat "${body}")" + + local txn_table_uuid="" + txn_table_uuid="$(grep -o '"table-uuid"[[:space:]]*:[[:space:]]*"[^"]*"' "${body}" | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/')" + [[ -n "${txn_table_uuid}" ]] \ + || fail "REST create of '${txn_table}' carries no metadata.table-uuid: $(cat "${body}")" + + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"table-changes":[{"identifier":{"namespace":["'"${namespace}"'"],"name":"'"${txn_table}"'"},"requirements":[{"type":"assert-table-uuid","uuid":"'"${txn_table_uuid}"'"}],"updates":[{"action":"set-properties","updates":{"txn":"yes"}}]}]}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/transactions/commit")" + [[ "${code}" == "204" ]] || fail "REST transaction commit of '${txn_table}' returned HTTP ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${namespace}/tables/${txn_table}" "${body}")" + [[ "${code}" == "200" ]] \ + || fail "REST load of '${txn_table}' after transaction commit returned HTTP ${code}: $(cat "${body}")" + + local txn_commit_metadata_location="" + txn_commit_metadata_location="$(grep -o '"metadata-location"[[:space:]]*:[[:space:]]*"[^"]*"' "${body}" | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/')" + [[ -n "${txn_commit_metadata_location}" ]] \ + || fail "REST load of '${txn_table}' after transaction commit carries no metadata-location: $(cat "${body}")" + [[ "${txn_commit_metadata_location}" != "${txn_create_metadata_location}" ]] \ + || fail "REST transaction commit of '${txn_table}' did not write a new metadata file: metadata-location is still '${txn_commit_metadata_location}'" + + code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/tables/${txn_table}" "${body}")" + [[ "${code}" =~ ^2 ]] || fail "REST drop of '${txn_table}' returned HTTP ${code}: $(cat "${body}")" else log "skipping REST write round trip because HMS_SMOKE_REST_WRITE_TABLE is not set" fi diff --git a/smoke-stand/README.md b/smoke-stand/README.md index 03cba66..a5518e3 100644 --- a/smoke-stand/README.md +++ b/smoke-stand/README.md @@ -184,11 +184,13 @@ docker logs stand-proxy 2>&1 | grep 'requires a Hortonworks backend runtime' ## Iceberg REST catalog front door The plain profile also enables the proxy's Iceberg REST listener (`rest-catalog.*` in -`proxy/hms-proxy.properties`, host port 19183). It is read-only and serves the default -catalog (`hdp`) only. `--scenario rest` (or the REST step of `--scenario all`) drives it -with curl from the host: config discovery, namespace and table listings, a table load, and -the negative shapes — unknown prefix, unknown table, and a write route, all of which must -fail cleanly. +`proxy/hms-proxy.properties`, host port 19183). Every write route — table, view and namespace +DDL, plus multi-table transaction commit — is served, but only for the default catalog (`hdp`): +its tables are backed by a real HMS lock, while every other catalog is served by the synthetic +lock shim and refuses writes with `403`. `--scenario rest` (or the REST step of `--scenario all`) +drives it with curl from the host: config discovery, namespace and table listings, a table load, +the full write round trips, and the negative shapes — unknown prefix, unknown table, and a write +route on a non-default catalog, all of which must fail cleanly. The load-table check needs a real Iceberg table. The stand registers a minimal one by hand — a hand-written `metadata.json` in HDFS plus a Hive table shell that points at it: diff --git a/smoke-stand/README.ru.md b/smoke-stand/README.ru.md index 8781906..4efd50c 100644 --- a/smoke-stand/README.ru.md +++ b/smoke-stand/README.ru.md @@ -182,11 +182,14 @@ docker logs stand-proxy 2>&1 | grep 'requires a Hortonworks backend runtime' ## Iceberg REST catalog front door Plain-профиль включает и Iceberg REST listener прокси (`rest-catalog.*` в -`proxy/hms-proxy.properties`, host-порт 19183). Он read-only и обслуживает только -default-каталог (`hdp`). `--scenario rest` (или REST-шаг `--scenario all`) гоняет его -curl'ом с хоста: discovery конфигурации, листинги namespace и таблиц, load таблицы и -негативные формы — неизвестный prefix, неизвестная таблица и write-роут; все должны -падать чисто. +`proxy/hms-proxy.properties`, host-порт 19183). Он обслуживает весь write-роут — +table, view и namespace DDL, а также multi-table transaction commit, — но только для +default-каталога (`hdp`): его таблицы подкреплены реальным HMS-локом, а любой другой +каталог обслуживает synthetic lock shim и отказывает write с `403`. `--scenario rest` +(или REST-шаг `--scenario all`) гоняет его curl'ом с хоста: discovery конфигурации, +листинги namespace и таблиц, load таблицы, полные write round trip'ы и негативные +формы — неизвестный prefix, неизвестная таблица и write-роут на non-default каталоге; +все должны падать чисто. Проверке load-table нужна настоящая Iceberg-таблица. Стенд регистрирует минимальную вручную — написанный руками `metadata.json` в HDFS плюс Hive-оболочка таблицы, которая diff --git a/smoke-stand/TEST-MATRIX.md b/smoke-stand/TEST-MATRIX.md index 80f8211..9bca03e 100644 --- a/smoke-stand/TEST-MATRIX.md +++ b/smoke-stand/TEST-MATRIX.md @@ -125,6 +125,9 @@ stay `n/a` until they are. | G29 | `POST /v1/{prefix}/namespaces` with a federated name (`apache__zzz_smoke`) refused with `403` | ✅ | n/a | | G30 | `POST /v1/{prefix}/tables/rename` with a federated *destination* namespace (source table still under its current name) refused with `403` - proves the destination side of the gate, not just the source | ✅ | n/a | | G31 | A request without `--negotiate` is rejected `401` with a `WWW-Authenticate: Negotiate` challenge and an empty body | n/a | ✅ | +| G32 | Namespace DDL round trip: `POST .../namespaces` create (`200`), `GET` load (`200`), `POST .../properties` update (`200`) with a follow-up `GET` confirming the property is actually present, `DELETE` (`204`), `GET` afterward (`404`) - genuinely new: `RoutingMetaStoreClient` did not implement `createDatabase`/`alterDatabase`/`dropDatabase` before this phase, so this is the first time namespace DDL reached a real metastore | ✅ | ✅ | +| G33 | View write round trip: `POST .../views` create answers `200` with a real `metadata-location`, `GET .../views` lists the new view, `DELETE` answers `204` | ✅ | ✅ | +| G34 | `POST /v1/{prefix}/transactions/commit` against a freshly created table: answers `204`, and the table's `metadata-location` afterward differs from create's - proof the multi-table commit path actually wrote a new metadata file, not a silent no-op | ✅ | n/a | ## F. Not covered, and why @@ -248,6 +251,41 @@ executed is claimed; a row not listed was not repeated and its ✅ stands on the of those entries. The remaining Kerberos-column read-only rows (G2-G22, G27-G30) were not re-run and stay `n/a`. +- **2026-07-28** (third entry), jar `1.0.49-2b778592` (phase 5b: namespace DDL in + `RoutingMetaStoreClient` and the full-write-surface `GET /v1/config` advertising). Before this + run the stand was still on a pre-phase jar, and probing it directly showed `POST + /v1/{prefix}/namespaces` answering `406` ("does not support `IMetaStoreClient.createDatabase`") + - namespace DDL had never actually been validated against a real metastore. Added rows G32-G34 + for the three new round trips (namespace DDL, view writes, transaction commit via + `POST /v1/{prefix}/transactions/commit`); the per-table commit route (G26) was already covered + and stayed green, unaffected by this phase's changes. + After rebuilding the fat jar and restaging (`./prepare.sh && docker compose up -d --build`, + plain profile), `--scenario rest` and `--scenario all` both re-ran green, this time actually + exercising namespace DDL for the first time: `POST /v1/hdp/namespaces` created + `smoke_rest_ns` (`200`), `GET` loaded it back, `POST .../properties` set `smoke=yes` (`200`) + and a follow-up `GET` confirmed the property was actually present, `DELETE` answered `204` and + a final `GET` answered `404`. The view round trip created `smoke_rest_view` (`200`, a real + `metadata-location`), listed it, and dropped it (`204`). The transaction round trip created a + table, committed it through `POST /v1/hdp/transactions/commit` (`204`), and confirmed the + table's `metadata-location` moved from a `00000-...` file to a `00001-...` one on reload - + manual curl round trips against the running stand captured the same verbatim responses outside + the smoke script, for the record. + Step 4 of the task proved the new transaction assertion actually discriminates: the check was + temporarily inverted to demand the `metadata-location` stay unchanged, `--scenario rest` was + rerun and failed with "did not write a new metadata file: metadata-location is still + '...00001-...'" as expected, then the assertion was restored and both `--scenario rest` and + `--scenario all` re-ran green. + The stand was then switched to the Kerberos profile + (`docker compose --env-file .env.kerberos --profile kerberos up -d --build`) and, from inside + `stand-proxy` after `kinit -kt /keytabs/smoke-user.keytab smoke-user@SMOKE.LOCAL`, curl + `--negotiate` drove G32 (namespace DDL) and G33 (view writes) by hand per the task brief, which + scoped the Kerberos re-run to those two round trips only. Both passed identically to the plain + profile - same status codes, same effects - and `hms-proxy-audit.log` showed genuine + `create_database`/`alter_database`/`drop_database` entries with + `"authenticatedUser":"smoke-user@SMOKE.LOCAL"`, confirming namespace DDL reached the real HDP + backend under Kerberos too. G34 (transaction commit) was not re-run under Kerberos and stays + `n/a`, matching the task's scope. + ## Two caveats on faithfulness - The Kerberos profile is complete end to end — client → HiveServer2 → proxy → metastores → HDFS, diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index de155b2..98f69b9 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -124,6 +124,9 @@ Thrift front door), и против него прогнали write round trip, | G29 | `POST /v1/{prefix}/namespaces` с federated-именем (`apache__zzz_smoke`) отклонён с `403` | ✅ | n/a | | G30 | `POST /v1/{prefix}/tables/rename` с federated destination-namespace (source-таблица ещё под текущим именем) отклонён с `403` — доказывает проверку именно destination-стороны gate, а не только source | ✅ | n/a | | G31 | Запрос без `--negotiate` отклоняется `401` с вызовом `WWW-Authenticate: Negotiate` и пустым телом | n/a | ✅ | +| G32 | Namespace DDL round trip: `POST .../namespaces` create (`200`), `GET` load (`200`), `POST .../properties` update (`200`) с последующим `GET`, подтверждающим, что property реально появилось, `DELETE` (`204`), `GET` после этого (`404`) — по-настоящему новое: `RoutingMetaStoreClient` не реализовывал `createDatabase`/`alterDatabase`/`dropDatabase` до этой фазы, так что namespace DDL впервые дошёл до реального metastore | ✅ | ✅ | +| G33 | View write round trip: `POST .../views` create отвечает `200` с реальным `metadata-location`, `GET .../views` листит новый view, `DELETE` отвечает `204` | ✅ | ✅ | +| G34 | `POST /v1/{prefix}/transactions/commit` против только что созданной таблицы: отвечает `204`, и `metadata-location` таблицы после этого отличается от того, что дал create — доказательство, что multi-table commit реально записал новый metadata-файл, а не тихий no-op | ✅ | n/a | ## F. Что не покрыто и почему @@ -248,6 +251,40 @@ Thrift front door), и против него прогнали write round trip, `"authenticatedUser":"smoke-user@SMOKE.LOCAL"` в каждой из этих записей. Остальные read-only строки Kerberos-колонки (G2-G22, G27-G30) не перепрогонялись и остаются `n/a`. +- **2026-07-28** (третья запись), jar `1.0.49-2b778592` (фаза 5b: namespace DDL в + `RoutingMetaStoreClient` и объявление полного write-роута в `GET /v1/config`). До этого прогона + стенд ещё стоял на jar'е до фазы, и прямая проверка показала, что `POST + /v1/{prefix}/namespaces` отвечает `406` ("does not support `IMetaStoreClient.createDatabase`") + — namespace DDL ни разу ещё не проверялся против настоящего metastore. Добавлены строки + G32-G34 для трёх новых round trip'ов (namespace DDL, view write, transaction commit через + `POST /v1/{prefix}/transactions/commit`); роут per-table commit (G26) уже был покрыт и остался + зелёным, эта фаза его не затронула. + После пересборки fat jar и рестейджа (`./prepare.sh && docker compose up -d --build`, + plain-профиль) `--scenario rest` и `--scenario all` оба перепрогнаны зелёными, на этот раз + реально прогоняя namespace DDL впервые: `POST /v1/hdp/namespaces` создал `smoke_rest_ns` + (`200`), `GET` загрузил его обратно, `POST .../properties` выставил `smoke=yes` (`200`), и + последующий `GET` подтвердил, что property реально появилось, `DELETE` отвечал `204`, а + финальный `GET` — `404`. View round trip создал `smoke_rest_view` (`200`, реальный + `metadata-location`), листнул его и удалил (`204`). Transaction round trip создал таблицу, + закоммитил её через `POST /v1/hdp/transactions/commit` (`204`) и подтвердил, что + `metadata-location` таблицы при перезагрузке сместился с файла `00000-...` на `00001-...` — + ручные curl round trip'ы против работающего стенда зафиксировали те же verbatim-ответы вне + smoke-скрипта, для протокола. + Шаг 4 задачи доказал, что новая transaction-проверка реально различающая: проверка была + временно инвертирована — потребовать, чтобы `metadata-location` НЕ менялся, `--scenario rest` + перепрогнан и упал с "did not write a new metadata file: metadata-location is still + '...00001-...'", как и ожидалось, затем проверка восстановлена и оба сценария (`rest` и `all`) + перепрогнаны зелёными. + Затем стенд переключён на Kerberos-профиль + (`docker compose --env-file .env.kerberos --profile kerberos up -d --build`), и изнутри + `stand-proxy` после `kinit -kt /keytabs/smoke-user.keytab smoke-user@SMOKE.LOCAL` curl с + `--negotiate` вручную прогнал G32 (namespace DDL) и G33 (view write) — именно так задача и + ограничила Kerberos-перепрогон. Оба прошли идентично plain-профилю — те же статусы, тот же + эффект, — а `hms-proxy-audit.log` показал настоящие записи `create_database`/`alter_database`/ + `drop_database` с `"authenticatedUser":"smoke-user@SMOKE.LOCAL"`, подтверждая, что namespace + DDL под Kerberos тоже дошёл до реального HDP-бэкенда. G34 (transaction commit) под Kerberos не + перепрогонялся и остаётся `n/a` — в рамках заявленного объёма задачи. + ## Две оговорки честности - Kerberos-профиль полный сквозняком — клиент → HiveServer2 → прокси → метасторы → HDFS, ни один diff --git a/smoke-stand/env/simple.env b/smoke-stand/env/simple.env index 49a3b6f..b47a317 100644 --- a/smoke-stand/env/simple.env +++ b/smoke-stand/env/simple.env @@ -97,11 +97,14 @@ HMS_SMOKE_REST_SECOND_PREFIX=apache HMS_SMOKE_REST_SECOND_ICEBERG_TABLE=smoke_iceberg_tbl_ap HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE=smoke_read_ap -# Table write round trip (create/load/drop) plus the two negatives (direct create under the -# non-default prefix, and create under the federated name of that prefix's namespace under the +# Table write round trip (create/commit/rename/drop) plus the two negatives (direct create under +# the non-default prefix, and create under the federated name of that prefix's namespace under the # default prefix) - both must answer 403, since only the default catalog's commit path reaches a # real backend lock. The name is distinct from every other table already on the stand so a rerun -# cannot collide with a leftover from a previous pass. +# cannot collide with a leftover from a previous pass. Also gates a namespace DDL round trip +# (smoke_rest_ns), a view round trip (smoke_rest_view, in HMS_SMOKE_REST_NAMESPACE) and a +# multi-table transaction commit round trip (_txn) - the same default-catalog +# restriction applies to all of them. HMS_SMOKE_REST_WRITE_TABLE=smoke_rest_written # Management metrics endpoint, reachable from the host. When set, the REST smoke also checks it diff --git a/smoke-stand/proxy/hms-proxy-kerberos.properties b/smoke-stand/proxy/hms-proxy-kerberos.properties index 3f8edfd..a0474db 100644 --- a/smoke-stand/proxy/hms-proxy-kerberos.properties +++ b/smoke-stand/proxy/hms-proxy-kerberos.properties @@ -20,7 +20,8 @@ additional-frontends.hdp.port=9084 additional-frontends.hdp.frontend-profile=HORTONWORKS_3_1_0_3_1_0_78 additional-frontends.hdp.standalone-metastore-jar=/opt/hms-proxy/hive-metastore/hive-standalone-metastore-3.1.0.3.1.0.0-78.jar -# Iceberg REST catalog front door (read-only, experimental) over SPNEGO. The KDC container's +# Iceberg REST catalog front door over SPNEGO, same served write surface as the plain profile +# (table, view and namespace DDL, transaction commit - default catalog only). The KDC container's # curl is GSS-capable (mit-krb5), so the round trip is exercised with --negotiate the same way # the plain profile exercises it unauthenticated. rest-catalog.enabled=true diff --git a/smoke-stand/proxy/hms-proxy.properties b/smoke-stand/proxy/hms-proxy.properties index 3b78217..9ed77e3 100644 --- a/smoke-stand/proxy/hms-proxy.properties +++ b/smoke-stand/proxy/hms-proxy.properties @@ -23,9 +23,9 @@ additional-frontends.hdp.port=9084 additional-frontends.hdp.frontend-profile=HORTONWORKS_3_1_0_3_1_0_78 additional-frontends.hdp.standalone-metastore-jar=/opt/hms-proxy/hive-metastore/hive-standalone-metastore-3.1.0.3.1.0.0-78.jar -# Iceberg REST catalog front door (read-only, experimental). Plain profile only: the SPNEGO -# variant is covered by SpnegoIntegrationTest in-JVM, and a curl-based smoke would need a -# GSS-enabled curl inside the network. +# Iceberg REST catalog front door. Serves the full write surface (table, view and namespace DDL, +# multi-table transaction commit) for the default catalog only; the Kerberos profile now runs it +# too (hms-proxy-kerberos.properties), exercised with curl --negotiate from inside the stand. rest-catalog.enabled=true rest-catalog.port=9183 From 33bfdaadc5fed855585807dd9437d0c72f5ae096 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 15:14:35 +0300 Subject: [PATCH 48/50] Drive view rename/update and cover four more write-gate negatives in the smoke Two advertised REST routes (view rename, view property update) were never exercised by the smoke, and WriteRouteGate had thirteen write routes but only five wire-level negatives. Extend the existing view round trip to update and rename the view, asserting the property actually stuck and that the rename moved the view (200 under the new name, 404 under the old) rather than copying it. Add four more 403 negatives against a federated namespace under the default prefix: CREATE_VIEW (using the full valid view body, since a stub body 400s before the gate is even consulted), DROP_VIEW, DROP_NAMESPACE and UPDATE_NAMESPACE. --- scripts/run-real-installation-smoke.sh | 74 ++++++++++++++++++++++++-- smoke-stand/TEST-MATRIX.md | 26 +++++++-- smoke-stand/TEST-MATRIX.ru.md | 24 ++++++++- 3 files changed, 114 insertions(+), 10 deletions(-) diff --git a/scripts/run-real-installation-smoke.sh b/scripts/run-real-installation-smoke.sh index af0d420..ab90711 100755 --- a/scripts/run-real-installation-smoke.sh +++ b/scripts/run-real-installation-smoke.sh @@ -1224,6 +1224,33 @@ run_rest_smoke() { "${HMS_SMOKE_REST_URL}/v1/${prefix}/tables/rename")" [[ "${code}" == "403" ]] \ || fail "REST rename with federated destination namespace '${write_fed_ns}' expected HTTP 403, got ${code}: $(cat "${body}")" + + # CREATE_VIEW with a federated namespace. Needs the FULL valid view body, not a stub - + # an earlier attempt with a minimal body got HTTP 400 because the body failed to parse + # before the gate was ever consulted. A 400 from this assertion means the request body is + # malformed, not that the gate let the write through. + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"name":"zzz_smoke_view","schema":{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"}]},"view-version":{"version-id":1,"timestamp-ms":1753700000000,"schema-id":0,"summary":{"operation":"create"},"default-namespace":["'"${write_fed_ns}"'"],"representations":[{"type":"sql","sql":"select 1","dialect":"hive"}]},"properties":{}}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${write_fed_ns}/views")" + [[ "${code}" == "403" ]] \ + || fail "REST CREATE_VIEW under federated namespace '${write_fed_ns}' expected HTTP 403, got ${code} (400 would mean the request body is malformed, not that the gate refused it): $(cat "${body}")" + + # DROP_VIEW with a federated namespace. + code="$(rest_request DELETE "/v1/${prefix}/namespaces/${write_fed_ns}/views/whatever" "${body}")" + [[ "${code}" == "403" ]] \ + || fail "REST DROP_VIEW under federated namespace '${write_fed_ns}' expected HTTP 403, got ${code}: $(cat "${body}")" + + # DROP_NAMESPACE of a federated namespace. + code="$(rest_request DELETE "/v1/${prefix}/namespaces/${write_fed_ns}" "${body}")" + [[ "${code}" == "403" ]] \ + || fail "REST DROP_NAMESPACE of federated namespace '${write_fed_ns}' expected HTTP 403, got ${code}: $(cat "${body}")" + + # UPDATE_NAMESPACE (properties) of a federated namespace. + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"removals":[],"updates":{"x":"y"}}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${write_fed_ns}/properties")" + [[ "${code}" == "403" ]] \ + || fail "REST UPDATE_NAMESPACE of federated namespace '${write_fed_ns}' expected HTTP 403, got ${code}: $(cat "${body}")" else log "skipping REST write-gate negative checks because HMS_SMOKE_REST_SECOND_PREFIX is not set" fi @@ -1274,11 +1301,16 @@ run_rest_smoke() { [[ "${code}" == "404" ]] \ || fail "REST namespace load of dropped '${ns_smoke}' expected HTTP 404, got ${code}: $(cat "${body}")" - # View write round trip: create (asserting a real metadata-location), list, and drop. View - # writes were already reachable after phase 5a's commit path; this smoke makes that officially - # covered rather than merely advertised, under the same default-catalog-only restriction. + # View write round trip: create (asserting a real metadata-location), list, update and + # rename (asserting effects, not just status codes), and drop. View writes were already + # reachable after phase 5a's commit path; this smoke makes that officially covered rather + # than merely advertised, under the same default-catalog-only restriction. Both the + # original and the renamed name are dropped defensively first so a rerun on a dirty stand + # cannot half-fail. local view_smoke="smoke_rest_view" + local renamed_view_smoke="${view_smoke}_renamed" discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/views/${view_smoke}" "${body}")" + discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/views/${renamed_view_smoke}" "${body}")" local view_create_body='{"name":"'"${view_smoke}"'","schema":{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"}]},"view-version":{"version-id":1,"timestamp-ms":1753700000000,"schema-id":0,"summary":{"operation":"create"},"default-namespace":["'"${namespace}"'"],"representations":[{"type":"sql","sql":"select 1","dialect":"hive"}]},"properties":{}}' code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ @@ -1287,13 +1319,45 @@ run_rest_smoke() { grep -q '"metadata-location"' "${body}" \ || fail "REST view create of '${view_smoke}' carries no metadata-location: $(cat "${body}")" + local view_uuid="" + view_uuid="$(grep -o '"view-uuid"[[:space:]]*:[[:space:]]*"[^"]*"' "${body}" | head -n 1 | sed 's/.*"\([^"]*\)"$/\1/')" + [[ -n "${view_uuid}" ]] \ + || fail "REST view create of '${view_smoke}' carries no metadata.view-uuid: $(cat "${body}")" + code="$(rest_request GET "/v1/${prefix}/namespaces/${namespace}/views" "${body}")" [[ "${code}" == "200" ]] || fail "REST view listing of '${namespace}' returned HTTP ${code}: $(cat "${body}")" grep -q "\"name\"[[:space:]]*:[[:space:]]*\"${view_smoke}\"" "${body}" \ || fail "view '${view_smoke}' missing from the REST view listing: $(cat "${body}")" - code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/views/${view_smoke}" "${body}")" - [[ "${code}" == "204" ]] || fail "REST view drop of '${view_smoke}' returned HTTP ${code}: $(cat "${body}")" + # Update: the effect that matters is the property actually landing, not merely a 200 status - + # a no-op update route would still answer 200. + local view_update_body='{"requirements":[{"type":"assert-view-uuid","uuid":"'"${view_uuid}"'"}],"updates":[{"action":"set-properties","updates":{"smoke":"updated"}}]}' + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data "${view_update_body}" "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces/${namespace}/views/${view_smoke}")" + [[ "${code}" == "200" ]] || fail "REST view update of '${view_smoke}' returned HTTP ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${namespace}/views/${view_smoke}" "${body}")" + [[ "${code}" == "200" ]] || fail "REST view reload of '${view_smoke}' returned HTTP ${code}: $(cat "${body}")" + grep -q '"smoke"[[:space:]]*:[[:space:]]*"updated"' "${body}" \ + || fail "REST view update of '${view_smoke}' did not stick: $(cat "${body}")" + + # Rename: the pair below (200 under the new name, 404 under the old) is what proves the + # rename moved the view rather than copying it - a bare 204 status would not distinguish + # the two. + code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ + --data '{"source":{"namespace":["'"${namespace}"'"],"name":"'"${view_smoke}"'"},"destination":{"namespace":["'"${namespace}"'"],"name":"'"${renamed_view_smoke}"'"}}' \ + "${HMS_SMOKE_REST_URL}/v1/${prefix}/views/rename")" + [[ "${code}" == "204" ]] || fail "REST rename of '${view_smoke}' returned HTTP ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${namespace}/views/${renamed_view_smoke}" "${body}")" + [[ "${code}" == "200" ]] || fail "REST load of renamed '${renamed_view_smoke}' returned HTTP ${code}: $(cat "${body}")" + + code="$(rest_request GET "/v1/${prefix}/namespaces/${namespace}/views/${view_smoke}" "${body}")" + [[ "${code}" == "404" ]] \ + || fail "REST load of pre-rename view name '${view_smoke}' expected HTTP 404, got ${code}: $(cat "${body}")" + + code="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/views/${renamed_view_smoke}" "${body}")" + [[ "${code}" == "204" ]] || fail "REST view drop of '${renamed_view_smoke}' returned HTTP ${code}: $(cat "${body}")" # Transaction commit round trip: a fresh table committed through # POST /v1/{prefix}/transactions/commit rather than the per-table commit route already diff --git a/smoke-stand/TEST-MATRIX.md b/smoke-stand/TEST-MATRIX.md index 9bca03e..7eb1bf3 100644 --- a/smoke-stand/TEST-MATRIX.md +++ b/smoke-stand/TEST-MATRIX.md @@ -89,8 +89,8 @@ because SPNEGO needed a GSS-capable curl inside the network; once that stopped b listener was turned on there too (`rest-catalog.kerberos.principal=HTTP/proxy@SMOKE.LOCAL`, same keytab as the Thrift front door) and the write round trip, the write gate and an unauthenticated-request check were run against it - see the 2026-07-28 kerberos entry below. -The read-only rows (G2-G22, G27-G30) have not yet been re-run against the Kerberos profile and -stay `n/a` until they are. +The read-only rows (G2-G22, G27-G30, G35-G38) have not yet been re-run against the Kerberos +profile and stay `n/a` until they are. | # | Check | plain | kerberos | | --- | --- | --- | --- | @@ -126,8 +126,12 @@ stay `n/a` until they are. | G30 | `POST /v1/{prefix}/tables/rename` with a federated *destination* namespace (source table still under its current name) refused with `403` - proves the destination side of the gate, not just the source | ✅ | n/a | | G31 | A request without `--negotiate` is rejected `401` with a `WWW-Authenticate: Negotiate` challenge and an empty body | n/a | ✅ | | G32 | Namespace DDL round trip: `POST .../namespaces` create (`200`), `GET` load (`200`), `POST .../properties` update (`200`) with a follow-up `GET` confirming the property is actually present, `DELETE` (`204`), `GET` afterward (`404`) - genuinely new: `RoutingMetaStoreClient` did not implement `createDatabase`/`alterDatabase`/`dropDatabase` before this phase, so this is the first time namespace DDL reached a real metastore | ✅ | ✅ | -| G33 | View write round trip: `POST .../views` create answers `200` with a real `metadata-location`, `GET .../views` lists the new view, `DELETE` answers `204` | ✅ | ✅ | +| G33 | View write round trip: `POST .../views` create answers `200` with a real `metadata-location`, `GET .../views` lists the new view, `POST .../views/{view}` update (`assert-view-uuid` requirement + `set-properties`) answers `200` with a follow-up `GET` confirming the property is actually present, `POST /v1/{prefix}/views/rename` answers `204` and the view loads back `200` under the new name while the old name answers `404` - the pair that proves the rename moved it rather than copying it, `DELETE` answers `204` | ✅ | ✅ | | G34 | `POST /v1/{prefix}/transactions/commit` against a freshly created table: answers `204`, and the table's `metadata-location` afterward differs from create's - proof the multi-table commit path actually wrote a new metadata file, not a silent no-op | ✅ | n/a | +| G35 | `POST .../views` (CREATE_VIEW, full valid view body) into the federated `apache__default` namespace refused with `403` - a minimal body instead gets `400` because it fails to parse before the gate is even consulted, so `400` here would mean the request is malformed, not that the gate let it through | ✅ | n/a | +| G36 | `DELETE .../views/{view}` (DROP_VIEW) under the federated `apache__default` namespace refused with `403` | ✅ | n/a | +| G37 | `DELETE /v1/{prefix}/namespaces/{ns}` (DROP_NAMESPACE) of the federated `apache__default` namespace refused with `403` | ✅ | n/a | +| G38 | `POST .../properties` (UPDATE_NAMESPACE) of the federated `apache__default` namespace refused with `403` | ✅ | n/a | ## F. Not covered, and why @@ -285,6 +289,22 @@ executed is claimed; a row not listed was not repeated and its ✅ stands on the `"authenticatedUser":"smoke-user@SMOKE.LOCAL"`, confirming namespace DDL reached the real HDP backend under Kerberos too. G34 (transaction commit) was not re-run under Kerberos and stays `n/a`, matching the task's scope. + Later still (same jar, script-only change, back on the plain profile): the view round trip + (G33) was extended to drive the two advertised view routes it had never exercised - update + (`assert-view-uuid` requirement, `POST .../views/{view}`) and rename (`POST + /v1/{prefix}/views/rename`) - and four more `WriteRouteGate` negatives were added (G35-G38: + CREATE_VIEW, DROP_VIEW, DROP_NAMESPACE and UPDATE_NAMESPACE, all against the federated + `apache__default` namespace under the default prefix). `--scenario rest` and `--scenario all` + both re-ran green: the view update's property reload confirmed `"smoke":"updated"` actually + stuck, the rename answered `204` and the view loaded back `200` under the new name while the + old name answered `404`, and all four new negatives answered `403` (the CREATE_VIEW one with + the full valid view body, since a stub body had earlier answered `400` before the gate was even + reached). The new rename-effect assertion was proven to discriminate by temporarily flipping its + expected status from `404` to `200` (i.e. asserting the old view name is still reachable after + the rename); the rerun failed - the pre-rename name still answered its real `404`, which the + inverted assertion now rejected - confirming the check would catch a rename that copies the view + instead of moving it; the assertion was restored and both `--scenario rest` and `--scenario all` + re-ran green. ## Two caveats on faithfulness diff --git a/smoke-stand/TEST-MATRIX.ru.md b/smoke-stand/TEST-MATRIX.ru.md index 98f69b9..3c74517 100644 --- a/smoke-stand/TEST-MATRIX.ru.md +++ b/smoke-stand/TEST-MATRIX.ru.md @@ -89,7 +89,7 @@ HDP-клиент не может пользоваться Apache-listener — Th и там тоже (`rest-catalog.kerberos.principal=HTTP/proxy@SMOKE.LOCAL`, тот же keytab, что и у Thrift front door), и против него прогнали write round trip, write gate и проверку неаутентифицированного запроса — см. вторую запись за 2026-07-28 в журнале ниже. Read-only строки -(G2-G22, G27-G30) на Kerberos-профиле пока не перепрогонялись и остаются `n/a`. +(G2-G22, G27-G30, G35-G38) на Kerberos-профиле пока не перепрогонялись и остаются `n/a`. | # | Проверка | plain | kerberos | | --- | --- | --- | --- | @@ -125,8 +125,12 @@ Thrift front door), и против него прогнали write round trip, | G30 | `POST /v1/{prefix}/tables/rename` с federated destination-namespace (source-таблица ещё под текущим именем) отклонён с `403` — доказывает проверку именно destination-стороны gate, а не только source | ✅ | n/a | | G31 | Запрос без `--negotiate` отклоняется `401` с вызовом `WWW-Authenticate: Negotiate` и пустым телом | n/a | ✅ | | G32 | Namespace DDL round trip: `POST .../namespaces` create (`200`), `GET` load (`200`), `POST .../properties` update (`200`) с последующим `GET`, подтверждающим, что property реально появилось, `DELETE` (`204`), `GET` после этого (`404`) — по-настоящему новое: `RoutingMetaStoreClient` не реализовывал `createDatabase`/`alterDatabase`/`dropDatabase` до этой фазы, так что namespace DDL впервые дошёл до реального metastore | ✅ | ✅ | -| G33 | View write round trip: `POST .../views` create отвечает `200` с реальным `metadata-location`, `GET .../views` листит новый view, `DELETE` отвечает `204` | ✅ | ✅ | +| G33 | View write round trip: `POST .../views` create отвечает `200` с реальным `metadata-location`, `GET .../views` листит новый view, `POST .../views/{view}` update (requirement `assert-view-uuid` + `set-properties`) отвечает `200`, и последующий `GET` подтверждает, что property реально появилось, `POST /v1/{prefix}/views/rename` отвечает `204`, view загружается обратно `200` под новым именем, а под старым именем отвечает `404` — именно эта пара доказывает, что rename переместил view, а не скопировал его, `DELETE` отвечает `204` | ✅ | ✅ | | G34 | `POST /v1/{prefix}/transactions/commit` против только что созданной таблицы: отвечает `204`, и `metadata-location` таблицы после этого отличается от того, что дал create — доказательство, что multi-table commit реально записал новый metadata-файл, а не тихий no-op | ✅ | n/a | +| G35 | `POST .../views` (CREATE_VIEW, полное валидное тело view) в federated-namespace `apache__default` отклонён с `403` — минимальное тело вместо этого получает `400`, потому что не парсится ещё до того, как gate вообще проверяется, так что `400` здесь означал бы, что тело запроса некорректно, а не что gate пропустил write | ✅ | n/a | +| G36 | `DELETE .../views/{view}` (DROP_VIEW) под federated-namespace `apache__default` отклонён с `403` | ✅ | n/a | +| G37 | `DELETE /v1/{prefix}/namespaces/{ns}` (DROP_NAMESPACE) federated-namespace `apache__default` отклонён с `403` | ✅ | n/a | +| G38 | `POST .../properties` (UPDATE_NAMESPACE) federated-namespace `apache__default` отклонён с `403` | ✅ | n/a | ## F. Что не покрыто и почему @@ -284,6 +288,22 @@ Thrift front door), и против него прогнали write round trip, `drop_database` с `"authenticatedUser":"smoke-user@SMOKE.LOCAL"`, подтверждая, что namespace DDL под Kerberos тоже дошёл до реального HDP-бэкенда. G34 (transaction commit) под Kerberos не перепрогонялся и остаётся `n/a` — в рамках заявленного объёма задачи. + Ещё позже (тот же jar, изменение только в скрипте, снова на plain-профиле): view round trip + (G33) расширен, чтобы прогнать два объявленных view-роута, которые он до сих пор ни разу не + прогонял, — update (requirement `assert-view-uuid`, `POST .../views/{view}`) и rename (`POST + /v1/{prefix}/views/rename`), — и добавлены ещё четыре негатива `WriteRouteGate` (G35-G38: + CREATE_VIEW, DROP_VIEW, DROP_NAMESPACE и UPDATE_NAMESPACE — все против federated-namespace + `apache__default` под default-prefix). `--scenario rest` и `--scenario all` оба перепрогнаны + зелёными: перезагрузка view после update подтвердила, что `"smoke":"updated"` реально + закрепилось, rename ответил `204`, view загрузилось обратно `200` под новым именем, а под + старым именем ответило `404`, и все четыре новых негатива ответили `403` (у CREATE_VIEW — + с полным валидным телом view, поскольку заглушка ранее отвечала `400` ещё до того, как gate + вообще был достигнут). Новая проверка эффекта rename доказала свою различающую способность: + ожидаемый статус был временно перевёрнут с `404` на `200` (то есть утверждалось, что старое + имя view остаётся доступным после rename); перепрогон упал — старое имя по-прежнему честно + отвечало `404`, что инвертированная проверка теперь отвергала, — подтвердив, что проверка + поймает rename, который копирует view вместо того, чтобы его переместить; проверка + восстановлена, оба сценария (`rest` и `all`) перепрогнаны зелёными. ## Две оговорки честности From 5f84d4eb7f0396cda21f04787fc192cf78bda35e Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 15:30:53 +0300 Subject: [PATCH 49/50] Fail closed on unresolved write namespaces, guard write-route/endpoint drift WriteRouteGate previously permitted a write when the namespace's owning catalog could not be resolved; make that case a refusal instead, since an unknown catalog is exactly the ambiguous case the synthetic lock shim's lack of conflict checking makes unsafe to guess about (currently unreachable in production, but the gate's shape should not rely on that). Add a test tying WriteRouteGate.WRITE_ROUTES to IcebergRestService.WRITE_ENDPOINTS one-to-one so a future write route added upstream cannot silently lag in the advertised endpoint list, and pin the integration test's positive endpoint assertions to the exact JSON-quoted rendering so a dropped route constant cannot hide behind a sibling whose longer rendering happens to share the same prefix (e.g. CREATE_VIEW vs UPDATE_VIEW). Also reorder IcebergHttpHandler.handle()'s finally block so the per-request thread-local context is restored before the metric is recorded, so a throwing recordRestRequest can no longer leak remoteAddress/remoteUser into the next request served by that pool thread. --- .../restcatalog/IcebergHttpHandler.java | 11 ++- .../restcatalog/IcebergRestService.java | 8 +++ .../hmsproxy/restcatalog/WriteRouteGate.java | 17 ++++- .../IcebergRestEndpointIntegrationTest.java | 30 ++++++-- .../restcatalog/WriteRouteGateTest.java | 70 +++++++++++++++++++ 5 files changed, 124 insertions(+), 12 deletions(-) diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java index 351db21..4c3c5b5 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergHttpHandler.java @@ -81,10 +81,17 @@ public void handle(HttpExchange exchange) throws IOException { try { doHandle(exchange, outcome); } finally { + // Restore the thread-local request context before recording the metric, not after: this + // handler instance is shared across the HTTP executor's threads, so if + // recordRestRequest ever threw, an ordering with the restore last would leak + // remoteAddress/remoteUser into whatever request that pool thread serves next. + try { + ClientRequestContext.restoreRemoteAddress(previousAddress); + } finally { + ClientRequestContext.restoreRemoteUser(previousUser); + } double durationSeconds = (System.nanoTime() - startNanos) / 1_000_000_000.0; metrics.recordRestRequest(outcome.prefix, outcome.route, outcome.status, durationSeconds); - ClientRequestContext.restoreRemoteAddress(previousAddress); - ClientRequestContext.restoreRemoteUser(previousUser); } } diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java index 5b26a20..0eee568 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestService.java @@ -77,6 +77,14 @@ public final class IcebergRestService implements AutoCloseable { private static final List DEFAULT_CATALOG_ENDPOINTS = Stream.concat(READ_ENDPOINTS.stream(), WRITE_ENDPOINTS.stream()).toList(); + /** + * Test-only accessor so a test can assert {@link #WRITE_ENDPOINTS} corresponds one-to-one with + * {@link WriteRouteGate}'s gated write routes - never widen this beyond package-private. + */ + static List writeEndpointsForTesting() { + return WRITE_ENDPOINTS; + } + private final String catalogName; private final RoutingHiveCatalog catalog; private final RESTCatalogAdapter adapter; diff --git a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGate.java b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGate.java index edb0fc7..7621b29 100644 --- a/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGate.java +++ b/src/main/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGate.java @@ -28,8 +28,11 @@ * resolves an external database name to the catalog that owns it; for the default catalog's own * service that is the federated name as-is, and for a name-translated service {@link * IcebergRestService} composes it with that service's own translation so it always answers with - * that service's own (non-default) catalog. A namespace that does not resolve at all is not this - * gate's business - {@code null} is returned so the normal dispatch can produce its usual 404. + * that service's own (non-default) catalog. This gate fails closed: a namespace whose owning + * catalog cannot be determined is refused, not allowed - an unknown catalog ownership is exactly + * the ambiguous case the synthetic lock shim's lack of conflict checking makes unsafe to guess + * about. Today {@code CatalogRouter.resolveDatabase} is total and never actually produces this + * case, but the gate does not rely on that holding forever. */ final class WriteRouteGate { // Every write route RESTCatalogAdapter.Route exposes. Read-only routes (LIST_*, LOAD_*, @@ -117,7 +120,15 @@ private String refusalFor(String externalDbName) { return null; } String resolvedCatalog = catalogForNamespace.apply(externalDbName); - if (resolvedCatalog == null || resolvedCatalog.equals(defaultCatalogName)) { + if (resolvedCatalog == null) { + // Fail closed: an unresolved namespace's owning catalog - and so whether it is backed by + // a real HMS lock or the synthetic shim - cannot be determined, so the write is refused + // rather than permitted by default. + return "Writes are only supported in the default catalog '" + defaultCatalogName + + "'; namespace '" + externalDbName + "' could not be resolved to any catalog, so " + + "whether it is safe to write could not be determined."; + } + if (resolvedCatalog.equals(defaultCatalogName)) { return null; } return "Writes are only supported in the default catalog '" + defaultCatalogName diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java index d4c9164..a9ac3f3 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/IcebergRestEndpointIntegrationTest.java @@ -352,6 +352,22 @@ public void commitTransactionMixingFederatedTableIsRefused() throws Exception { Assert.assertTrue(response.body(), response.body().contains("ForbiddenException")); } + // Iceberg's Endpoint#toString() renders " ", and several write routes' + // renderings are a plain prefix of a sibling route's rendering - e.g. V1_CREATE_TABLE's + // "POST /v1/{prefix}/namespaces/{namespace}/tables" is a prefix of V1_UPDATE_TABLE's + // ".../tables/{table}", and V1_CREATE_NAMESPACE's bare "POST /v1/{prefix}/namespaces" is a + // prefix of nearly every other namespace/table/view write route. A plain body.contains(...) on + // one of these bare renderings would still pass even if that exact Endpoint constant were + // dropped from IcebergRestService.WRITE_ENDPOINTS, as long as the sibling with the longer + // rendering stayed - it would not pin the route it claims to. Printing the actual + // ConfigResponse JSON (via ConfigResponseParser, e.g. {"endpoints":["POST /v1/{prefix}/namespaces", + // "POST /v1/{prefix}/namespaces/{namespace}/tables", ...]}) confirms each endpoint is emitted as + // its own JSON string, so wrapping the expected rendering in its JSON quotes pins it to exactly + // the one Endpoint constant that produces that full quoted token. + private static String jsonEndpoint(String httpMethodAndPath) { + return "\"" + httpMethodAndPath + "\""; + } + @Test public void configAdvertisesOnlyServedEndpoints() throws Exception { // /v1/config with no warehouse resolves to the default catalog (catalog1 in this fixture), @@ -365,24 +381,24 @@ public void configAdvertisesOnlyServedEndpoints() throws Exception { Assert.assertTrue(body, body.contains("GET /v1/{prefix}/namespaces")); Assert.assertTrue(body, body.contains("HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}")); Assert.assertTrue("default catalog must advertise table writes: " + body, - body.contains("POST /v1/{prefix}/namespaces/{namespace}/tables")); + body.contains(jsonEndpoint("POST /v1/{prefix}/namespaces/{namespace}/tables"))); Assert.assertTrue("default catalog must advertise table deletes: " + body, body.contains("DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}")); Assert.assertTrue( "default catalog must advertise view writes: " + body, - body.contains("POST /v1/{prefix}/namespaces/{namespace}/views")); + body.contains(jsonEndpoint("POST /v1/{prefix}/namespaces/{namespace}/views"))); Assert.assertTrue( "default catalog must advertise view renames: " + body, body.contains("POST /v1/{prefix}/views/rename")); Assert.assertTrue( "default catalog must advertise namespace creation: " + body, - body.contains("POST /v1/{prefix}/namespaces")); + body.contains(jsonEndpoint("POST /v1/{prefix}/namespaces"))); Assert.assertTrue( "default catalog must advertise namespace property updates: " + body, body.contains("POST /v1/{prefix}/namespaces/{namespace}/properties")); Assert.assertTrue( "default catalog must advertise namespace deletes: " + body, - body.contains("DELETE /v1/{prefix}/namespaces/{namespace}")); + body.contains(jsonEndpoint("DELETE /v1/{prefix}/namespaces/{namespace}"))); Assert.assertTrue( "default catalog must advertise the transaction commit: " + body, body.contains("POST /v1/{prefix}/transactions/commit")); @@ -391,8 +407,8 @@ public void configAdvertisesOnlyServedEndpoints() throws Exception { @Test public void defaultCatalogAdvertisesViewAndNamespaceWrites() throws Exception { String body = get("/v1/" + CATALOG_NAME + "/config").body(); - Assert.assertTrue(body, body.contains("POST /v1/{prefix}/namespaces/{namespace}/views")); - Assert.assertTrue(body, body.contains("POST /v1/{prefix}/namespaces")); + Assert.assertTrue(body, body.contains(jsonEndpoint("POST /v1/{prefix}/namespaces/{namespace}/views"))); + Assert.assertTrue(body, body.contains(jsonEndpoint("POST /v1/{prefix}/namespaces"))); Assert.assertTrue(body, body.contains("POST /v1/{prefix}/transactions/commit")); } @@ -407,7 +423,7 @@ public void nonDefaultCatalogAdvertisesNoWritesAtAll() throws Exception { @Test public void defaultCatalogConfigAdvertisesWrites() throws Exception { String body = get("/v1/" + CATALOG_NAME + "/config").body(); - Assert.assertTrue(body, body.contains("POST /v1/{prefix}/namespaces/{namespace}/tables")); + Assert.assertTrue(body, body.contains(jsonEndpoint("POST /v1/{prefix}/namespaces/{namespace}/tables"))); } @Test diff --git a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGateTest.java b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGateTest.java index e5483ab..9530e20 100644 --- a/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGateTest.java +++ b/src/test/java/io/github/mmalykhin/hmsproxy/restcatalog/WriteRouteGateTest.java @@ -1,5 +1,6 @@ package io.github.mmalykhin.hmsproxy.restcatalog; +import java.util.EnumMap; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -8,6 +9,7 @@ import org.apache.iceberg.UpdateRequirement; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.Endpoint; import org.apache.iceberg.rest.RESTCatalogAdapter.Route; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; @@ -47,6 +49,62 @@ public class WriteRouteGateTest { Route.REPORT_METRICS, Route.LIST_VIEWS, Route.VIEW_EXISTS, Route.LOAD_VIEW); + // Hand-maintained Route -> Endpoint correspondence for the write surface: WriteRouteGate gates + // on org.apache.iceberg.rest.RESTCatalogAdapter.Route, while IcebergRestService advertises + // discovery through the unrelated org.apache.iceberg.rest.Endpoint type, so nothing in the + // production code ties the two enumerations together. This table is that tie: it must be kept + // in sync with both WriteRouteGate.WRITE_ROUTES and IcebergRestService.WRITE_ENDPOINTS whenever + // either changes, and everyGatedWriteRouteHasAnAdvertisedEndpointAndViceVersa below fails until + // it is. + private static final Map WRITE_ROUTE_TO_ENDPOINT = new EnumMap<>(Route.class); + + static { + WRITE_ROUTE_TO_ENDPOINT.put(Route.CREATE_TABLE, Endpoint.V1_CREATE_TABLE); + WRITE_ROUTE_TO_ENDPOINT.put(Route.UPDATE_TABLE, Endpoint.V1_UPDATE_TABLE); + WRITE_ROUTE_TO_ENDPOINT.put(Route.DROP_TABLE, Endpoint.V1_DELETE_TABLE); + WRITE_ROUTE_TO_ENDPOINT.put(Route.RENAME_TABLE, Endpoint.V1_RENAME_TABLE); + WRITE_ROUTE_TO_ENDPOINT.put(Route.REGISTER_TABLE, Endpoint.V1_REGISTER_TABLE); + WRITE_ROUTE_TO_ENDPOINT.put(Route.CREATE_VIEW, Endpoint.V1_CREATE_VIEW); + WRITE_ROUTE_TO_ENDPOINT.put(Route.UPDATE_VIEW, Endpoint.V1_UPDATE_VIEW); + WRITE_ROUTE_TO_ENDPOINT.put(Route.DROP_VIEW, Endpoint.V1_DELETE_VIEW); + WRITE_ROUTE_TO_ENDPOINT.put(Route.RENAME_VIEW, Endpoint.V1_RENAME_VIEW); + WRITE_ROUTE_TO_ENDPOINT.put(Route.CREATE_NAMESPACE, Endpoint.V1_CREATE_NAMESPACE); + WRITE_ROUTE_TO_ENDPOINT.put(Route.UPDATE_NAMESPACE, Endpoint.V1_UPDATE_NAMESPACE); + WRITE_ROUTE_TO_ENDPOINT.put(Route.DROP_NAMESPACE, Endpoint.V1_DELETE_NAMESPACE); + WRITE_ROUTE_TO_ENDPOINT.put(Route.COMMIT_TRANSACTION, Endpoint.V1_COMMIT_TRANSACTION); + } + + @Test + public void everyGatedWriteRouteHasAnAdvertisedEndpointAndViceVersa() { + Set gatedRoutes = WriteRouteGate.writeRoutesForTesting(); + List advertisedEndpoints = IcebergRestService.writeEndpointsForTesting(); + String updateBothMessage = "WriteRouteGate.WRITE_ROUTES, IcebergRestService.WRITE_ENDPOINTS and " + + "this test's WRITE_ROUTE_TO_ENDPOINT table have drifted apart - update all three together."; + + Assert.assertEquals( + "every gated write Route must have exactly one corresponding advertised Endpoint mapped " + + "in this test's WRITE_ROUTE_TO_ENDPOINT; " + updateBothMessage, + gatedRoutes, WRITE_ROUTE_TO_ENDPOINT.keySet()); + + for (Map.Entry entry : WRITE_ROUTE_TO_ENDPOINT.entrySet()) { + Assert.assertTrue( + "Route." + entry.getKey().name() + " is gated as a write but its mapped Endpoint " + + entry.getValue() + " is not in IcebergRestService.WRITE_ENDPOINTS; " + + updateBothMessage, + advertisedEndpoints.contains(entry.getValue())); + } + for (Endpoint endpoint : advertisedEndpoints) { + Assert.assertTrue( + "IcebergRestService advertises write endpoint " + endpoint + " that no gated write " + + "Route maps to in this test's WRITE_ROUTE_TO_ENDPOINT; " + updateBothMessage, + WRITE_ROUTE_TO_ENDPOINT.containsValue(endpoint)); + } + Assert.assertEquals( + "IcebergRestService.WRITE_ENDPOINTS has a duplicate or is missing an entry relative to " + + "the gated write routes; " + updateBothMessage, + WRITE_ROUTE_TO_ENDPOINT.size(), advertisedEndpoints.size()); + } + private static String catalogForNamespace(String externalDbName) { if (externalDbName != null && externalDbName.startsWith(OTHER_CATALOG + SEPARATOR)) { return OTHER_CATALOG; @@ -91,6 +149,18 @@ public void writeToDefaultCatalogNamespaceIsAllowed() { Assert.assertNull(GATE.check(Route.CREATE_TABLE, vars, null)); } + @Test + public void writeToUnresolvableNamespaceIsRefused() { + // The gate must fail closed: if the resolver cannot say which catalog owns this namespace, + // it is refused rather than permitted - an unknown catalog is exactly the ambiguous case the + // synthetic lock shim's lack of conflict checking makes unsafe to guess about. + WriteRouteGate gateWithUnresolvingCatalogLookup = new WriteRouteGate(DEFAULT_CATALOG, externalDbName -> null); + Map vars = Map.of("namespace", "mystery"); + String refusal = gateWithUnresolvingCatalogLookup.check(Route.CREATE_TABLE, vars, null); + Assert.assertNotNull("an unresolvable namespace must be refused, not allowed", refusal); + Assert.assertTrue(refusal, refusal.contains("could not be determined")); + } + @Test public void writeToFederatedNamespaceUnderDefaultPrefixIsRefused() { Map vars = Map.of("namespace", OTHER_CATALOG + SEPARATOR + "default"); From 86c81e2fe5173328da4dc4a78fb4606bd13319c7 Mon Sep 17 00:00:00 2001 From: Maksim Malykhin Date: Tue, 28 Jul 2026 15:41:35 +0300 Subject: [PATCH 50/50] Make the REST smoke's namespace/view names configurable and safer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Iceberg REST write round trip hardcoded the namespace ("smoke_rest_ns") and view ("smoke_rest_view") names it creates and defensively pre-deletes, unlike the write table which was already configurable via HMS_SMOKE_REST_WRITE_TABLE. Against a real installation this could silently drop a pre-existing namespace sharing the default name. Add HMS_SMOKE_REST_WRITE_NAMESPACE and HMS_SMOKE_REST_WRITE_VIEW (defaulting to the previous hardcoded values), document them in the runner's usage text and env example, and set them explicitly in the stand's simple.env. The defensive namespace pre-delete now only runs when the namespace is absent or already empty (no tables, no views), refusing with a clear error otherwise instead of destroying real content. Also fix mktemp templates that put a literal suffix after the X's (...XXXXXX.json / .sql / .out): BSD mktemp on macOS does not randomize X's followed by a suffix, so every run reused the same filename, and a leftover from an interrupted run (fail() bypasses the cleanup trap) broke the next run. Put the X's last and append the suffix with mv after creation. Finally, reword "write-роут" back to "write-поверхность"/"все write-роуты" in three RU-only spots (README.ru.md, CHANGELOG.ru.md, smoke-stand/README.ru.md) where "write surface" had been mistranslated as a single route rather than the whole set of write routes. --- CHANGELOG.ru.md | 4 +- README.ru.md | 6 +- ...real-installation-smoke.simple.env.example | 16 +++- scripts/run-real-installation-smoke.sh | 91 +++++++++++++++---- smoke-stand/README.ru.md | 2 +- smoke-stand/env/simple.env | 10 +- 6 files changed, 97 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.ru.md b/CHANGELOG.ru.md index 6766e54..a679c1f 100644 --- a/CHANGELOG.ru.md +++ b/CHANGELOG.ru.md @@ -52,8 +52,8 @@ English version: [CHANGELOG.md](CHANGELOG.md). независимо от каталога. Имена транслируются через существующий `CatalogNameTranslation`, а payload `Database`, передаваемый в create/alter, транслируется на копии, а не мутацией объекта вызывающего. -- `GET /v1/config` и `GET /v1/{prefix}/config` теперь объявляют весь - обслуживаемый write-роут целиком: view CRUD/rename и namespace CRUD уже +- `GET /v1/config` и `GET /v1/{prefix}/config` теперь объявляют все + обслуживаемые write-роуты: view CRUD/rename и namespace CRUD уже были достижимы через тот же общий dispatch-путь, которым пользуется write таблиц, и `WriteRouteGate` уже гейтил все тринадцать write-роутов — отставали только discovery и smoke. В `endpoints` дефолтного каталога теперь diff --git a/README.ru.md b/README.ru.md index c5b0251..40940e5 100644 --- a/README.ru.md +++ b/README.ru.md @@ -983,11 +983,11 @@ catalog.catalog2.conf.hms.proxy.external-table-drop-purge.allowed-prefixes=hdfs: Proxy дополнительно умеет поднять параллельный HTTP listener со спецификацией Iceberg REST Catalog, использующий тот же routing/federation pipeline что и -Thrift HMS front door. Статус: **экспериментально**; весь write-роут, который -выставляет `RESTCatalogAdapter`, — write таблиц (create, commit, drop, +Thrift HMS front door. Статус: **экспериментально**; вся write-поверхность, +которую выставляет `RESTCatalogAdapter`, — write таблиц (create, commit, drop, rename, register), write view (create, commit, drop, rename) и namespace DDL (create, update properties, drop), а также multi-table transaction commit — -поддержан, но **только когда целевой namespace резолвится в +поддержана, но **только когда целевой namespace резолвится в `routing.default-catalog`**. Iceberg-клиенты (PyIceberg, Spark `iceberg-rest`, Trino `iceberg-rest`) могут discover и load Iceberg-таблицы, хранящиеся в HMS через стандартный параметр `metadata_location`. diff --git a/scripts/hms-real-installation-smoke.simple.env.example b/scripts/hms-real-installation-smoke.simple.env.example index 180cdb1..9a2f304 100644 --- a/scripts/hms-real-installation-smoke.simple.env.example +++ b/scripts/hms-real-installation-smoke.simple.env.example @@ -129,9 +129,19 @@ HMS_SMOKE_PARTITION_LOCK_TRANSACTIONAL=false # with a federated name, and a rename with a federated destination - proving the gate is enforced # on the resolved catalog, not the request's own prefix. Use a name distinct from every other table # already on the target so a rerun cannot collide with a leftover. Also gates a namespace DDL round -# trip (create/load/update-property/drop of namespace "smoke_rest_ns"), a view round trip -# (create/list/drop of view "smoke_rest_view" in HMS_SMOKE_REST_NAMESPACE) and a multi-table +# trip (create/load/update-property/drop of HMS_SMOKE_REST_WRITE_NAMESPACE), a view round trip +# (create/list/drop of HMS_SMOKE_REST_WRITE_VIEW in HMS_SMOKE_REST_NAMESPACE) and a multi-table # transaction commit round trip (create "_txn", commit through # POST /v1/{prefix}/transactions/commit, assert its metadata-location changed, drop) - the same -# default-catalog restriction applies to every one of these routes. +# default-catalog restriction applies to every one of these routes. All three objects (the table, +# the namespace and the view) are created and destroyed by this smoke run itself; the namespace's +# defensive pre-delete only goes through when it does not exist yet or is already empty (no +# tables, no views), so a pre-existing namespace of the same name on a real installation is +# refused rather than silently dropped. # HMS_SMOKE_REST_WRITE_TABLE=smoke_rest_written +# Namespace created/destroyed by the write round trip above. Default: smoke_rest_ns. Override this +# on a real installation if a namespace with the default name may already exist. +# HMS_SMOKE_REST_WRITE_NAMESPACE=smoke_rest_ns +# View created/destroyed by the write round trip above, in HMS_SMOKE_REST_NAMESPACE. Default: +# smoke_rest_view. +# HMS_SMOKE_REST_WRITE_VIEW=smoke_rest_view diff --git a/scripts/run-real-installation-smoke.sh b/scripts/run-real-installation-smoke.sh index ab90711..f88e53b 100755 --- a/scripts/run-real-installation-smoke.sh +++ b/scripts/run-real-installation-smoke.sh @@ -89,8 +89,8 @@ Optional Iceberg REST env vars: HMS_SMOKE_REST_WRITE_TABLE table name to create, commit, rename and drop through the REST write routes; skipped when unset. Also gates a namespace DDL round trip (create/load/update-property/ - drop of namespace "smoke_rest_ns"), a view round trip - (create/list/drop of view "smoke_rest_view" in + drop of HMS_SMOKE_REST_WRITE_NAMESPACE), a view round + trip (create/list/drop of HMS_SMOKE_REST_WRITE_VIEW in HMS_SMOKE_REST_NAMESPACE) and a multi-table transaction commit round trip (create "_txn", commit through POST /v1/{prefix}/transactions/commit, assert its @@ -99,14 +99,24 @@ Optional Iceberg REST env vars: Writes only work when the advertised default-prefix catalog resolves to the real backend (non-default catalogs are served by the synthetic lock shim and refuse - writes with 403). The block drops leftovers under every + writes with 403). All three objects (the table, the + namespace and the view) are created and destroyed by this + smoke run itself. The block drops leftovers under every name it can create defensively before creating, so a - rerun on a dirty stand cannot half-fail. Requires - HMS_SMOKE_REST_SECOND_PREFIX to also exercise the gate - negatives: CREATE_TABLE under that prefix and under its - federated name, COMMIT_TRANSACTION naming a federated - table, CREATE_NAMESPACE with a federated name, and RENAME - with a federated destination - all expected 403 + rerun on a dirty stand cannot half-fail; the namespace + drop only goes through when the namespace does not exist + yet or is already empty (no tables, no views), so a + pre-existing namespace of the same name on a real + installation is refused rather than silently destroyed. + Requires HMS_SMOKE_REST_SECOND_PREFIX to also exercise + the gate negatives: CREATE_TABLE under that prefix and + under its federated name, COMMIT_TRANSACTION naming a + federated table, CREATE_NAMESPACE with a federated name, + and RENAME with a federated destination - all expected 403 + HMS_SMOKE_REST_WRITE_NAMESPACE namespace created/destroyed by the write round trip + above; default: smoke_rest_ns + HMS_SMOKE_REST_WRITE_VIEW view created/destroyed by the write round trip above, + in HMS_SMOKE_REST_NAMESPACE; default: smoke_rest_view EOF cat <<'EOF' @@ -671,8 +681,16 @@ run_sql_smoke() { local cross_db_table="smoke_cross_db_tbl_${run_id}" local sql_file="" local output_file="" - sql_file="$(mktemp "${TMPDIR:-/tmp}/hms-proxy-sql-smoke.XXXXXX.sql")" - output_file="$(mktemp "${TMPDIR:-/tmp}/hms-proxy-sql-smoke.XXXXXX.out")" + # The X's must be the last characters of the template: BSD mktemp (macOS) only randomizes a + # trailing run of X's and leaves a literal suffix untouched, so every run would otherwise reuse + # the same filename - and a leftover from an interrupted run (fail() exits before the cleanup + # trap runs) then breaks the next one. The .sql/.out suffix is appended after creation instead. + sql_file="$(mktemp "${TMPDIR:-/tmp}/hms-proxy-sql-smoke.XXXXXX")" + mv "${sql_file}" "${sql_file}.sql" + sql_file="${sql_file}.sql" + output_file="$(mktemp "${TMPDIR:-/tmp}/hms-proxy-sql-smoke.XXXXXX")" + mv "${output_file}" "${output_file}.out" + output_file="${output_file}.out" # ${var:-} guards matter: the RETURN trap stays installed after this function returns and # fires again for enclosing functions, where these locals no longer exist and set -u would # kill the whole run. @@ -942,7 +960,13 @@ run_rest_smoke() { local iceberg_table="${HMS_SMOKE_REST_ICEBERG_TABLE:-}" local non_iceberg_table="${HMS_SMOKE_REST_NON_ICEBERG_TABLE:-}" local body="" - body="$(mktemp "${TMPDIR:-/tmp}/hms-proxy-rest-smoke.XXXXXX.json")" + # The X's must trail the template with nothing after them: BSD mktemp (macOS) does not + # randomize X's followed by a literal suffix, so a fixed ".json" suffix here would make every + # run reuse the same filename - and a leftover from an interrupted run (fail() exits before the + # cleanup trap runs) then breaks the next one. The suffix is appended after creation instead. + body="$(mktemp "${TMPDIR:-/tmp}/hms-proxy-rest-smoke.XXXXXX")" + mv "${body}" "${body}.json" + body="${body}.json" trap 'rm -f "${body:-}"' RETURN log "running Iceberg REST smoke against ${HMS_SMOKE_REST_URL}" @@ -1270,10 +1294,35 @@ run_rest_smoke() { # Namespace DDL round trip: create, load, update a property and drop - genuinely new since # RoutingMetaStoreClient only gained createDatabase/alterDatabase/dropDatabase this phase. # Same default-catalog-only restriction as table writes (WriteRouteGate covers CREATE_NAMESPACE - # by namespace, not by the request's own prefix). Dropped defensively first so a rerun on a - # dirty stand cannot half-fail. - local ns_smoke="smoke_rest_ns" - discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${ns_smoke}" "${body}")" + # by namespace, not by the request's own prefix). This namespace (like the write table above, + # and the view below) is created and destroyed by the smoke itself. + local ns_smoke="${HMS_SMOKE_REST_WRITE_NAMESPACE:-smoke_rest_ns}" + # Dropped defensively first so a rerun on a dirty stand cannot half-fail - but only when it is + # either absent or already empty. A real installation may already have a namespace under this + # exact name (especially the default), and blindly dropping it would destroy real content; + # emptiness (no tables, no views) is the cheap signal that whatever is there was left by an + # earlier interrupted run of this same smoke rather than being genuine. + local ns_precheck_code="" + ns_precheck_code="$(rest_request GET "/v1/${prefix}/namespaces/${ns_smoke}" "${body}")" + if [[ "${ns_precheck_code}" == "200" ]]; then + local ns_precheck_tables_code="" + ns_precheck_tables_code="$(rest_request GET "/v1/${prefix}/namespaces/${ns_smoke}/tables" "${body}")" + [[ "${ns_precheck_tables_code}" == "200" ]] \ + || fail "GET tables of '${ns_smoke}' before the defensive namespace drop returned HTTP ${ns_precheck_tables_code}: $(cat "${body}")" + grep -q '"identifiers"[[:space:]]*:[[:space:]]*\[\]' "${body}" \ + || fail "namespace '${ns_smoke}' already exists and has tables; refusing to drop it defensively. Set HMS_SMOKE_REST_WRITE_NAMESPACE to a namespace name that is not in use on this installation." + + local ns_precheck_views_code="" + ns_precheck_views_code="$(rest_request GET "/v1/${prefix}/namespaces/${ns_smoke}/views" "${body}")" + [[ "${ns_precheck_views_code}" == "200" ]] \ + || fail "GET views of '${ns_smoke}' before the defensive namespace drop returned HTTP ${ns_precheck_views_code}: $(cat "${body}")" + grep -q '"identifiers"[[:space:]]*:[[:space:]]*\[\]' "${body}" \ + || fail "namespace '${ns_smoke}' already exists and has views; refusing to drop it defensively. Set HMS_SMOKE_REST_WRITE_NAMESPACE to a namespace name that is not in use on this installation." + + discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${ns_smoke}" "${body}")" + elif [[ "${ns_precheck_code}" != "404" ]]; then + fail "GET namespace '${ns_smoke}' before the defensive namespace drop returned HTTP ${ns_precheck_code}: $(cat "${body}")" + fi code="$(curl -sS -o "${body}" -w '%{http_code}' -X POST -H 'Content-Type: application/json' \ --data '{"namespace":["'"${ns_smoke}"'"]}' "${HMS_SMOKE_REST_URL}/v1/${prefix}/namespaces")" @@ -1304,10 +1353,12 @@ run_rest_smoke() { # View write round trip: create (asserting a real metadata-location), list, update and # rename (asserting effects, not just status codes), and drop. View writes were already # reachable after phase 5a's commit path; this smoke makes that officially covered rather - # than merely advertised, under the same default-catalog-only restriction. Both the - # original and the renamed name are dropped defensively first so a rerun on a dirty stand - # cannot half-fail. - local view_smoke="smoke_rest_view" + # than merely advertised, under the same default-catalog-only restriction. This view is + # created and destroyed by the smoke itself. Both the original and the renamed name are + # dropped defensively first so a rerun on a dirty stand cannot half-fail - a single named + # view is a narrower blast radius than a whole namespace, so no emptiness check is needed + # here the way there is for the namespace above. + local view_smoke="${HMS_SMOKE_REST_WRITE_VIEW:-smoke_rest_view}" local renamed_view_smoke="${view_smoke}_renamed" discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/views/${view_smoke}" "${body}")" discard="$(rest_request DELETE "/v1/${prefix}/namespaces/${namespace}/views/${renamed_view_smoke}" "${body}")" diff --git a/smoke-stand/README.ru.md b/smoke-stand/README.ru.md index 4efd50c..99debef 100644 --- a/smoke-stand/README.ru.md +++ b/smoke-stand/README.ru.md @@ -182,7 +182,7 @@ docker logs stand-proxy 2>&1 | grep 'requires a Hortonworks backend runtime' ## Iceberg REST catalog front door Plain-профиль включает и Iceberg REST listener прокси (`rest-catalog.*` в -`proxy/hms-proxy.properties`, host-порт 19183). Он обслуживает весь write-роут — +`proxy/hms-proxy.properties`, host-порт 19183). Он обслуживает все write-роуты — table, view и namespace DDL, а также multi-table transaction commit, — но только для default-каталога (`hdp`): его таблицы подкреплены реальным HMS-локом, а любой другой каталог обслуживает synthetic lock shim и отказывает write с `403`. `--scenario rest` diff --git a/smoke-stand/env/simple.env b/smoke-stand/env/simple.env index b47a317..5dbb4ae 100644 --- a/smoke-stand/env/simple.env +++ b/smoke-stand/env/simple.env @@ -102,10 +102,14 @@ HMS_SMOKE_REST_SECOND_NON_ICEBERG_TABLE=smoke_read_ap # default prefix) - both must answer 403, since only the default catalog's commit path reaches a # real backend lock. The name is distinct from every other table already on the stand so a rerun # cannot collide with a leftover from a previous pass. Also gates a namespace DDL round trip -# (smoke_rest_ns), a view round trip (smoke_rest_view, in HMS_SMOKE_REST_NAMESPACE) and a -# multi-table transaction commit round trip (_txn) - the same default-catalog -# restriction applies to all of them. +# (HMS_SMOKE_REST_WRITE_NAMESPACE), a view round trip (HMS_SMOKE_REST_WRITE_VIEW, in +# HMS_SMOKE_REST_NAMESPACE) and a multi-table transaction commit round trip (_txn) - +# the same default-catalog restriction applies to all of them. All three objects are created and +# destroyed by this smoke run itself; set explicitly here (matching the defaults) for clarity even +# though the runner would use the same names unset. HMS_SMOKE_REST_WRITE_TABLE=smoke_rest_written +HMS_SMOKE_REST_WRITE_NAMESPACE=smoke_rest_ns +HMS_SMOKE_REST_WRITE_VIEW=smoke_rest_view # Management metrics endpoint, reachable from the host. When set, the REST smoke also checks it # carries the hms_proxy_rest_requests_total and hms_proxy_rest_listener_info series.