diff --git a/.circleci/config.yml b/.circleci/config.yml index 42ccd67f4f..aca64cb11e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -389,7 +389,7 @@ jobs: } build_and_push cmd sqrl-cli --build-arg BASE_TAG=$BASE_TAG - build_and_push sqrl-server sqrl-server/sqrl-server-vertx --build-arg DUCKDB_EXTENSIONS_TAG=$DUCKDB_EXTENSIONS_TAG + build_and_push sqrl-server sqrl-server/sqrl-server-spring --build-arg DUCKDB_EXTENSIONS_TAG=$DUCKDB_EXTENSIONS_TAG docker-build-duckdb-extensions: machine: diff --git a/CLAUDE.md b/CLAUDE.md index 3d4d646442..ec42b4be8b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ mvn clean install -P quickbuild mvn -P dev initialize # Server-specific builds -mvn clean package # Build fat JAR (vertx-server.jar) +mvn clean package # Build fat JAR (spring-server.jar) mvn clean package -Pskip-shade-plugin # Build without fat JAR mvn clean package -Pinstrument # Build with JaCoCo instrumentation ``` @@ -94,8 +94,7 @@ This is a multi-module Maven project with the following key components: **sqrl-server/** - GraphQL API server implementation that translates GraphQL queries, mutations, and subscriptions into database calls: - `sqrl-server-core/` - Core interfaces and models (GraphQL schema, execution coordinates) -- `sqrl-server-vertx-base/` - Full Vert.x implementation with database clients and Kafka integration -- `sqrl-server-vertx/` - Standalone deployment module with Docker support +- `sqrl-server-spring/` - Spring Boot implementation with R2DBC/JDBC database clients and Kafka integration **sqrl-tools/** - Command-line tools and utilities: - `sqrl-cli/` - Main CLI interface (entry point: `com.datasqrl.cli.DatasqrlCli`) @@ -111,11 +110,11 @@ This is a multi-module Maven project with the following key components: - **Java 17** with Maven build system - **Apache Flink 1.19.2** for stream processing - **Apache Calcite 1.27.0** for SQL parsing and optimization -- **Vert.x 5.0.0** for API server +- **Spring Boot 4.0** for API server (WebFlux reactive) - **GraphQL Java 19.2** for API generation - **Apache Kafka 3.4.0** for streaming - **PostgreSQL 42.7.7** for storage -- **JUnit 5** with Testcontainers for testing +- **JUnit 6** with Testcontainers for testing ## Development Workflow @@ -142,7 +141,7 @@ All dependency versions should be centralized as properties in the root pom.xml ```xml 2.19.1 - 5.0.1 + 4.0.2 3.4.0 1.19.3 4.5.14 @@ -280,28 +279,27 @@ If tests fail due to Flink memory issues, uncomment the configuration line in `E ### Key Design Patterns - **Visitor Pattern**: Extensively used for processing GraphQL model (`RootVisitor`, `QueryCoordVisitor`, `SchemaVisitor`) -- **Reactive Architecture**: Built on Vert.x event loop with CompletableFuture for async operations -- **Schema-First**: GraphQL schema loaded from `server-model.json` at runtime with pre-compiled execution paths +- **Reactive Architecture**: Built on Spring WebFlux with CompletableFuture for async operations +- **Schema-First**: GraphQL schema loaded from `vertx.json` at runtime with pre-compiled execution paths ### Runtime Model -The server operates on a compiled model where the DataSQRL compiler generates `server-model.json` containing all GraphQL execution metadata. The server loads this at startup and creates optimized execution paths - no runtime SQL generation occurs. +The server operates on a compiled model where the DataSQRL compiler generates `vertx.json` containing all GraphQL execution metadata. The server loads this at startup and creates optimized execution paths - no runtime SQL generation occurs. ### Database Abstraction -Multi-database support through `SqlClient` interface: -- PostgreSQL: Native Vert.x client with pipelining -- DuckDB: JDBC-based connection +Multi-database support through `SpringJdbcClient` interface: +- PostgreSQL: R2DBC async client for reactive database access +- DuckDB: JDBC-based connection with Virtual Threads - Snowflake: JDBC-based connection with specialized configuration ### Server Configuration Files -- `server-model.json`: Runtime GraphQL model and execution coordinates -- `server-config.json`: Server configuration (ports, database connections) +- `vertx.json`: Runtime GraphQL model and execution coordinates +- `application.yaml`: Spring Boot configuration (ports, database connections) - `snowflake-config.json`: Optional Snowflake-specific configuration -- `log4j2.properties`: Logging configuration ## Important Development Notes ### Module Dependencies -Always check existing dependencies in `pom.xml` files before adding new libraries. The project uses specific versions of Vert.x, GraphQL-Java, and database drivers. +Always check existing dependencies in `pom.xml` files before adding new libraries. The project uses specific versions of Spring Boot, GraphQL-Java, and database drivers. ### Database Operations All database operations are asynchronous and non-blocking. Use the appropriate `SqlClient` implementation for the target database system. diff --git a/pom.xml b/pom.xml index d0cbeecf58..914e98d1ed 100644 --- a/pom.xml +++ b/pom.xml @@ -66,10 +66,10 @@ - sqrl-cli sqrl-discovery sqrl-planner sqrl-server + sqrl-cli sqrl-testing @@ -129,9 +129,8 @@ 25.0 1.0.1 33.5.0-jre - 4.2.3 - 7.0.0 2.4.240 + 6.2.3 7.0.2 @@ -141,10 +140,11 @@ 2.21.0 0.8.14 3.0.0 + 2.0.1 2.0.1.Final 0.13.0 2.0.0 - 6.0.2 + 6.0.2 3.9.1 2.25.3 1.18.42 @@ -165,7 +165,9 @@ 2.5 2.2.42 2.0.3 - 5.0.7 + 4.0.2 + 1.0.7.RELEASE + 1.3.23 1.4.13 @@ -194,12 +196,7 @@ com.datasqrl - sqrl-server-vertx-base - ${project.version} - - - com.datasqrl - sqrl-server-vertx + sqrl-server-spring ${project.version} @@ -326,27 +323,64 @@ ${graphql-java-extended-scalars.version} + + + com.fasterxml.jackson + jackson-bom + ${jackson.version} + pom + import + + + + + org.junit + junit-bom + ${junit.version} + pom + import + + + - io.vertx - vertx-dependencies - ${vertx.version} + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} pom import - + + + com.fasterxml.jackson.module + jackson-module-scala_2.12 + ${jackson.version} + + + + + org.postgresql + r2dbc-postgresql + ${r2dbc-postgresql.version} + + + + + io.projectreactor.kafka + reactor-kafka + ${reactor-kafka.version} + + io.projectreactor reactor-core ${projectreactor.version} - io.micrometer micrometer-registry-prometheus ${micrometer.version} - io.agroal agroal-pool @@ -555,14 +589,6 @@ import - - com.fasterxml.jackson - jackson-bom - ${jackson.version} - pom - import - - com.networknt @@ -606,13 +632,6 @@ testcontainers-junit-jupiter ${testcontainers.version} test - - - - junit - junit - - org.testcontainers @@ -632,13 +651,6 @@ ${testcontainers.version} test - - org.junit - junit-bom - ${junit.jupiter.version} - pom - import - @@ -674,6 +686,16 @@ flink-test-utils ${flink.version} test + + + org.junit.jupiter + * + + + org.junit.vintage + * + + @@ -762,6 +784,11 @@ junit-jupiter-api test + + junit + junit + test + org.assertj @@ -792,11 +819,6 @@ maven-shade-plugin 3.6.1 - - io.reactiverse - vertx-maven-plugin - 2.0.2 - org.apache.maven.plugins maven-javadoc-plugin @@ -1077,13 +1099,6 @@ - - io.reactiverse - vertx-maven-plugin - - true - - org.apache.maven.plugins maven-shade-plugin @@ -1183,7 +1198,7 @@ - sqrl-server-vertx + sqrl-server-spring sqrl-cli sqrl-test sqrl-run @@ -1361,13 +1376,6 @@ - - io.reactiverse - vertx-maven-plugin - - true - - org.apache.maven.plugins maven-shade-plugin @@ -1399,13 +1407,6 @@ true - - io.reactiverse - vertx-maven-plugin - - true - - org.apache.maven.plugins maven-shade-plugin diff --git a/sqrl-cli/pom.xml b/sqrl-cli/pom.xml index fbc6037597..e6d28d7836 100644 --- a/sqrl-cli/pom.xml +++ b/sqrl-cli/pom.xml @@ -43,7 +43,17 @@ com.datasqrl - sqrl-server-vertx-base + sqrl-server-core + + + com.datasqrl + sqrl-server-spring + + + org.springframework.boot + spring-boot-starter-logging + + @@ -164,6 +174,16 @@ compile + + + org.apache.logging.log4j + log4j-api + + + org.apache.logging.log4j + log4j-core + + org.apache.hadoop @@ -303,22 +323,6 @@ mockito-junit-jupiter test - - io.vertx - vertx-junit5 - test - - - - junit - junit - - - - - io.vertx - vertx-web-client - com.datasqrl sqrl-planner @@ -350,6 +354,7 @@ sqrl-cli true false + false com.datasqrl.cli.DatasqrlCli @@ -383,10 +388,6 @@ jakarta. sqrl.jakarta. - - io.vertx. - sqrl.vertx. - com.sun.el. sqrl.el. diff --git a/sqrl-cli/src/main/java/com/datasqrl/cli/AbstractCompileCmd.java b/sqrl-cli/src/main/java/com/datasqrl/cli/AbstractCompileCmd.java index 95d13985cd..7b13592363 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/cli/AbstractCompileCmd.java +++ b/sqrl-cli/src/main/java/com/datasqrl/cli/AbstractCompileCmd.java @@ -32,13 +32,13 @@ import com.datasqrl.util.ConfigLoaderUtils; import com.datasqrl.util.FlinkCompileException; import com.datasqrl.util.SqrlInjector; -import com.google.inject.Guice; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.tuple.Pair; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; public abstract class AbstractCompileCmd extends BasePackageConfCmd { @@ -60,23 +60,23 @@ protected void compile(ErrorCollector errors) { errors.checkFatal( Files.isDirectory(cli.rootDir), "Not a valid root directory: %s", cli.rootDir); - var injector = - Guice.createInjector( - new SqrlInjector( - errors, - cli.rootDir, - getTargetFolder(), - sqrlConfig, - getGoal(), - cli.internalTestExec)); - - var engineHolder = injector.getInstance(ExecutionEnginesHolder.class); + var context = new AnnotationConfigApplicationContext(); + context.registerBean(ErrorCollector.class, () -> errors); + context.registerBean("rootDir", Path.class, () -> cli.rootDir); + context.registerBean("targetDir", Path.class, () -> getTargetFolder()); + context.registerBean(PackageJson.class, () -> sqrlConfig); + context.registerBean(ExecutionGoal.class, () -> getGoal()); + context.registerBean("internalTestExec", Boolean.class, () -> cli.internalTestExec); + context.register(SqrlInjector.class); + context.refresh(); + + var engineHolder = context.getBean(ExecutionEnginesHolder.class); engineHolder.initEnabledEngines(); if (getGoal() == ExecutionGoal.COMPILE) { formatter.phaseStart("Processing dependencies"); } - var packager = injector.getInstance(Packager.class); + var packager = context.getBean(Packager.class); packager.preprocess(errors.withLocation(ErrorPrefix.CONFIG.resolve(PACKAGE_JSON))); if (errors.hasErrors()) { return; @@ -85,7 +85,7 @@ protected void compile(ErrorCollector errors) { if (getGoal() == ExecutionGoal.COMPILE) { formatter.phaseStart("Compiling SQRL script"); } - var compilationProcess = injector.getInstance(CompilationProcess.class); + var compilationProcess = context.getBean(CompilationProcess.class); var testDir = sqrlConfig.getTestConfig().getTestDir(cli.rootDir); testDir.ifPresent(this::validateTestPath); diff --git a/sqrl-cli/src/main/java/com/datasqrl/cli/BaseCmd.java b/sqrl-cli/src/main/java/com/datasqrl/cli/BaseCmd.java index a0821e3294..75dbcd80cc 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/cli/BaseCmd.java +++ b/sqrl-cli/src/main/java/com/datasqrl/cli/BaseCmd.java @@ -19,10 +19,10 @@ import com.datasqrl.error.CollectedException; import com.datasqrl.error.ErrorCollector; import com.datasqrl.error.ErrorPrinter; -import com.google.inject.ProvisionException; import java.nio.file.Path; import java.util.concurrent.atomic.AtomicInteger; import lombok.SneakyThrows; +import org.springframework.beans.factory.BeanCreationException; import picocli.CommandLine.IExitCodeGenerator; import picocli.CommandLine.ParentCommand; @@ -44,7 +44,7 @@ public void run() { runInternal(collector); cli.statusHook.onSuccess(collector); - } catch (ProvisionException | CollectedException e) { + } catch (BeanCreationException | CollectedException e) { var ce = unwrapCollectedException(e); if (ce.isInternalError()) { ce.printStackTrace(); @@ -87,26 +87,25 @@ public int getExitCode() { } /** - * Unwraps {@link CollectedException} from Guice {@link ProvisionException} wrappers to provide - * clean error messages. + * Unwraps {@link CollectedException} from Spring {@link BeanCreationException} wrappers to + * provide clean error messages. * *

Validation errors during dependency injection (e.g., in pipeline configuration) are thrown - * as {@link CollectedException} with clear messages. Guice wraps them in {@link - * ProvisionException}, obscuring the original error with verbose DI stack traces. + * as {@link CollectedException} with clear messages. Spring wraps them in multiple layers of + * {@link BeanCreationException}, obscuring the original error with verbose DI stack traces. * * @param e the runtime exception that may be a CollectedException or contain one as a cause * @return the unwrapped CollectedException * @throws RuntimeException the original exception if it doesn't contain a CollectedException */ private CollectedException unwrapCollectedException(RuntimeException e) { - if (e instanceof CollectedException ce) { - return ce; - } - - if (e.getCause() instanceof CollectedException ce) { - return ce; + Throwable current = e; + while (current != null) { + if (current instanceof CollectedException ce) { + return ce; + } + current = current.getCause(); } - throw e; } } diff --git a/sqrl-cli/src/main/java/com/datasqrl/cli/DatasqrlRun.java b/sqrl-cli/src/main/java/com/datasqrl/cli/DatasqrlRun.java index bbadc13504..bfb77b64e3 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/cli/DatasqrlRun.java +++ b/sqrl-cli/src/main/java/com/datasqrl/cli/DatasqrlRun.java @@ -21,20 +21,14 @@ import static com.datasqrl.env.EnvVariableNames.POSTGRES_USERNAME; import com.datasqrl.config.PackageJson; -import com.datasqrl.engine.server.VertxEngineFactory; +import com.datasqrl.env.GlobalEnvironmentStore; import com.datasqrl.flinkrunner.EnvVarResolver; import com.datasqrl.flinkrunner.SqrlRunner; -import com.datasqrl.graphql.HttpServerVerticle; import com.datasqrl.graphql.SqrlObjectMapper; -import com.datasqrl.graphql.config.ServerConfigUtil; +import com.datasqrl.graphql.SqrlServerApplication; import com.datasqrl.graphql.server.ModelContainer; import com.datasqrl.util.ConfigLoaderUtils; import com.fasterxml.jackson.databind.ObjectMapper; -import io.micrometer.prometheusmetrics.PrometheusConfig; -import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; -import io.vertx.core.Vertx; -import io.vertx.core.VertxOptions; -import io.vertx.micrometer.MicrometerMetricsFactory; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -69,6 +63,8 @@ import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.NewTopic; +import org.springframework.boot.SpringApplication; +import org.springframework.context.ConfigurableApplicationContext; @Slf4j public class DatasqrlRun { @@ -82,8 +78,8 @@ public class DatasqrlRun { private final ObjectMapper mapper; @Nullable private final CountDownLatch shutdownLatch; - private Vertx vertx; private TableResult tableResult; + private ConfigurableApplicationContext springContext; private DatasqrlRun( Path planDir, @@ -113,7 +109,7 @@ public TableResult run() { initPostgres(); initKafka(); - startVertx(); + startServer(); tableResult = runFlinkJob(); if (shutdownLatch != null) { @@ -166,8 +162,9 @@ private void closeCommon(Runnable closeFlinkJob) { // allow failure if job already ended } } - if (vertx != null) { - vertx.close(); + + if (springContext != null) { + springContext.close(); } // Signal shutdown to release the hold @@ -274,7 +271,7 @@ private String getenv(String key) { } @SneakyThrows - private void startVertx() { + private void startServer() { var vertxJson = planDir.resolve("vertx.json").toFile(); if (!vertxJson.exists()) { return; @@ -282,36 +279,20 @@ private void startVertx() { var rootGraphqlModel = mapper.readValue(vertxJson, ModelContainer.class).models; if (rootGraphqlModel == null || rootGraphqlModel.isEmpty()) { - return; // no graphql server queries + return; } - var vertxConfigJson = planDir.resolve("vertx-config.json").toFile(); - if (!vertxConfigJson.exists()) { - throw new IllegalStateException( - "Server config JSON '%s' does not exist".formatted(vertxConfigJson)); - } + GlobalEnvironmentStore.putAll(env); - Map json = mapper.readValue(vertxConfigJson, Map.class); - var baseServerConfig = ServerConfigUtil.fromConfigMap(json); - - var serverConfig = ServerConfigUtil.mergeConfigs(baseServerConfig, vertxConfig()); - var serverVerticle = new HttpServerVerticle(serverConfig, rootGraphqlModel, planDir); - var prometheusMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); - var metricsOptions = - new MicrometerMetricsFactory(prometheusMeterRegistry).newOptions().setEnabled(true); - - vertx = Vertx.vertx(new VertxOptions().setMetricsOptions(metricsOptions)); - vertx - .deployVerticle(serverVerticle) - .onComplete( - res -> { - if (res.succeeded()) { - log.info("Vertx deployment succeeded. ID: {}", res.result()); - } else { - log.error("Vertx deployment failed", res.cause()); - vertx.close().onComplete(v -> System.exit(1)); - } - }); + var app = new SpringApplication(SqrlServerApplication.class); + app.setDefaultProperties( + Map.of( + "spring.main.banner-mode", "off", + "logging.level.root", "WARN", + "logging.level.com.datasqrl", "INFO")); + + springContext = app.run(); + log.info("Spring Boot GraphQL server started"); } @SneakyThrows @@ -351,12 +332,4 @@ private Tuple2 attachCreationTime(Path path) { BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); return Tuple2.of(path, attrs.creationTime()); } - - private Map vertxConfig() { - return sqrlConfig - .getEngines() - .getEngineConfig(VertxEngineFactory.ENGINE_NAME) - .map(PackageJson.EngineConfig::getConfig) - .orElse(null); - } } diff --git a/sqrl-cli/src/main/java/com/datasqrl/cli/SubscriptionClient.java b/sqrl-cli/src/main/java/com/datasqrl/cli/SubscriptionClient.java index 0ff4dc80eb..29124c839e 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/cli/SubscriptionClient.java +++ b/sqrl-cli/src/main/java/com/datasqrl/cli/SubscriptionClient.java @@ -17,22 +17,20 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import io.vertx.core.Future; -import io.vertx.core.MultiMap; -import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.http.WebSocket; -import io.vertx.core.http.WebSocketConnectOptions; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -// Simplified example WebSocket client code using Vert.x @RequiredArgsConstructor @Slf4j public class SubscriptionClient implements AutoCloseable { @@ -40,7 +38,7 @@ public class SubscriptionClient implements AutoCloseable { private static final long INITIAL_DELAY_MS = 100; private final ObjectMapper objectMapper = new ObjectMapper(); - private final Vertx vertx = Vertx.vertx(); + private final HttpClient httpClient = HttpClient.newHttpClient(); private final CompletableFuture connectedFuture = new CompletableFuture<>(); @Getter private final List messages = new ArrayList<>(); @@ -73,88 +71,72 @@ private void attemptConnection(int attempt) { MAX_RETRIES, name, delay); - vertx.setTimer(delay, id -> connectWebSocket(attempt)); - } else { - connectWebSocket(attempt); + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + connectedFuture.completeExceptionally(e); + return; + } } + + connectWebSocket(attempt); } private void connectWebSocket(int attempt) { - /* 1. Collect handshake headers */ - var headerMap = MultiMap.caseInsensitiveMultiMap(); - headers.forEach(headerMap::add); - - /* 2. Describe the connection */ - var opts = - new WebSocketConnectOptions() - .setHost("localhost") - .setPort(8888) - .setURI("/%s/graphql".formatted(version)) - .addSubProtocol("graphql-transport-ws") // or "graphql-ws" - .setHeaders(headerMap); - - /* 3. Open the socket with the new WebSocketClient API */ - var wsClient = vertx.createWebSocketClient(); - wsClient - .connect(opts) - .onSuccess( - ws -> { - this.webSocket = ws; - log.info("WebSocket opened for subscription: {}", name); - - // Set a message handler for incoming messages - ws.handler(this::handleTextMessage); - - // Send initial connection message - sendConnectionInit() - .onComplete(success -> connectedFuture.complete(null)) - .onFailure(connectedFuture::completeExceptionally); - }) - .onFailure( - throwable -> { - if (attempt < MAX_RETRIES - 1) { - log.warn( - "Failed to open WebSocket for subscription: {} (attempt {}/{}), retrying...", - name, - attempt + 1, - MAX_RETRIES, - throwable); - attemptConnection(attempt + 1); + var uri = URI.create("ws://localhost:8888/%s/graphql".formatted(version)); + var builder = httpClient.newWebSocketBuilder(); + builder.subprotocols("graphql-transport-ws"); + headers.forEach(builder::header); + + builder + .buildAsync(uri, new WebSocketListener()) + .whenComplete( + (ws, throwable) -> { + if (throwable != null) { + if (attempt < MAX_RETRIES - 1) { + log.warn( + "Failed to open WebSocket for subscription: {} (attempt {}/{}), retrying...", + name, + attempt + 1, + MAX_RETRIES, + throwable); + attemptConnection(attempt + 1); + } else { + log.error( + "Failed to open WebSocket for subscription: {} after {} attempts", + name, + MAX_RETRIES, + throwable); + connectedFuture.completeExceptionally(throwable); + } } else { - log.error( - "Failed to open WebSocket for subscription: {} after {} attempts", - name, - MAX_RETRIES, - throwable); - connectedFuture.completeExceptionally(throwable); + this.webSocket = ws; + log.info("WebSocket opened for subscription: {}", name); + sendConnectionInit(); } }); } - private Future sendConnectionInit() { - return sendMessage(Map.of("type", "connection_init")); + private void sendConnectionInit() { + sendMessage(Map.of("type", "connection_init")); } - private Future sendSubscribe() { - Map payload = - Map.of( - // "operationName", "breakMe", - "query", query); + private void sendSubscribe() { + Map payload = Map.of("query", query); Map message = Map.of("id", System.nanoTime(), "type", "subscribe", "payload", payload); - return sendMessage(message); + sendMessage(message); } @SneakyThrows - private Future sendMessage(Map message) { + private void sendMessage(Map message) { String json = objectMapper.writeValueAsString(message); System.out.println("Sending: " + json); - return webSocket.writeTextMessage(json); + webSocket.sendText(json, true); } - private void handleTextMessage(Buffer buffer) { - var data = buffer.toString(); - // Handle the incoming messages + private void handleTextMessage(String data) { System.out.println("Data: " + data); Map message; try { @@ -175,13 +157,11 @@ private void handleTextMessage(Buffer buffer) { var type = (String) message.get("type"); if ("connection_ack".equals(type)) { - // Connection acknowledged, send the subscribe message sendSubscribe(); connectedFuture.complete(null); } else if ("complete".equals(type)) { // Subscription complete } else if ("error".equals(type)) { - // Handle error System.err.println("Error message received: " + data); throw new RuntimeException("Error data: " + data); } else { @@ -191,16 +171,46 @@ private void handleTextMessage(Buffer buffer) { @Override public void close() throws Exception { - if (webSocket != null && !webSocket.isClosed()) { - // Send 'complete' message to close the subscription properly - waitCompletion(sendMessage(Map.of("id", System.nanoTime(), "type", "complete"))); - waitCompletion(webSocket.close()); + if (webSocket != null) { + sendMessage(Map.of("id", System.nanoTime(), "type", "complete")); + webSocket.sendClose(WebSocket.NORMAL_CLOSURE, "closing").join(); } - waitCompletion(vertx.close()); } - @SneakyThrows - private void waitCompletion(Future future) { - future.toCompletionStage().toCompletableFuture().get(); + private class WebSocketListener implements WebSocket.Listener { + private final StringBuilder textBuffer = new StringBuilder(); + + @Override + public void onOpen(WebSocket webSocket) { + webSocket.request(1); + } + + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + textBuffer.append(data); + if (last) { + handleTextMessage(textBuffer.toString()); + textBuffer.setLength(0); + } + webSocket.request(1); + return null; + } + + @Override + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + webSocket.request(1); + return null; + } + + @Override + public CompletionStage onClose(WebSocket webSocket, int statusCode, String reason) { + log.info("WebSocket closed: {} - {}", statusCode, reason); + return null; + } + + @Override + public void onError(WebSocket webSocket, Throwable error) { + log.error("WebSocket error for subscription: {}", name, error); + } } } diff --git a/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java b/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java index 4777b2c237..6a7b33ff32 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java +++ b/sqrl-cli/src/main/java/com/datasqrl/compile/CompilationProcess.java @@ -35,13 +35,17 @@ import com.datasqrl.planner.Sqrl2FlinkSQLTranslator; import com.datasqrl.planner.dag.DAGPlanner; import com.datasqrl.util.ServiceLoaderDiscovery; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.nio.file.Path; import java.util.List; import java.util.Optional; import lombok.AllArgsConstructor; import org.apache.commons.lang3.tuple.Pair; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +@Component +@Lazy @AllArgsConstructor(onConstructor_ = @Inject) public class CompilationProcess { diff --git a/sqrl-cli/src/main/java/com/datasqrl/compile/DagWriter.java b/sqrl-cli/src/main/java/com/datasqrl/compile/DagWriter.java index d57f95b044..2a7d40ea28 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/compile/DagWriter.java +++ b/sqrl-cli/src/main/java/com/datasqrl/compile/DagWriter.java @@ -22,7 +22,7 @@ import com.datasqrl.planner.dag.PipelineDAG; import com.datasqrl.serializer.Deserializer; import com.google.common.io.Resources; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -32,7 +32,9 @@ import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.SneakyThrows; +import org.springframework.stereotype.Component; +@Component @AllArgsConstructor(onConstructor_ = @Inject) public class DagWriter { diff --git a/sqrl-cli/src/main/java/com/datasqrl/packager/FilePreprocessingPipeline.java b/sqrl-cli/src/main/java/com/datasqrl/packager/FilePreprocessingPipeline.java index 31e974cdb2..d76c19d032 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/packager/FilePreprocessingPipeline.java +++ b/sqrl-cli/src/main/java/com/datasqrl/packager/FilePreprocessingPipeline.java @@ -25,7 +25,7 @@ import com.datasqrl.packager.preprocess.Preprocessor; import com.datasqrl.util.FilenameAnalyzer; import com.datasqrl.util.NameUtil; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.io.IOException; import java.nio.file.FileVisitOption; import java.nio.file.FileVisitResult; @@ -42,12 +42,14 @@ import lombok.AllArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; +import org.springframework.stereotype.Component; /** * Helps to preprocess files which means 1) copying all relevant files into the build directory * (preserving relative paths) 2) running all registered preprocessors for more elaborate * preprocessing than just copying files. */ +@Component @SuppressWarnings("NullableProblems") @AllArgsConstructor(onConstructor_ = @Inject) public class FilePreprocessingPipeline { diff --git a/sqrl-cli/src/main/java/com/datasqrl/packager/Packager.java b/sqrl-cli/src/main/java/com/datasqrl/packager/Packager.java index d6a1d4be8a..01f0516e6b 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/packager/Packager.java +++ b/sqrl-cli/src/main/java/com/datasqrl/packager/Packager.java @@ -35,7 +35,7 @@ import com.github.mustachejava.DefaultMustacheFactory; import com.github.mustachejava.Mustache; import com.github.mustachejava.MustacheFactory; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.io.File; import java.io.IOException; import java.io.ObjectOutputStream; @@ -53,7 +53,9 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; +@Component @Getter @AllArgsConstructor(onConstructor_ = @Inject) public class Packager { diff --git a/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/CopyStaticDataPreprocessor.java b/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/CopyStaticDataPreprocessor.java index 6935049b69..02dbbb435d 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/CopyStaticDataPreprocessor.java +++ b/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/CopyStaticDataPreprocessor.java @@ -32,11 +32,13 @@ import java.util.Set; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; /** * Copies {@code .jsonl}, {@code .csv}, and {@code .avro} files (optionally with compression) to the * data folder, so they can be useb by Flink. */ +@Component @Slf4j public class CopyStaticDataPreprocessor implements Preprocessor { diff --git a/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/JBangPreprocessor.java b/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/JBangPreprocessor.java index 2813aa6264..bb5253c601 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/JBangPreprocessor.java +++ b/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/JBangPreprocessor.java @@ -17,7 +17,7 @@ import com.datasqrl.packager.FilePreprocessingPipeline; import com.datasqrl.util.JBangRunner; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -27,7 +27,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.exec.ExecuteException; import org.apache.commons.io.FilenameUtils; +import org.springframework.stereotype.Component; +@Component @RequiredArgsConstructor(onConstructor_ = @Inject) @Slf4j public class JBangPreprocessor extends UdfManifestPreprocessor { diff --git a/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/JarPreprocessor.java b/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/JarPreprocessor.java index ce5b61df34..3a440f497a 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/JarPreprocessor.java +++ b/sqrl-cli/src/main/java/com/datasqrl/packager/preprocess/JarPreprocessor.java @@ -18,10 +18,12 @@ import com.datasqrl.packager.FilePreprocessingPipeline; import java.nio.file.Path; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; /* * Reads a jar and creates sqrl manifest entries in the build directory */ +@Component @Slf4j public class JarPreprocessor extends UdfManifestPreprocessor { diff --git a/sqrl-cli/src/main/java/com/datasqrl/util/SqrlInjector.java b/sqrl-cli/src/main/java/com/datasqrl/util/SqrlInjector.java index 7004d54f33..b3838836eb 100644 --- a/sqrl-cli/src/main/java/com/datasqrl/util/SqrlInjector.java +++ b/sqrl-cli/src/main/java/com/datasqrl/util/SqrlInjector.java @@ -15,133 +15,71 @@ */ package com.datasqrl.util; -import com.datasqrl.MainScriptImpl; -import com.datasqrl.calcite.type.TypeFactory; import com.datasqrl.canonicalizer.NameCanonicalizer; -import com.datasqrl.config.ConnectorFactoryFactory; -import com.datasqrl.config.ConnectorFactoryFactoryImpl; +import com.datasqrl.config.BuildPath; import com.datasqrl.config.ExecutionEnginesHolder; import com.datasqrl.config.PackageJson; -import com.datasqrl.config.PackageJson.CompilerConfig; -import com.datasqrl.config.QueryEngineConfigConverter; -import com.datasqrl.config.QueryEngineConfigConverterImpl; -import com.datasqrl.config.SqrlCompilerConfiguration; -import com.datasqrl.config.SqrlConfigPipeline; +import com.datasqrl.config.RootPath; import com.datasqrl.config.SqrlConstants; -import com.datasqrl.engine.pipeline.ExecutionPipeline; -import com.datasqrl.error.ErrorCollector; -import com.datasqrl.loaders.ModuleLoader; -import com.datasqrl.loaders.ModuleLoaderImpl; +import com.datasqrl.config.TargetPath; import com.datasqrl.loaders.resolver.FileResourceResolver; import com.datasqrl.loaders.resolver.ResourceResolver; -import com.datasqrl.packager.preprocess.CopyStaticDataPreprocessor; -import com.datasqrl.packager.preprocess.JBangPreprocessor; -import com.datasqrl.packager.preprocess.JarPreprocessor; -import com.datasqrl.packager.preprocess.Preprocessor; -import com.datasqrl.plan.MainScript; import com.datasqrl.plan.validate.ExecutionGoal; -import com.google.inject.AbstractModule; -import com.google.inject.Injector; -import com.google.inject.Provides; -import com.google.inject.Singleton; -import com.google.inject.multibindings.Multibinder; -import com.google.inject.name.Named; import java.nio.file.Path; -import org.apache.calcite.rel.type.RelDataTypeFactory; - -public class SqrlInjector extends AbstractModule { - - private final ErrorCollector errors; - private final Path rootDir; - private final Path buildDir; - private final Path targetDir; - private final PackageJson sqrlConfig; - private final ExecutionGoal goal; - private final boolean internalTestExec; - - public SqrlInjector( - ErrorCollector errors, - Path rootDir, - Path targetDir, - PackageJson sqrlConfig, - ExecutionGoal goal, - boolean internalTestExec) { - this.errors = errors; - this.rootDir = rootDir; - this.buildDir = rootDir.resolve(SqrlConstants.BUILD_DIR_NAME); - this.targetDir = targetDir; - this.sqrlConfig = sqrlConfig; - this.goal = goal; - this.internalTestExec = internalTestExec; - } - - @Override - public void configure() { - bind(RelDataTypeFactory.class).to(TypeFactory.class); - bind(MainScript.class).to(MainScriptImpl.class); - bind(ExecutionPipeline.class).to(SqrlConfigPipeline.class); - bind(ModuleLoader.class).to(ModuleLoaderImpl.class); - bind(CompilerConfig.class).to(SqrlCompilerConfiguration.class); - bind(ConnectorFactoryFactory.class).to(ConnectorFactoryFactoryImpl.class); - bind(QueryEngineConfigConverter.class).to(QueryEngineConfigConverterImpl.class); - - Multibinder binder = Multibinder.newSetBinder(binder(), Preprocessor.class); - binder.addBinding().to(CopyStaticDataPreprocessor.class); - binder.addBinding().to(JBangPreprocessor.class); - binder.addBinding().to(JarPreprocessor.class); +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ComponentScan(basePackages = "com.datasqrl") +public class SqrlInjector { + + @Bean + @Qualifier("buildDir") + public Path buildDir(@Qualifier("rootDir") Path rootDir) { + return rootDir.resolve(SqrlConstants.BUILD_DIR_NAME); } - @Provides - @Named("buildDir") - public Path provideBuildDir() { - return buildDir; + @Bean + public BuildPath buildPath( + @Qualifier("buildDir") Path buildDir, @Qualifier("targetDir") Path targetDir) { + return new BuildPath(buildDir, targetDir); } - @Provides - @Named("rootDir") - public Path provideRootDir() { - return rootDir; - } - - @Provides - @Named("targetDir") - public Path provideTargetDir() { - return targetDir; - } - - @Provides - public ResourceResolver provideResourceResolver() { + @Bean + public ResourceResolver resourceResolver(@Qualifier("buildDir") Path buildDir) { return new FileResourceResolver(buildDir); } - @Provides - public NameCanonicalizer provideNameCanonicalizer() { + @Bean + public NameCanonicalizer nameCanonicalizer() { return NameCanonicalizer.SYSTEM; } - @Provides - @Singleton - public JBangRunner provideJBangRunner() { + @Bean + public JBangRunner jBangRunner(@Qualifier("internalTestExec") Boolean internalTestExec) { return internalTestExec ? JBangRunner.disabled() : JBangRunner.create(); } - @Provides - public PackageJson provideSqrlConfig() { - return sqrlConfig; - } - - @Provides - public ExecutionGoal provideExecutionGoal() { - return goal; + @Bean + public ExecutionEnginesHolder executionEnginesHolder( + com.datasqrl.error.ErrorCollector errors, + ApplicationContext applicationContext, + PackageJson sqrlConfig, + ExecutionGoal goal) { + return new ExecutionEnginesHolder( + errors, applicationContext, sqrlConfig, goal == ExecutionGoal.TEST); } - @Provides - public ErrorCollector provideErrorCollector() { - return errors; + @Bean + public RootPath rootPath(@Qualifier("rootDir") Path rootDir) { + return new RootPath(rootDir); } - @Provides - public ExecutionEnginesHolder provideExecutionEnginesHolder(Injector injector) { - return new ExecutionEnginesHolder(errors, injector, sqrlConfig, goal == ExecutionGoal.TEST); + @Bean + public TargetPath targetPath(@Qualifier("targetDir") Path targetDir) { + return new TargetPath(targetDir); } } diff --git a/sqrl-cli/src/test/java/com/datasqrl/engine/server/GenericJavaServerEngineTest.java b/sqrl-cli/src/test/java/com/datasqrl/engine/server/GenericJavaServerEngineTest.java deleted file mode 100644 index 6d586d5e8a..0000000000 --- a/sqrl-cli/src/test/java/com/datasqrl/engine/server/GenericJavaServerEngineTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.engine.server; - -import static com.datasqrl.graphql.SqrlObjectMapper.MAPPER; -import static org.assertj.core.api.Assertions.assertThat; - -import com.datasqrl.config.PackageJson.EmptyEngineConfig; -import com.datasqrl.config.QueryEngineConfigConverter; -import com.datasqrl.graphql.config.ServerConfigUtil; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.List; -import java.util.Map; -import lombok.SneakyThrows; -import org.junit.jupiter.api.Test; - -class GenericJavaServerEngineTest { - - GenericJavaServerEngine underTest = - new GenericJavaServerEngine("", new EmptyEngineConfig(""), new DummyConverter()) {}; - - @Test - void test() { - // innermost object: a single pub-/sec key - var config = getConfigMap(); - - var defaultConfig = underTest.readDefaultConfig(); - assertThat(defaultConfig.getJwtAuth()).isNull(); - - var result = ServerConfigUtil.mergeConfigs(defaultConfig, config); - - assertThat(result).isNotNull(); - assertThat(result.getJwtAuth()).isNotNull(); - assertThat(result.getJwtAuth().getPubSecKeys()).isNotNull().isNotEmpty(); - assertThat(result.getJwtAuth().getJWTOptions()).isNotNull(); - assertThat(result.getJwtAuth().getJWTOptions().getIssuer()).isEqualTo("my-test-issuer"); - } - - @Test - @SneakyThrows - void givenJwtConfiguration_whenConfigMerged_thenBuffersAreStringValues() { - // Create JWT configuration with buffer as simple string (not complex object) - var config = getConfigMap(); - - // Test the configuration merging that would happen during serverConfig() generation - var defaultConfig = underTest.readDefaultConfig(); - var mergedConfig = ServerConfigUtil.mergeConfigs(defaultConfig, config); - - // Serialize the merged configuration to JSON string and parse back - var serializedJson = MAPPER.writeValueAsString(mergedConfig); - JsonNode configNode = MAPPER.readTree(serializedJson); - JsonNode pubSecKeysNode = configNode.path("jwtAuth").path("pubSecKeys"); - - assertThat(pubSecKeysNode.isArray()).isTrue(); - assertThat(pubSecKeysNode.size()).isEqualTo(1); - - JsonNode firstKey = pubSecKeysNode.get(0); - JsonNode bufferNode = firstKey.path("buffer"); - - // Verify buffer is a simple string value, not a complex object with "bytes" field - assertThat(bufferNode.isTextual()).isTrue(); - // Note: VertxModule may process base64 buffers, but it should still be a string, not a complex - // object - assertThat(bufferNode.asText()).contains("dGVzdFNlY3JldA"); - - // Ensure buffer is not a complex object (would have been corrupted by old VertxModule usage) - assertThat(bufferNode.isObject()).isFalse(); - assertThat(bufferNode.has("bytes")).isFalse(); - } - - private static Map getConfigMap() { - var pubSecKey = - Map.of( - "algorithm", "HS256", - "buffer", "dGVzdFNlY3JldA=="); - - var jwtOptions = - Map.of( - "issuer", - "my-test-issuer", - "audience", - List.of("my-test-audience"), - "expiresInSeconds", - 3600, - "leeway", - 30); - - var jwtAuth = Map.of("pubSecKeys", List.of(pubSecKey), "jwtOptions", jwtOptions); - - return Map.of("jwtAuth", jwtAuth); - } - - private static class DummyConverter implements QueryEngineConfigConverter { - - @Override - public List convertConfigsToJson() { - return List.of(); - } - } -} diff --git a/sqrl-cli/src/test/java/com/datasqrl/packager/config/ServerConfigTemplateTest.java b/sqrl-cli/src/test/java/com/datasqrl/packager/config/ServerConfigTemplateTest.java deleted file mode 100644 index 0c079f4197..0000000000 --- a/sqrl-cli/src/test/java/com/datasqrl/packager/config/ServerConfigTemplateTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.packager.config; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.datasqrl.graphql.SqrlObjectMapper; -import com.datasqrl.graphql.config.ServerConfigUtil; -import com.fasterxml.jackson.databind.MapperFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.File; -import java.util.Comparator; -import java.util.Map; -import java.util.Objects; -import lombok.SneakyThrows; -import org.junit.jupiter.api.Test; - -class ServerConfigTemplateTest { - private static final File TEMPLATE = new File("src/main/resources/templates/server-config.json"); - - private static ObjectMapper mapper = SqrlObjectMapper.MAPPER; - - @SuppressWarnings("unchecked") - @Test - @SneakyThrows - void test() { - var original = mapper.readValue(TEMPLATE, Map.class); - var afterParsing = ServerConfigUtil.fromConfigMap(original); - assertThat(mapper.convertValue(afterParsing, Map.class)) - .usingRecursiveComparison() - .withComparatorForType(numberComparatorIgnoringType(), Number.class) - .isEqualTo(original); - } - - private static Comparator numberComparatorIgnoringType() { - return (n1, n2) -> { - if (n1 == null && n2 == null) return 0; - if (n1 == null) return -1; - if (n2 == null) return 1; - return Double.compare(n1.doubleValue(), n2.doubleValue()); - }; - } - - @SneakyThrows - public static void main(String[] args) { - var original = mapper.readValue(TEMPLATE, Map.class); - var afterParsing = ServerConfigUtil.fromConfigMap(original); - - if (!Objects.equals(original, mapper.convertValue(afterParsing, Map.class))) { - mapper - .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) - .writerWithDefaultPrettyPrinter() - .writeValue(TEMPLATE, afterParsing); - } - } -} diff --git a/sqrl-planner/pom.xml b/sqrl-planner/pom.xml index a2375781fc..7a56dcc2af 100644 --- a/sqrl-planner/pom.xml +++ b/sqrl-planner/pom.xml @@ -37,6 +37,11 @@ jakarta.annotation-api ${jakarta.annotation.version} + + jakarta.inject + jakarta.inject-api + ${jakarta.inject.version} + org.glassfish javax.el @@ -107,7 +112,7 @@ com.datasqrl - sqrl-server-vertx-base + sqrl-server-core compile @@ -146,14 +151,9 @@ - com.google.inject - guice - ${guice.version} - - - com.google.inject.extensions - guice-multibindings - ${guice-multibindings.version} + org.springframework + spring-context + ${spring-framework.version} com.fasterxml.jackson.datatype diff --git a/sqrl-planner/src/main/java/com/datasqrl/MainScriptImpl.java b/sqrl-planner/src/main/java/com/datasqrl/MainScriptImpl.java index c237d2a1d2..a6f4af6cf2 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/MainScriptImpl.java +++ b/sqrl-planner/src/main/java/com/datasqrl/MainScriptImpl.java @@ -19,11 +19,13 @@ import com.datasqrl.loaders.resolver.ResourceResolver; import com.datasqrl.plan.MainScript; import com.datasqrl.util.FileUtil; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.nio.file.Path; import java.util.Optional; import lombok.AllArgsConstructor; +import org.springframework.stereotype.Component; +@Component @AllArgsConstructor(onConstructor_ = @Inject) public class MainScriptImpl implements MainScript { diff --git a/sqrl-planner/src/main/java/com/datasqrl/calcite/type/TypeFactory.java b/sqrl-planner/src/main/java/com/datasqrl/calcite/type/TypeFactory.java index 236d69a653..a266d78e17 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/calcite/type/TypeFactory.java +++ b/sqrl-planner/src/main/java/com/datasqrl/calcite/type/TypeFactory.java @@ -15,14 +15,16 @@ */ package com.datasqrl.calcite.type; -import com.google.inject.Singleton; +import jakarta.inject.Singleton; import java.lang.reflect.Type; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.calcite.FlinkTypeSystem; +import org.springframework.stereotype.Component; +@Component @Singleton public class TypeFactory extends FlinkTypeFactory { diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/BuildPath.java b/sqrl-planner/src/main/java/com/datasqrl/config/BuildPath.java index d6744f429f..4389e7e7ee 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/BuildPath.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/BuildPath.java @@ -15,8 +15,8 @@ */ package com.datasqrl.config; -import com.google.inject.Inject; -import com.google.inject.name.Named; +import jakarta.inject.Inject; +import jakarta.inject.Named; import java.nio.file.Path; public record BuildPath(Path buildDir, Path targetDir) { diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/ConnectorFactoryFactoryImpl.java b/sqrl-planner/src/main/java/com/datasqrl/config/ConnectorFactoryFactoryImpl.java index c6d51652f2..fce5d0e452 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/ConnectorFactoryFactoryImpl.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/ConnectorFactoryFactoryImpl.java @@ -15,11 +15,13 @@ */ package com.datasqrl.config; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.util.Optional; import lombok.AllArgsConstructor; +import org.springframework.stereotype.Component; /** Placeholder for future templated connector handling */ +@Component @AllArgsConstructor(onConstructor_ = @Inject) public class ConnectorFactoryFactoryImpl implements ConnectorFactoryFactory { diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/ExecutionEnginesHolder.java b/sqrl-planner/src/main/java/com/datasqrl/config/ExecutionEnginesHolder.java index cfe5b99c1e..3c42beec7d 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/ExecutionEnginesHolder.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/ExecutionEnginesHolder.java @@ -28,9 +28,8 @@ import com.datasqrl.error.ErrorCollector; import com.datasqrl.util.ServiceLoaderDiscovery; import com.datasqrl.util.StreamUtil; -import com.google.inject.Injector; -import com.google.inject.Singleton; import jakarta.annotation.Nullable; +import jakarta.inject.Singleton; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -38,6 +37,7 @@ import java.util.Set; import java.util.TreeMap; import lombok.RequiredArgsConstructor; +import org.springframework.context.ApplicationContext; /** Configuration for the engines */ @RequiredArgsConstructor @@ -52,7 +52,7 @@ public class ExecutionEnginesHolder { private static final String VERTX_NAME = VertxEngineFactory.ENGINE_NAME; private final ErrorCollector errors; - private final Injector injector; + private final ApplicationContext applicationContext; private final PackageJson packageJson; private final boolean testExecution; @@ -119,8 +119,8 @@ public Map getEngines( ExecutionEngine getEngineInstance(String engineName) { var engineFactory = ServiceLoaderDiscovery.get(EngineFactory.class, EngineFactory::getEngineName, engineName); - - return (ExecutionEngine) injector.getInstance(engineFactory.getFactoryClass()); + var beanFactory = applicationContext.getAutowireCapableBeanFactory(); + return (ExecutionEngine) beanFactory.createBean(engineFactory.getFactoryClass()); } Map adaptToTest(Map engines) { diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java b/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java index 57994dbe18..619812a365 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/GraphqlSourceLoader.java @@ -23,14 +23,18 @@ import com.datasqrl.loaders.resolver.ResourceResolver; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; import lombok.SneakyThrows; import lombok.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +@Component +@Lazy @Value public class GraphqlSourceLoader { diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/PipelineFactory.java b/sqrl-planner/src/main/java/com/datasqrl/config/PipelineFactory.java index 184ad219d5..99f4ede67e 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/PipelineFactory.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/PipelineFactory.java @@ -18,17 +18,17 @@ import com.datasqrl.engine.pipeline.ExecutionPipeline; import com.datasqrl.engine.pipeline.SimplePipeline; import com.datasqrl.error.ErrorCollector; -import com.google.inject.Injector; import lombok.RequiredArgsConstructor; +import org.springframework.context.ApplicationContext; @RequiredArgsConstructor public class PipelineFactory { - private final Injector injector; + private final ApplicationContext applicationContext; public ExecutionPipeline createPipeline() { - var engineHolder = injector.getInstance(ExecutionEnginesHolder.class); - var errorCollector = injector.getInstance(ErrorCollector.class); + var engineHolder = applicationContext.getBean(ExecutionEnginesHolder.class); + var errorCollector = applicationContext.getBean(ErrorCollector.class); return SimplePipeline.of(engineHolder.getEngines(), errorCollector); } diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/QueryEngineConfigConverterImpl.java b/sqrl-planner/src/main/java/com/datasqrl/config/QueryEngineConfigConverterImpl.java index a4b88a858f..363878c83c 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/QueryEngineConfigConverterImpl.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/QueryEngineConfigConverterImpl.java @@ -19,11 +19,13 @@ import com.datasqrl.engine.database.QueryEngine; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.util.ArrayList; import java.util.List; import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +@Component @RequiredArgsConstructor(onConstructor_ = @Inject) public class QueryEngineConfigConverterImpl implements QueryEngineConfigConverter { diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/RootPath.java b/sqrl-planner/src/main/java/com/datasqrl/config/RootPath.java index e3bf632be6..3899dfaf99 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/RootPath.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/RootPath.java @@ -15,8 +15,8 @@ */ package com.datasqrl.config; -import com.google.inject.Inject; -import com.google.inject.name.Named; +import jakarta.inject.Inject; +import jakarta.inject.Named; import java.nio.file.Path; public record RootPath(Path rootDir) { diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/SqrlCompilerConfiguration.java b/sqrl-planner/src/main/java/com/datasqrl/config/SqrlCompilerConfiguration.java index 193be4b3e7..e8f7df3306 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/SqrlCompilerConfiguration.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/SqrlCompilerConfiguration.java @@ -15,9 +15,11 @@ */ package com.datasqrl.config; -import com.google.inject.Inject; +import jakarta.inject.Inject; import lombok.experimental.Delegate; +import org.springframework.stereotype.Component; +@Component public class SqrlCompilerConfiguration implements PackageJson.CompilerConfig { @Delegate private final PackageJson.CompilerConfig compilerConfig; diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/SqrlConfigPipeline.java b/sqrl-planner/src/main/java/com/datasqrl/config/SqrlConfigPipeline.java index e09254493f..351aea7f16 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/SqrlConfigPipeline.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/SqrlConfigPipeline.java @@ -16,18 +16,22 @@ package com.datasqrl.config; import com.datasqrl.engine.pipeline.ExecutionPipeline; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.Singleton; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; import lombok.experimental.Delegate; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; +@Component +@Lazy @Singleton public class SqrlConfigPipeline implements ExecutionPipeline { @Delegate ExecutionPipeline pipeline; @Inject - public SqrlConfigPipeline(Injector injector) { - this.pipeline = new PipelineFactory(injector).createPipeline(); + public SqrlConfigPipeline(ApplicationContext applicationContext) { + this.pipeline = new PipelineFactory(applicationContext).createPipeline(); } } diff --git a/sqrl-planner/src/main/java/com/datasqrl/config/TargetPath.java b/sqrl-planner/src/main/java/com/datasqrl/config/TargetPath.java index a6f76ae2df..db6925ed03 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/config/TargetPath.java +++ b/sqrl-planner/src/main/java/com/datasqrl/config/TargetPath.java @@ -15,8 +15,8 @@ */ package com.datasqrl.config; -import com.google.inject.Inject; -import com.google.inject.name.Named; +import jakarta.inject.Inject; +import jakarta.inject.Named; import java.nio.file.Path; public record TargetPath(Path targetDir) { diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/DuckDBEngine.java b/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/DuckDBEngine.java index 1d6e747ecb..8e682ee901 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/DuckDBEngine.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/DuckDBEngine.java @@ -19,7 +19,7 @@ import com.datasqrl.config.JdbcDialect; import com.datasqrl.config.PackageJson; import com.datasqrl.graphql.jdbc.DatabaseType; -import com.google.inject.Inject; +import jakarta.inject.Inject; import lombok.NonNull; public class DuckDBEngine extends AbstractJDBCQueryEngine { diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/IcebergEngine.java b/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/IcebergEngine.java index 712454f917..b166f4bf68 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/IcebergEngine.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/IcebergEngine.java @@ -27,7 +27,7 @@ import com.datasqrl.datatype.flink.iceberg.IcebergDataTypeMapper; import com.datasqrl.engine.database.QueryEngine; import com.datasqrl.planner.tables.FlinkTableBuilder; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.util.Map; import java.util.TreeMap; import java.util.regex.Pattern; diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/PostgresJdbcEngine.java b/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/PostgresJdbcEngine.java index 86ff8dcb59..d31c59fe7b 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/PostgresJdbcEngine.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/PostgresJdbcEngine.java @@ -22,7 +22,7 @@ import com.datasqrl.datatype.flink.jdbc.FlinkSqrlPostgresDataTypeMapper; import com.datasqrl.engine.database.relational.ddl.PostgresDDLFactory; import com.datasqrl.graphql.jdbc.DatabaseType; -import com.google.inject.Inject; +import jakarta.inject.Inject; import lombok.NonNull; public class PostgresJdbcEngine extends AbstractJDBCDatabaseEngine { diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/SnowflakeEngine.java b/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/SnowflakeEngine.java index ace23b554f..2ba4eb4264 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/SnowflakeEngine.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/database/relational/SnowflakeEngine.java @@ -19,7 +19,7 @@ import com.datasqrl.config.JdbcDialect; import com.datasqrl.config.PackageJson; import com.datasqrl.graphql.jdbc.DatabaseType; -import com.google.inject.Inject; +import jakarta.inject.Inject; import lombok.NonNull; public class SnowflakeEngine extends AbstractJDBCQueryEngine { diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/export/PrintEngine.java b/sqrl-planner/src/main/java/com/datasqrl/engine/export/PrintEngine.java index 0978901256..5704b0382e 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/export/PrintEngine.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/export/PrintEngine.java @@ -25,7 +25,7 @@ import com.datasqrl.engine.pipeline.ExecutionStage; import com.datasqrl.planner.analyzer.TableAnalysis; import com.datasqrl.planner.tables.FlinkTableBuilder; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.util.Optional; import org.apache.calcite.rel.type.RelDataType; diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/log/kafka/KafkaLogEngine.java b/sqrl-planner/src/main/java/com/datasqrl/engine/log/kafka/KafkaLogEngine.java index cb5d8afb6a..1334434516 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/log/kafka/KafkaLogEngine.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/log/kafka/KafkaLogEngine.java @@ -43,7 +43,7 @@ import com.datasqrl.util.StreamUtil; import com.google.common.base.Preconditions; import com.google.common.collect.Streams; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.time.Duration; import java.util.EnumSet; import java.util.HashMap; diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/log/postgres/PostgresLogEngine.java b/sqrl-planner/src/main/java/com/datasqrl/engine/log/postgres/PostgresLogEngine.java index 0579789e6e..1a0e88564f 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/log/postgres/PostgresLogEngine.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/log/postgres/PostgresLogEngine.java @@ -32,7 +32,7 @@ import com.datasqrl.planner.analyzer.TableAnalysis; import com.datasqrl.planner.dag.plan.MaterializationStagePlan; import com.datasqrl.planner.tables.FlinkTableBuilder; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.util.EnumSet; import java.util.Optional; import lombok.Getter; diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/server/GenericJavaServerEngine.java b/sqrl-planner/src/main/java/com/datasqrl/engine/server/GenericJavaServerEngine.java index 996a1d3cce..185a3aaf35 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/server/GenericJavaServerEngine.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/server/GenericJavaServerEngine.java @@ -17,7 +17,6 @@ import static com.datasqrl.engine.EngineFeature.NO_CAPABILITIES; import static com.datasqrl.graphql.SqrlObjectMapper.MAPPER; -import static com.datasqrl.graphql.config.ServerConfigUtil.mergeConfigs; import com.datasqrl.config.EngineType; import com.datasqrl.config.PackageJson.EngineConfig; @@ -26,9 +25,9 @@ import com.datasqrl.engine.EnginePhysicalPlan.ArtifactType; import com.datasqrl.engine.EnginePhysicalPlan.DeploymentArtifact; import com.datasqrl.engine.ExecutionEngine; -import com.datasqrl.graphql.config.ServerConfig; import com.datasqrl.planner.dag.plan.ServerStagePlan; import com.datasqrl.planner.tables.SqrlTableFunction; +import com.datasqrl.util.JsonMergeUtils; import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; import java.util.List; @@ -78,17 +77,20 @@ private void validateFunctions(List functions) { @SneakyThrows private String serverConfig() { - var mergedConfig = mergeConfigs(readDefaultConfig(), engineConfig.getConfig()); - return MAPPER.copy().writer(new PrettyPrinter()).writeValueAsString(mergedConfig); + var baseConfig = readDefaultConfig(); + var overrides = engineConfig.getConfig(); + if (overrides != null && !overrides.isEmpty()) { + JsonMergeUtils.merge(baseConfig, MAPPER.valueToTree(overrides)); + } + return MAPPER.copy().writer(new PrettyPrinter()).writeValueAsString(baseConfig); } @SneakyThrows - ServerConfig readDefaultConfig() { + ObjectNode readDefaultConfig() { try (var input = getClass().getResourceAsStream("/templates/server-config.json")) { var serverConfNode = (ObjectNode) MAPPER.readTree(input); configConverter.convertConfigsToJson().forEach(serverConfNode::setAll); - - return MAPPER.treeToValue(serverConfNode, ServerConfig.class).validated(); + return serverConfNode; } } } diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/server/VertxEngineFactory.java b/sqrl-planner/src/main/java/com/datasqrl/engine/server/VertxEngineFactory.java index d88465f857..1cf3b8a87e 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/server/VertxEngineFactory.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/server/VertxEngineFactory.java @@ -20,7 +20,7 @@ import com.datasqrl.config.QueryEngineConfigConverter; import com.datasqrl.engine.IExecutionEngine; import com.google.auto.service.AutoService; -import com.google.inject.Inject; +import jakarta.inject.Inject; @AutoService(EngineFactory.class) public class VertxEngineFactory extends GenericJavaServerEngineFactory { diff --git a/sqrl-planner/src/main/java/com/datasqrl/engine/stream/flink/FlinkStreamEngine.java b/sqrl-planner/src/main/java/com/datasqrl/engine/stream/flink/FlinkStreamEngine.java index 05707e788f..a4b8e60f94 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/engine/stream/flink/FlinkStreamEngine.java +++ b/sqrl-planner/src/main/java/com/datasqrl/engine/stream/flink/FlinkStreamEngine.java @@ -26,7 +26,7 @@ import com.datasqrl.engine.EngineFeature; import com.datasqrl.engine.ExecutionEngine; import com.datasqrl.engine.stream.StreamEngine; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.io.IOException; import java.time.Duration; import java.util.EnumSet; diff --git a/sqrl-planner/src/main/java/com/datasqrl/graphql/GenerateServerModel.java b/sqrl-planner/src/main/java/com/datasqrl/graphql/GenerateServerModel.java index 00491a59a8..0a5250cbd1 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/graphql/GenerateServerModel.java +++ b/sqrl-planner/src/main/java/com/datasqrl/graphql/GenerateServerModel.java @@ -23,11 +23,13 @@ import com.datasqrl.graphql.server.RootGraphqlModel; import com.datasqrl.graphql.server.RootGraphqlModel.StringSchema; import com.datasqrl.graphql.server.operation.ApiOperation; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.util.ArrayList; import lombok.AllArgsConstructor; +import org.springframework.stereotype.Component; /** Generates the model for the server */ +@Component @AllArgsConstructor(onConstructor_ = @Inject) public class GenerateServerModel { diff --git a/sqrl-planner/src/main/java/com/datasqrl/graphql/GraphqlSchemaFactory.java b/sqrl-planner/src/main/java/com/datasqrl/graphql/GraphqlSchemaFactory.java index 89bc207980..24b88e6a08 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/graphql/GraphqlSchemaFactory.java +++ b/sqrl-planner/src/main/java/com/datasqrl/graphql/GraphqlSchemaFactory.java @@ -35,7 +35,6 @@ import com.datasqrl.planner.parser.AccessModifier; import com.datasqrl.planner.tables.SqrlFunctionParameter; import com.datasqrl.planner.tables.SqrlTableFunction; -import com.google.inject.Inject; import graphql.Scalars; import graphql.introspection.Introspection; import graphql.language.IntValue; @@ -47,6 +46,7 @@ import graphql.schema.GraphQLOutputType; import graphql.schema.GraphQLSchema; import graphql.schema.GraphQLTypeReference; +import jakarta.inject.Inject; import java.util.ArrayList; import java.util.HashSet; import java.util.LinkedHashMap; @@ -60,8 +60,10 @@ import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.schema.FunctionParameter; import org.apache.commons.collections.ListUtils; +import org.springframework.stereotype.Component; /** Creates a default graphql schema based on the SQRL schema */ +@Component @Slf4j public class GraphqlSchemaFactory { diff --git a/sqrl-planner/src/main/java/com/datasqrl/graphql/GraphqlSchemaValidator.java b/sqrl-planner/src/main/java/com/datasqrl/graphql/GraphqlSchemaValidator.java index 104545877e..01f540d781 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/graphql/GraphqlSchemaValidator.java +++ b/sqrl-planner/src/main/java/com/datasqrl/graphql/GraphqlSchemaValidator.java @@ -28,7 +28,6 @@ import com.datasqrl.planner.dag.plan.MutationQuery; import com.datasqrl.planner.tables.SqrlFunctionParameter; import com.datasqrl.planner.tables.SqrlTableFunction; -import com.google.inject.Inject; import graphql.language.EnumTypeDefinition; import graphql.language.FieldDefinition; import graphql.language.InputObjectTypeDefinition; @@ -44,6 +43,7 @@ import graphql.schema.GraphQLNonNull; import graphql.schema.idl.SchemaParser; import graphql.schema.idl.TypeDefinitionRegistry; +import jakarta.inject.Inject; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; diff --git a/sqrl-planner/src/main/java/com/datasqrl/graphql/InferGraphqlSchema.java b/sqrl-planner/src/main/java/com/datasqrl/graphql/InferGraphqlSchema.java index 2c94a1134f..3f7df29e15 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/graphql/InferGraphqlSchema.java +++ b/sqrl-planner/src/main/java/com/datasqrl/graphql/InferGraphqlSchema.java @@ -19,14 +19,16 @@ import com.datasqrl.engine.server.ServerPhysicalPlan; import com.datasqrl.error.ErrorCollector; -import com.google.inject.Inject; import graphql.schema.GraphQLSchema; import graphql.schema.GraphqlTypeComparatorRegistry; import graphql.schema.idl.SchemaPrinter; +import jakarta.inject.Inject; import lombok.AllArgsConstructor; import lombok.SneakyThrows; +import org.springframework.stereotype.Component; /** Creates new table functions from the GraphQL schema. */ +@Component @AllArgsConstructor(onConstructor_ = @Inject) public class InferGraphqlSchema { diff --git a/sqrl-planner/src/main/java/com/datasqrl/graphql/ScriptFiles.java b/sqrl-planner/src/main/java/com/datasqrl/graphql/ScriptFiles.java index c94962589f..d7b52c1e29 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/graphql/ScriptFiles.java +++ b/sqrl-planner/src/main/java/com/datasqrl/graphql/ScriptFiles.java @@ -18,11 +18,13 @@ import com.datasqrl.config.PackageJson; import com.datasqrl.config.PackageJson.ScriptApiConfig; import com.datasqrl.config.PackageJson.ScriptConfig; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.util.List; import java.util.Optional; import lombok.Getter; +import org.springframework.stereotype.Component; +@Component @Getter public class ScriptFiles { diff --git a/sqrl-planner/src/main/java/com/datasqrl/graphql/converter/GraphQLSchemaConverter.java b/sqrl-planner/src/main/java/com/datasqrl/graphql/converter/GraphQLSchemaConverter.java index 2da10de061..f045d5a8ae 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/graphql/converter/GraphQLSchemaConverter.java +++ b/sqrl-planner/src/main/java/com/datasqrl/graphql/converter/GraphQLSchemaConverter.java @@ -76,11 +76,13 @@ import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; /** * Converts a given GraphQL Schema to a tools configuration for the function backend. It extracts * all queries and mutations and converts them into {@link ApiOperation}. */ +@Component @RequiredArgsConstructor @Slf4j public class GraphQLSchemaConverter { diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/CodeGenBridge.java b/sqrl-planner/src/main/java/com/datasqrl/graphql/exec/CodeGenBridge.java similarity index 100% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/CodeGenBridge.java rename to sqrl-planner/src/main/java/com/datasqrl/graphql/exec/CodeGenBridge.java diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunctionFactory.java b/sqrl-planner/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunctionFactory.java similarity index 100% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunctionFactory.java rename to sqrl-planner/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunctionFactory.java diff --git a/sqrl-planner/src/main/java/com/datasqrl/loaders/ModuleLoaderImpl.java b/sqrl-planner/src/main/java/com/datasqrl/loaders/ModuleLoaderImpl.java index 462a1a4ac1..0ae1a4e708 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/loaders/ModuleLoaderImpl.java +++ b/sqrl-planner/src/main/java/com/datasqrl/loaders/ModuleLoaderImpl.java @@ -32,7 +32,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Files; @@ -44,7 +44,9 @@ import lombok.AllArgsConstructor; import lombok.SneakyThrows; import org.apache.flink.table.functions.UserDefinedFunction; +import org.springframework.stereotype.Component; +@Component @AllArgsConstructor(access = AccessLevel.PRIVATE) public class ModuleLoaderImpl implements ModuleLoader { diff --git a/sqrl-planner/src/main/java/com/datasqrl/planner/SqlScriptPlanner.java b/sqrl-planner/src/main/java/com/datasqrl/planner/SqlScriptPlanner.java index 763f9ffada..ce2886fd01 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/planner/SqlScriptPlanner.java +++ b/sqrl-planner/src/main/java/com/datasqrl/planner/SqlScriptPlanner.java @@ -99,7 +99,7 @@ import com.datasqrl.util.FunctionUtil; import com.datasqrl.util.StringUtil; import com.google.common.base.Preconditions; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; @@ -128,6 +128,8 @@ import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.functions.UserDefinedFunction; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; /** * This is the main class for planning SQRL scripts. It relies on the {@link SqrlStatementParser} @@ -137,6 +139,8 @@ *

In planning the SQRL statements, it uses produces a {@link TableAnalysis} that has the * information needed to build the computation DAG via {@link DAGBuilder}. */ +@Component +@Lazy public class SqlScriptPlanner { public static final String EXPORT_SUFFIX = "_ex"; diff --git a/sqrl-planner/src/main/java/com/datasqrl/planner/dag/DAGPlanner.java b/sqrl-planner/src/main/java/com/datasqrl/planner/dag/DAGPlanner.java index 2028bcdae3..d20301f993 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/planner/dag/DAGPlanner.java +++ b/sqrl-planner/src/main/java/com/datasqrl/planner/dag/DAGPlanner.java @@ -51,7 +51,7 @@ import com.datasqrl.util.FlinkCompileException; import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; -import com.google.inject.Inject; +import jakarta.inject.Inject; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -89,8 +89,12 @@ import org.apache.flink.table.planner.plan.schema.FlinkPreparingTableBase; import org.apache.flink.table.planner.plan.schema.TableSourceTable; import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; /** Optimizes the DAG and produces the physical plan after DAG cutting */ +@Component +@Lazy @AllArgsConstructor(onConstructor_ = @Inject) public class DAGPlanner { diff --git a/sqrl-planner/src/main/java/com/datasqrl/planner/parser/SqrlStatementParser.java b/sqrl-planner/src/main/java/com/datasqrl/planner/parser/SqrlStatementParser.java index 6ac52e4351..9fe22f0f7a 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/planner/parser/SqrlStatementParser.java +++ b/sqrl-planner/src/main/java/com/datasqrl/planner/parser/SqrlStatementParser.java @@ -31,6 +31,7 @@ import java.util.regex.Pattern; import org.apache.commons.lang3.tuple.Pair; import org.apache.flink.table.catalog.ObjectIdentifier; +import org.springframework.stereotype.Component; /** * A lightweight REGEX based parser to identify and parse SQRL specific SQL statements. @@ -47,6 +48,7 @@ * the code and complexity in this implementation is due to that. We use {@link ParsedObject} to * keep file locations. This is handled through {@link ParsedObject}. */ +@Component public class SqrlStatementParser { public static final String IDENTIFIER_REGEX = "[\\w\\.`*-]+?"; diff --git a/sqrl-planner/src/main/java/com/datasqrl/serializer/Deserializer.java b/sqrl-planner/src/main/java/com/datasqrl/serializer/Deserializer.java index c3e672fe18..86d9969d02 100644 --- a/sqrl-planner/src/main/java/com/datasqrl/serializer/Deserializer.java +++ b/sqrl-planner/src/main/java/com/datasqrl/serializer/Deserializer.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import io.vertx.core.json.jackson.VertxModule; import java.io.IOException; import java.nio.file.Path; import lombok.Getter; @@ -42,7 +41,6 @@ protected Deserializer() { new ObjectMapper() .registerModule(new Jdk8Module()) .registerModule(new JavaTimeModule()) - .registerModule(new VertxModule()) .registerModule(module) .setSerializationInclusion(JsonInclude.Include.NON_NULL) .addMixIn(LogicalType.class, AlphabeticMixin.class) diff --git a/sqrl-planner/src/test/java/com/datasqrl/MockSqrlInjector.java b/sqrl-planner/src/test/java/com/datasqrl/MockSqrlInjector.java index 93dd563094..868e9f7c38 100644 --- a/sqrl-planner/src/test/java/com/datasqrl/MockSqrlInjector.java +++ b/sqrl-planner/src/test/java/com/datasqrl/MockSqrlInjector.java @@ -15,110 +15,56 @@ */ package com.datasqrl; -import com.datasqrl.calcite.type.TypeFactory; import com.datasqrl.canonicalizer.NameCanonicalizer; -import com.datasqrl.canonicalizer.NamePath; -import com.datasqrl.config.PackageJson; -import com.datasqrl.config.PackageJson.CompilerConfig; -import com.datasqrl.config.SqrlCompilerConfiguration; -import com.datasqrl.config.SqrlConfigPipeline; -import com.datasqrl.engine.pipeline.ExecutionPipeline; -import com.datasqrl.error.ErrorCollector; -import com.datasqrl.loaders.ModuleLoader; -import com.datasqrl.loaders.ModuleLoaderImpl; -import com.datasqrl.loaders.SqrlModule; +import com.datasqrl.config.BuildPath; import com.datasqrl.loaders.resolver.FileResourceResolver; import com.datasqrl.loaders.resolver.ResourceResolver; -import com.datasqrl.plan.MainScript; -import com.google.inject.AbstractModule; -import com.google.inject.Provides; -import com.google.inject.name.Named; import java.nio.file.Path; -import java.util.Map; -import java.util.Optional; -import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; -public class MockSqrlInjector extends AbstractModule { +@Configuration +@ComponentScan( + basePackages = { + "com.datasqrl.calcite.type", + "com.datasqrl.config", + "com.datasqrl.graphql", + "com.datasqrl.loaders", + "com.datasqrl.planner" + }) +public class MockSqrlInjector { - private final ErrorCollector errors; - private final PackageJson config; - private final Path rootDir; - private final Map addlModules; - private final Optional errorDir; - - public MockSqrlInjector( - ErrorCollector errors, - PackageJson config, - Optional errorDir, - Path rootDir, - Map addlModules) { - this.errors = errors; - this.config = config; - this.rootDir = rootDir; - this.errorDir = errorDir; - this.addlModules = addlModules; - } - - @Override - public void configure() { - bind(RelDataTypeFactory.class).to(TypeFactory.class); - bind(MainScript.class).to(MainScriptImpl.class); - bind(ExecutionPipeline.class).to(SqrlConfigPipeline.class); - bind(CompilerConfig.class).to(SqrlCompilerConfiguration.class); - bind(ModuleLoader.class).to(ModuleLoaderImpl.class); - } - - @Provides - public ErrorCollector provideErrorCollector() { - return errors; - } - - @Provides - public NameCanonicalizer provideNameCanonicalizer() { + @Bean + public NameCanonicalizer nameCanonicalizer() { return NameCanonicalizer.SYSTEM; } - @Provides - @Named("rootDir") - public Path provideRootDir() { - return rootDir; - } - - @Provides - @Named("buildDir") - public Path provideBuildDir() { + @Bean + @Qualifier("buildDir") + public Path buildDir(@Qualifier("rootDir") Path rootDir) { return rootDir.resolve("build"); } - @Provides - @Named("targetDir") - public Path provideTargetDir() { + @Bean + @Qualifier("targetDir") + public Path targetDir(@Qualifier("rootDir") Path rootDir) { return rootDir.resolve("build").resolve("deploy"); } - @Provides - @Named("errorDir") - public Optional provideErrorDir() { - return errorDir; - } - - @Provides - @Named("addlModules") - public Map provideAddlModules() { - return addlModules; + @Bean + public BuildPath buildPath( + @Qualifier("buildDir") Path buildDir, @Qualifier("targetDir") Path targetDir) { + return new BuildPath(buildDir, targetDir); } - @Provides - public ResourceResolver provideResourceResolver() { + @Bean + public ResourceResolver resourceResolver(@Qualifier("rootDir") Path rootDir) { if (rootDir == null) { return new FileResourceResolver( Path.of("../sqrl-testing/sqrl-testing-integration/src/test/resources/dagplanner")); } return new FileResourceResolver(rootDir); } - - @Provides - public PackageJson provideSqrlConfig() { - return config; - } } diff --git a/sqrl-server/pom.xml b/sqrl-server/pom.xml index 3982862b3c..d137b52b44 100644 --- a/sqrl-server/pom.xml +++ b/sqrl-server/pom.xml @@ -30,7 +30,6 @@ sqrl-server-core - sqrl-server-vertx-base - sqrl-server-vertx + sqrl-server-spring diff --git a/sqrl-server/sqrl-server-core/pom.xml b/sqrl-server/sqrl-server-core/pom.xml index ab1b9742b0..ea63cececa 100644 --- a/sqrl-server/sqrl-server-core/pom.xml +++ b/sqrl-server/sqrl-server-core/pom.xml @@ -75,5 +75,20 @@ jackson-databind + + jakarta.annotation + jakarta.annotation-api + + + + + org.apache.flink + flink-table-common + + + org.apache.flink + flink-table-runtime + + diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/JsonEnvVarDeserializer.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/JsonEnvVarDeserializer.java similarity index 100% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/JsonEnvVarDeserializer.java rename to sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/JsonEnvVarDeserializer.java diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/SqrlObjectMapper.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/SqrlObjectMapper.java similarity index 94% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/SqrlObjectMapper.java rename to sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/SqrlObjectMapper.java index 0f6207c425..b75ab48400 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/SqrlObjectMapper.java +++ b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/SqrlObjectMapper.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import io.vertx.core.json.jackson.VertxModule; import jakarta.annotation.Nullable; import java.util.Map; @@ -31,7 +30,6 @@ public class SqrlObjectMapper { static { MAPPER.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); MAPPER.registerModule(new JavaTimeModule()); - MAPPER.registerModule(new VertxModule()); } public static ObjectMapper getMapperWithEnvVarResolver(@Nullable Map env) { diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunction.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunction.java similarity index 100% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunction.java rename to sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunction.java diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunctionPlan.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunctionPlan.java similarity index 100% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunctionPlan.java rename to sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/graphql/exec/FlinkExecFunctionPlan.java diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/util/JsonMergeUtils.java b/sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/util/JsonMergeUtils.java similarity index 100% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/util/JsonMergeUtils.java rename to sqrl-server/sqrl-server-core/src/main/java/com/datasqrl/util/JsonMergeUtils.java diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/resources/log4j2-test.properties b/sqrl-server/sqrl-server-spring/Dockerfile similarity index 64% rename from sqrl-server/sqrl-server-vertx-base/src/test/resources/log4j2-test.properties rename to sqrl-server/sqrl-server-spring/Dockerfile index c75b64e9a0..adec849ab9 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/test/resources/log4j2-test.properties +++ b/sqrl-server/sqrl-server-spring/Dockerfile @@ -14,13 +14,15 @@ # limitations under the License. # -# Console appender -appender.console.type = Console -appender.console.name = Console -appender.console.target = SYSTEM_OUT -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n\ +FROM eclipse-temurin:17-jre-alpine -# Root logger -rootLogger.level = WARN -rootLogger.appenderRef.console.ref = Console +WORKDIR /app + +COPY target/spring-server-exec.jar /app/spring-server.jar + +EXPOSE 8888 + +HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ + CMD wget -q --spider http://localhost:8888/actuator/health || exit 1 + +ENTRYPOINT ["java", "-jar", "/app/spring-server.jar"] diff --git a/sqrl-server/sqrl-server-vertx-base/pom.xml b/sqrl-server/sqrl-server-spring/pom.xml similarity index 58% rename from sqrl-server/sqrl-server-vertx-base/pom.xml rename to sqrl-server/sqrl-server-spring/pom.xml index 8cec4f003c..7ff552198d 100644 --- a/sqrl-server/sqrl-server-vertx-base/pom.xml +++ b/sqrl-server/sqrl-server-spring/pom.xml @@ -24,57 +24,91 @@ 1.0-SNAPSHOT - sqrl-server-vertx-base - SQRL :: Server :: Vert.x Base + sqrl-server-spring + SQRL :: Server :: Spring Boot + + + datasqrl/sqrl-server + + + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} + pom + import + + + + com.datasqrl sqrl-server-core - + - io.vertx - vertx-core + org.springframework.boot + spring-boot-starter-webflux - io.vertx - vertx-config + org.springframework.boot + spring-boot-starter-actuator - io.vertx - vertx-auth-jwt + org.springframework.boot + spring-boot-starter-security - io.vertx - vertx-auth-properties + org.springframework.boot + spring-boot-starter-oauth2-resource-server - io.vertx - vertx-auth-oauth2 + org.springframework.boot + spring-boot-starter-data-r2dbc - jakarta.annotation - jakarta.annotation-api - ${jakarta.annotation.version} + org.springframework.boot + spring-boot-starter-jdbc - io.vertx - vertx-jdbc-client + org.springframework.boot + spring-boot-configuration-processor + true + + - io.vertx - vertx-web + org.postgresql + r2dbc-postgresql + + - io.vertx - vertx-health-check + org.postgresql + postgresql + + - io.vertx - vertx-micrometer-metrics + org.springframework.kafka + spring-kafka + + io.projectreactor.kafka + reactor-kafka + + + software.amazon.msk + aws-msk-iam-auth + ${aws-msk-iam-auth.version} + + + io.micrometer micrometer-registry-prometheus @@ -85,44 +119,39 @@ ${graphql-micrometer.version} + - io.vertx - vertx-junit5 - test - - - - junit - junit - - - - - io.vertx - vertx-web-client + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 - io.vertx - vertx-web-graphql + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + + - io.vertx - vertx-pg-client + org.duckdb + duckdb_jdbc + ${duckdb.version} - io.agroal - agroal-pool + net.snowflake + snowflake-jdbc + ${snowflake-jdbc.version} + + com.zaxxer HikariCP + org.apache.flink flink-table-planner_2.12 - org.apache.flink flink-streaming-java @@ -206,64 +235,70 @@ - + - io.vertx - vertx-kafka-client + com.networknt + json-schema-validator + + - software.amazon.msk - aws-msk-iam-auth - ${aws-msk-iam-auth.version} + io.swagger.core.v3 + swagger-core + ${swagger-core.version} - io.projectreactor - reactor-core + io.swagger.core.v3 + swagger-models + ${swagger-core.version} - - io.netty - netty-codec + io.swagger.core.v3 + swagger-annotations + ${swagger-core.version} + + - org.duckdb - duckdb_jdbc - ${duckdb.version} + io.jsonwebtoken + jjwt-api + ${jjwt.version} - net.snowflake - snowflake-jdbc - ${snowflake-jdbc.version} + io.jsonwebtoken + jjwt-impl + ${jjwt.version} + runtime - com.fasterxml.jackson.datatype - jackson-datatype-jsr310 + io.jsonwebtoken + jjwt-jackson + ${jjwt.version} + runtime - - + + org.slf4j slf4j-api + + - org.apache.logging.log4j - log4j-api + org.springframework.boot + spring-boot-starter-test + test - org.apache.logging.log4j - log4j-core + io.projectreactor + reactor-test + test - org.apache.logging.log4j - log4j-slf4j-impl + org.springframework.security + spring-security-test + test - - - org.postgresql - postgresql - - - org.testcontainers testcontainers @@ -285,49 +320,100 @@ test - org.mockito - mockito-core + org.awaitility + awaitility + ${awaitility.version} test - - - - com.networknt - json-schema-validator - - - - - io.swagger.core.v3 - swagger-core - ${swagger-core.version} - - - io.swagger.core.v3 - swagger-models - ${swagger-core.version} - - - io.swagger.core.v3 - swagger-annotations - ${swagger-core.version} - + spring-server - io.reactiverse - vertx-maven-plugin + org.apache.maven.plugins + maven-enforcer-plugin - initialize + ban-vertx-dependencies - initialize + enforce + + + + + io.vertx:* + + Vert.x dependencies are not allowed in the Spring Boot server module + + + true + + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + com.datasqrl.graphql.SqrlServerApplication + exec + + + + + repackage + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + + build + + package + + + + instrument + + + + com.marvinformatics.jacoco + easy-jacoco-maven-plugin + ${easy-jacoco-maven-plugin.version} + + + instrument-uber-jar + + instrument-jar + + + ${project.build.directory}/spring-server.jar + ${project.build.directory}/spring-server.jar + + com/datasqrl/* + + + + + + + + + diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/JsonEnvVarDeserializer.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/JsonEnvVarDeserializer.java new file mode 100644 index 0000000000..a61745be10 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/JsonEnvVarDeserializer.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql; + +import com.datasqrl.env.GlobalEnvironmentStore; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import jakarta.annotation.Nullable; +import java.io.IOException; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** Custom deserializer to replace environment variables in JSON strings. */ +public class JsonEnvVarDeserializer extends JsonDeserializer { + + private final Map env; + + public JsonEnvVarDeserializer() { + this(null); + } + + public JsonEnvVarDeserializer(@Nullable Map env) { + this.env = env != null ? env : GlobalEnvironmentStore.getAll(); + } + + @Override + public String deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + var value = p.getText(); + return replaceWithEnv(this.env, value); + } + + public String replaceWithEnv(Map env, String value) { + var pattern = Pattern.compile("\\$\\{(.+?)\\}"); + var matcher = pattern.matcher(value); + var result = new StringBuffer(); + while (matcher.find()) { + var key = matcher.group(1); + var envVarValue = env.get(key); + if (envVarValue != null) { + matcher.appendReplacement(result, Matcher.quoteReplacement(envVarValue)); + } + } + matcher.appendTail(result); + + return result.toString(); + } +} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/VertxContext.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/SpringContext.java similarity index 84% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/VertxContext.java rename to sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/SpringContext.java index 72f51a9df0..1ac2c58f76 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/VertxContext.java +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/SpringContext.java @@ -18,14 +18,13 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.datasqrl.graphql.jdbc.JdbcClient; -import com.datasqrl.graphql.jdbc.VertxJdbcClient; -import com.datasqrl.graphql.jdbc.VertxQueryExecutionContext; +import com.datasqrl.graphql.jdbc.SpringJdbcClient; +import com.datasqrl.graphql.jdbc.SpringQueryExecutionContext; import com.datasqrl.graphql.server.Context; import com.datasqrl.graphql.server.FunctionExecutor; import com.datasqrl.graphql.server.GraphQLEngineBuilder; import com.datasqrl.graphql.server.MetadataReader; import com.datasqrl.graphql.server.MetadataType; -import com.datasqrl.graphql.server.QueryExecutionContext; import com.datasqrl.graphql.server.RootGraphqlModel; import com.datasqrl.graphql.server.RootGraphqlModel.Argument; import com.datasqrl.graphql.server.RootGraphqlModel.ResolvedQuery; @@ -38,12 +37,12 @@ import lombok.Value; import lombok.extern.slf4j.Slf4j; -/** Implements {@link Context} for Vert.x, providing SQL clients and data fetchers. */ +/** Spring-based implementation of {@link Context}. Replaces the Vert.x-based VertxContext. */ @Slf4j @Value -public class VertxContext implements Context { +public class SpringContext implements Context { - VertxJdbcClient sqlClient; + SpringJdbcClient sqlClient; Map metadataReaders; FunctionExecutor functionExecutor; @@ -67,8 +66,7 @@ public DataFetcher createArgumentLookupFetcher( var cf = new CompletableFuture<>(); - // Execute - QueryExecutionContext context = new VertxQueryExecutionContext(this, env, argumentSet, cf); + var context = new SpringQueryExecutionContext(this, env, argumentSet, cf); resolvedQuery.accept(server, context); return cf; diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/SqrlServerApplication.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/SqrlServerApplication.java new file mode 100644 index 0000000000..9834b9dc94 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/SqrlServerApplication.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql; + +import com.datasqrl.env.GlobalEnvironmentStore; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.web.server.context.WebServerInitializedEvent; +import org.springframework.context.event.EventListener; + +/** + * Main entry point for the Spring Boot GraphQL server application. Replaces the Vert.x-based + * SqrlLauncher. + */ +@Slf4j +@SpringBootApplication +@EnableConfigurationProperties +public class SqrlServerApplication { + + public static void main(String[] args) { + GlobalEnvironmentStore.putAll(System.getenv()); + SpringApplication.run(SqrlServerApplication.class, args); + } + + @EventListener + public void onApplicationEvent(WebServerInitializedEvent event) { + int port = event.getWebServer().getPort(); + log.info("HTTP server listening on port {}", port); + } +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/DatabaseConfiguration.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/DatabaseConfiguration.java new file mode 100644 index 0000000000..d91faa40b8 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/DatabaseConfiguration.java @@ -0,0 +1,156 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.config; + +import com.datasqrl.graphql.jdbc.DatabaseType; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import io.r2dbc.pool.ConnectionPool; +import io.r2dbc.pool.ConnectionPoolConfiguration; +import io.r2dbc.postgresql.PostgresqlConnectionConfiguration; +import io.r2dbc.postgresql.PostgresqlConnectionFactory; +import io.r2dbc.spi.ConnectionFactory; +import java.time.Duration; +import java.util.EnumMap; +import java.util.Map; +import javax.sql.DataSource; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +/** + * Database configuration for the Spring Boot GraphQL server. Sets up R2DBC for PostgreSQL (async) + * and JDBC for DuckDB/Snowflake. + */ +@Slf4j +@Configuration +@RequiredArgsConstructor +public class DatabaseConfiguration { + + private final ServerConfigProperties config; + + @Bean + @Primary + public ConnectionFactory postgresConnectionFactory() { + var pgConfig = config.getPostgres(); + + var connectionConfig = + PostgresqlConnectionConfiguration.builder() + .host(pgConfig.getHost()) + .port(pgConfig.getPort()) + .database(pgConfig.getDatabase()) + .username(pgConfig.getUser()) + .password(pgConfig.getPassword()) + .build(); + + var connectionFactory = new PostgresqlConnectionFactory(connectionConfig); + + var poolConfig = + ConnectionPoolConfiguration.builder(connectionFactory) + .maxSize(pgConfig.getPoolSize()) + .maxIdleTime(Duration.ofMillis(pgConfig.getMaxIdleTime())) + .build(); + + log.info( + "Creating R2DBC PostgreSQL connection pool: {}:{}/{}", + pgConfig.getHost(), + pgConfig.getPort(), + pgConfig.getDatabase()); + + return new ConnectionPool(poolConfig); + } + + @Bean(name = "duckDbDataSource") + @ConditionalOnProperty(prefix = "sqrl.server.duck-db", name = "path") + public DataSource duckDbDataSource() { + var duckDbConfig = config.getDuckDb(); + if (duckDbConfig == null) { + return null; + } + + var hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl("jdbc:duckdb:" + duckDbConfig.getPath()); + hikariConfig.setMaximumPoolSize(duckDbConfig.getPoolSize()); + hikariConfig.setDriverClassName("org.duckdb.DuckDBDriver"); + + log.info("Creating DuckDB connection pool: {}", duckDbConfig.getPath()); + + return new HikariDataSource(hikariConfig); + } + + @Bean(name = "snowflakeDataSource") + @ConditionalOnProperty(prefix = "sqrl.server.snowflake", name = "url") + public DataSource snowflakeDataSource() { + var snowflakeConfig = config.getSnowflake(); + if (snowflakeConfig == null) { + return null; + } + + var hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(snowflakeConfig.getUrl()); + hikariConfig.setUsername(snowflakeConfig.getUser()); + hikariConfig.setPassword(snowflakeConfig.getPassword()); + hikariConfig.setDriverClassName("net.snowflake.client.jdbc.SnowflakeDriver"); + + if (snowflakeConfig.getDatabase() != null) { + hikariConfig.addDataSourceProperty("db", snowflakeConfig.getDatabase()); + } + if (snowflakeConfig.getSchema() != null) { + hikariConfig.addDataSourceProperty("schema", snowflakeConfig.getSchema()); + } + if (snowflakeConfig.getWarehouse() != null) { + hikariConfig.addDataSourceProperty("warehouse", snowflakeConfig.getWarehouse()); + } + + if (snowflakeConfig.getProperties() != null) { + snowflakeConfig.getProperties().forEach(hikariConfig::addDataSourceProperty); + } + + log.info("Creating Snowflake connection pool: {}", snowflakeConfig.getUrl()); + + return new HikariDataSource(hikariConfig); + } + + @Bean + public Map databaseClients( + ConnectionFactory postgresConnectionFactory, + @org.springframework.beans.factory.annotation.Autowired(required = false) + @org.springframework.beans.factory.annotation.Qualifier("duckDbDataSource") + DataSource duckDbDataSource, + @org.springframework.beans.factory.annotation.Autowired(required = false) + @org.springframework.beans.factory.annotation.Qualifier("snowflakeDataSource") + DataSource snowflakeDataSource) { + + Map clients = new EnumMap<>(DatabaseType.class); + + clients.put(DatabaseType.POSTGRES, postgresConnectionFactory); + + if (duckDbDataSource != null) { + clients.put(DatabaseType.DUCKDB, duckDbDataSource); + } + + if (snowflakeDataSource != null) { + clients.put(DatabaseType.SNOWFLAKE, snowflakeDataSource); + } + + log.info("Configured database clients: {}", clients.keySet()); + + return clients; + } +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/GraphQLConfiguration.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/GraphQLConfiguration.java new file mode 100644 index 0000000000..ffbdf6bc03 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/GraphQLConfiguration.java @@ -0,0 +1,180 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.config; + +import com.datasqrl.graphql.SpringContext; +import com.datasqrl.graphql.jdbc.DatabaseType; +import com.datasqrl.graphql.jdbc.SpringJdbcClient; +import com.datasqrl.graphql.kafka.SpringMutationConfiguration; +import com.datasqrl.graphql.kafka.SpringSubscriptionConfiguration; +import com.datasqrl.graphql.server.CustomScalars; +import com.datasqrl.graphql.server.FunctionExecutor; +import com.datasqrl.graphql.server.GraphQLEngineBuilder; +import com.datasqrl.graphql.server.MetadataReader; +import com.datasqrl.graphql.server.MetadataType; +import com.datasqrl.graphql.server.ModelContainer; +import com.datasqrl.graphql.server.RootGraphqlModel; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import com.symbaloo.graphqlmicrometer.MicrometerInstrumentation; +import graphql.GraphQL; +import io.micrometer.core.instrument.MeterRegistry; +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * GraphQL configuration for Spring Boot. Creates and configures the GraphQL engine with all + * necessary dependencies. + */ +@Slf4j +@Configuration +@RequiredArgsConstructor +public class GraphQLConfiguration { + + private final ServerConfigProperties config; + private final MeterRegistry meterRegistry; + + @Bean + @SneakyThrows + public Map rootGraphqlModels(ObjectMapper objectMapper) { + var modelFile = new File("vertx.json"); + if (!modelFile.exists()) { + log.warn("vertx.json not found, GraphQL models will be empty"); + return Map.of(); + } + + return objectMapper.readValue(modelFile, ModelContainer.class).models; + } + + @Bean + public SpringJdbcClient springJdbcClient(Map databaseClients) { + var asyncMode = config.getExecutionMode() == ServerConfigProperties.ExecutionMode.ASYNC; + return new SpringJdbcClient(databaseClients, asyncMode); + } + + @Bean + public SpringMutationConfiguration springMutationConfiguration() { + return new SpringMutationConfiguration(config); + } + + @Bean + public SpringSubscriptionConfiguration springSubscriptionConfiguration() { + return new SpringSubscriptionConfiguration(config); + } + + @Bean + public Map graphQLEngines( + Map models, + SpringJdbcClient springJdbcClient, + SpringMutationConfiguration mutationConfiguration, + SpringSubscriptionConfiguration subscriptionConfiguration) { + + Map engines = new HashMap<>(); + + for (var entry : models.entrySet()) { + var modelVersion = entry.getKey(); + var model = entry.getValue(); + + var engine = + createGraphQL( + model, + springJdbcClient, + mutationConfiguration, + subscriptionConfiguration, + createMetadataReaders(), + createFunctionExecutor()); + + engines.put(modelVersion, engine); + log.info("Created GraphQL engine for model version: {}", modelVersion); + } + + return engines; + } + + private GraphQL createGraphQL( + RootGraphqlModel model, + SpringJdbcClient jdbcClient, + SpringMutationConfiguration mutationConfiguration, + SpringSubscriptionConfiguration subscriptionConfiguration, + Map metadataReaders, + FunctionExecutor functionExecutor) { + + try { + var context = new SpringContext(jdbcClient, metadataReaders, functionExecutor); + + var graphQL = + model.accept( + new GraphQLEngineBuilder.Builder() + .withMutationConfiguration(mutationConfiguration) + .withSubscriptionConfiguration(subscriptionConfiguration) + .withExtendedScalarTypes(CustomScalars.getExtendedScalars()) + .build(), + context); + + if (meterRegistry != null) { + graphQL.instrumentation(new MicrometerInstrumentation(meterRegistry)); + } + + return graphQL.build(); + } catch (Exception e) { + log.error("Unable to create GraphQL engine", e); + throw e; + } + } + + private Map createMetadataReaders() { + var readers = ImmutableMap.builder(); + + if (config.getJwt() != null || config.getOauth() != null) { + log.debug("Configuring authentication metadata reader"); + readers.put(MetadataType.AUTH, new SpringAuthMetadataReader()); + } + + return readers.build(); + } + + private FunctionExecutor createFunctionExecutor() { + return (functionId, arguments) -> { + throw new UnsupportedOperationException( + "Function execution not yet implemented in Spring version"); + }; + } + + private static class SpringAuthMetadataReader implements MetadataReader { + @Override + public Object read(graphql.schema.DataFetchingEnvironment env, String name, boolean required) { + var graphqlContext = env.getGraphQlContext(); + var claims = graphqlContext.get("claims"); + if (claims instanceof Map claimsMap) { + var value = claimsMap.get(name); + if (value == null && required) { + throw new RuntimeException("Required claim not found: " + name); + } + return value; + } + if (required) { + throw new RuntimeException("Authentication claims not available"); + } + return null; + } + } +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/SecurityConfiguration.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/SecurityConfiguration.java new file mode 100644 index 0000000000..bce1f4b5df --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/SecurityConfiguration.java @@ -0,0 +1,104 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.config; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; +import org.springframework.security.config.web.server.ServerHttpSecurity; +import org.springframework.security.oauth2.jwt.ReactiveJwtDecoder; +import org.springframework.security.oauth2.jwt.ReactiveJwtDecoders; +import org.springframework.security.web.server.SecurityWebFilterChain; + +/** + * Spring Security configuration for the GraphQL server. Configures JWT/OAuth2 authentication when + * configured. + */ +@Slf4j +@Configuration +@EnableWebFluxSecurity +@RequiredArgsConstructor +public class SecurityConfiguration { + + private final ServerConfigProperties config; + + @Bean + public SecurityWebFilterChain securityWebFilterChain(ServerHttpSecurity http) { + http.csrf(ServerHttpSecurity.CsrfSpec::disable); + + http.cors(cors -> {}); + + var jwtConfig = config.getJwt(); + var oauthConfig = config.getOauth(); + + if (jwtConfig != null && jwtConfig.getJwksUri() != null) { + log.info("Configuring JWT authentication with JWKS URI: {}", jwtConfig.getJwksUri()); + http.oauth2ResourceServer(oauth2 -> oauth2.jwt(jwt -> jwt.jwtDecoder(jwtDecoder()))); + + http.authorizeExchange( + exchanges -> + exchanges + .pathMatchers("/health/**", "/metrics", "/actuator/**") + .permitAll() + .pathMatchers("/graphiql/**", "/swagger**", "/**/swagger**") + .permitAll() + .anyExchange() + .authenticated()); + } else if (oauthConfig != null && oauthConfig.getSite() != null) { + log.info("Configuring OAuth2 authentication with site: {}", oauthConfig.getSite()); + http.oauth2ResourceServer(oauth2 -> oauth2.jwt(jwt -> jwt.jwtDecoder(oauthJwtDecoder()))); + + http.authorizeExchange( + exchanges -> + exchanges + .pathMatchers("/health/**", "/metrics", "/actuator/**") + .permitAll() + .pathMatchers("/graphiql/**", "/swagger**", "/**/swagger**") + .permitAll() + .anyExchange() + .authenticated()); + } else { + log.info("No authentication configured, allowing all requests"); + http.authorizeExchange(exchanges -> exchanges.anyExchange().permitAll()); + } + + return http.build(); + } + + @Bean + public ReactiveJwtDecoder jwtDecoder() { + var jwtConfig = config.getJwt(); + if (jwtConfig != null && jwtConfig.getJwksUri() != null) { + return ReactiveJwtDecoders.fromIssuerLocation(jwtConfig.getIssuer()); + } + return null; + } + + private ReactiveJwtDecoder oauthJwtDecoder() { + var oauthConfig = config.getOauth(); + if (oauthConfig != null && oauthConfig.getSite() != null) { + var jwksUri = + oauthConfig.getSite() + + (oauthConfig.getJwksPath() != null + ? oauthConfig.getJwksPath() + : "/.well-known/jwks.json"); + return ReactiveJwtDecoders.fromIssuerLocation(oauthConfig.getSite()); + } + return null; + } +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/ServerConfigProperties.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/ServerConfigProperties.java new file mode 100644 index 0000000000..8d0b8e942e --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/ServerConfigProperties.java @@ -0,0 +1,175 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.config; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Spring Boot configuration properties for the SQRL GraphQL server. Replaces the Vert.x-based + * ServerConfig. + */ +@Data +@Component +@ConfigurationProperties(prefix = "sqrl.server") +public class ServerConfigProperties { + + private ExecutionMode executionMode = ExecutionMode.ASYNC; + private ServletConfig servletConfig = new ServletConfig(); + private PostgresConfig postgres = new PostgresConfig(); + private KafkaMutationConfig kafkaMutation; + private KafkaSubscriptionConfig kafkaSubscription; + private DuckDbConfig duckDb; + private SnowflakeConfig snowflake; + private CorsConfig cors = new CorsConfig(); + private SwaggerConfig swagger = new SwaggerConfig(); + private GraphiQLConfig graphiql; + private JwtConfig jwt; + private OAuthConfig oauth; + + public enum ExecutionMode { + ASYNC, + SYNC + } + + @Data + public static class ServletConfig { + private String graphqlEndpoint = "/graphql"; + private String graphiqlEndpoint = "/graphiql/*"; + private String restEndpoint = "/rest"; + private String mcpEndpoint = "/mcp"; + + public String getGraphQLEndpoint(String modelVersion) { + return modelVersion.isEmpty() ? graphqlEndpoint : "/" + modelVersion + graphqlEndpoint; + } + + public String getGraphiQLEndpoint(String modelVersion) { + return modelVersion.isEmpty() ? graphiqlEndpoint : "/" + modelVersion + graphiqlEndpoint; + } + + public String getRestEndpoint(String modelVersion) { + return modelVersion.isEmpty() ? restEndpoint : "/" + modelVersion + restEndpoint; + } + + public String getMcpEndpoint(String modelVersion) { + return modelVersion.isEmpty() ? mcpEndpoint : "/" + modelVersion + mcpEndpoint; + } + } + + @Data + public static class PostgresConfig { + private String host = "localhost"; + private int port = 5432; + private String database = "datasqrl"; + private String user = "postgres"; + private String password = ""; + private int poolSize = 10; + private int maxIdleTime = 30000; + } + + @Data + public static class KafkaMutationConfig { + private Map properties; + private boolean transactional = false; + + public Map asMap(boolean transactional) { + return properties; + } + } + + @Data + public static class KafkaSubscriptionConfig { + private Map properties; + + public Map asMap() { + return properties; + } + } + + @Data + public static class DuckDbConfig { + private String path = ":memory:"; + private int poolSize = 5; + private List extensions; + } + + @Data + public static class SnowflakeConfig { + private String url; + private String user; + private String password; + private String database; + private String schema; + private String warehouse; + private Map properties; + } + + @Data + public static class CorsConfig { + private String allowedOrigin; + private List allowedOrigins; + private Set allowedMethods = Set.of("GET", "POST", "OPTIONS"); + private Set allowedHeaders = Set.of("*"); + private Set exposedHeaders = Set.of(); + private boolean allowCredentials = false; + private int maxAgeSeconds = 3600; + private boolean allowPrivateNetwork = false; + } + + @Data + public static class SwaggerConfig { + private boolean enabled = false; + private String title = "SQRL GraphQL API"; + private String description = "Auto-generated REST API from GraphQL schema"; + private String version = "1.0.0"; + + public String getEndpoint(String modelVersion) { + return modelVersion.isEmpty() ? "/swagger.json" : "/" + modelVersion + "/swagger.json"; + } + + public String getUiEndpoint(String modelVersion) { + return modelVersion.isEmpty() ? "/swagger-ui" : "/" + modelVersion + "/swagger-ui"; + } + } + + @Data + public static class GraphiQLConfig { + private boolean enabled = true; + private String graphqlEndpoint = "/graphql"; + } + + @Data + public static class JwtConfig { + private String publicKey; + private String publicKeyPath; + private String jwksUri; + private String issuer; + private List audience; + } + + @Data + public static class OAuthConfig { + private String site; + private String clientId; + private String clientSecret; + private String introspectionPath; + private String jwksPath; + } +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/WebConfiguration.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/WebConfiguration.java new file mode 100644 index 0000000000..a135d6622e --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/config/WebConfiguration.java @@ -0,0 +1,94 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.config; + +import com.datasqrl.graphql.JsonEnvVarDeserializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.web.reactive.config.CorsRegistry; +import org.springframework.web.reactive.config.EnableWebFlux; +import org.springframework.web.reactive.config.WebFluxConfigurer; + +/** + * Web configuration for the Spring Boot GraphQL server. Configures CORS and other web-related + * settings. + */ +@Slf4j +@Configuration +@EnableWebFlux +@RequiredArgsConstructor +public class WebConfiguration implements WebFluxConfigurer { + + private final ServerConfigProperties config; + + @Override + public void addCorsMappings(CorsRegistry registry) { + var corsConfig = config.getCors(); + if (corsConfig == null) { + log.info("No CORS configuration found, using defaults"); + return; + } + + var corsRegistration = registry.addMapping("/**"); + + if (corsConfig.getAllowedOrigin() != null) { + corsRegistration.allowedOrigins(corsConfig.getAllowedOrigin()); + } + + if (corsConfig.getAllowedOrigins() != null && !corsConfig.getAllowedOrigins().isEmpty()) { + corsRegistration.allowedOrigins(corsConfig.getAllowedOrigins().toArray(String[]::new)); + } + + if (corsConfig.getAllowedMethods() != null) { + corsRegistration.allowedMethods(corsConfig.getAllowedMethods().toArray(String[]::new)); + } + + if (corsConfig.getAllowedHeaders() != null) { + corsRegistration.allowedHeaders(corsConfig.getAllowedHeaders().toArray(String[]::new)); + } + + if (corsConfig.getExposedHeaders() != null) { + corsRegistration.exposedHeaders(corsConfig.getExposedHeaders().toArray(String[]::new)); + } + + corsRegistration.allowCredentials(corsConfig.isAllowCredentials()); + corsRegistration.maxAge(corsConfig.getMaxAgeSeconds()); + + log.info( + "CORS configured: origins={}, methods={}", + corsConfig.getAllowedOrigins(), + corsConfig.getAllowedMethods()); + } + + @Bean + @Primary + public ObjectMapper objectMapper() { + var objectMapper = new ObjectMapper(); + var module = new SimpleModule(); + module.addDeserializer(String.class, new JsonEnvVarDeserializer()); + objectMapper.registerModule(module); + objectMapper.registerModule(new JavaTimeModule()); + objectMapper.registerModule(new Jdk8Module()); + return objectMapper; + } +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/controller/GraphQLController.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/controller/GraphQLController.java new file mode 100644 index 0000000000..97e3535e57 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/controller/GraphQLController.java @@ -0,0 +1,135 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.controller; + +import com.datasqrl.graphql.config.ServerConfigProperties; +import graphql.ExecutionInput; +import graphql.ExecutionResult; +import graphql.GraphQL; +import java.security.Principal; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.MediaType; +import org.springframework.security.core.Authentication; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import reactor.core.publisher.Mono; + +/** + * GraphQL HTTP endpoint controller. Handles GraphQL queries, mutations, and subscriptions over + * HTTP. + */ +@Slf4j +@RestController +@RequiredArgsConstructor +public class GraphQLController { + + private final Map graphQLEngines; + private final ServerConfigProperties config; + + @PostMapping( + value = "/graphql", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) + public Mono> executeGraphQL( + @RequestBody GraphQLRequest request, Principal principal) { + return executeQuery("", request, principal); + } + + @PostMapping( + value = "/{version}/graphql", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) + public Mono> executeVersionedGraphQL( + @PathVariable String version, @RequestBody GraphQLRequest request, Principal principal) { + return executeQuery(version, request, principal); + } + + private Mono> executeQuery( + String version, GraphQLRequest request, Principal principal) { + + var graphQL = graphQLEngines.get(version); + if (graphQL == null) { + var errorResponse = new HashMap(); + errorResponse.put( + "errors", + java.util.List.of(Map.of("message", "GraphQL engine not found for version: " + version))); + return Mono.just(errorResponse); + } + + var executionInput = + ExecutionInput.newExecutionInput() + .query(request.query()) + .operationName(request.operationName()) + .variables(request.variables() != null ? request.variables() : Map.of()) + .graphQLContext( + builder -> { + if (principal instanceof Authentication auth + && auth.getPrincipal() instanceof Jwt jwt) { + builder.of("claims", jwt.getClaims()); + } + }) + .build(); + + CompletableFuture futureResult = graphQL.executeAsync(executionInput); + + return Mono.fromFuture(futureResult).map(this::toSpecificationResult); + } + + private Map toSpecificationResult(ExecutionResult executionResult) { + var result = new HashMap(); + + if (executionResult.getData() != null) { + result.put("data", executionResult.getData()); + } + + if (!executionResult.getErrors().isEmpty()) { + result.put( + "errors", + executionResult.getErrors().stream() + .map( + error -> { + var errorMap = new HashMap(); + errorMap.put("message", error.getMessage()); + if (error.getPath() != null) { + errorMap.put("path", error.getPath()); + } + if (error.getLocations() != null) { + errorMap.put("locations", error.getLocations()); + } + if (error.getExtensions() != null) { + errorMap.put("extensions", error.getExtensions()); + } + return errorMap; + }) + .toList()); + } + + if (executionResult.getExtensions() != null && !executionResult.getExtensions().isEmpty()) { + result.put("extensions", executionResult.getExtensions()); + } + + return result; + } + + public record GraphQLRequest(String query, String operationName, Map variables) {} +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/controller/McpBridgeController.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/controller/McpBridgeController.java new file mode 100644 index 0000000000..c1f7e729a7 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/controller/McpBridgeController.java @@ -0,0 +1,460 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.controller; + +import com.datasqrl.graphql.config.ServerConfigProperties; +import com.datasqrl.graphql.server.RootGraphqlModel; +import com.datasqrl.graphql.server.operation.ApiOperation; +import com.datasqrl.graphql.server.operation.McpMethodType; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import graphql.ExecutionInput; +import graphql.ExecutionResult; +import graphql.GraphQL; +import java.security.Principal; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.codec.ServerSentEvent; +import org.springframework.security.core.Authentication; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.publisher.Sinks; + +/** + * MCP (Model Context Protocol) bridge controller. Implements JSON-RPC 2.0 with SSE for + * server-to-client streaming. Replaces the Vert.x-based McpBridgeVerticle. + */ +@Slf4j +@RestController +@RequiredArgsConstructor +public class McpBridgeController { + + private static final String JSONRPC_VERSION = "2.0"; + private static final String PROTOCOL_VERSION = "2024-11-05"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final Map graphQLEngines; + private final Map rootGraphqlModels; + private final ServerConfigProperties config; + + private final ConcurrentHashMap sseConnections = new ConcurrentHashMap<>(); + + @PostMapping( + value = "/mcp", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) + public Mono>> handleMcpPost( + @RequestBody JsonNode payload, Principal principal) { + return handleMcpRequest("", payload, principal); + } + + @PostMapping( + value = "/{version}/mcp", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) + public Mono>> handleVersionedMcpPost( + @PathVariable String version, @RequestBody JsonNode payload, Principal principal) { + return handleMcpRequest(version, payload, principal); + } + + @GetMapping(value = "/mcp/sse", produces = MediaType.TEXT_EVENT_STREAM_VALUE) + public Flux> handleMcpSse() { + return createSseFlux(""); + } + + @GetMapping(value = "/{version}/mcp/sse", produces = MediaType.TEXT_EVENT_STREAM_VALUE) + public Flux> handleVersionedMcpSse(@PathVariable String version) { + return createSseFlux(version); + } + + private Mono>> handleMcpRequest( + String version, JsonNode payload, Principal principal) { + + var graphQL = graphQLEngines.get(version); + var model = rootGraphqlModels.get(version); + + if (graphQL == null || model == null) { + return Mono.just( + ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(createErrorResponse(null, -32601, "MCP not found for version: " + version))); + } + + // Handle batch requests + if (payload.isArray()) { + return handleBatchRequest(version, payload, principal, graphQL, model); + } + + return handleSingleRequest(version, payload, principal, graphQL, model); + } + + private Mono>> handleSingleRequest( + String version, + JsonNode payload, + Principal principal, + GraphQL graphQL, + RootGraphqlModel model) { + + var method = payload.has("method") ? payload.get("method").asText() : null; + var id = payload.has("id") ? payload.get("id") : null; + var params = payload.has("params") ? payload.get("params") : null; + + log.debug("MCP request: method={}, id={}", method, id); + + if (method == null) { + return Mono.just( + ResponseEntity.ok(createErrorResponse(id, -32601, "Method not found: null"))); + } + + return switch (method) { + case "initialize" -> handleInitialize(id); + case "tools/list" -> handleToolsList(id, model); + case "tools/call" -> handleToolCall(id, params, principal, graphQL, model); + case "resources/list" -> handleResourcesList(id, model); + case "resources/templates/list" -> handleResourceTemplatesList(id, model); + case "resources/read" -> handleResourceRead(id, params, principal, graphQL, model); + case "ping" -> handlePing(id); + default -> + Mono.just( + ResponseEntity.ok(createErrorResponse(id, -32601, "Method not found: " + method))); + }; + } + + private Mono>> handleBatchRequest( + String version, + JsonNode payload, + Principal principal, + GraphQL graphQL, + RootGraphqlModel model) { + // For simplicity, just handle the first request in batch + if (payload.size() > 0) { + return handleSingleRequest(version, payload.get(0), principal, graphQL, model); + } + return Mono.just( + ResponseEntity.badRequest().body(createErrorResponse(null, -32600, "Invalid Request"))); + } + + private Mono>> handleInitialize(JsonNode id) { + var result = + Map.of( + "protocolVersion", PROTOCOL_VERSION, + "capabilities", + Map.of( + "tools", Map.of("listChanged", false), + "resources", Map.of("subscribe", false, "listChanged", false)), + "serverInfo", + Map.of( + "name", "sqrl-server", + "version", "1.0.0")); + return Mono.just(ResponseEntity.ok(createSuccessResponse(id, result))); + } + + private Mono>> handleToolsList( + JsonNode id, RootGraphqlModel model) { + var tools = + model.getOperations().stream() + .filter(op -> op.getMcpMethod() == McpMethodType.TOOL) + .map(this::operationToTool) + .toList(); + + var result = Map.of("tools", tools); + return Mono.just(ResponseEntity.ok(createSuccessResponse(id, result))); + } + + private Mono>> handleToolCall( + JsonNode id, JsonNode params, Principal principal, GraphQL graphQL, RootGraphqlModel model) { + + if (params == null || !params.has("name")) { + return Mono.just( + ResponseEntity.ok(createErrorResponse(id, -32602, "Invalid params: missing tool name"))); + } + + var toolName = params.get("name").asText(); + var arguments = params.has("arguments") ? params.get("arguments") : null; + + var tools = + model.getOperations().stream() + .filter(op -> op.getMcpMethod() == McpMethodType.TOOL) + .collect(Collectors.toMap(ApiOperation::getId, Function.identity())); + + var tool = tools.get(toolName); + if (tool == null) { + return Mono.just( + ResponseEntity.ok(createErrorResponse(id, -32602, "Tool not found: " + toolName))); + } + + Map variables = new HashMap<>(); + if (arguments != null) { + try { + variables = + OBJECT_MAPPER.convertValue( + arguments, + new com.fasterxml.jackson.core.type.TypeReference>() {}); + } catch (Exception e) { + return Mono.just( + ResponseEntity.ok( + createErrorResponse(id, -32602, "Invalid arguments: " + e.getMessage()))); + } + } + + return executeGraphQL(graphQL, tool, variables, principal) + .map( + executionResult -> { + if (!executionResult.getErrors().isEmpty()) { + var errorMsg = + executionResult.getErrors().stream() + .map(err -> err.getMessage()) + .collect(Collectors.joining("; ")); + return ResponseEntity.ok(createErrorResponse(id, -32603, errorMsg)); + } + + var data = getExecutionData(executionResult, tool); + var content = List.of(Map.of("type", "text", "text", serializeResult(data))); + var result = Map.of("content", content, "isError", false); + return ResponseEntity.ok(createSuccessResponse(id, result)); + }); + } + + private Mono>> handleResourcesList( + JsonNode id, RootGraphqlModel model) { + var resources = + model.getOperations().stream() + .filter(op -> op.getMcpMethod() == McpMethodType.RESOURCE) + .filter( + op -> + op.getFunction().getParameters() == null + || op.getFunction().getParameters().getProperties() == null + || op.getFunction().getParameters().getProperties().isEmpty()) + .map(this::operationToResource) + .toList(); + + var result = Map.of("resources", resources); + return Mono.just(ResponseEntity.ok(createSuccessResponse(id, result))); + } + + private Mono>> handleResourceTemplatesList( + JsonNode id, RootGraphqlModel model) { + var templates = + model.getOperations().stream() + .filter(op -> op.getMcpMethod() == McpMethodType.RESOURCE) + .filter( + op -> + op.getFunction().getParameters() != null + && op.getFunction().getParameters().getProperties() != null + && !op.getFunction().getParameters().getProperties().isEmpty()) + .map(this::operationToResourceTemplate) + .toList(); + + var result = Map.of("resourceTemplates", templates); + return Mono.just(ResponseEntity.ok(createSuccessResponse(id, result))); + } + + private Mono>> handleResourceRead( + JsonNode id, JsonNode params, Principal principal, GraphQL graphQL, RootGraphqlModel model) { + + if (params == null || !params.has("uri")) { + return Mono.just( + ResponseEntity.ok(createErrorResponse(id, -32602, "Invalid params: missing uri"))); + } + + var uri = params.get("uri").asText(); + // Find matching resource + var resources = + model.getOperations().stream() + .filter(op -> op.getMcpMethod() == McpMethodType.RESOURCE) + .toList(); + + for (var resource : resources) { + if (matchesResourceUri(resource, uri)) { + Map variables = extractUriParameters(resource, uri); + + return executeGraphQL(graphQL, resource, variables, principal) + .map( + executionResult -> { + if (!executionResult.getErrors().isEmpty()) { + var errorMsg = + executionResult.getErrors().stream() + .map(err -> err.getMessage()) + .collect(Collectors.joining("; ")); + return ResponseEntity.ok(createErrorResponse(id, -32603, errorMsg)); + } + + var data = getExecutionData(executionResult, resource); + var content = + List.of( + Map.of( + "uri", + uri, + "mimeType", + "application/json", + "text", + serializeResult(data))); + var result = Map.of("contents", content); + return ResponseEntity.ok(createSuccessResponse(id, result)); + }); + } + } + + return Mono.just( + ResponseEntity.ok(createErrorResponse(id, -32602, "Resource not found: " + uri))); + } + + private Mono>> handlePing(JsonNode id) { + return Mono.just(ResponseEntity.ok(createSuccessResponse(id, Map.of()))); + } + + private Flux> createSseFlux(String version) { + var sessionId = UUID.randomUUID().toString(); + var sink = Sinks.many().multicast().onBackpressureBuffer(); + + var connection = new SseConnection(sessionId, sink); + sseConnections.put(sessionId, connection); + + log.info("New SSE connection: {}", sessionId); + + // Send endpoint event + var mcpEndpoint = config.getServletConfig().getMcpEndpoint(version); + sink.tryEmitNext("{\"sessionId\":\"" + sessionId + "\",\"endpoint\":\"" + mcpEndpoint + "\"}"); + + return sink.asFlux() + .map(data -> ServerSentEvent.builder().event("message").data(data).build()) + .doOnCancel( + () -> { + sseConnections.remove(sessionId); + log.info("SSE connection closed: {}", sessionId); + }); + } + + private Map operationToTool(ApiOperation op) { + var tool = new HashMap(); + tool.put("name", op.getId()); + var description = op.getFunction().getDescription(); + tool.put("description", description != null ? description : op.getName()); + + if (op.getFunction().getParameters() != null) { + tool.put("inputSchema", op.getFunction().getParameters()); + } else { + tool.put("inputSchema", Map.of("type", "object", "properties", Map.of())); + } + + return tool; + } + + private Map operationToResource(ApiOperation op) { + var resource = new HashMap(); + resource.put("uri", op.getUriTemplate()); + resource.put("name", op.getName()); + resource.put("description", op.getFunction().getDescription()); + resource.put("mimeType", "application/json"); + return resource; + } + + private Map operationToResourceTemplate(ApiOperation op) { + var template = new HashMap(); + template.put("uriTemplate", op.getUriTemplate()); + template.put("name", op.getName()); + template.put("description", op.getFunction().getDescription()); + template.put("mimeType", "application/json"); + return template; + } + + private boolean matchesResourceUri(ApiOperation resource, String uri) { + var template = resource.getUriTemplate(); + if (template == null) return false; + var pattern = template.replaceAll("\\{[^}]+}", "[^/]+"); + return uri.matches(pattern); + } + + private Map extractUriParameters(ApiOperation resource, String uri) { + // Simple extraction - could be improved + return Map.of(); + } + + private Mono executeGraphQL( + GraphQL graphQL, ApiOperation operation, Map variables, Principal principal) { + + var executionInput = + ExecutionInput.newExecutionInput() + .query(operation.getApiQuery().query()) + .operationName(operation.getApiQuery().queryName()) + .variables(variables) + .graphQLContext( + builder -> { + if (principal instanceof Authentication auth + && auth.getPrincipal() instanceof Jwt jwt) { + builder.put("claims", jwt.getClaims()); + } + }) + .build(); + + return Mono.fromFuture(graphQL.executeAsync(executionInput)); + } + + private static Object getExecutionData(ExecutionResult executionResult, ApiOperation operation) { + var result = executionResult.getData(); + if (result instanceof Map resultMap && operation.removeNesting()) { + if (resultMap.size() == 1) { + result = resultMap.values().iterator().next(); + } + } + return result; + } + + private String serializeResult(Object data) { + try { + return OBJECT_MAPPER.writeValueAsString(data); + } catch (Exception e) { + return String.valueOf(data); + } + } + + private Map createSuccessResponse(JsonNode id, Object result) { + var response = new HashMap(); + response.put("jsonrpc", JSONRPC_VERSION); + if (id != null) { + response.put("id", id.isTextual() ? id.asText() : id.asInt()); + } + response.put("result", result); + return response; + } + + private Map createErrorResponse(JsonNode id, int code, String message) { + var response = new HashMap(); + response.put("jsonrpc", JSONRPC_VERSION); + if (id != null) { + response.put("id", id.isTextual() ? id.asText() : id.asInt()); + } + response.put("error", Map.of("code", code, "message", message)); + return response; + } + + private record SseConnection(String sessionId, Sinks.Many sink) {} +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/controller/RestBridgeController.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/controller/RestBridgeController.java new file mode 100644 index 0000000000..3a440320dc --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/controller/RestBridgeController.java @@ -0,0 +1,392 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.controller; + +import com.datasqrl.graphql.config.ServerConfigProperties; +import com.datasqrl.graphql.server.RootGraphqlModel; +import com.datasqrl.graphql.server.operation.ApiOperation; +import com.datasqrl.graphql.server.operation.FunctionDefinition; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.networknt.schema.SchemaRegistry; +import com.networknt.schema.dialect.Dialects; +import graphql.ExecutionInput; +import graphql.ExecutionResult; +import graphql.GraphQL; +import java.security.Principal; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.security.oauth2.jwt.Jwt; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import reactor.core.publisher.Mono; + +/** REST-to-GraphQL bridge controller. Replaces the Vert.x-based RestBridgeVerticle. */ +@Slf4j +@RestController +@RequiredArgsConstructor +public class RestBridgeController { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final String RESULT_DATA_KEY = "data"; + + private final Map graphQLEngines; + private final Map rootGraphqlModels; + private final ServerConfigProperties config; + + @GetMapping(value = "/rest/**", produces = MediaType.APPLICATION_JSON_VALUE) + public Mono>> handleRestGet( + @RequestParam Map queryParams, + Principal principal, + org.springframework.web.server.ServerWebExchange exchange) { + return handleRestRequest("", queryParams, null, principal, exchange, true); + } + + @PostMapping( + value = "/rest/**", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) + public Mono>> handleRestPost( + @RequestBody Map body, + Principal principal, + org.springframework.web.server.ServerWebExchange exchange) { + return handleRestRequest("", Map.of(), body, principal, exchange, false); + } + + @GetMapping(value = "/{version}/rest/**", produces = MediaType.APPLICATION_JSON_VALUE) + public Mono>> handleVersionedRestGet( + @PathVariable String version, + @RequestParam Map queryParams, + Principal principal, + org.springframework.web.server.ServerWebExchange exchange) { + return handleRestRequest(version, queryParams, null, principal, exchange, true); + } + + @PostMapping( + value = "/{version}/rest/**", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) + public Mono>> handleVersionedRestPost( + @PathVariable String version, + @RequestBody Map body, + Principal principal, + org.springframework.web.server.ServerWebExchange exchange) { + return handleRestRequest(version, Map.of(), body, principal, exchange, false); + } + + private Mono>> handleRestRequest( + String version, + Map queryParams, + Map body, + Principal principal, + org.springframework.web.server.ServerWebExchange exchange, + boolean isGet) { + + var graphQL = graphQLEngines.get(version); + var model = rootGraphqlModels.get(version); + + if (graphQL == null || model == null) { + var errorResponse = + Map.of("error", "REST API not found for version: " + version); + return Mono.just(ResponseEntity.status(HttpStatus.NOT_FOUND).body(errorResponse)); + } + + // Extract the operation path from the request URI + var requestPath = exchange.getRequest().getPath().value(); + var restPrefix = config.getServletConfig().getRestEndpoint(version); + var operationPath = + requestPath.substring(requestPath.indexOf(restPrefix) + restPrefix.length()); + if (operationPath.startsWith("/")) { + operationPath = operationPath.substring(1); + } + + // Find matching operation + var operation = findOperation(model, operationPath, isGet); + if (operation == null) { + var errorResponse = Map.of("error", "Operation not found: " + operationPath); + return Mono.just(ResponseEntity.status(HttpStatus.NOT_FOUND).body(errorResponse)); + } + + // Extract parameters + Map variables = new HashMap<>(); + if (isGet) { + extractGetParameters(queryParams, exchange, operation, variables); + } else if (body != null) { + variables.putAll(body); + } + + // Validate parameters + try { + validateParameters(variables, operation); + } catch (ValidationException e) { + var errorResponse = + Map.of("error", "Invalid parameters", "message", e.getMessage()); + return Mono.just(ResponseEntity.badRequest().body(errorResponse)); + } + + // Execute GraphQL + return executeGraphQL(graphQL, operation, variables, principal) + .map( + executionResult -> { + if (!executionResult.getErrors().isEmpty()) { + var errorResponse = new HashMap(); + errorResponse.put( + "errors", + executionResult.getErrors().stream() + .map( + err -> + Map.of( + "message", + err.getMessage(), + "path", + err.getPath() != null ? err.getPath() : java.util.List.of())) + .toList()); + return ResponseEntity.badRequest().body(errorResponse); + } + + var result = getExecutionData(executionResult, operation); + var response = Map.of(RESULT_DATA_KEY, result); + return ResponseEntity.ok(response); + }); + } + + private ApiOperation findOperation(RootGraphqlModel model, String path, boolean isGet) { + return model.getOperations().stream() + .filter(op -> op.isRestEndpoint()) + .filter(op -> matchesPath(op.getUriTemplate(), path)) + .filter( + op -> + isGet + ? op.getRestMethod().name().equals("GET") + : op.getRestMethod().name().equals("POST")) + .findFirst() + .orElse(null); + } + + private boolean matchesPath(String uriTemplate, String path) { + // Remove query params pattern {?param1,param2} + var templatePath = uriTemplate.replaceAll("\\{\\?[^}]+}", ""); + // Convert {param} to regex pattern + var pattern = templatePath.replaceAll("\\{[^}]+}", "[^/]+"); + return path.matches(pattern); + } + + private void extractGetParameters( + Map queryParams, + org.springframework.web.server.ServerWebExchange exchange, + ApiOperation operation, + Map variables) { + + var functionParams = operation.getFunction().getParameters(); + if (functionParams == null || functionParams.getProperties() == null) { + return; + } + + var properties = functionParams.getProperties(); + var pathParams = exchange.getRequest().getPath().pathWithinApplication().value().split("/"); + + for (var entry : properties.entrySet()) { + var paramName = entry.getKey(); + if (queryParams.containsKey(paramName)) { + var value = queryParams.get(paramName); + var convertedValue = convertParameterValue(value, entry.getValue()); + variables.put(paramName, convertedValue); + } + } + } + + private Object convertParameterValue(String value, FunctionDefinition.Argument argumentDef) { + if (argumentDef == null || argumentDef.getType() == null) { + return value; + } + + return switch (argumentDef.getType()) { + case "integer" -> { + try { + yield Long.parseLong(value); + } catch (NumberFormatException e) { + yield value; + } + } + case "number" -> { + try { + yield Double.parseDouble(value); + } catch (NumberFormatException e) { + yield value; + } + } + case "boolean" -> Boolean.parseBoolean(value); + default -> value; + }; + } + + private void validateParameters(Map variables, ApiOperation operation) + throws ValidationException { + var parameters = operation.getFunction().getParameters(); + if (parameters == null) { + return; + } + + try { + var schemaMapper = + OBJECT_MAPPER.copy().setSerializationInclusion(JsonInclude.Include.NON_NULL); + var schemaText = schemaMapper.writeValueAsString(parameters); + var schemaRegistry = SchemaRegistry.withDefaultDialect(Dialects.getDraft202012()); + var schema = schemaRegistry.getSchema(schemaText); + + JsonNode arguments; + if (variables == null || variables.isEmpty()) { + arguments = OBJECT_MAPPER.readTree("{}"); + } else { + arguments = OBJECT_MAPPER.valueToTree(variables); + } + + var schemaErrors = schema.validate(arguments); + if (!schemaErrors.isEmpty()) { + var schemaErrorsText = + schemaErrors.stream().map(Object::toString).collect(Collectors.joining("; ")); + throw new ValidationException("Invalid Schema: " + schemaErrorsText); + } + } catch (ValidationException e) { + throw e; + } catch (Exception e) { + throw new ValidationException("Could not parse parameter JSON: " + e.getMessage()); + } + } + + private Mono executeGraphQL( + GraphQL graphQL, ApiOperation operation, Map variables, Principal principal) { + + var executionInput = + ExecutionInput.newExecutionInput() + .query(operation.getApiQuery().query()) + .operationName(operation.getApiQuery().queryName()) + .variables(variables) + .graphQLContext( + builder -> { + if (principal instanceof Authentication auth + && auth.getPrincipal() instanceof Jwt jwt) { + builder.put("claims", jwt.getClaims()); + } + }) + .build(); + + return Mono.fromFuture(graphQL.executeAsync(executionInput)); + } + + private static Object getExecutionData(ExecutionResult executionResult, ApiOperation operation) { + var result = executionResult.getData(); + if (result instanceof Map resultMap && operation.removeNesting()) { + if (resultMap.size() == 1) { + result = resultMap.values().iterator().next(); + } + } + return result; + } + + public static class ValidationException extends Exception { + public ValidationException(String message) { + super(message); + } + } + + public static String convertUriTemplateToPath(String uriTemplate) { + // Remove query params pattern {?param1,param2} + return uriTemplate.replaceAll("\\{\\?[^}]+}", ""); + } + + public static void extractAndConvertQueryParams( + Map queryParams, ApiOperation operation, Map parameters) { + + var functionParams = operation.getFunction().getParameters(); + if (functionParams == null || functionParams.getProperties() == null) { + return; + } + + var properties = functionParams.getProperties(); + for (var entry : properties.entrySet()) { + var paramName = entry.getKey(); + if (queryParams.containsKey(paramName)) { + var value = queryParams.get(paramName); + var convertedValue = convertStaticParameterValue(value, entry.getValue()); + parameters.put(paramName, convertedValue); + } + } + } + + public static void extractAndConvertPathParams( + Map pathParams, ApiOperation operation, Map parameters) { + + var functionParams = operation.getFunction().getParameters(); + if (functionParams == null || functionParams.getProperties() == null) { + return; + } + + var properties = functionParams.getProperties(); + for (var entry : properties.entrySet()) { + var paramName = entry.getKey(); + if (pathParams.containsKey(paramName)) { + var value = pathParams.get(paramName); + var convertedValue = convertStaticParameterValue(value, entry.getValue()); + parameters.put(paramName, convertedValue); + } + } + } + + public static void extractBody(Map body, Map parameters) { + if (body != null) { + parameters.putAll(body); + } + } + + private static Object convertStaticParameterValue( + String value, FunctionDefinition.Argument argumentDef) { + if (argumentDef == null || argumentDef.getType() == null) { + return value; + } + + return switch (argumentDef.getType()) { + case "integer" -> { + try { + yield Long.parseLong(value); + } catch (NumberFormatException e) { + yield value; + } + } + case "number" -> { + try { + yield Double.parseDouble(value); + } catch (NumberFormatException e) { + yield value; + } + } + case "boolean" -> Boolean.parseBoolean(value); + default -> value; + }; + } +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/jdbc/SpringJdbcClient.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/jdbc/SpringJdbcClient.java new file mode 100644 index 0000000000..ba60bc4423 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/jdbc/SpringJdbcClient.java @@ -0,0 +1,161 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.jdbc; + +import com.datasqrl.graphql.server.Context; +import com.datasqrl.graphql.server.RootGraphqlModel.PreparedSqrlQuery; +import com.datasqrl.graphql.server.RootGraphqlModel.ResolvedQuery; +import com.datasqrl.graphql.server.RootGraphqlModel.ResolvedSqlQuery; +import com.datasqrl.graphql.server.RootGraphqlModel.SqlQuery; +import io.r2dbc.spi.Connection; +import io.r2dbc.spi.ConnectionFactory; +import io.r2dbc.spi.Row; +import io.r2dbc.spi.RowMetadata; +import io.r2dbc.spi.Statement; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import javax.sql.DataSource; +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * Spring-based JDBC client that supports both R2DBC (async) and JDBC (sync with virtual threads). + * Replaces the Vert.x-based VertxJdbcClient. + */ +@Slf4j +public record SpringJdbcClient(Map clients, boolean asyncMode) + implements JdbcClient { + + @Override + public ResolvedQuery prepareQuery(SqlQuery query, Context context) { + var client = clients.get(query.getDatabase()); + if (client == null) { + throw new RuntimeException("Could not find database engine: " + query.getDatabase()); + } + + return new ResolvedSqlQuery(query, new PreparedQueryWrapper(query.getSql())); + } + + @Override + public ResolvedQuery unpreparedQuery(SqlQuery sqlQuery, Context context) { + return new ResolvedSqlQuery(sqlQuery, null); + } + + public CompletableFuture>> execute( + DatabaseType database, String sql, List params) { + + var client = clients.get(database); + if (client == null) { + return CompletableFuture.failedFuture( + new RuntimeException("Could not find database engine: " + database)); + } + + if (client instanceof ConnectionFactory connectionFactory) { + return executeR2dbc(connectionFactory, sql, params); + } else if (client instanceof DataSource dataSource) { + return executeJdbc(dataSource, sql, params); + } else { + return CompletableFuture.failedFuture( + new RuntimeException("Unknown client type: " + client.getClass())); + } + } + + private CompletableFuture>> executeR2dbc( + ConnectionFactory connectionFactory, String sql, List params) { + + return Mono.usingWhen( + connectionFactory.create(), + connection -> executeStatement(connection, sql, params), + Connection::close) + .toFuture(); + } + + private Mono>> executeStatement( + Connection connection, String sql, List params) { + + Statement statement = connection.createStatement(sql); + + for (int i = 0; i < params.size(); i++) { + Object param = params.get(i); + if (param == null) { + statement.bindNull(i, Object.class); + } else { + statement.bind(i, param); + } + } + + return Flux.from(statement.execute()).flatMap(result -> result.map(this::mapRow)).collectList(); + } + + private Map mapRow(Row row, RowMetadata metadata) { + var result = new HashMap(); + for (var columnMetadata : metadata.getColumnMetadatas()) { + var columnName = columnMetadata.getName(); + result.put(columnName, row.get(columnName)); + } + return result; + } + + private CompletableFuture>> executeJdbc( + DataSource dataSource, String sql, List params) { + + if (asyncMode) { + return CompletableFuture.supplyAsync(() -> executeJdbcSync(dataSource, sql, params)); + } else { + return CompletableFuture.completedFuture(executeJdbcSync(dataSource, sql, params)); + } + } + + private List> executeJdbcSync( + DataSource dataSource, String sql, List params) { + + try (var connection = dataSource.getConnection(); + var statement = connection.prepareStatement(sql)) { + + for (int i = 0; i < params.size(); i++) { + statement.setObject(i + 1, params.get(i)); + } + + try (var resultSet = statement.executeQuery()) { + var metadata = resultSet.getMetaData(); + var columnCount = metadata.getColumnCount(); + var results = new java.util.ArrayList>(); + + while (resultSet.next()) { + var row = new HashMap(); + for (int i = 1; i <= columnCount; i++) { + row.put(metadata.getColumnLabel(i), resultSet.getObject(i)); + } + results.add(row); + } + + return results; + } + } catch (Exception e) { + throw new RuntimeException("Error executing JDBC query", e); + } + } + + public record PreparedQueryWrapper(String sql) implements PreparedSqrlQuery { + @Override + public String preparedQuery() { + return sql; + } + } +} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/jdbc/VertxQueryExecutionContext.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/jdbc/SpringQueryExecutionContext.java similarity index 57% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/jdbc/VertxQueryExecutionContext.java rename to sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/jdbc/SpringQueryExecutionContext.java index 3e86056b18..9099e3eb0b 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/jdbc/VertxQueryExecutionContext.java +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/jdbc/SpringQueryExecutionContext.java @@ -18,34 +18,30 @@ import static com.datasqrl.graphql.jdbc.SchemaConstants.LIMIT; import static com.datasqrl.graphql.jdbc.SchemaConstants.OFFSET; -import com.datasqrl.graphql.VertxContext; -import com.datasqrl.graphql.jdbc.VertxJdbcClient.PreparedSqrlQueryImpl; +import com.datasqrl.graphql.SpringContext; +import com.datasqrl.graphql.jdbc.SpringJdbcClient.PreparedQueryWrapper; import com.datasqrl.graphql.server.RootGraphqlModel.Argument; import com.datasqrl.graphql.server.RootGraphqlModel.ResolvedSqlQuery; import graphql.schema.DataFetchingEnvironment; -import io.vertx.core.Future; -import io.vertx.core.json.JsonArray; -import io.vertx.sqlclient.Row; -import io.vertx.sqlclient.RowSet; -import io.vertx.sqlclient.Tuple; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.stream.StreamSupport; +import lombok.extern.slf4j.Slf4j; /** - * It is the ExecutionContext per servlet type. It is responsible for executing the resolved SQL - * queries (paginated or not) in Vert.x and mapping the database resultSet to json for using in - * GraphQL responses. It also implements the parameters and arguments visitors for the {@link - * com.datasqrl.graphql.server.RootGraphqlModel} visitors + * Spring-based query execution context that executes resolved SQL queries. Replaces the + * Vert.x-based VertxQueryExecutionContext. */ -public class VertxQueryExecutionContext extends AbstractQueryExecutionContext { +@Slf4j +public class SpringQueryExecutionContext extends AbstractQueryExecutionContext { - final CompletableFuture cf; + private final CompletableFuture cf; - public VertxQueryExecutionContext( - VertxContext context, + public SpringQueryExecutionContext( + SpringContext context, DataFetchingEnvironment environment, Set arguments, CompletableFuture cf) { @@ -73,21 +69,16 @@ protected Object mapParamArgumentType(Object param) { if (param instanceof List l) { return l.toArray(); } - - if (param instanceof JsonArray arr) { - // Unwrap JsonArray to plain Java array to avoid pgclient treating it as JSONB - return arr.getList().toArray(); - } - return param; } - private CompletableFuture runQueryInternal( + private void runQueryInternal( ResolvedSqlQuery resolvedQuery, boolean isList, List paramObj) { - var preparedQueryContainer = (PreparedSqrlQueryImpl) resolvedQuery.getPreparedQueryContainer(); var query = resolvedQuery.getQuery(); + var preparedQueryContainer = (PreparedQueryWrapper) resolvedQuery.getPreparedQueryContainer(); var unpreparedSqlQuery = query.getSql(); + switch (query.getPagination()) { case NONE: break; @@ -95,10 +86,7 @@ private CompletableFuture runQueryInternal( Optional limit = Optional.ofNullable(getEnvironment().getArgument(LIMIT)); Optional offset = Optional.ofNullable(getEnvironment().getArgument(OFFSET)); - // special case where database doesn't support binding for limit/offset => need - // to execute dynamically if (!query.getDatabase().supportsLimitOffsetBinding) { - assert preparedQueryContainer == null; unpreparedSqlQuery = AbstractQueryExecutionContext.addLimitOffsetToQuery( unpreparedSqlQuery, @@ -113,33 +101,26 @@ private CompletableFuture runQueryInternal( throw new UnsupportedOperationException("Unsupported pagination: " + query.getPagination()); } - // execute the preparedQuery with the arguments extracted above - Future> future; - var params = Tuple.from(paramObj); + var sqlClient = getContext().getSqlClient(); var database = resolvedQuery.getQuery().getDatabase(); + var finalSql = + preparedQueryContainer != null ? preparedQueryContainer.sql() : unpreparedSqlQuery; - if (preparedQueryContainer == null) { - future = getContext().getSqlClient().execute(database, unpreparedSqlQuery, params); - } else { - var preparedQuery = preparedQueryContainer.preparedQuery(); - future = getContext().getSqlClient().execute(preparedQuery, params); - } - - // map the resultSet to json for GraphQL response - future - .map(r -> resultMapper(r, isList)) - .onSuccess(cf::complete) - .onFailure( - f -> { - f.printStackTrace(); - cf.completeExceptionally(f); + sqlClient + .execute(database, finalSql, new ArrayList<>(paramObj)) + .thenApply(results -> resultMapper(results, isList)) + .whenComplete( + (result, throwable) -> { + if (throwable != null) { + log.error("Query execution failed", throwable); + cf.completeExceptionally(throwable); + } else { + cf.complete(result); + } }); - return cf; } - private Object resultMapper(RowSet r, boolean isList) { - var o = StreamSupport.stream(r.spliterator(), false).map(Row::toJson).toList(); - - return unboxList(o, isList); + private Object resultMapper(List> rows, boolean isList) { + return unboxList(rows, isList); } } diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/kafka/SpringMutationConfiguration.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/kafka/SpringMutationConfiguration.java new file mode 100644 index 0000000000..8c2c5774f6 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/kafka/SpringMutationConfiguration.java @@ -0,0 +1,263 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.kafka; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.datasqrl.graphql.config.ServerConfigProperties; +import com.datasqrl.graphql.server.Context; +import com.datasqrl.graphql.server.MetadataReader; +import com.datasqrl.graphql.server.MetadataType; +import com.datasqrl.graphql.server.MutationConfiguration; +import com.datasqrl.graphql.server.RootGraphqlModel.MutationCoordsVisitor; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.kafka.core.KafkaTemplate; + +/** + * Spring-based mutation configuration that uses Spring Kafka to send messages. Replaces the + * Vert.x-based MutationConfigurationImpl. + */ +@Slf4j +@RequiredArgsConstructor +public class SpringMutationConfiguration implements MutationConfiguration> { + + private final ServerConfigProperties config; + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + public MutationCoordsVisitor, Context> createSinkFetcherVisitor() { + return (coords, context) -> { + var kafkaConfig = config.getKafkaMutation(); + if (kafkaConfig == null) { + throw new RuntimeException("Kafka mutation configuration is not set"); + } + + var kafkaTemplate = createKafkaTemplate(kafkaConfig, coords.isTransactional()); + var topic = coords.getTopic(); + + var computedInputColumns = new HashMap(); + coords + .getComputedColumns() + .forEach( + (colName, metadata) -> { + ComputeInputColumns fct = + switch (metadata.metadataType()) { + case UUID -> (env -> UUID.randomUUID()); + case AUTH -> { + MetadataReader metadataReader = + context.getMetadataReader(metadata.metadataType()); + yield (env -> + metadataReader.read(env, metadata.name(), metadata.required())); + } + default -> null; + }; + if (fct != null) { + computedInputColumns.put(colName, fct); + } + }); + + var timestampColumns = + coords.getComputedColumns().entrySet().stream() + .filter(e -> e.getValue().metadataType() == MetadataType.TIMESTAMP) + .map(Entry::getKey) + .toList(); + + checkNotNull(topic, "Could not find topic for field: %s", coords.getFieldName()); + + return env -> { + var entries = getEntries(env, computedInputColumns); + var cf = new CompletableFuture(); + + if (coords.isTransactional()) { + sendMessagesTransactionally( + kafkaTemplate, topic, entries, timestampColumns, cf, coords.isReturnList()); + } else { + sendMessagesNonTransactionally( + kafkaTemplate, topic, entries, timestampColumns, cf, coords.isReturnList()); + } + + return cf; + }; + }; + } + + @FunctionalInterface + interface ComputeInputColumns { + Object compute(DataFetchingEnvironment env); + } + + @SuppressWarnings("unchecked") + private List> getEntries( + DataFetchingEnvironment env, Map computedColumns) { + + var args = env.getArguments(); + var argument = args.entrySet().stream().findFirst().map(Entry::getValue).orElse(null); + + List> entries; + if (argument instanceof List) { + entries = (List>) argument; + } else { + entries = List.of((Map) argument); + } + + if (!computedColumns.isEmpty()) { + entries.forEach( + entry -> computedColumns.forEach((colName, fct) -> entry.put(colName, fct.compute(env)))); + } + + return entries; + } + + private KafkaTemplate createKafkaTemplate( + ServerConfigProperties.KafkaMutationConfig kafkaConfig, boolean transactional) { + + Map props = new HashMap<>(); + kafkaConfig.getProperties().forEach(props::put); + if (transactional) { + props.put("transactional.id", UUID.randomUUID().toString()); + } + + var producerFactory = + new org.springframework.kafka.core.DefaultKafkaProducerFactory(props); + return new KafkaTemplate<>(producerFactory); + } + + private void sendMessagesNonTransactionally( + KafkaTemplate kafkaTemplate, + String topic, + List> entries, + List timestampColumns, + CompletableFuture cf, + boolean returnList) { + + var futures = + entries.stream() + .map(entry -> sendMessage(kafkaTemplate, topic, entry, timestampColumns)) + .toList(); + + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .whenComplete( + (v, throwable) -> { + if (throwable != null) { + cf.completeExceptionally(throwable); + } else { + var results = + futures.stream().map(CompletableFuture::join).collect(Collectors.toList()); + completeWithResults(cf, results, returnList); + } + }); + } + + private void sendMessagesTransactionally( + KafkaTemplate kafkaTemplate, + String topic, + List> entries, + List timestampColumns, + CompletableFuture cf, + boolean returnList) { + + try { + kafkaTemplate.executeInTransaction( + operations -> { + List> results = + entries.stream() + .map(entry -> sendMessageSync(operations, topic, entry, timestampColumns)) + .collect(Collectors.toList()); + completeWithResults(cf, results, returnList); + return null; + }); + } catch (Exception e) { + cf.completeExceptionally(e); + } + } + + private CompletableFuture> sendMessage( + KafkaTemplate kafkaTemplate, + String topic, + Map entry, + List timestampColumns) { + + try { + var json = objectMapper.writeValueAsString(entry); + var record = new ProducerRecord(topic, null, json); + + var cf = new CompletableFuture>(); + kafkaTemplate + .send(record) + .whenComplete( + (result, throwable) -> { + if (throwable != null) { + cf.completeExceptionally(throwable); + } else { + var timestamp = result.getRecordMetadata().timestamp(); + var dateTime = + ZonedDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC); + timestampColumns.forEach( + colName -> entry.put(colName, dateTime.toOffsetDateTime())); + cf.complete(entry); + } + }); + return cf; + } catch (JsonProcessingException e) { + return CompletableFuture.failedFuture(e); + } + } + + private Map sendMessageSync( + org.springframework.kafka.core.KafkaOperations operations, + String topic, + Map entry, + List timestampColumns) { + + try { + var json = objectMapper.writeValueAsString(entry); + var record = new ProducerRecord(topic, null, json); + var result = operations.send(record).get(); + var timestamp = result.getRecordMetadata().timestamp(); + var dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(timestamp), ZoneOffset.UTC); + timestampColumns.forEach(colName -> entry.put(colName, dateTime.toOffsetDateTime())); + return entry; + } catch (Exception e) { + throw new RuntimeException("Failed to send Kafka message", e); + } + } + + private void completeWithResults( + CompletableFuture cf, List> results, boolean returnList) { + + if (returnList) { + cf.complete(results); + } else { + cf.complete(results.get(0)); + } + } +} diff --git a/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/kafka/SpringSubscriptionConfiguration.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/kafka/SpringSubscriptionConfiguration.java new file mode 100644 index 0000000000..e05b2d39ff --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/kafka/SpringSubscriptionConfiguration.java @@ -0,0 +1,131 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.kafka; + +import com.datasqrl.graphql.config.ServerConfigProperties; +import com.datasqrl.graphql.server.Context; +import com.datasqrl.graphql.server.RootGraphqlModel.KafkaSubscriptionCoords; +import com.datasqrl.graphql.server.RootGraphqlModel.SubscriptionCoordsVisitor; +import com.datasqrl.graphql.server.SubscriptionConfiguration; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import graphql.schema.DataFetcher; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import reactor.kafka.receiver.KafkaReceiver; +import reactor.kafka.receiver.ReceiverOptions; + +/** + * Spring-based subscription configuration that uses Reactor Kafka for subscriptions. Replaces the + * Vert.x-based SubscriptionConfigurationImpl. + */ +@Slf4j +@RequiredArgsConstructor +public class SpringSubscriptionConfiguration implements SubscriptionConfiguration> { + + private final ServerConfigProperties config; + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + public SubscriptionCoordsVisitor, Context> createSubscriptionFetcherVisitor() { + return (coords, context) -> { + var kafkaConfig = config.getKafkaSubscription(); + if (kafkaConfig == null) { + throw new RuntimeException("Kafka subscription configuration is not set"); + } + + return env -> { + var topic = coords.getTopic(); + var receiver = createKafkaReceiver(kafkaConfig, topic); + + return receiver + .receive() + .map( + record -> { + try { + var value = record.value(); + record.receiverOffset().acknowledge(); + return objectMapper.readValue( + value, new TypeReference>() {}); + } catch (Exception e) { + log.error("Error deserializing Kafka message", e); + return Map.of("error", e.getMessage()); + } + }) + .filter(data -> matchesEqualityConditions(data, coords, env.getArguments())); + }; + }; + } + + private KafkaReceiver createKafkaReceiver( + ServerConfigProperties.KafkaSubscriptionConfig kafkaConfig, String topic) { + + var props = new HashMap(); + kafkaConfig.getProperties().forEach(props::put); + + // Ensure unique consumer group for each subscription + props.put( + ConsumerConfig.GROUP_ID_CONFIG, + props.getOrDefault(ConsumerConfig.GROUP_ID_CONFIG, "sqrl-subscription") + + "-" + + UUID.randomUUID()); + + var receiverOptions = + ReceiverOptions.create(props).subscription(Collections.singleton(topic)); + + log.info("Creating Kafka subscription receiver for topic: {}", topic); + + return KafkaReceiver.create(receiverOptions); + } + + private boolean matchesEqualityConditions( + Map data, KafkaSubscriptionCoords coords, Map arguments) { + + var conditions = coords.getEqualityConditions(); + if (conditions == null || conditions.isEmpty()) { + return true; + } + + for (var entry : conditions.entrySet()) { + var fieldName = entry.getKey(); + var fieldValue = data.get(fieldName); + + var paramHandler = entry.getValue(); + var expectedValue = resolveParameterValue(paramHandler, arguments, data); + + if (expectedValue != null && !expectedValue.equals(fieldValue)) { + return false; + } + } + + return true; + } + + private Object resolveParameterValue( + Object paramHandler, Map arguments, Map data) { + // This is a simplified implementation + // The full implementation would need to handle various parameter types + if (paramHandler instanceof String paramName) { + return arguments.get(paramName); + } + return null; + } +} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcher.java b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcher.java similarity index 75% rename from sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcher.java rename to sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcher.java index 030c13ef33..721972b684 100644 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcher.java +++ b/sqrl-server/sqrl-server-spring/src/main/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcher.java @@ -18,10 +18,13 @@ import graphql.schema.DataFetchingEnvironment; import graphql.schema.GraphQLFieldDefinition; import graphql.schema.PropertyDataFetcher; -import io.vertx.core.json.JsonObject; import java.util.Map; import java.util.function.Supplier; +/** + * A DataFetcher that performs case-insensitive property lookup. This is useful when database + * drivers may not preserve column name case sensitivity. + */ public class CaseInsensitiveJsonDataFetcher extends PropertyDataFetcher { public CaseInsensitiveJsonDataFetcher(String propertyName) { @@ -31,8 +34,8 @@ public CaseInsensitiveJsonDataFetcher(String propertyName) { @Override public Object get(DataFetchingEnvironment environment) { var source = environment.getSource(); - if (source instanceof JsonObject jsonObj) { - return fetchJsonObject(jsonObj); + if (source instanceof Map map) { + return fetchFromMap(map); } return super.get(environment); @@ -43,21 +46,21 @@ public Object get( GraphQLFieldDefinition fieldDef, Object source, Supplier envSupplier) throws Exception { - if (source instanceof JsonObject jsonObj) { - return fetchJsonObject(jsonObj); + if (source instanceof Map map) { + return fetchFromMap(map); } return super.get(fieldDef, source, envSupplier); } - Object fetchJsonObject(JsonObject jsonObj) { - var value = jsonObj.getValue(getPropertyName()); + Object fetchFromMap(Map map) { + var value = map.get(getPropertyName()); if (value != null) { return value; } - // Case-insensitive lookup for drivers that may not preserve sensitivity - return jsonObj.getMap().entrySet().stream() - .filter(e -> e.getKey().equalsIgnoreCase(getPropertyName())) + + return map.entrySet().stream() + .filter(e -> e.getKey() instanceof String key && key.equalsIgnoreCase(getPropertyName())) .filter(e -> e.getValue() != null) .map(Map.Entry::getValue) .findAny() diff --git a/sqrl-server/sqrl-server-spring/src/main/resources/application.yaml b/sqrl-server/sqrl-server-spring/src/main/resources/application.yaml new file mode 100644 index 0000000000..8cae5da962 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/main/resources/application.yaml @@ -0,0 +1,64 @@ +# +# Copyright © 2021 DataSQRL (contact@datasqrl.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + application: + name: sqrl-server + main: + web-application-type: reactive + +server: + port: ${SERVER_PORT:8888} + +sqrl: + server: + execution-mode: ${SQRL_EXECUTION_MODE:async} + servlet-config: + graphql-endpoint: /graphql + graphiql-endpoint: /graphiql/* + rest-endpoint: /rest + mcp-endpoint: /mcp + postgres: + host: ${PGHOST:localhost} + port: ${PGPORT:5432} + database: ${PGDATABASE:datasqrl} + user: ${PGUSER:postgres} + password: ${PGPASSWORD:} + pool-size: ${PG_POOL_SIZE:10} + cors: + allowed-origin: "*" + allowed-methods: + - GET + - POST + - OPTIONS + allowed-headers: + - "*" + allow-credentials: false + max-age-seconds: 3600 + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + show-details: always + +logging: + level: + root: INFO + com.datasqrl: DEBUG diff --git a/sqrl-server/sqrl-server-spring/src/test/java/com/datasqrl/graphql/SqrlServerApplicationTest.java b/sqrl-server/sqrl-server-spring/src/test/java/com/datasqrl/graphql/SqrlServerApplicationTest.java new file mode 100644 index 0000000000..2353edee7d --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/test/java/com/datasqrl/graphql/SqrlServerApplicationTest.java @@ -0,0 +1,52 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.datasqrl.graphql.config.ServerConfigProperties; +import org.junit.jupiter.api.Test; + +class SqrlServerApplicationTest { + + @Test + void givenDefaultConfig_whenGetExecutionMode_thenReturnsAsync() { + var config = new ServerConfigProperties(); + assertThat(config.getExecutionMode()).isEqualTo(ServerConfigProperties.ExecutionMode.ASYNC); + } + + @Test + void givenDefaultConfig_whenGetServletConfig_thenReturnsDefaults() { + var config = new ServerConfigProperties(); + var servletConfig = config.getServletConfig(); + + assertThat(servletConfig).isNotNull(); + assertThat(servletConfig.getGraphqlEndpoint()).isEqualTo("/graphql"); + assertThat(servletConfig.getGraphiqlEndpoint()).isEqualTo("/graphiql/*"); + assertThat(servletConfig.getRestEndpoint()).isEqualTo("/rest"); + assertThat(servletConfig.getMcpEndpoint()).isEqualTo("/mcp"); + } + + @Test + void givenVersionedEndpoint_whenGetEndpoint_thenReturnsVersionedPath() { + var config = new ServerConfigProperties(); + var servletConfig = config.getServletConfig(); + + assertThat(servletConfig.getGraphQLEndpoint("v1")).isEqualTo("/v1/graphql"); + assertThat(servletConfig.getRestEndpoint("v2")).isEqualTo("/v2/rest"); + assertThat(servletConfig.getMcpEndpoint("")).isEqualTo("/mcp"); + } +} diff --git a/sqrl-server/sqrl-server-spring/src/test/java/com/datasqrl/graphql/controller/RestBridgeControllerTest.java b/sqrl-server/sqrl-server-spring/src/test/java/com/datasqrl/graphql/controller/RestBridgeControllerTest.java new file mode 100644 index 0000000000..5ea0006e18 --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/test/java/com/datasqrl/graphql/controller/RestBridgeControllerTest.java @@ -0,0 +1,150 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.controller; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.datasqrl.graphql.server.ModelContainer; +import com.datasqrl.graphql.server.RootGraphqlModel; +import com.datasqrl.graphql.server.operation.ApiOperation; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.io.Resources; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.SneakyThrows; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class RestBridgeControllerTest { + + private RootGraphqlModel model; + + @BeforeEach + @SneakyThrows + void givenTestModel_whenSetup_thenModelIsLoaded() { + model = + new ObjectMapper() + .readValue(Resources.getResource("testdata/spring-rest.json"), ModelContainer.class) + .models + .get("v1"); + } + + @Test + void givenQueryParamsInUri_whenConvertUriTemplateToPath_thenQueryParamsRemoved() { + var uriTemplate = "queries/HighTempAlert{?offset,limit}"; + var path = RestBridgeController.convertUriTemplateToPath(uriTemplate); + + assertThat(path).isEqualTo("queries/HighTempAlert"); + } + + @Test + void givenPathParamsInUri_whenConvertUriTemplateToPath_thenPathParamsConverted() { + var uriTemplate = "queries/{sensorid}/maxTemp{?offset,limit}"; + var path = RestBridgeController.convertUriTemplateToPath(uriTemplate); + + assertThat(path).isEqualTo("queries/{sensorid}/maxTemp"); + } + + @Test + void givenSimpleUri_whenConvertUriTemplateToPath_thenPathUnchanged() { + var uriTemplate = "mutations/SensorReading"; + var path = RestBridgeController.convertUriTemplateToPath(uriTemplate); + + assertThat(path).isEqualTo("mutations/SensorReading"); + } + + @Test + void givenIntegerProperty_whenExtractAndConvertQueryParams_thenConvertsToLong() { + var operation = getHighTempOperation(); + Map queryParams = new HashMap<>(); + queryParams.put("offset", "10"); + queryParams.put("limit", "20"); + + Map parameters = new HashMap<>(); + RestBridgeController.extractAndConvertQueryParams(queryParams, operation, parameters); + + assertThat(parameters).containsEntry("offset", 10L); + assertThat(parameters).containsEntry("limit", 20L); + } + + @Test + void givenPathParams_whenExtractAndConvertPathParams_thenConvertsCorrectly() { + var operation = getSensorMaxOperation(); + Map pathParams = new HashMap<>(); + pathParams.put("sensorid", "123"); + + Map parameters = new HashMap<>(); + RestBridgeController.extractAndConvertPathParams(pathParams, operation, parameters); + + assertThat(parameters).containsEntry("sensorid", 123L); + } + + @Test + void givenBodyWithNestedObject_whenExtractBody_thenExtractsCorrectly() { + Map body = new HashMap<>(); + body.put( + "event", + List.of( + Map.of("sensorid", 1, "temperature", 25.5), + Map.of("sensorid", 2, "temperature", 30.0))); + + Map parameters = new HashMap<>(); + RestBridgeController.extractBody(body, parameters); + + assertThat(parameters).containsKey("event"); + assertThat(parameters.get("event")).isInstanceOf(List.class); + @SuppressWarnings("unchecked") + List> eventList = (List>) parameters.get("event"); + assertThat(eventList).hasSize(2); + } + + @Test + void givenModel_whenGetOperations_thenReturnsAllOperations() { + assertThat(model.getOperations()).hasSize(4); + } + + @Test + void givenGetOperation_whenCheckRestMethod_thenReturnsGet() { + var operation = getHighTempOperation(); + assertThat(operation.getRestMethod().name()).isEqualTo("GET"); + } + + @Test + void givenPostOperation_whenCheckRestMethod_thenReturnsPost() { + var operation = addSensorReadingOperation(); + assertThat(operation.getRestMethod().name()).isEqualTo("POST"); + } + + private ApiOperation getOperationByName(String name) { + return model.getOperations().stream() + .filter(op -> op.getName().equalsIgnoreCase(name)) + .findFirst() + .orElseThrow(() -> new RuntimeException("Operation not found: " + name)); + } + + private ApiOperation getHighTempOperation() { + return getOperationByName("GetHighTempAlert"); + } + + private ApiOperation getSensorMaxOperation() { + return getOperationByName("GetSensorMaxTemp"); + } + + private ApiOperation addSensorReadingOperation() { + return getOperationByName("AddSensorReading"); + } +} diff --git a/sqrl-server/sqrl-server-spring/src/test/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcherTest.java b/sqrl-server/sqrl-server-spring/src/test/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcherTest.java new file mode 100644 index 0000000000..f16311560e --- /dev/null +++ b/sqrl-server/sqrl-server-spring/src/test/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcherTest.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2021 DataSQRL (contact@datasqrl.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datasqrl.graphql.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class CaseInsensitiveJsonDataFetcherTest { + + private final CaseInsensitiveJsonDataFetcher fetcher = + new CaseInsensitiveJsonDataFetcher("testkey"); + + @Test + void givenMapWithDifferentCaseKey_whenFetchFromMap_thenReturnsValue() { + Map map = new HashMap<>(); + map.put("TestKey", "TestValue"); + + assertThat(fetcher.fetchFromMap(map)).isEqualTo("TestValue"); + } + + @Test + void givenMapWithExactCaseKey_whenFetchFromMap_thenReturnsValue() { + Map map = new HashMap<>(); + map.put("testkey", "TestValue"); + + assertThat(fetcher.fetchFromMap(map)).isEqualTo("TestValue"); + } + + @Test + void givenMapWithNullValue_whenFetchFromMap_thenReturnsNull() { + Map map = new HashMap<>(); + map.put("TestKey", null); + + assertThat(fetcher.fetchFromMap(map)).isNull(); + } + + @Test + void givenMapWithoutMatchingKey_whenFetchFromMap_thenReturnsNull() { + Map map = new HashMap<>(); + map.put("AnotherKey", "SomeValue"); + + assertThat(fetcher.fetchFromMap(map)).isNull(); + } + + @Test + void givenMapWithUppercaseKey_whenFetchFromMap_thenReturnsValue() { + Map map = new HashMap<>(); + map.put("TESTKEY", "TestValue"); + + assertThat(fetcher.fetchFromMap(map)).isEqualTo("TestValue"); + } +} diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/resources/testdata/vertx-rest.json b/sqrl-server/sqrl-server-spring/src/test/resources/testdata/spring-rest.json similarity index 100% rename from sqrl-server/sqrl-server-vertx-base/src/test/resources/testdata/vertx-rest.json rename to sqrl-server/sqrl-server-spring/src/test/resources/testdata/spring-rest.json diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/AbstractBridgeVerticle.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/AbstractBridgeVerticle.java deleted file mode 100644 index d2c0ecc22a..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/AbstractBridgeVerticle.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.operation.ApiOperation; -import com.datasqrl.graphql.server.operation.FunctionDefinition; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.networknt.schema.Error; -import com.networknt.schema.Schema; -import com.networknt.schema.SchemaRegistry; -import com.networknt.schema.dialect.Dialects; -import graphql.ExecutionInput; -import graphql.ExecutionResult; -import io.vertx.core.AbstractVerticle; -import io.vertx.core.Future; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.auth.authentication.AuthenticationProvider; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.RoutingContext; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import lombok.AccessLevel; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** Abstract Verticle that maps requests to GraphQL queries */ -@RequiredArgsConstructor(access = AccessLevel.PROTECTED) -@Slf4j -public abstract class AbstractBridgeVerticle extends AbstractVerticle { - - protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - // Reusable JSON field names - protected static final String JSON_ERROR = "error"; - protected static final String JSON_MESSAGE = "message"; - - protected final Router router; - protected final ServerConfig config; - protected final String modelVersion; - protected final RootGraphqlModel model; - protected final List authProviders; - protected final GraphQLServerVerticle graphQLServerVerticle; - - protected void handleError( - Throwable err, RoutingContext ctx, int statusCode, String errorMessage) { - if (statusCode == 500) { - log.error(errorMessage, err); - } else { - log.info(errorMessage, err); - } - - ctx.response() - .setStatusCode(statusCode) - .putHeader("content-type", "application/json") - .end( - new JsonObject() - .put(JSON_ERROR, errorMessage) - .put(JSON_MESSAGE, err.getMessage()) - .encode()); - } - - protected Future bridgeRequestToGraphQL( - RoutingContext ctx, ApiOperation operation, Map variables) - throws ValidationException { - // Validate parameters - validateParameters(variables, operation); - - // Execute GraphQL query directly with ExecutionInput - return executeGraphQLAsync(ctx, operation, variables); - } - - protected void validateParameters(Map variables, ApiOperation operation) - throws ValidationException { - var parameters = operation.getFunction().getParameters(); - if (parameters == null) { - return; // No validation required - } - final JsonNode arguments; - final Schema schema; - try { - // Build a JSON Schema from the parameters definition - var schemaText = getSchemaMapper().writeValueAsString(parameters); - var schemaRegistry = SchemaRegistry.withDefaultDialect(Dialects.getDraft202012()); - schema = schemaRegistry.getSchema(schemaText); - - // Convert the collected variables to a JsonNode - if (variables == null || variables.isEmpty()) { - arguments = OBJECT_MAPPER.readTree("{}"); - } else { - arguments = OBJECT_MAPPER.valueToTree(variables); - } - } catch (JsonProcessingException e) { - throw new ValidationException("Could not parse parameter JSON:" + e.getMessage()); - } - - // Validate against the schema - var schemaErrors = schema.validate(arguments); - if (!schemaErrors.isEmpty()) { - var schemaErrorsText = - schemaErrors.stream().map(Error::toString).collect(Collectors.joining("; ")); - log.info("Function call had schema errors: {}", schemaErrorsText); - throw new ValidationException("Invalid Schema: " + schemaErrorsText); - } - } - - protected ObjectMapper getSchemaMapper() { - return OBJECT_MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); - } - - protected Future executeGraphQLAsync( - RoutingContext ctx, ApiOperation operation, Map variables) { - - var graphQLEngine = graphQLServerVerticle.getGraphQLEngine(); - - // Build the ExecutionInput - var execInput = - ExecutionInput.newExecutionInput() - .query(operation.getApiQuery().query()) - .operationName(operation.getApiQuery().queryName()) - .variables(variables) - .graphQLContext(builder -> builder.put(RoutingContext.class, ctx)) - .build(); - - // Kick off async execution (GraphQL Java spawns its own executor) - return Future.fromCompletionStage(graphQLEngine.executeAsync(execInput)); - } - - protected static Object getExecutionData( - ExecutionResult executionResult, ApiOperation operation) { - var result = executionResult.getData(); - if (result instanceof Map resultMap && operation.removeNesting()) { - if (resultMap.size() == 1) { - result = resultMap.values().iterator().next(); // Get only element - } - } - return result; - } - - protected static void extractGetParameters( - RoutingContext ctx, ApiOperation operation, Map variables) { - var request = ctx.request(); - var functionParams = operation.getFunction().getParameters(); - - if (functionParams == null || functionParams.getProperties() == null) { - return; - } - - var properties = functionParams.getProperties(); - var queryParams = request.params(); - var pathParams = ctx.pathParams(); - - // Merge query and path parameters, giving precedence to path params - Map combinedParams = new HashMap<>(); - for (String key : queryParams.names()) { - combinedParams.put(key, queryParams.get(key)); - } - combinedParams.putAll(pathParams); // path params take precedence - - for (Map.Entry entry : properties.entrySet()) { - var paramName = entry.getKey(); - if (combinedParams.containsKey(paramName)) { - var value = combinedParams.get(paramName); - var convertedValue = convertParameterValue(value, entry.getValue()); - variables.put(paramName, convertedValue); - } - } - } - - protected static void extractPostParameters(RoutingContext ctx, Map variables) { - JsonObject body = ctx.body().asJsonObject(); - if (body != null) { - variables.putAll(body.getMap()); - } - } - - protected static Object convertParameterValue( - String value, FunctionDefinition.Argument argumentDef) { - if (argumentDef == null || argumentDef.getType() == null) { - return value; - } - - return switch (argumentDef.getType()) { - case "integer" -> { - try { - yield Long.parseLong(value); - } catch (NumberFormatException e) { - yield value; // Let validation catch this - } - } - case "number" -> { - try { - yield Double.parseDouble(value); - } catch (NumberFormatException e) { - yield value; // Let validation catch this - } - } - case "boolean" -> Boolean.parseBoolean(value); - default -> value; - }; - } - - /** Custom exception for parameter validation errors */ - public static class ValidationException extends Exception { - public ValidationException(String message) { - super(message); - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/DetailedRequestTracer.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/DetailedRequestTracer.java deleted file mode 100644 index 8f0a6efbdb..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/DetailedRequestTracer.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import io.vertx.core.Future; -import io.vertx.core.Handler; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.http.HttpServerRequest; -import io.vertx.core.http.HttpServerResponse; -import io.vertx.ext.web.RoutingContext; -import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; -import lombok.experimental.Delegate; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class DetailedRequestTracer implements Handler { - - public DetailedRequestTracer() { - log.info("DetailedRequestTracer enabled"); - } - - @Override - public void handle(RoutingContext context) { - var request = context.request(); - var response = context.response(); - var startTime = System.currentTimeMillis(); - var requestId = generateRequestId(); - - // Capture request data - var requestData = captureRequestData(request, context.body()); - - // Capture response body by wrapping the response - var responseBodyCapture = Buffer.buffer(); - var wrappedResponse = - new ResponseCapturingWrapper( - response, responseBodyCapture, requestId, requestData, startTime); - - // Replace the response in the routing context using reflection - try { - var implClass = context.getClass(); - var field = implClass.getDeclaredField("response"); - field.setAccessible(true); - field.set(context, wrappedResponse); - } catch (Exception e) { - log.warn("[{}] Could not wrap response for body capture: {}", requestId, e.getMessage()); - // Fall back to logging without response body - response.endHandler( - v -> { - var duration = System.currentTimeMillis() - startTime; - logCompleteTrace(requestId, requestData, response, responseBodyCapture, duration); - }); - } - - context.next(); - } - - private RequestData captureRequestData( - HttpServerRequest request, io.vertx.ext.web.RequestBody body) { - var headers = - request.headers().entries().stream() - .map(entry -> entry.getKey() + ": " + entry.getValue()) - .collect(Collectors.joining(", ")); - - var params = - request.params().entries().stream() - .map(entry -> entry.getKey() + "=" + entry.getValue()) - .collect(Collectors.joining("&")); - - String requestBody = null; - Integer requestBodySize = null; - if (body != null && body.buffer() != null) { - var buffer = body.buffer(); - requestBodySize = buffer.length(); - requestBody = buffer.toString(); - if (requestBody.length() > 10000) { - requestBody = requestBody.substring(0, 10000) + "... [TRUNCATED]"; - } - } - - return new RequestData( - request.method().toString(), - request.uri(), - headers.isEmpty() ? "none" : headers, - params.isEmpty() ? "none" : params, - request.remoteAddress() != null ? request.remoteAddress().toString() : "unknown", - requestBody, - requestBodySize); - } - - private void logCompleteTrace( - String requestId, - RequestData requestData, - HttpServerResponse response, - Buffer responseBodyCapture, - long durationMs) { - var responseHeaders = - response.headers().entries().stream() - .map(entry -> entry.getKey() + ": " + entry.getValue()) - .collect(Collectors.joining(", ")); - - String responseBody = null; - Integer responseBodySize = null; - if (responseBodyCapture.length() > 0) { - responseBodySize = responseBodyCapture.length(); - responseBody = responseBodyCapture.toString(); - if (responseBody.length() > 10000) { - responseBody = responseBody.substring(0, 10000) + "... [TRUNCATED]"; - } - } - - log.debug( - "[{}] {} {} | Status: {} | Duration: {}ms | RemoteAddr: {} | ReqHeaders: [{}] | ReqParams: [{}] | ReqBody: {} bytes {} | RespHeaders: [{}] | RespBody: {} bytes {}", - requestId, - requestData.method, - requestData.uri, - response.getStatusCode(), - durationMs, - requestData.remoteAddress, - requestData.headers, - requestData.params, - requestData.requestBodySize != null ? requestData.requestBodySize : 0, - requestData.requestBody != null ? "- " + requestData.requestBody : "- [EMPTY]", - responseHeaders.isEmpty() ? "none" : responseHeaders, - responseBodySize != null ? responseBodySize : 0, - responseBody != null ? "- " + responseBody : "- [EMPTY]"); - } - - private String generateRequestId() { - return "REQ-" + System.nanoTime() + "-" + Thread.currentThread().getId(); - } - - private record RequestData( - String method, - String uri, - String headers, - String params, - String remoteAddress, - String requestBody, - Integer requestBodySize) {} - - @RequiredArgsConstructor - private class ResponseCapturingWrapper implements HttpServerResponse { - @Delegate(excludes = BodyCaptureMethods.class) - private final HttpServerResponse delegate; - - private final Buffer capture; - private final String requestId; - private final RequestData requestData; - private final long startTime; - private boolean logged = false; - - @Override - public Future write(Buffer data) { - if (data != null) { - capture.appendBuffer(data); - } - return delegate.write(data); - } - - @Override - public Future write(String chunk) { - if (chunk != null) { - capture.appendString(chunk); - } - return delegate.write(chunk); - } - - @Override - public Future write(String chunk, String enc) { - if (chunk != null) { - capture.appendString(chunk, enc); - } - return delegate.write(chunk, enc); - } - - @Override - public Future end() { - logIfNotLogged(); - return delegate.end(); - } - - @Override - public Future end(Buffer chunk) { - if (chunk != null) { - capture.appendBuffer(chunk); - } - logIfNotLogged(); - return delegate.end(chunk); - } - - @Override - public Future end(String chunk) { - if (chunk != null) { - capture.appendString(chunk); - } - logIfNotLogged(); - return delegate.end(chunk); - } - - @Override - public Future end(String chunk, String enc) { - if (chunk != null) { - capture.appendString(chunk, enc); - } - logIfNotLogged(); - return delegate.end(chunk, enc); - } - - private void logIfNotLogged() { - if (!logged) { - logged = true; - var duration = System.currentTimeMillis() - startTime; - logCompleteTrace(requestId, requestData, delegate, capture, duration); - } - } - - @SuppressWarnings("unused") - private interface BodyCaptureMethods { - Future write(Buffer data); - - Future write(String chunk); - - Future write(String chunk, String enc); - - Future end(); - - Future end(Buffer chunk); - - Future end(String chunk); - - Future end(String chunk, String enc); - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/GraphQLServerVerticle.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/GraphQLServerVerticle.java deleted file mode 100644 index 77ede156e7..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/GraphQLServerVerticle.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import static com.datasqrl.graphql.config.ServerConfigUtil.createVersionedGraphiQLHandlerOptions; - -import com.datasqrl.graphql.auth.AuthMetadataReader; -import com.datasqrl.graphql.auth.JwtFailureHandler; -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.exec.FlinkExecFunctionPlan; -import com.datasqrl.graphql.exec.FlinkFunctionExecutor; -import com.datasqrl.graphql.jdbc.DatabaseType; -import com.datasqrl.graphql.jdbc.JdbcClientsConfig; -import com.datasqrl.graphql.jdbc.VertxJdbcClient; -import com.datasqrl.graphql.server.CustomScalars; -import com.datasqrl.graphql.server.FunctionExecutor; -import com.datasqrl.graphql.server.GraphQLEngineBuilder; -import com.datasqrl.graphql.server.MetadataReader; -import com.datasqrl.graphql.server.MetadataType; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.google.common.collect.ImmutableMap; -import com.symbaloo.graphqlmicrometer.MicrometerInstrumentation; -import graphql.GraphQL; -import io.vertx.core.AbstractVerticle; -import io.vertx.core.Promise; -import io.vertx.ext.auth.authentication.AuthenticationProvider; -import io.vertx.ext.auth.jwt.JWTAuth; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.handler.AuthenticationHandler; -import io.vertx.ext.web.handler.ChainAuthHandler; -import io.vertx.ext.web.handler.JWTAuthHandler; -import io.vertx.ext.web.handler.graphql.GraphQLHandler; -import io.vertx.ext.web.handler.graphql.GraphiQLHandler; -import io.vertx.ext.web.handler.graphql.ws.GraphQLWSHandler; -import io.vertx.micrometer.backends.BackendRegistries; -import io.vertx.sqlclient.SqlClient; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * GraphQL-specific verticle that configures GraphQL handlers on a shared router. This verticle is - * instantiated by HttpServerVerticle with shared config, model, and JWT auth provider. - */ -@Slf4j -@RequiredArgsConstructor -public class GraphQLServerVerticle extends AbstractVerticle { - - private final Router router; - private final ServerConfig config; - private final String modelVersion; - private final RootGraphqlModel model; - private final List authProviders; - private final Optional execFunctionPlan; - - private GraphQL graphQLEngine; - - @Override - public void start(Promise startPromise) { - try { - setupGraphQLRoutes(startPromise); - } catch (Exception e) { - log.error("Could not setup GraphQL routes", e); - startPromise.fail(e); - } - } - - /** - * Sets up GraphQL routes including authentication handlers, GraphiQL interface, and the main - * GraphQL endpoint with WebSocket support. - * - * @param startPromise the promise to complete when setup is finished - */ - protected void setupGraphQLRoutes(Promise startPromise) { - // Setup GraphiQL handler if configured - if (this.config.getGraphiQLHandlerOptions() != null) { - var versionedOptions = - createVersionedGraphiQLHandlerOptions(modelVersion, config.getGraphiQLHandlerOptions()); - - var graphiQlHandler = GraphiQLHandler.builder(vertx).with(versionedOptions).build(); - router - .route(this.config.getServletConfig().getGraphiQLEndpoint(modelVersion)) - .subRouter(graphiQlHandler.router()); - } - - // Setup database clients - var jdbcConfig = new JdbcClientsConfig(vertx, config); - var dbClients = jdbcConfig.createClients(); - - // Setup GraphQL endpoint with auth if configured - var handler = router.route(this.config.getServletConfig().getGraphQLEndpoint(modelVersion)); - if (!authProviders.isEmpty()) { - log.info( - "Applying {} authentication provider(s) to GraphQL endpoint: {}", - authProviders.size(), - this.config.getServletConfig().getGraphQLEndpoint(modelVersion)); - // Required for adding auth on ws handler - System.setProperty("io.vertx.web.router.setup.lenient", "true"); - handler.handler(createChainAuthHandler()); - handler.failureHandler(new JwtFailureHandler()); - } - - // Create subscription configuration to track async subscription setups - var subscriptionConfig = new SubscriptionConfigurationImpl(vertx, config); - - // Create GraphQL engine - this.graphQLEngine = - createGraphQL( - dbClients, - subscriptionConfig, - createMetadataReaders(), - new FlinkFunctionExecutor(vertx, execFunctionPlan)); - - handler - .handler(GraphQLWSHandler.create(this.graphQLEngine)) - .handler(GraphQLHandler.create(this.graphQLEngine, this.config.getGraphQLHandlerOptions())); - - // Wait for all subscriptions to be set up before completing the promise - subscriptionConfig - .getAllSubscriptionsFuture() - .onSuccess(v -> startPromise.complete()) - .onFailure(startPromise::fail); - } - - private Map createMetadataReaders() { - var readers = ImmutableMap.builder(); - if (!authProviders.isEmpty()) { - log.debug("Configuring authentication metadata reader"); - readers.put(MetadataType.AUTH, new AuthMetadataReader()); - } - - return readers.build(); - } - - /** - * Creates a chained authentication handler that accepts any of the configured auth methods. Uses - * ChainAuthHandler.any() so a request is authenticated if ANY provider succeeds. - */ - private AuthenticationHandler createChainAuthHandler() { - if (authProviders.size() == 1) { - return createAuthHandler(authProviders.get(0)); - } - - var chain = ChainAuthHandler.any(); - for (var provider : authProviders) { - chain.add(createAuthHandler(provider)); - } - return chain; - } - - private AuthenticationHandler createAuthHandler(AuthenticationProvider auth) { - if (auth instanceof JWTAuth jwtAuth) { - return JWTAuthHandler.create(jwtAuth); - } else { - throw new IllegalArgumentException("Unsupported auth provider type: " + auth.getClass()); - } - } - - /** - * Returns the GraphQL engine instance for internal use by other verticles. - * - * @return the GraphQL engine, or null if not yet initialized - */ - public GraphQL getGraphQLEngine() { - return this.graphQLEngine; - } - - public GraphQL createGraphQL( - Map clients, - SubscriptionConfigurationImpl subscriptionConfig, - Map metadataReaders, - FunctionExecutor functionExecutor) { - try { - var vertxJdbcClient = new VertxJdbcClient(clients); - var graphQL = - model.accept( - new GraphQLEngineBuilder.Builder() - .withMutationConfiguration(new MutationConfigurationImpl(vertx, config)) - .withSubscriptionConfiguration(subscriptionConfig) - .withExtendedScalarTypes(CustomScalars.getExtendedScalars()) - .build(), - new VertxContext(vertxJdbcClient, metadataReaders, functionExecutor)); - var meterRegistry = BackendRegistries.getDefaultNow(); - if (meterRegistry != null) { - graphQL.instrumentation(new MicrometerInstrumentation(meterRegistry)); - } - return graphQL.build(); - } catch (Exception e) { - log.error("Unable to create GraphQL", e); - throw e; - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/HttpServerVerticle.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/HttpServerVerticle.java deleted file mode 100644 index 5c1e7494e8..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/HttpServerVerticle.java +++ /dev/null @@ -1,424 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import com.datasqrl.graphql.auth.OAuth2AuthFactory; -import com.datasqrl.graphql.auth.OAuthDiscoveryHandler; -import com.datasqrl.graphql.config.CorsHandlerOptions; -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.config.ServerConfigUtil; -import com.datasqrl.graphql.exec.FlinkExecFunctionPlan; -import com.datasqrl.graphql.server.ModelContainer; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.operation.ApiOperation; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.google.common.collect.MoreCollectors; -import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; -import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; -import io.micrometer.core.instrument.binder.system.UptimeMetrics; -import io.micrometer.core.instrument.composite.CompositeMeterRegistry; -import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; -import io.vertx.core.*; -import io.vertx.core.http.HttpMethod; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.auth.authentication.AuthenticationProvider; -import io.vertx.ext.auth.jwt.JWTAuth; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.handler.BodyHandler; -import io.vertx.ext.web.handler.CorsHandler; -import io.vertx.ext.web.handler.LoggerHandler; -import io.vertx.ext.web.healthchecks.HealthCheckHandler; -import io.vertx.micrometer.backends.BackendRegistries; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import javax.annotation.Nullable; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -/** - * Bootstrap verticle that owns the HTTP server, root router and all cross-cutting handlers. - * Protocol-specific verticles (GraphQL, REST façade, etc.) are deployed and receive the shared - * {@link Router}. - */ -@Slf4j -public class HttpServerVerticle extends AbstractVerticle { - - /** Resources to close */ - private final List closeables = new ArrayList<>(); - - // Configuration is loaded once and shared with child verticles - /** Server configuration */ - private ServerConfig config; - - /** Server model */ - private Map models; - - @Nullable private Path configDir; - - // --------------------------------------------------------------------------- - // Lifecyle - // --------------------------------------------------------------------------- - - @SuppressWarnings("unused") - public HttpServerVerticle() { - this.config = null; - this.models = null; - this.configDir = null; - } - - public HttpServerVerticle( - ServerConfig config, Map models, @Nullable Path configDir) { - this.config = config; - this.models = models; - this.configDir = configDir; - } - - @Override - public void start(Promise startPromise) { - if (this.models == null) { - try { - this.models = loadModel(); - } catch (Exception e) { - startPromise.fail(e); - return; - } - } - - if (this.config == null) { - loadConfig() - .onFailure(startPromise::fail) - .onSuccess( - raw -> { - this.config = ServerConfigUtil.fromConfigMap(raw.getMap()); - try { - bootstrap(startPromise); - } catch (Exception e) { - startPromise.fail(e); - } - }); - } else { - try { - bootstrap(startPromise); - } catch (Exception e) { - startPromise.fail(e); - } - } - } - - @Override - public void stop(Promise stopPromise) throws Exception { - try { - for (AutoCloseable closeable : closeables) { - closeable.close(); - } - stopPromise.complete(); - - } catch (Exception e) { - stopPromise.fail(e); - } - } - - // --------------------------------------------------------------------------- - // Bootstrap - // --------------------------------------------------------------------------- - - private void bootstrap(Promise startPromise) { - var router = Router.router(vertx); - - // ── Metrics ─────────────────────────────────────────────────────────────── - var meterRegistry = findMeterRegistry(); - meterRegistry.ifPresent( - registry -> { - registerJvmMetrics(registry); - - router - .route("/metrics") - .handler( - ctx -> - ctx.response() - .putHeader("content-type", "text/plain") - .end(registry.scrape())); - }); - - // ── Global handlers (CORS, body, etc.) ──────────────────────────────────── - router.route().handler(toCorsHandler(config.getCorsHandlerOptions())); - router.route().handler(BodyHandler.create()); - - // Use detailed tracing if enabled, otherwise use standard logging (must be after BodyHandler) - if (System.getenv("SQRL_DEBUG") != null) { - router.route().handler(new DetailedRequestTracer()); - } else { - router.route().handler(LoggerHandler.create()); - } - - // ── Health checks ──────────────────────────────────────────────────────── - router.get("/health*").handler(HealthCheckHandler.create(vertx)); - - // ── OAuth Discovery Endpoints (RFC 9728) ───────────────────────────────── - var oauthDiscovery = new OAuthDiscoveryHandler(config); - oauthDiscovery.registerRoutes(router); - - // Deploy GraphQL verticles and collect futures - var deploymentFutures = new ArrayList>(); - for (var modelEntry : models.entrySet()) { - var deploymentFuture = - deployVersionedModel(router, modelEntry.getKey(), modelEntry.getValue()); - deploymentFutures.add(deploymentFuture); - } - - // Wait for all GraphQL verticles to deploy, then start HTTP server - Future.all(deploymentFutures) - .compose( - compositeFuture -> { - // ── Start the HTTP server ──────────────────────────────────────────────── - return vertx - .createHttpServer(config.getHttpServerOptions()) - .requestHandler(router) - .listen(config.getHttpServerOptions().getPort()); - }) - .onSuccess( - server -> { - log.info("HTTP server listening on port {}", server.actualPort()); - startPromise.complete(); - }) - .onFailure( - err -> { - log.error("Failed to start application", err); - startPromise.fail(err); - }); - } - - private Optional findMeterRegistry() { - var registry = BackendRegistries.getDefaultNow(); - if (registry == null) { - return Optional.empty(); - } - - log.info("Found registry: {}", registry.getClass().getSimpleName()); - - if (registry instanceof PrometheusMeterRegistry meterRegistry) { - return Optional.of(meterRegistry); - } - - if (registry instanceof CompositeMeterRegistry compositeRegistry) { - return compositeRegistry.getRegistries().stream() - .filter(PrometheusMeterRegistry.class::isInstance) - .map(PrometheusMeterRegistry.class::cast) - .collect(MoreCollectors.toOptional()); - } - - throw new IllegalStateException( - "Unable to register metrics to: " + registry.getClass().getSimpleName()); - } - - private Future deployVersionedModel( - Router router, String modelVersion, RootGraphqlModel model) { - var childOpts = new DeploymentOptions().setInstances(1); - var hasMcp = model.getOperations().stream().anyMatch(ApiOperation::isMcpEndpoint); - var hasRest = model.getOperations().stream().anyMatch(ApiOperation::isRestEndpoint); - - return createAuthProviders() - .compose( - authProviders -> { - var execFunctionPlan = loadExecFunctionPlan(); - if (execFunctionPlan.isPresent()) { - log.info("Exec function plan loaded"); - } - - var graphQLVerticle = - new GraphQLServerVerticle( - router, config, modelVersion, model, authProviders, execFunctionPlan); - - return vertx - .deployVerticle(graphQLVerticle, childOpts) - .onSuccess( - graphQLDeploymentId -> { - log.info("GraphQL verticle deployed successfully: {}", graphQLDeploymentId); - if (hasMcp) { - var mcpBridgeVerticle = - new McpBridgeVerticle( - router, - config, - modelVersion, - model, - authProviders, - graphQLVerticle); - vertx - .deployVerticle(mcpBridgeVerticle, childOpts) - .onSuccess( - id -> - log.info("MCP bridge verticle deployed successfully: {}", id)) - .onFailure( - err -> log.error("Failed to deploy MCP bridge verticle", err)); - } - if (hasRest) { - var restBridgeVerticle = - new RestBridgeVerticle( - router, - config, - modelVersion, - model, - authProviders, - graphQLVerticle); - vertx - .deployVerticle(restBridgeVerticle, childOpts) - .onSuccess( - id -> - log.info( - "REST bridge verticle deployed successfully: {}", id)) - .onFailure( - err -> log.error("Failed to deploy REST bridge verticle", err)); - } - }) - .onFailure( - err -> - log.error( - "Failed to deploy GraphQL verticle, will trigger orderly shutdown", - err)); - }); - } - - /** - * Creates authentication providers for all configured auth methods. Supports both OAuth and JWT - * simultaneously when both are configured. - */ - private Future> createAuthProviders() { - List> providerFutures = new ArrayList<>(); - - if (config.getOauthConfig() != null && config.getOauthConfig().getSite() != null) { - log.info("Configuring OAuth authentication with JWKS"); - providerFutures.add( - OAuth2AuthFactory.createAuthProvider(vertx, config.getOauthConfig()) - .map(provider -> (AuthenticationProvider) provider)); - } - - if (config.getJwtAuth() != null) { - log.info("Configuring JWT authentication"); - providerFutures.add( - Future.succeededFuture( - (AuthenticationProvider) JWTAuth.create(vertx, config.getJwtAuth()))); - } - - if (providerFutures.isEmpty()) { - log.info("No authentication configured"); - return Future.succeededFuture(List.of()); - } - - return Future.all(providerFutures) - .map( - composite -> { - List providers = new ArrayList<>(); - for (int i = 0; i < composite.size(); i++) { - providers.add(composite.resultAt(i)); - } - log.info("Configured {} authentication provider(s)", providers.size()); - return providers; - }); - } - - // --------------------------------------------------------------------------- - // Helpers - // --------------------------------------------------------------------------- - - /** Async-read server-config.json and return as {@link JsonObject}. */ - private Future loadConfig() { - Promise promise = Promise.promise(); - vertx - .fileSystem() - .readFile("vertx-config.json") - .onComplete( - result -> { - if (result.succeeded()) { - try { - var config = - new JsonObject( - getObjectMapper().readValue(result.result().toString(), Map.class)); - promise.complete(config); - } catch (Exception e) { - e.printStackTrace(); - promise.fail(e); - } - } else { - promise.fail(result.cause()); - } - }); - return promise.future(); - } - - private Optional loadExecFunctionPlan() { - var parent = configDir != null ? configDir : Path.of("."); - var planFile = parent.resolve("vertx-exec-functions.ser"); - - if (!Files.exists(planFile)) { - return Optional.empty(); - } - - return Optional.of(FlinkExecFunctionPlan.deserialize(planFile)); - } - - private void registerJvmMetrics(PrometheusMeterRegistry registry) { - new UptimeMetrics().bindTo(registry); - new JvmMemoryMetrics().bindTo(registry); - new JvmThreadMetrics().bindTo(registry); - - var jvmGcMetrics = new JvmGcMetrics(); - jvmGcMetrics.bindTo(registry); - closeables.add(jvmGcMetrics); - } - - @SneakyThrows - private static Map loadModel() { - return getObjectMapper() - .readValue(new File("vertx.json").getAbsoluteFile(), ModelContainer.class) - .models; - } - - public static ObjectMapper getObjectMapper() { - var objectMapper = new ObjectMapper(); - var module = new SimpleModule(); - module.addDeserializer(String.class, new JsonEnvVarDeserializer()); - objectMapper.registerModule(module); - return objectMapper; - } - - /** Build a Vert.x {@link CorsHandler} from our own options DTO. */ - private CorsHandler toCorsHandler(CorsHandlerOptions opts) { - var ch = - opts.getAllowedOrigin() != null - ? CorsHandler.create().addOrigin(opts.getAllowedOrigin()) - : CorsHandler.create(); - - if (opts.getAllowedOrigins() != null) { - ch.addOrigins(opts.getAllowedOrigins()); - } - - return ch.allowedMethods( - opts.getAllowedMethods().stream().map(HttpMethod::valueOf).collect(Collectors.toSet())) - .allowedHeaders(opts.getAllowedHeaders()) - .exposedHeaders(opts.getExposedHeaders()) - .allowCredentials(opts.isAllowCredentials()) - .maxAgeSeconds(opts.getMaxAgeSeconds()) - .allowPrivateNetwork(opts.isAllowPrivateNetwork()); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/McpBridgeVerticle.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/McpBridgeVerticle.java deleted file mode 100644 index 9a75006ede..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/McpBridgeVerticle.java +++ /dev/null @@ -1,688 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import com.datasqrl.graphql.auth.OAuthFailureHandler; -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.operation.ApiOperation; -import com.datasqrl.graphql.server.operation.McpMethodType; -import com.datasqrl.util.ProjectConstants; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import graphql.language.OperationDefinition.Operation; -import io.vertx.core.Future; -import io.vertx.core.Promise; -import io.vertx.core.http.HttpHeaders; -import io.vertx.core.http.HttpServerResponse; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.auth.authentication.AuthenticationProvider; -import io.vertx.ext.auth.jwt.JWTAuth; -import io.vertx.ext.auth.oauth2.OAuth2Auth; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.RoutingContext; -import io.vertx.ext.web.handler.AuthenticationHandler; -import io.vertx.ext.web.handler.ChainAuthHandler; -import io.vertx.ext.web.handler.JWTAuthHandler; -import io.vertx.ext.web.handler.OAuth2AuthHandler; -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class McpBridgeVerticle extends AbstractBridgeVerticle { - - private static final String JSONRPC_VERSION = "2.0"; - private static final String PROTOCOL_VERSION = "2024-11-05"; - private static final String CT_JSON = "application/json"; - private static final String CT_SSE = "text/event-stream"; - - private static final String RESOURCES_RESULT_KEY = "resources"; - private static final String RESOURCE_TEMPLATES_RESULT_KEY = "resourceTemplates"; - - private final ConcurrentHashMap sseConnections = new ConcurrentHashMap<>(); - - private final Map tools; - private final JsonObject toolsList; - private final List resources; - private final JsonObject resourceList; - private final JsonObject resourceTemplatesList; - - public McpBridgeVerticle( - Router router, - ServerConfig config, - String modelVersion, - RootGraphqlModel model, - List authProviders, - GraphQLServerVerticle graphQLServerVerticle) { - super(router, config, modelVersion, model, authProviders, graphQLServerVerticle); - this.tools = - model.getOperations().stream() - .filter(op -> op.getMcpMethod() == McpMethodType.TOOL) - .collect(Collectors.toMap(ApiOperation::getId, Function.identity())); - this.toolsList = getToolsList(this.tools.values()); - this.resources = - model.getOperations().stream() - .filter(op -> op.getMcpMethod() == McpMethodType.RESOURCE) - .toList(); - this.resourceList = - getResourceList( - RESOURCES_RESULT_KEY, - resources.stream() - .filter(op -> op.getFunction().getParameters().getProperties().isEmpty()) - .collect(Collectors.toList())); - this.resourceTemplatesList = - getResourceList( - RESOURCE_TEMPLATES_RESULT_KEY, - resources.stream() - .filter(op -> !op.getFunction().getParameters().getProperties().isEmpty()) - .collect(Collectors.toList())); - } - - @Override - public void start(Promise startPromise) { - var mcpRoutePrefix = config.getServletConfig().getMcpEndpoint(modelVersion); - log.info("Starting McpBridgeVerticle with endpoint prefix: {}", mcpRoutePrefix); - log.info("Available tools: {}", tools.keySet()); - log.info("Available resources: {}", resources.size()); - - // Debug: Log ALL incoming requests to help identify what the MCP client is doing - router - .route("/*") - .handler( - ctx -> { - String path = ctx.request().path(); - if (path.contains("/mcp") || path.contains("/v1")) { - log.info( - "Request received: {} {} - Headers: {}", - ctx.request().method(), - path, - ctx.request().headers().names()); - } - ctx.next(); - }); - - // Add CORS support for MCP endpoint - router - .route(mcpRoutePrefix + "*") - .handler( - ctx -> { - log.info("MCP route matched: {} {}", ctx.request().method(), ctx.request().path()); - ctx.response() - .putHeader("Access-Control-Allow-Origin", "*") - .putHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS") - .putHeader("Access-Control-Allow-Headers", "Content-Type, Accept, Authorization"); - ctx.next(); - }); - - // Handle preflight OPTIONS requests - router - .options(mcpRoutePrefix) - .handler( - ctx -> { - log.debug("Handling OPTIONS request for MCP endpoint"); - ctx.response().setStatusCode(204).end(); - }); - - // Add a catch-all GET handler for MCP endpoint to help with debugging - router - .get(mcpRoutePrefix) - .handler( - ctx -> { - log.info( - "Received GET request to MCP endpoint - this might be a client capability check"); - ctx.response() - .setStatusCode(405) - .putHeader("Content-Type", "application/json") - .putHeader("Allow", "POST, OPTIONS") - .end( - "{\"jsonrpc\":\"2.0\",\"error\":{\"code\":-32601,\"message\":\"Method not allowed. Use POST for MCP requests.\"}}"); - }); - - // MCP endpoint - handles POST for messages - var postRoute = router.post(mcpRoutePrefix); - - // Add JWT/OAuth auth if configured - if (!authProviders.isEmpty()) { - log.info( - "Applying {} authentication provider(s) to MCP endpoint: {}", - authProviders.size(), - mcpRoutePrefix); - postRoute.handler(createChainAuthHandler()); - postRoute.failureHandler(new OAuthFailureHandler(config)); - } - - postRoute.handler( - ctx -> { - log.debug("Received POST request to MCP endpoint: {}", mcpRoutePrefix); - try { - var payload = OBJECT_MAPPER.readTree(ctx.body().buffer().getBytes()); - log.debug("Parsed payload: {}", payload); - processIncomingMessages(ctx, payload); - } catch (IOException e) { // TODO: improve error handling - log.error("Failed to parse MCP request payload", e); - ctx.response() - .setStatusCode(400) - .putHeader("Content-Type", "application/json") - .end( - "{\"jsonrpc\":\"2.0\",\"error\":{\"code\":-32700,\"message\":\"Parse error\"}}"); - } - }); - - // SSE endpoint for server-to-client streaming - var sseRoute = router.get(mcpRoutePrefix + "/sse"); - - // Add JWT/OAuth auth to SSE endpoint if configured - if (!authProviders.isEmpty()) { - log.info( - "Applying {} authentication provider(s) to MCP SSE endpoint: {}/sse", - authProviders.size(), - mcpRoutePrefix); - sseRoute.handler(createChainAuthHandler()); - sseRoute.failureHandler(new OAuthFailureHandler(config)); - } - - sseRoute.handler( - ctx -> { - log.info("Received SSE connection request to: {}/sse", mcpRoutePrefix); - if (!accepts(ctx, CT_SSE)) { - log.warn("SSE request rejected - client doesn't accept text/event-stream"); - ctx.response().setStatusCode(406).end(); - return; - } - var response = ctx.response(); - - // Set up SSE headers - response - .putHeader("Content-Type", "text/event-stream") - .putHeader("Cache-Control", "no-cache") - .putHeader("Connection", "keep-alive") - .putHeader("Access-Control-Allow-Origin", "*") - .setChunked(true); - - var connectionId = UUID.randomUUID().toString(); - var connection = new SseConnection(connectionId, response); - sseConnections.put(connectionId, connection); - log.info("New SSE connection established: {}", connectionId); - - // Send initial connection event - sendSseMessage( - connection, "connected", new JsonObject().put("connectionId", connectionId)); - - // Send heartbeat every 30 seconds - var timerId = - vertx.setPeriodic( - 30000, - id -> { - if (sseConnections.containsKey(connectionId)) { - sendSseMessage( - connection, - "heartbeat", - new JsonObject().put("timestamp", System.currentTimeMillis())); - } - }); - - // Handle connection close - response.closeHandler( - v -> { - vertx.cancelTimer(timerId); - sseConnections.remove(connectionId); - log.info("SSE connection closed: {}", connectionId); - }); - - // Handle client disconnect - response.exceptionHandler( - throwable -> { - vertx.cancelTimer(timerId); - sseConnections.remove(connectionId); - log.warn("SSE connection error: {} - {}", connectionId, throwable.getMessage()); - }); - }); - startPromise.complete(); - } - - private void sendSseMessage(SseConnection connection, String event, JsonObject data) { - try { - String message = String.format("event: %s\ndata: %s\n\n", event, data.encode()); - connection.response.write(message); - } catch (Exception e) { - log.error("Error sending SSE message: {}", e.getMessage()); - sseConnections.remove(connection.id); - } - } - - // Broadcast a message to all connected SSE clients - public void broadcast(String event, JsonObject data) { - sseConnections.values().forEach(connection -> sendSseMessage(connection, event, data)); - } - - public Future handleRequest(RoutingContext ctx, JsonNode request) { - var method = request.get("method").asText(); - var id = request.get("id"); - var params = request.get("params"); - - log.info( - "Handling MCP request - method: {}, id: {}", method, id != null ? id.asText() : "null"); - log.debug("Request params: {}", params); - - // Handle notifications (messages without id) - these should not return responses - if (id == null) { - log.debug("Processing notification for method: {}", method); - return handleNotification(method, params); - } - - try { - Future resultFuture = - switch (method) { - case "initialize" -> { - log.info("Handling initialize request"); - yield Future.succeededFuture(handleInitialize(params)); - } - case "tools/list" -> { - log.info("Handling tools/list request - returning {} tools", tools.size()); - yield Future.succeededFuture(toolsList); - } - case "tools/call" -> { - log.info("Handling tools/call request"); - yield handleCallTool(ctx, params); - } - case "resources/list" -> { - log.info( - "Handling resources/list request - returning {} resources", resources.size()); - yield Future.succeededFuture(resourceList); - } - case "resources/templates/list" -> { - log.info("Handling resources/templates/list request"); - yield Future.succeededFuture(resourceTemplatesList); - } - case "resources/read" -> { - log.info("Handling resources/read request"); - yield handleReadResource(ctx, params); - } - case "ping" -> { - log.debug("Handling ping request"); - yield Future.succeededFuture(new JsonObject()); - } - default -> { - log.warn("Unknown MCP method: {}", method); - yield Future.failedFuture(new McpException(-32601, "Method not found")); - } - }; - - return resultFuture - .map( - result -> { - log.debug("Request {} completed successfully", method); - return createResponse(id, result); - }) - .recover( - error -> { - log.error( - "Error handling MCP request method '{}': {}", - method, - error.getMessage(), - error); - var errorResponse = - error instanceof McpException e - ? createErrorResponse(id, e.code, e.getMessage()) - : createErrorResponse(id, -32603, "Internal error: " + error.getMessage()); - - // Enhanced debugging for tool call errors - if ("tools/call".equals(method)) { - log.error("Tool call error response: {}", errorResponse.encode()); - } - - return Future.succeededFuture(errorResponse); - }); - } catch (ValidationException e) { - var json = new JsonObject(); - json.put( - "content", - List.of( - new JsonObject() - .put("type", "text") - .put("text", "Validation error: " + e.getMessage()))); - json.put("isError", true); - var response = createResponse(id, json); - return Future.succeededFuture(response); - } catch (Exception e) { - var errorResponse = createErrorResponse(id, -32603, "Internal error: " + e.getMessage()); - log.error("Request error: {}", errorResponse.encode()); - return Future.succeededFuture(errorResponse); - } - } - - /** - * Handle notifications (messages without id field). Notifications should not return responses. - */ - private Future handleNotification(String method, JsonNode params) { - switch (method) { - case "notifications/initialized": - // Handle the initialized notification - just log it - log.info("Client initialized notification received"); - break; - case "notifications/cancelled": - // Handle cancellation notifications if needed - log.info("Request cancelled notification received"); - break; - default: - // Unknown notifications are ignored (as per JSON-RPC spec) - log.warn("Unknown notification: {}", method); - break; - } - - // Return null to indicate no response should be sent for notifications - return Future.succeededFuture(null); - } - - private JsonObject handleInitialize(JsonNode params) { - log.info("Initializing MCP server with protocol version: {}", PROTOCOL_VERSION); - var capabilities = - new JsonObject() - .put("tools", new JsonObject()) - .put("resources", new JsonObject().put("subscribe", false).put("listChanged", false)); - - var serverInfo = - new JsonObject() - .put("name", "datasqrl-mcp-server") - .put("version", ProjectConstants.SQRL_VERSION); - - var response = - new JsonObject() - .put("protocolVersion", PROTOCOL_VERSION) - .put("capabilities", capabilities) - .put("serverInfo", serverInfo); - - log.info("Initialize response: {}", response.encodePrettily()); - return response; - } - - /** - * Dummy implementation for now - * - * @param params - * @return - */ - private Future handleReadResource(RoutingContext ctx, JsonNode params) { - var uri = params.get("uri").asText(); - if (uri == null) { - return Future.failedFuture(new McpException(-32602, "URI parameter required")); - } - if (uri.isBlank()) - return Future.failedFuture(new McpException(-32602, "Resource not found: " + uri)); - var contents = - new JsonArray() - .add( - new JsonObject() - .put("uri", uri) - .put("mimeType", "application/json") - .put("text", new JsonObject().encodePrettily())); - return Future.succeededFuture(new JsonObject().put("contents", contents)); - } - - private Future handleCallTool(RoutingContext ctx, JsonNode params) - throws ValidationException { - var toolName = params.get("name").asText(); - var arguments = params.get("arguments"); - - var tool = tools.get(toolName); - if (tool == null) { - return Future.failedFuture(new McpException(-32602, "Tool not found: " + toolName)); - } - var variables = - OBJECT_MAPPER.>convertValue(arguments, new TypeReference<>() {}); - return bridgeRequestToGraphQL(ctx, tool, variables) - .map( - executionResult -> { - var json = new JsonObject(); - if (!executionResult.getErrors().isEmpty()) { - json.put( - "content", - executionResult.getErrors().stream() - .map( - err -> - new JsonObject() - .put("type", "text") - .put( - "text", - "Tool Error[" + err.getPath() + "]: " + err.getMessage())) - .toList()); - json.put("isError", true); - } else { - var result = getExecutionData(executionResult, tool); - try { - json.put( - "content", - List.of( - new JsonObject() - .put("type", "text") - .put("text", OBJECT_MAPPER.writeValueAsString(result)))); - json.put("isError", false); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - return json; - }) - .recover( - error -> { - // Return tool error within the result (not as MCP protocol error) - var errorContent = - new JsonArray() - .add( - new JsonObject() - .put("type", "text") - .put("text", "Tool error: " + error.getMessage())); - - var errorResult = new JsonObject().put("content", errorContent).put("isError", true); - - return Future.succeededFuture(errorResult); - }); - } - - private JsonObject getToolsList(Collection toolOperations) { - var toolsArray = new JsonArray(); - - for (var tool : toolOperations) { - var inputSchema = - getSchemaMapper() - .convertValue( - tool.getFunction().getParameters(), new TypeReference>() {}); - var description = tool.getFunction().getDescription(); - if (description == null) { - description = - "Invokes %s %s" - .formatted( - tool.getFunction().getName(), - tool.getApiQuery().operationType().name().toLowerCase()); - } - var isReadOnly = tool.getApiQuery().operationType() != Operation.MUTATION; - var toolInfo = - new JsonObject() - .put("name", tool.getName()) - .put("description", description) - .put("inputSchema", inputSchema) - .put("annotations", new JsonObject().put("readOnlyHint", isReadOnly)); - toolsArray.add(toolInfo); - } - return new JsonObject().put("tools", toolsArray); - } - - private JsonObject getResourceList( - String resultKey, Collection resourceOperations) { - var resourcesArray = new JsonArray(); - for (var resource : resourceOperations) { - var description = resource.getFunction().getDescription(); - if (description == null) { - description = "Returns %s resource".formatted(resource.getFunction().getName()); - } - var resourceDef = - new JsonObject() - .put("uri", resource.getUriTemplate()) - .put("name", resource.getName()) - .put("description", description) - .put("mimeType", "application/json"); - resourcesArray.add(resourceDef); - } - return new JsonObject().put(resultKey, resourcesArray); - } - - private JsonObject createResponse(Object id, JsonObject result) { - return new JsonObject().put("jsonrpc", JSONRPC_VERSION).put("id", id).put("result", result); - } - - private JsonObject createErrorResponse(Object id, int code, String message) { - var error = new JsonObject().put("code", code).put("message", message); - return new JsonObject().put("jsonrpc", JSONRPC_VERSION).put("id", id).put("error", error); - } - - // --------------------------------------------------------------------------- - // MCP Streamable‑HTTP helpers - // --------------------------------------------------------------------------- - - private void processIncomingMessages(RoutingContext ctx, JsonNode payload) { - var containsRequest = containsRequestObjects(payload); - - // Pure notifications (no "method" requests) → 202 Accepted - if (!containsRequest) { - ctx.response().setStatusCode(202).end(); - return; - } - - var clientAcceptsSse = - accepts(ctx, CT_SSE) && !accepts(ctx, CT_JSON); // prefer JSON if both present - - if (clientAcceptsSse && requestNeedsStreaming(payload)) { - streamResponse(ctx, payload); - } else { - singleJsonResponse(ctx, payload); - } - } - - private void singleJsonResponse(RoutingContext ctx, JsonNode payload) { - if (!payload.isObject()) { - ctx.fail(400, new IllegalArgumentException("Batch requests not supported yet")); - return; - } - handleRequest(ctx, payload) - .onSuccess( - result -> { - log.debug("Request completed successfully"); - if (result != null) { - ctx.response().putHeader(HttpHeaders.CONTENT_TYPE, CT_JSON).end(result.encode()); - } else { - ctx.response().setStatusCode(204).end(); - } - }) - .onFailure( - error -> { - log.error("Request failed: {}", error.getMessage(), error); - ctx.response() - .setStatusCode(500) - .putHeader("Content-Type", "application/json") - .end( - "{\"jsonrpc\":\"2.0\",\"error\":{\"code\":-32603,\"message\":\"Internal error\"}}"); - }); - } - - private void streamResponse(RoutingContext ctx, JsonNode payload) { - if (!payload.isObject()) { - ctx.fail(400, new IllegalArgumentException("Batch requests not supported in streaming mode")); - return; - } - var res = ctx.response(); - res.setChunked(true) - .putHeader(HttpHeaders.CONTENT_TYPE, CT_SSE) - .putHeader("Cache-Control", "no-cache"); - - handleRequest(ctx, payload) - .onSuccess( - result -> { - // send single chunk – for true multi‑chunk you'd wire this to reactive execution - res.write("data: " + result.encode() + "\n\n").onComplete(ar -> res.end()); - }) - .onFailure( - err -> { - res.write( - "data: " + new JsonObject().put("error", err.getMessage()).encode() + "\n\n") - .onComplete(ar -> res.end()); - }); - } - - private static boolean containsRequestObjects(JsonNode node) { - if (node.isArray()) { - for (var n : node) { - if (n.hasNonNull("method")) return true; - } - return false; - } - return node.hasNonNull("method"); - } - - /** Minimal heuristic: initialise & streamed tool calls need SSE */ - private static boolean requestNeedsStreaming(JsonNode payload) { - if (!payload.isObject()) return false; - var method = payload.path("method").asText(""); - return "initialize".equals(method) - || ("tools/call".equals(method) && payload.path("params").path("stream").asBoolean(false)); - } - - private static boolean accepts(RoutingContext ctx, String mime) { - var accept = ctx.request().headers().getAll(HttpHeaders.ACCEPT); - return accept.stream().anyMatch(h -> h.contains(mime)); - } - - private AuthenticationHandler createChainAuthHandler() { - if (authProviders.size() == 1) { - return createAuthHandler(authProviders.get(0)); - } - - var chain = ChainAuthHandler.any(); - for (var provider : authProviders) { - chain.add(createAuthHandler(provider)); - } - return chain; - } - - private AuthenticationHandler createAuthHandler(AuthenticationProvider auth) { - if (auth instanceof JWTAuth jwtAuth) { - return JWTAuthHandler.create(jwtAuth); - } else if (auth instanceof OAuth2Auth oauth2Auth) { - return OAuth2AuthHandler.create(vertx, oauth2Auth); - } else { - throw new IllegalArgumentException("Unsupported auth provider type: " + auth.getClass()); - } - } - - private record SseConnection(String id, HttpServerResponse response) {} - - static class McpException extends RuntimeException { - final int code; - - McpException(int code, String message) { - super(message); - this.code = code; - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/MutationConfigurationImpl.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/MutationConfigurationImpl.java deleted file mode 100644 index 7425af9ddb..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/MutationConfigurationImpl.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.io.SinkProducer; -import com.datasqrl.graphql.kafka.KafkaSinkProducer; -import com.datasqrl.graphql.server.Context; -import com.datasqrl.graphql.server.MetadataReader; -import com.datasqrl.graphql.server.MetadataType; -import com.datasqrl.graphql.server.MutationConfiguration; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.RootGraphqlModel.MutationCoordsVisitor; -import graphql.schema.DataFetcher; -import graphql.schema.DataFetchingEnvironment; -import io.vertx.core.CompositeFuture; -import io.vertx.core.Future; -import io.vertx.core.Vertx; -import io.vertx.kafka.client.producer.KafkaProducer; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.UUID; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * Purpose: Configures data fetchers for GraphQL mutations and executes the mutations (kafka - * messages sending and SQL inserting) Collaboration: Uses {@link RootGraphqlModel} to get mutation - * coordinates and creates data fetchers for Kafka and PostgreSQL. - */ -@Slf4j -@AllArgsConstructor -public class MutationConfigurationImpl implements MutationConfiguration> { - - private Vertx vertx; - private ServerConfig config; - - @Override - public MutationCoordsVisitor, Context> createSinkFetcherVisitor() { - return (coords, context) -> { - KafkaProducer producer = - KafkaProducer.create( - vertx, config.getKafkaMutationConfig().asMap(coords.isTransactional())); - SinkProducer emitter = new KafkaSinkProducer<>(coords.getTopic(), producer); - final Map computedInputColumns = new HashMap<>(); - coords - .getComputedColumns() - .forEach( - (colName, metadata) -> { - ComputeInputColumns fct = - switch (metadata.metadataType()) { - case UUID -> (env -> UUID.randomUUID()); - case AUTH -> { - MetadataReader metadataReader = - context.getMetadataReader(metadata.metadataType()); - yield (env -> - metadataReader.read(env, metadata.name(), metadata.required())); - } - default -> null; - }; - if (fct != null) computedInputColumns.put(colName, fct); - }); - final List timestampColumns = - coords.getComputedColumns().entrySet().stream() - .filter(e -> e.getValue().metadataType() == MetadataType.TIMESTAMP) - .map(Entry::getKey) - .collect(Collectors.toList()); - - checkNotNull(emitter, "Could not find sink for field: %s", coords.getFieldName()); - return env -> { - var entries = getEntries(env, computedInputColumns); - var cf = new CompletableFuture(); - - Future> sendFuture = - coords.isTransactional() - ? sendMessagesTransactionally(producer, entries, emitter, timestampColumns) - : sendMessagesNonTransactionally( - createSendFutures(entries, emitter, timestampColumns)); - - sendFuture - .onSuccess(results -> completeWithResults(cf, results, coords.isReturnList())) - .onFailure(cf::completeExceptionally); - - return cf; - }; - }; - } - - @FunctionalInterface - interface ComputeInputColumns { - - Object compute(DataFetchingEnvironment env); - } - - private List getEntries( - DataFetchingEnvironment env, Map computedColumns) { - // Rules: - // - Only one argument is allowed, it doesn't matter the name - // - input argument cannot be null. - var args = env.getArguments(); - - var argument = args.entrySet().stream().findFirst().map(Entry::getValue).get(); - - List entries; - if (argument instanceof List) { - entries = (List) argument; - } else { - entries = List.of((Map) argument); - } - - if (!computedColumns.isEmpty()) { - // Add UUID for event for the computed uuid columns - entries.forEach( - entry -> { - computedColumns.forEach( - (colName, fct) -> { - var value = fct.compute(env); - entry.put(colName, value); - }); - }); - } - return entries; - } - - private List> createSendFutures( - List entries, SinkProducer emitter, List timestampColumns) { - return entries.stream() - .map( - entry -> - emitter - .send(entry) - .map( - sinkResult -> { - // Add timestamp from sink to result - var dateTime = - ZonedDateTime.ofInstant(sinkResult.sourceTime(), ZoneOffset.UTC); - timestampColumns.forEach( - colName -> entry.put(colName, dateTime.toOffsetDateTime())); - return entry; - })) - .collect(Collectors.toList()); - } - - private void completeWithResults( - CompletableFuture cf, List results, boolean returnList) { - if (returnList) { - cf.complete(results); - } else { - cf.complete(results.get(0)); - } - } - - private Future> sendMessagesTransactionally( - KafkaProducer producer, - List entries, - SinkProducer emitter, - List timestampColumns) { - return producer - .initTransactions() - .compose(v -> producer.beginTransaction()) - .compose( - v -> { - // Create send futures only after transaction is initialized and begun - var futures = createSendFutures(entries, emitter, timestampColumns); - return Future.join(futures); - }) - .compose(compositeFuture -> producer.commitTransaction().map(v -> compositeFuture.list())) - .recover( - throwable -> producer.abortTransaction().compose(v -> Future.failedFuture(throwable))); - } - - private Future> sendMessagesNonTransactionally(List> futures) { - return Future.join(futures).map(CompositeFuture::list); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/RestBridgeVerticle.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/RestBridgeVerticle.java deleted file mode 100644 index ad7cd28d7e..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/RestBridgeVerticle.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import com.datasqrl.graphql.auth.JwtFailureHandler; -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.operation.ApiOperation; -import com.datasqrl.graphql.server.operation.RestMethodType; -import com.datasqrl.graphql.swagger.SwaggerService; -import io.vertx.core.Promise; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.auth.authentication.AuthenticationProvider; -import io.vertx.ext.auth.jwt.JWTAuth; -import io.vertx.ext.auth.oauth2.OAuth2Auth; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.RoutingContext; -import io.vertx.ext.web.handler.AuthenticationHandler; -import io.vertx.ext.web.handler.ChainAuthHandler; -import io.vertx.ext.web.handler.JWTAuthHandler; -import io.vertx.ext.web.handler.OAuth2AuthHandler; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; -import lombok.extern.slf4j.Slf4j; - -/** - * REST-to-GraphQL bridge verticle that creates REST endpoints for API operations with - * restEndpoint=true. Handles URI template expansion, parameter validation, and GraphQL query - * routing. - */ -@Slf4j -public class RestBridgeVerticle extends AbstractBridgeVerticle { - - private static final String RESULT_DATA_KEY = "data"; - - // Pattern for RFC 6570 URI template query parameters: {?param1,param2} - private static final Pattern QUERY_PARAMS_PATTERN = Pattern.compile("\\{\\?([^}]+)\\}"); - // Pattern for RFC 6570 URI template path parameters: {param} - private static final Pattern PATH_PARAMS_PATTERN = Pattern.compile("\\{([^}?]+)\\}"); - - private SwaggerService swaggerService; - - public RestBridgeVerticle( - Router router, - ServerConfig config, - String modelVersion, - RootGraphqlModel model, - List authProviders, - GraphQLServerVerticle graphQLServerVerticle) { - super(router, config, modelVersion, model, authProviders, graphQLServerVerticle); - } - - @Override - public void start(Promise startPromise) { - try { - setupRestEndpoints(); - setupSwaggerEndpoints(); - startPromise.complete(); - } catch (Exception e) { - log.error("Could not setup REST endpoints", e); - startPromise.fail(e); - } - } - - private void setupRestEndpoints() { - // Process each API operation and create REST endpoints for those with restEndpoint=true - for (ApiOperation operation : model.getOperations()) { - if (operation.isRestEndpoint()) { - createRestEndpoint(operation); - } - } - } - - private void setupSwaggerEndpoints() { - if (config.getSwaggerConfig() == null || !config.getSwaggerConfig().isEnabled()) { - log.info("Swagger is disabled, skipping Swagger endpoints setup"); - return; - } - - // Initialize Swagger service - swaggerService = - new SwaggerService( - config.getSwaggerConfig(), - model, - modelVersion, - config.getServletConfig().getRestEndpoint(modelVersion)); - - // Setup Swagger JSON endpoint - var swaggerJsonEndpoint = config.getSwaggerConfig().getEndpoint(modelVersion); - router - .get(swaggerJsonEndpoint) - .handler( - ctx -> { - try { - // Extract request host and port for dynamic server URL - var requestHost = getRequestBaseUrl(ctx); - var swaggerJson = swaggerService.generateSwaggerJson(requestHost); - ctx.response().putHeader("content-type", "application/json").end(swaggerJson); - } catch (Exception e) { - log.error("Failed to generate Swagger JSON", e); - ctx.response() - .setStatusCode(500) - .putHeader("content-type", "application/json") - .end("{\"error\": \"Failed to generate Swagger documentation\"}"); - } - }); - - // Setup Swagger UI endpoint - var swaggerUIEndpoint = config.getSwaggerConfig().getUiEndpoint(modelVersion); - router - .get(swaggerUIEndpoint) - .handler( - ctx -> { - try { - var swaggerUIHtml = swaggerService.generateSwaggerUI(); - ctx.response().putHeader("content-type", "text/html").end(swaggerUIHtml); - } catch (Exception e) { - log.error("Failed to generate Swagger UI", e); - ctx.response() - .setStatusCode(500) - .putHeader("content-type", "text/html") - .end("

Error: Failed to generate Swagger UI

"); - } - }); - - log.info("Swagger endpoints setup completed:"); - log.info(" Swagger JSON: {}", swaggerJsonEndpoint); - log.info(" Swagger UI: {}", swaggerUIEndpoint); - } - - private void createRestEndpoint(ApiOperation operation) { - var uriTemplate = operation.getUriTemplate(); - var httpMethod = operation.getRestMethod(); - - if (uriTemplate == null || httpMethod == null) { - log.warn( - "Skipping REST endpoint for operation {} - missing uriTemplate or httpMethod", - operation.getName()); - return; - } - - // Convert URI template to Vert.x route pattern - var routePattern = convertUriTemplateToRoute(uriTemplate); - - log.info( - "Creating REST endpoint: {} {} -> GraphQL {}", - httpMethod, - routePattern, - operation.getName()); - - // Create route based on HTTP method - var route = - switch (httpMethod) { - case GET -> router.get(routePattern); - case POST -> router.post(routePattern); - case NONE -> throw new UnsupportedOperationException("Should not be called"); - }; - - // Add auth if configured - if (!authProviders.isEmpty()) { - route.handler(createChainAuthHandler()); - route.failureHandler(new JwtFailureHandler()); - } - - // Add the REST handler - route.handler( - ctx -> { - var variables = extractParameters(ctx, operation); - try { - var fut = bridgeRequestToGraphQL(ctx, operation, variables); - fut.onSuccess( - executionResult -> { - var res = ctx.response(); - var statusCode = res.getStatusCode(); - - // Preserve status code if already set to non-200 - // Otherwise use 200 for success, 400 for errors - if (statusCode == 200) { - res.setStatusCode(executionResult.getErrors().isEmpty() ? 200 : 400); - } - - res.putHeader("content-type", "application/json"); - - if (!executionResult.getErrors().isEmpty()) { - var json = new JsonObject(); - json.put( - "errors", - executionResult.getErrors().stream() - .map( - err -> - new JsonObject() - .put(JSON_MESSAGE, err.getMessage()) - .put("path", err.getPath()) - .put("extensions", err.getExtensions())) - .toList()); - ctx.end(json.encode()); - } else { - var result = getExecutionData(executionResult, operation); - ctx.end(new JsonObject().put(RESULT_DATA_KEY, result).encode()); - } - }) - .onFailure(err -> handleError(err, ctx, 500, "Error in query processing")); - } catch (ValidationException e) { - handleError(e, ctx, 400, "Parameters are invalid"); - } catch (Exception e) { - handleError(e, ctx, 500, "Error in REST query processing"); - } - }); - } - - /** - * Converts RFC 6570 URI template to Vert.x route pattern. Examples: - - * "queries/HighTempAlert{?offset,limit}" -> "/queries/HighTempAlert" - "mutations/SensorReading" - * -> "/mutations/SensorReading" - "users/{userId}/posts" -> "/users/:userId/posts" - */ - private String convertUriTemplateToRoute(String uriTemplate) { - // Remove query parameters pattern {?param1,param2} - var route = QUERY_PARAMS_PATTERN.matcher(uriTemplate).replaceAll(""); - - // Convert path parameters {param} to :param - route = PATH_PARAMS_PATTERN.matcher(route).replaceAll(":$1"); - - // Ensure route starts with / - if (!route.startsWith("/")) { - route = "/" + route; - } - // Ensure route starts with REST prefix - route = config.getServletConfig().getRestEndpoint(modelVersion) + route; - return route; - } - - protected static Map extractParameters( - RoutingContext ctx, ApiOperation operation) { - var variables = new HashMap(); - - if (operation.getRestMethod() == RestMethodType.GET) { - // For GET requests, extract parameters from URL query parameters and path parameters - extractGetParameters(ctx, operation, variables); - } else { - // For POST/PUT requests, use the JSON body as variables - extractPostParameters(ctx, variables); - } - - return variables; - } - - /** Extract the base URL from the current request including scheme, host, and port */ - private String getRequestBaseUrl(RoutingContext ctx) { - var scheme = ctx.request().isSSL() ? "https" : "http"; - var host = ctx.request().getHeader("Host"); - - // If host header includes port, use it as-is - if (host != null && host.contains(":")) { - return scheme + "://" + host; - } else { - // Use the actual server port from the request - var port = ctx.request().localAddress().port(); - return scheme + "://" + (host != null ? host : "localhost") + ":" + port; - } - } - - private AuthenticationHandler createChainAuthHandler() { - if (authProviders.size() == 1) { - return createAuthHandler(authProviders.get(0)); - } - - var chain = ChainAuthHandler.any(); - for (var provider : authProviders) { - chain.add(createAuthHandler(provider)); - } - return chain; - } - - private AuthenticationHandler createAuthHandler(AuthenticationProvider auth) { - if (auth instanceof JWTAuth jwtAuth) { - return JWTAuthHandler.create(jwtAuth); - } else if (auth instanceof OAuth2Auth oauth2Auth) { - return OAuth2AuthHandler.create(vertx, oauth2Auth); - } else { - throw new IllegalArgumentException("Unsupported auth provider type: " + auth.getClass()); - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/SubscriptionConfigurationImpl.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/SubscriptionConfigurationImpl.java deleted file mode 100644 index 9b6ba3b283..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/SubscriptionConfigurationImpl.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.kafka.KafkaDataFetcherFactory; -import com.datasqrl.graphql.kafka.KafkaSinkConsumer; -import com.datasqrl.graphql.server.Context; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.RootGraphqlModel.SubscriptionCoordsVisitor; -import com.datasqrl.graphql.server.SubscriptionConfiguration; -import graphql.schema.DataFetcher; -import io.vertx.core.Future; -import io.vertx.core.Vertx; -import io.vertx.kafka.client.consumer.KafkaConsumer; -import java.util.ArrayList; -import java.util.List; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * Purpose: Configures {@link DataFetcher}s for GraphQL subscriptions the subscriptions (kafka - * messages subscription and postgreSQL listen/notify mechanism) that embed the code for executing - * the subscriptions. - * - *

Collaboration: Uses {@link RootGraphqlModel} to get subscription coordinates and create data - * fetchers for Kafka and PostgreSQL. - */ -@Slf4j -@RequiredArgsConstructor -public class SubscriptionConfigurationImpl implements SubscriptionConfiguration> { - - private final Vertx vertx; - private final ServerConfig config; - private final List> subscriptionFutures = new ArrayList<>(); - - @Override - public SubscriptionCoordsVisitor, Context> createSubscriptionFetcherVisitor() { - return (coords, context) -> { - KafkaConsumer consumer = - KafkaConsumer.create(vertx, config.getKafkaSubscriptionConfig().asMap()); - var subscriptionFuture = - consumer - .subscribe(coords.getTopic()) - .onSuccess(v -> log.info("Subscribed to topic: {}", coords.getTopic())) - .onFailure( - err -> log.error("Failed to subscribe to topic: {}", coords.getTopic(), err)); - subscriptionFutures.add(subscriptionFuture); - return KafkaDataFetcherFactory.create(new KafkaSinkConsumer<>(consumer), coords, context); - }; - } - - /** - * Returns a composite future that completes when all subscriptions have been set up successfully, - * or fails if any subscription fails. - * - * @return a future that tracks all subscription setups - */ - public Future getAllSubscriptionsFuture() { - return subscriptionFutures.isEmpty() - ? Future.succeededFuture() - : Future.all(subscriptionFutures).mapEmpty(); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/AuthMetadataReader.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/AuthMetadataReader.java deleted file mode 100644 index 490f43dc42..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/AuthMetadataReader.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.auth; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.datasqrl.graphql.server.MetadataReader; -import graphql.schema.DataFetchingEnvironment; -import io.vertx.ext.web.RoutingContext; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class AuthMetadataReader implements MetadataReader { - - @Override - public Object read(DataFetchingEnvironment env, String name, boolean isRequired) { - RoutingContext rc = env.getGraphQlContext().get(RoutingContext.class); - var principal = rc.user(); - - if (principal == null) { - throw new IllegalStateException("Not authenticated"); - } - - if (isRequired && !principal.containsKey(name)) { - log.warn( - "Required claim '{}' is not present on authorization, attributes: {}", - name, - principal.attributes()); - - // Set the HTTP status to 403 directly - rc.response().setStatusCode(403); - rc.response() - .putHeader( - "WWW-Authenticate", - "Bearer error=\"insufficient_scope\", error_description=\"Required claim missing\""); - - throw new MissingRequiredClaimException( - name, - env.getField().getSourceLocation() != null - ? java.util.List.of(env.getField().getSourceLocation()) - : null, - env.getExecutionStepInfo().getPath()); - } - - var value = principal.get(name); - - if (isRequired) { - checkNotNull(value, "Claim '%s' must not be null", name); - } - - return value; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/JwtFailureHandler.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/JwtFailureHandler.java deleted file mode 100644 index 6ae980e0b1..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/JwtFailureHandler.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.auth; - -import io.vertx.core.Handler; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.RoutingContext; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.exception.ExceptionUtils; - -@Slf4j -public class JwtFailureHandler implements Handler { - - @Override - public void handle(RoutingContext ctx) { - var throwable = ctx.failure(); - var statusCode = ctx.statusCode(); - var errorMsg = "JWT auth failed"; - var errorCause = - throwable == null ? "Unknown error" : ExceptionUtils.getRootCauseMessage(throwable); - - log.warn( - "JWT authentication failed for request {} {}: {} (status: {})", - ctx.request().method(), - ctx.request().path(), - errorCause, - statusCode); - - if (throwable != null) { - log.debug("JWT authentication failure details", throwable); - } - - var response = - ctx.response() - .setStatusCode(statusCode != -1 ? statusCode : 401) - .putHeader("Content-Type", "application/json"); - - var json = new JsonObject().put("error", errorMsg).put("cause", errorCause); - response.end(json.encode()); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/MissingRequiredClaimException.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/MissingRequiredClaimException.java deleted file mode 100644 index 80de7a426d..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/MissingRequiredClaimException.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.auth; - -import graphql.GraphQLError; -import graphql.execution.ResultPath; -import graphql.language.SourceLocation; -import java.util.List; -import lombok.Getter; - -/** - * Exception thrown when a required JWT claim is missing or not present in the authorization token. - * This exception should result in an HTTP 403 Forbidden response. - */ -@Getter -public class MissingRequiredClaimException extends RuntimeException implements GraphQLError { - - private final String claimName; - private final List locations; - private final ResultPath path; - - public MissingRequiredClaimException(String claimName) { - super("Required claim missing"); - this.claimName = claimName; - this.locations = null; - this.path = null; - } - - public MissingRequiredClaimException( - String claimName, List locations, ResultPath path) { - super("Required claim missing"); - this.claimName = claimName; - this.locations = locations; - this.path = path; - } - - @Override - public String getMessage() { - return "Forbidden"; - } - - @Override - public List getLocations() { - return locations; - } - - @Override - public graphql.ErrorClassification getErrorType() { - return graphql.ErrorType.ValidationError; - } - - @Override - public List getPath() { - return path != null ? path.toList() : null; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/OAuth2AuthFactory.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/OAuth2AuthFactory.java deleted file mode 100644 index a59f731368..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/OAuth2AuthFactory.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.auth; - -import com.datasqrl.graphql.config.OAuthConfig; -import io.vertx.core.Future; -import io.vertx.core.Vertx; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.auth.authentication.AuthenticationProvider; -import io.vertx.ext.auth.jwt.JWTAuth; -import io.vertx.ext.auth.jwt.JWTAuthOptions; -import io.vertx.ext.web.client.WebClient; -import io.vertx.ext.web.client.WebClientOptions; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import lombok.extern.slf4j.Slf4j; - -/** - * Factory for creating JWT authentication providers with JWKS-based key discovery from OAuth/OIDC - * providers like Auth0 and Keycloak. Uses JWTAuth instead of OAuth2Auth for proper claim extraction - * from bearer tokens. - */ -@Slf4j -public class OAuth2AuthFactory { - - private static final String OIDC_DISCOVERY_PATH = "/.well-known/openid-configuration"; - private static final Set JWK_KEYS = - Set.of("kty", "use", "kid", "alg", "n", "e", "x", "y", "crv"); - - /** - * Creates a JWTAuth provider using JWKS discovered from an OpenID Connect provider. This enables - * automatic JWKS key discovery while using JWTAuth for proper JWT claim extraction. - * - * @param vertx the Vert.x instance - * @param oauthConfig OAuth configuration containing the OIDC site URL - * @return Future containing the AuthenticationProvider, or null if config is invalid - */ - public static Future createAuthProvider( - Vertx vertx, OAuthConfig oauthConfig) { - if (oauthConfig == null || oauthConfig.getOauth2Options() == null) { - return Future.succeededFuture(null); - } - - var oauth2Options = oauthConfig.getOauth2Options(); - var site = oauth2Options.getSite(); - if (site == null || site.isBlank()) { - log.warn("OAuth config present but no site configured"); - return Future.succeededFuture(null); - } - - var normalizedSite = site.endsWith("/") ? site.substring(0, site.length() - 1) : site; - log.info("Creating JWTAuth provider via OIDC JWKS discovery from: {}", normalizedSite); - - var webClient = - WebClient.create(vertx, new WebClientOptions().setSsl(normalizedSite.startsWith("https"))); - - return discoverJwksUri(webClient, normalizedSite) - .compose(jwksUri -> fetchJwks(webClient, jwksUri)) - .map( - jwks -> { - var jwtAuthOptions = new JWTAuthOptions().setJwks(jwks); - var jwtAuth = JWTAuth.create(vertx, jwtAuthOptions); - log.info( - "JWTAuth provider created successfully with {} keys from OIDC discovery", - jwks.size()); - return (AuthenticationProvider) jwtAuth; - }) - .recover( - err -> { - log.error( - "Failed to create JWTAuth from OIDC provider at {}: {}", - normalizedSite, - err.getMessage()); - return Future.failedFuture(err); - }); - } - - private static Future discoverJwksUri(WebClient webClient, String site) { - var discoveryUrl = site + OIDC_DISCOVERY_PATH; - log.debug("Fetching OIDC discovery document from: {}", discoveryUrl); - - return webClient - .getAbs(discoveryUrl) - .send() - .map( - response -> { - if (response.statusCode() != 200) { - throw new RuntimeException( - "OIDC discovery failed with status " + response.statusCode()); - } - var jwksUri = response.bodyAsJsonObject().getString("jwks_uri"); - if (jwksUri == null || jwksUri.isBlank()) { - throw new IllegalStateException("OIDC discovery document missing jwks_uri"); - } - log.debug("Discovered JWKS URI: {}", jwksUri); - return jwksUri; - }); - } - - private static Future> fetchJwks(WebClient webClient, String jwksUri) { - log.debug("Fetching JWKS from: {}", jwksUri); - - return webClient - .getAbs(jwksUri) - .send() - .map( - response -> { - if (response.statusCode() != 200) { - throw new RuntimeException( - "JWKS fetch failed with status " + response.statusCode()); - } - var keys = response.bodyAsJsonObject().getJsonArray("keys"); - if (keys == null || keys.isEmpty()) { - throw new IllegalStateException("JWKS document has no keys"); - } - log.debug("Fetched {} keys from JWKS", keys.size()); - return convertToJwkOptions(keys); - }); - } - - private static List convertToJwkOptions(JsonArray keys) { - var result = new ArrayList(); - for (int i = 0; i < keys.size(); i++) { - var source = keys.getJsonObject(i); - var jwk = new JsonObject(); - for (var key : JWK_KEYS) { - if (source.containsKey(key)) { - jwk.put(key, source.getValue(key)); - } - } - result.add(jwk); - } - return result; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/OAuthDiscoveryHandler.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/OAuthDiscoveryHandler.java deleted file mode 100644 index 04a4a3bbe0..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/OAuthDiscoveryHandler.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.auth; - -import com.datasqrl.graphql.config.OAuthConfig; -import com.datasqrl.graphql.config.ServerConfig; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.RoutingContext; -import java.util.List; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -/** - * Handler for OAuth 2.0 Protected Resource Metadata (RFC 9728). This enables MCP clients like - * Claude Code to discover the OAuth authorization server. - */ -@Slf4j -@RequiredArgsConstructor -public class OAuthDiscoveryHandler { - - private static final List DEFAULT_SCOPES = List.of("mcp:tools", "mcp:resources"); - - private final ServerConfig config; - - /** - * Registers OAuth discovery routes on the router. Only registers if OAuth configuration is - * present. - */ - public void registerRoutes(Router router) { - var oauthConfig = config.getOauthConfig(); - if (oauthConfig == null || oauthConfig.getSite() == null) { - log.debug("OAuth config not present or site not configured, skipping discovery endpoints"); - return; - } - - log.info("Registering OAuth discovery endpoints for site: {}", oauthConfig.getSite()); - - router - .get("/.well-known/oauth-protected-resource") - .handler(this::handleProtectedResourceMetadata); - } - - /** - * Returns OAuth 2.0 Protected Resource Metadata per RFC 9728. This tells MCP clients which - * authorization server to use. - */ - private void handleProtectedResourceMetadata(RoutingContext ctx) { - var oauthConfig = config.getOauthConfig(); - - var resource = determineResourceUri(ctx, oauthConfig); - var authServer = oauthConfig.getEffectiveAuthorizationServer(); - var scopes = getScopes(oauthConfig); - - var metadata = - new JsonObject() - .put("resource", resource) - .put("authorization_servers", new JsonArray().add(authServer)) - .put("scopes_supported", new JsonArray(scopes)) - .put("bearer_methods_supported", new JsonArray().add("header")); - - log.debug("Returning protected resource metadata: {}", metadata.encodePrettily()); - - ctx.response() - .putHeader("Content-Type", "application/json") - .putHeader("Cache-Control", "public, max-age=3600") - .end(metadata.encode()); - } - - private List getScopes(OAuthConfig oauthConfig) { - if (oauthConfig.getScopesSupported() != null) { - return oauthConfig.getScopesSupported(); - } - return DEFAULT_SCOPES; - } - - private String determineResourceUri(RoutingContext ctx, OAuthConfig oauthConfig) { - if (oauthConfig.getResource() != null && !oauthConfig.getResource().isBlank()) { - return oauthConfig.getResource(); - } - - var host = ctx.request().getHeader("Host"); - var forwardedProto = ctx.request().getHeader("X-Forwarded-Proto"); - var scheme = forwardedProto != null ? forwardedProto : ctx.request().scheme(); - var mcpEndpoint = config.getServletConfig().getMcpEndpoint("v1"); - - return scheme + "://" + host + mcpEndpoint; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/OAuthFailureHandler.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/OAuthFailureHandler.java deleted file mode 100644 index fd3fcc1784..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/auth/OAuthFailureHandler.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.auth; - -import com.datasqrl.graphql.config.ServerConfig; -import io.vertx.core.Handler; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.RoutingContext; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.exception.ExceptionUtils; - -/** - * OAuth-aware authentication failure handler. Returns 401 with WWW-Authenticate header for OAuth - * discovery (RFC 9728). - */ -@Slf4j -@RequiredArgsConstructor -public class OAuthFailureHandler implements Handler { - - private final ServerConfig config; - - @Override - public void handle(RoutingContext ctx) { - var throwable = ctx.failure(); - var statusCode = ctx.statusCode(); - var errorCause = - throwable == null ? "Unknown error" : ExceptionUtils.getRootCauseMessage(throwable); - - log.warn( - "Authentication failed for request {} {}: {} (status: {})", - ctx.request().method(), - ctx.request().path(), - errorCause, - statusCode); - - if (throwable != null) { - log.debug("Authentication failure details", throwable); - } - - var response = - ctx.response() - .setStatusCode(statusCode != -1 ? statusCode : 401) - .putHeader("Content-Type", "application/json"); - - var oauthConfig = config.getOauthConfig(); - if (oauthConfig != null && oauthConfig.getSite() != null) { - var resourceMetadataUrl = buildResourceMetadataUrl(ctx); - response.putHeader( - "WWW-Authenticate", - String.format("Bearer resource_metadata=\"%s\"", resourceMetadataUrl)); - } - - var json = new JsonObject().put("error", "authentication_required").put("cause", errorCause); - - response.end(json.encode()); - } - - private String buildResourceMetadataUrl(RoutingContext ctx) { - var host = ctx.request().getHeader("Host"); - var forwardedProto = ctx.request().getHeader("X-Forwarded-Proto"); - var scheme = forwardedProto != null ? forwardedProto : ctx.request().scheme(); - - return scheme + "://" + host + "/.well-known/oauth-protected-resource"; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/CorsHandlerOptions.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/CorsHandlerOptions.java deleted file mode 100644 index c3bbbce86e..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/CorsHandlerOptions.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@Getter -@Setter -@NoArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public class CorsHandlerOptions { - - private String allowedOrigin; - private List allowedOrigins; - private boolean allowCredentials = false; - private Integer maxAgeSeconds = -1; - private boolean allowPrivateNetwork = false; - private Set allowedMethods = new LinkedHashSet<>(); - private Set allowedHeaders = new LinkedHashSet<>(); - private Set exposedHeaders = new LinkedHashSet<>(); -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/JdbcConfig.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/JdbcConfig.java deleted file mode 100644 index 89a6131847..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/JdbcConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import static com.google.common.base.Preconditions.checkNotNull; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@Getter -@Setter -@NoArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public class JdbcConfig { - - private String url; - - public void validateConfig() { - checkNotNull(url, "The 'url' config must be set"); - } - - @Getter - @Setter - @NoArgsConstructor - public static class DuckDbConfig extends JdbcConfig { - - @JsonProperty("use-disk-cache") - private boolean useDiskCache; - - @JsonProperty("use-version-guessing") - private boolean useVersionGuessing; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/KafkaConfig.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/KafkaConfig.java deleted file mode 100644 index ab26b99673..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/KafkaConfig.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import static com.google.common.base.Preconditions.checkNotNull; -import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.TRANSACTIONAL_ID_CONFIG; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@NoArgsConstructor -@AllArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public abstract class KafkaConfig { - - protected Map config = new HashMap<>(); - - /** Validates that required configuration is present. Should be called after deserialization. */ - public void validateConfig() { - checkNotNull( - config.get(BOOTSTRAP_SERVERS_CONFIG), - "The '%s' config must be provided".formatted(BOOTSTRAP_SERVERS_CONFIG)); - } - - @JsonAnyGetter - public Map getConfig() { - return config; - } - - @JsonAnySetter - public void setConfig(String key, String value) { - config.put(key, value); - } - - @Getter - @Setter - @NoArgsConstructor - public static class KafkaSubscriptionConfig extends KafkaConfig { - - public KafkaSubscriptionConfig(Map config) { - super(config); - } - - public Map asMap() { - var finalConfig = new HashMap<>(config); - finalConfig.put(GROUP_ID_CONFIG, UUID.randomUUID().toString()); - - return Collections.unmodifiableMap(finalConfig); - } - } - - @Getter - @Setter - @NoArgsConstructor - public static class KafkaMutationConfig extends KafkaConfig { - - public KafkaMutationConfig(Map config) { - super(config); - } - - public Map asMap(boolean transactional) { - var finalConfig = new HashMap<>(config); - - if (transactional) { - finalConfig.put(TRANSACTIONAL_ID_CONFIG, "sqrl-mutation-" + UUID.randomUUID()); - finalConfig.put(ENABLE_IDEMPOTENCE_CONFIG, "true"); - } - - return Collections.unmodifiableMap(finalConfig); - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/OAuthConfig.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/OAuthConfig.java deleted file mode 100644 index 471feca67b..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/OAuthConfig.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonSetter; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.auth.oauth2.OAuth2Options; -import java.util.List; -import java.util.Map; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -/** - * OAuth 2.0 configuration combining Vert.x OAuth2Options with discovery metadata. This allows - * reusing Vert.x's OAuth2Options for authentication while adding fields needed for RFC 9728 - * Protected Resource Metadata. - */ -@Getter -@Setter -@NoArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public class OAuthConfig { - - /** Vert.x OAuth2Options for authentication provider creation. */ - private OAuth2Options oauth2Options; - - /** - * External authorization server URL for discovery metadata. Use when the internal OAuth2Options - * site URL is not reachable by external clients (e.g., Docker internal hostname vs localhost). - */ - private String authorizationServerUrl; - - /** Scopes supported by this resource server. */ - private List scopesSupported = List.of("mcp:tools", "mcp:resources"); - - /** Resource identifier for this server (derived from request if not set). */ - private String resource; - - @JsonSetter("oauth2Options") - public void setOauth2OptionsFromJson(Map options) { - this.oauth2Options = options == null ? null : new OAuth2Options(new JsonObject(options)); - } - - /** Returns the site URL from OAuth2Options, or null if not configured. */ - public String getSite() { - return oauth2Options != null ? oauth2Options.getSite() : null; - } - - /** Returns the authorization server URL for discovery, defaulting to site if not set. */ - public String getEffectiveAuthorizationServer() { - if (authorizationServerUrl != null && !authorizationServerUrl.isBlank()) { - return authorizationServerUrl; - } - var site = getSite(); - if (site == null) { - return null; - } - return site.endsWith("/") ? site : site + "/"; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/ServerConfig.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/ServerConfig.java deleted file mode 100644 index bc961182cd..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/ServerConfig.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import static com.datasqrl.graphql.SqrlObjectMapper.MAPPER; - -import com.datasqrl.env.EnvVariableNames; -import com.datasqrl.env.GlobalEnvironmentStore; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonSetter; -import io.vertx.core.http.HttpServerOptions; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.auth.jwt.JWTAuthOptions; -import io.vertx.ext.web.handler.graphql.GraphQLHandlerOptions; -import io.vertx.ext.web.handler.graphql.GraphiQLHandlerOptions; -import io.vertx.pgclient.PgConnectOptions; -import io.vertx.sqlclient.PoolOptions; -import java.util.Map; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@Getter -@Setter -@NoArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public class ServerConfig { - - private ServletConfig servletConfig = new ServletConfig(); - private GraphQLHandlerOptions graphQLHandlerOptions = new GraphQLHandlerOptions(); - private GraphiQLHandlerOptions graphiQLHandlerOptions; - private HttpServerOptions httpServerOptions = new HttpServerOptions(); - private PgConnectOptions pgConnectOptions = new PgConnectOptions(); - private PoolOptions poolOptions = new PoolOptions(); - private CorsHandlerOptions corsHandlerOptions = new CorsHandlerOptions(); - private SwaggerConfig swaggerConfig = new SwaggerConfig(); - private JWTAuthOptions jwtAuth; - private OAuthConfig oauthConfig; - - private KafkaConfig.KafkaMutationConfig kafkaMutationConfig; - private KafkaConfig.KafkaSubscriptionConfig kafkaSubscriptionConfig; - private JdbcConfig.DuckDbConfig duckDbConfig; - private JdbcConfig snowflakeConfig; - - /** - * Validates and applies post-deserialization logic to this ServerConfig instance. This method - * should be called after any Jackson deserialization. - * - * @return this ServerConfig instance for method chaining - */ - public ServerConfig validated() { - // Apply PostgreSQL port from environment variable if set - var pgPort = GlobalEnvironmentStore.get(EnvVariableNames.POSTGRES_PORT); - try { - if (pgPort != null && !pgPort.isEmpty()) { - pgConnectOptions.setPort(Integer.parseInt(pgPort)); - } - } catch (NumberFormatException ignored) { - // Ignore invalid port numbers - } - - // Validate Kafka configs if present - if (kafkaMutationConfig != null) { - kafkaMutationConfig.validateConfig(); - } - if (kafkaSubscriptionConfig != null) { - kafkaSubscriptionConfig.validateConfig(); - } - - // Validate DuckDB config if present - if (duckDbConfig != null) { - duckDbConfig.validateConfig(); - } - - return this; - } - - //////////////////////////////////////////////////////////////////////////////// - // Custom JSON setters for Jackson deserialization of Vert.x classes - //////////////////////////////////////////////////////////////////////////////// - - @JsonSetter("graphQLHandlerOptions") - public void setGraphQLHandlerOptionsFromJson(Map options) { - this.graphQLHandlerOptions = new GraphQLHandlerOptions(getJsonObjectOrEmpty(options)); - } - - @JsonSetter("httpServerOptions") - public void setHttpServerOptionsFromJson(Map options) { - this.httpServerOptions = new HttpServerOptions(getJsonObjectOrEmpty(options)); - } - - @JsonSetter("pgConnectOptions") - public void setPgConnectOptionsFromJson(Map options) { - this.pgConnectOptions = new PgConnectOptions(getJsonObjectOrEmpty(options)); - } - - @JsonSetter("poolOptions") - public void setPoolOptionsFromJson(Map options) { - this.poolOptions = new PoolOptions(getJsonObjectOrEmpty(options)); - } - - @JsonSetter("graphiQLHandlerOptions") - public void setGraphiQLHandlerOptionsFromJson(Map options) { - this.graphiQLHandlerOptions = - options == null ? null : new GraphiQLHandlerOptions(new JsonObject(options)); - } - - @JsonSetter("jwtAuth") - public void setJwtAuthFromJson(Map options) { - this.jwtAuth = options == null ? null : new JWTAuthOptions(new JsonObject(options)); - } - - //////////////////////////////////////////////////////////////////////////////// - // Custom JSON setters for Jackson deserialization of our own POJO classes - //////////////////////////////////////////////////////////////////////////////// - - @JsonSetter("servletConfig") - public void setServletConfigFromJson(Map options) { - this.servletConfig = - options == null ? new ServletConfig() : MAPPER.convertValue(options, ServletConfig.class); - } - - @JsonSetter("corsHandlerOptions") - public void setCorsHandlerOptionsFromJson(Map options) { - this.corsHandlerOptions = - options == null - ? new CorsHandlerOptions() - : MAPPER.convertValue(options, CorsHandlerOptions.class); - } - - @JsonSetter("swaggerConfig") - public void setSwaggerConfigFromJson(Map options) { - this.swaggerConfig = - options == null ? new SwaggerConfig() : MAPPER.convertValue(options, SwaggerConfig.class); - } - - private JsonObject getJsonObjectOrEmpty(Map options) { - return options == null ? new JsonObject() : new JsonObject(options); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/ServerConfigUtil.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/ServerConfigUtil.java deleted file mode 100644 index 7f3a3d3140..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/ServerConfigUtil.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import static com.datasqrl.graphql.SqrlObjectMapper.MAPPER; -import static org.apache.commons.lang3.ObjectUtils.isEmpty; - -import com.datasqrl.util.JsonMergeUtils; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.vertx.ext.web.handler.graphql.GraphiQLHandlerOptions; -import jakarta.annotation.Nullable; -import java.util.Map; -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; - -@UtilityClass -@Slf4j -public class ServerConfigUtil { - - /** - * Merges server configuration with override values using Jackson. - * - * @param serverConfig base server configuration - * @param configOverrides map of configuration overrides - * @return merged and validated server configuration - */ - @SneakyThrows - public static ServerConfig mergeConfigs( - ServerConfig serverConfig, Map configOverrides) { - if (isEmpty(configOverrides)) { - return serverConfig; - } - var config = ((ObjectNode) MAPPER.valueToTree(serverConfig)).deepCopy(); - JsonMergeUtils.merge(config, MAPPER.valueToTree(configOverrides)); - return MAPPER.treeToValue(config, ServerConfig.class).validated(); - } - - /** - * Creates a ServerConfig from a configuration map using Jackson deserialization. - * - * @param configMap map containing configuration values - * @return deserialized and validated ServerConfig instance - */ - @SneakyThrows - public static ServerConfig fromConfigMap(Map configMap) { - return MAPPER.convertValue(configMap, ServerConfig.class).validated(); - } - - /** - * Creates a copy of GraphiQL handler options with versioned URIs. - * - * @param version the version prefix to prepend to endpoints - * @param graphiQLHandlerOptions the original options to copy and version - * @return a new GraphiQL handler options instance with versioned endpoints, or null if input is - * null - */ - @Nullable - public static GraphiQLHandlerOptions createVersionedGraphiQLHandlerOptions( - String version, @Nullable GraphiQLHandlerOptions graphiQLHandlerOptions) { - if (graphiQLHandlerOptions == null) { - return null; - } - - var versionedOptions = new GraphiQLHandlerOptions(graphiQLHandlerOptions); - var versionedGraphQLUri = getVersionedEndpoint(version, versionedOptions.getGraphQLUri()); - versionedOptions.setGraphQLUri(versionedGraphQLUri); - - var versionedGraphQLWSUri = getVersionedEndpoint(version, versionedOptions.getGraphQLWSUri()); - versionedOptions.setGraphWSQLUri(versionedGraphQLWSUri); - - return versionedOptions; - } - - /** - * Prepends a version prefix to an endpoint path. Assumes {@code endpoint} start with '/'. - * - * @param version the version prefix to prepend - * @param endpoint the endpoint path to version - * @return the versioned endpoint as "/{version}{endpoint}", or null if endpoint is null - */ - @Nullable - public static String getVersionedEndpoint(String version, @Nullable String endpoint) { - return endpoint == null ? null : '/' + version + endpoint; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/ServletConfig.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/ServletConfig.java deleted file mode 100644 index 19ab36abd5..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/ServletConfig.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import static com.datasqrl.graphql.config.ServerConfigUtil.getVersionedEndpoint; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@Getter -@Setter -@NoArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public class ServletConfig { - - private String graphiQLEndpoint = "/graphiql*"; - private String graphQLEndpoint = "/graphql"; - private String restEndpoint = "/rest"; - private String mcpEndpoint = "/mcp"; - private boolean usePgPool = true; - - public String getGraphiQLEndpoint(String version) { - return getVersionedEndpoint(version, graphiQLEndpoint); - } - - public String getGraphQLEndpoint(String version) { - return getVersionedEndpoint(version, graphQLEndpoint); - } - - public String getRestEndpoint(String version) { - return getVersionedEndpoint(version, restEndpoint); - } - - public String getMcpEndpoint(String version) { - return getVersionedEndpoint(version, mcpEndpoint); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/SwaggerConfig.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/SwaggerConfig.java deleted file mode 100644 index 3a5ebd3df3..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/config/SwaggerConfig.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import static com.datasqrl.graphql.config.ServerConfigUtil.getVersionedEndpoint; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -@Getter -@Setter -@NoArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public class SwaggerConfig { - - private boolean enabled = true; - private String endpoint = "/swagger"; - private String uiEndpoint = "/swagger-ui"; - private String title = "DataSQRL REST API"; - private String description = "Auto-generated REST API documentation for DataSQRL endpoints"; - private String version = "1.0.0"; - private String contact = "DataSQRL"; - private String contactUrl = "https://datasqrl.com"; - private String contactEmail = "contact@datasqrl.com"; - private String license = "Apache License 2.0"; - private String licenseUrl = "https://www.apache.org/licenses/LICENSE-2.0"; - - public String getEndpoint(String version) { - return getVersionedEndpoint(version, endpoint); - } - - public String getUiEndpoint(String version) { - return getVersionedEndpoint(version, uiEndpoint); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/FlinkFunctionExecutor.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/FlinkFunctionExecutor.java deleted file mode 100644 index 38675ad8cf..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/FlinkFunctionExecutor.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.exec; - -import com.datasqrl.graphql.server.FunctionExecutor; -import graphql.schema.DataFetchingEnvironment; -import io.vertx.core.Vertx; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; -import org.apache.flink.table.data.GenericRowData; -import org.apache.flink.table.types.logical.RowType; - -@RequiredArgsConstructor -public class FlinkFunctionExecutor implements FunctionExecutor { - - private final Vertx vertx; - private final Optional optPlan; - - @Override - public CompletableFuture execute(DataFetchingEnvironment env, String functionId) { - var plan = optPlan.orElseThrow(() -> new IllegalStateException("Exec function plan not found")); - var fn = - plan.getFunction(functionId) - .orElseThrow( - () -> new IllegalArgumentException("Function " + functionId + " not found")); - - var inputType = fn.getInputType(); - validateInputFields(env, inputType); - - // Execute blocking function operations on Vert.x worker pool - return vertx - .executeBlocking( - () -> { - fn.instantiateFunction(Thread.currentThread().getContextClassLoader()); - - var mapper = new RowDataMapper(inputType); - var rowData = mapper.toRowData(env.getArguments()); - - var internalRes = fn.execute(rowData); - var res = mapper.fromRowData((GenericRowData) internalRes); - - return fn.isListOutput() ? res : res.get(0); - }, - false) // false -> unordered execution (better concurrency) - .toCompletionStage() - .toCompletableFuture(); - } - - private void validateInputFields(DataFetchingEnvironment env, RowType inputType) { - var missingFields = - inputType.getFieldNames().stream() - .filter(fieldName -> !env.getArguments().containsKey(fieldName)) - .collect(Collectors.toList()); - - if (!missingFields.isEmpty()) { - throw new IllegalArgumentException( - "Cannot execute function. Missing required input fields: " - + String.join(", ", missingFields)); - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/RowDataMapper.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/RowDataMapper.java deleted file mode 100644 index 4195f46f2c..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/exec/RowDataMapper.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.exec; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import org.apache.flink.table.data.GenericRowData; -import org.apache.flink.table.data.conversion.DataStructureConverter; -import org.apache.flink.table.data.conversion.DataStructureConverters; -import org.apache.flink.table.types.logical.RowType; -import org.apache.flink.table.types.utils.TypeConversions; - -/** - * Bidirectional mapper between external Java objects and Flink's internal RowData representation. - */ -public class RowDataMapper { - - private final RowType rowType; - private final List> fieldConverters; - - /** - * Creates a new RowDataMapper for the specified row type schema. - * - *

Initializes field converters for each column in the row type, preparing them for - * bidirectional conversion between external and internal representations. - * - * @param rowType the Flink logical row type defining the schema and data types for conversion - */ - public RowDataMapper(RowType rowType) { - this.rowType = rowType; - var rowDataType = TypeConversions.fromLogicalToDataType(rowType); - - this.fieldConverters = new ArrayList<>(rowType.getFieldCount()); - for (int i = 0; i < rowType.getFieldCount(); i++) { - var fieldDataType = rowDataType.getChildren().get(i); - var converter = DataStructureConverters.getConverter(fieldDataType); - converter.open(getClass().getClassLoader()); - - fieldConverters.add(converter); - } - } - - /** - * Converts a map of field names to values into Flink's internal RowData representation. - * - *

This method transforms external Java objects (e.g., from GraphQL arguments) into Flink's - * internal data format by applying appropriate type converters to each field value based on the - * row type schema. - * - * @param args a map where keys are field names and values are the external representation of - * field values - * @return a GenericRowData containing the converted internal representation of all fields - */ - public GenericRowData toRowData(Map args) { - var rowData = new GenericRowData(rowType.getFieldCount()); - - for (int i = 0; i < rowType.getFieldCount(); i++) { - var fieldName = rowType.getFieldNames().get(i); - var external = args.get(fieldName); - var internal = fieldConverters.get(i).toInternalOrNull(external); - - rowData.setField(i, internal); - } - - return rowData; - } - - /** - * Converts Flink's internal RowData representation back to a list of external Java objects. - * - *

This method transforms Flink's internal data format back into external Java objects by - * applying appropriate type converters to each field. The resulting list maintains the same field - * order as defined in the row type schema. - * - * @param rowData the internal Flink row data to convert - * @return a list of external values in the same order as the row type field definitions - */ - public List fromRowData(GenericRowData rowData) { - var res = new ArrayList<>(rowData.getArity()); - - for (int i = 0; i < rowData.getArity(); i++) { - var internal = rowData.getField(i); - var external = fieldConverters.get(i).toExternalOrNull(internal); - - res.add(external); - } - - return res; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/io/SinkConsumer.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/io/SinkConsumer.java deleted file mode 100644 index f879e4f02c..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/io/SinkConsumer.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.io; - -import java.util.function.Consumer; - -/** - * Consuming records from a sink (kafka topic or postGreSQL table). It is a sink from the SQRL - * pipeline perspective. - */ -public interface SinkConsumer { - - void listen( - Consumer listener, Consumer errorHandler, Consumer endOfStream); -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/io/SinkProducer.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/io/SinkProducer.java deleted file mode 100644 index 9b9e4b5590..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/io/SinkProducer.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.io; - -import io.vertx.core.Future; -import java.util.Map; - -/** Sending records to a sink (such as a Kafka topic) */ -public interface SinkProducer { - public Future send(Map entry); -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/io/SinkResult.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/io/SinkResult.java deleted file mode 100644 index c5dae8b3ad..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/io/SinkResult.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.io; - -import java.time.Instant; - -public record SinkResult(Instant sourceTime) {} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/jdbc/JdbcClientsConfig.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/jdbc/JdbcClientsConfig.java deleted file mode 100644 index c284fe38b7..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/jdbc/JdbcClientsConfig.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.jdbc; - -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.util.DuckDbExtensions; -import com.google.common.collect.ImmutableMap; -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; -import io.vertx.core.Vertx; -import io.vertx.jdbcclient.JDBCConnectOptions; -import io.vertx.jdbcclient.JDBCPool; -import io.vertx.sqlclient.Pool; -import io.vertx.sqlclient.PoolOptions; -import io.vertx.sqlclient.SqlClient; -import java.util.Map; -import java.util.Optional; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -/** - * Configuration class responsible for creating and managing database clients for different database - * types (PostgreSQL, DuckDB, Snowflake). - */ -@RequiredArgsConstructor -@Slf4j -public class JdbcClientsConfig { - - private final Vertx vertx; - private final ServerConfig config; - - /** Creates a map of database clients for all configured database types. */ - public Map createClients() { - var clientsMapBuilder = ImmutableMap.builder(); - - // PostgreSQL is always required and initializes synchronously - clientsMapBuilder.put(DatabaseType.POSTGRES, initPostgresSqlClient()); - - // DuckDB initializes synchronously - initDuckdbSqlClient().ifPresent(client -> clientsMapBuilder.put(DatabaseType.DUCKDB, client)); - - // Snowflake initializes synchronously - initSnowflakeClient() - .ifPresent(client -> clientsMapBuilder.put(DatabaseType.SNOWFLAKE, client)); - - return clientsMapBuilder.build(); - } - - @SneakyThrows - private Optional initSnowflakeClient() { - var snowflakeConf = config.getSnowflakeConfig(); - if (snowflakeConf == null) { - return Optional.empty(); - } - - // No need for Class.forName() - Snowflake JDBC driver auto-registers via JDBC 4.0+ - // ServiceLoader - var url = snowflakeConf.getUrl(); - url += "?CLIENT_SESSION_KEEP_ALIVE=true"; - - return Optional.of( - JDBCPool.pool( - vertx, - new JDBCConnectOptions().setJdbcUrl(url).setMetricsName("snowflake"), - new PoolOptions(this.config.getPoolOptions()).setName("snowflake-pool"))); - } - - @SneakyThrows - private Optional initDuckdbSqlClient() { - var duckDbConf = config.getDuckDbConfig(); - if (duckDbConf == null) { - return Optional.empty(); - } - - // No need for Class.forName() - DuckDB JDBC driver auto-registers via JDBC 4.0+ ServiceLoader - var url = duckDbConf.getUrl(); - var extensions = new DuckDbExtensions(duckDbConf); - var initSql = extensions.buildInitSql(); - - var hikariCfg = new HikariConfig(); - hikariCfg.setJdbcUrl(url); - initSql.ifPresent( - isql -> { - log.debug("DuckDB init SQL: {}", isql); - hikariCfg.setConnectionInitSql(isql); - }); - - var poolOptions = - new PoolOptions(this.config.getPoolOptions()) - .setName("duckdb-pool") - .setMaxSize(1) - .setMaxLifetime(0) - .setIdleTimeout(0); - - return Optional.of(JDBCPool.pool(vertx, new HikariDataSource(hikariCfg), poolOptions)); - } - - private SqlClient initPostgresSqlClient() { - var poolOptions = new PoolOptions(this.config.getPoolOptions()).setName("postgres-pool"); - // Note: setPipelined() method was removed in Vert.x 5, pipelining is now always enabled - return Pool.pool(vertx, this.config.getPgConnectOptions(), poolOptions); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/jdbc/VertxJdbcClient.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/jdbc/VertxJdbcClient.java deleted file mode 100644 index c56853e174..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/jdbc/VertxJdbcClient.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.jdbc; - -import com.datasqrl.graphql.VertxContext; -import com.datasqrl.graphql.server.Context; -import com.datasqrl.graphql.server.RootGraphqlModel.PreparedSqrlQuery; -import com.datasqrl.graphql.server.RootGraphqlModel.ResolvedQuery; -import com.datasqrl.graphql.server.RootGraphqlModel.ResolvedSqlQuery; -import com.datasqrl.graphql.server.RootGraphqlModel.SqlQuery; -import io.vertx.core.Future; -import io.vertx.sqlclient.PreparedQuery; -import io.vertx.sqlclient.Row; -import io.vertx.sqlclient.RowSet; -import io.vertx.sqlclient.SqlClient; -import io.vertx.sqlclient.Tuple; -import java.util.Map; -import lombok.extern.slf4j.Slf4j; - -/** - * Purpose: Manages SQL clients and executes queries. Collaboration: Used by {@link VertxContext} to - * prepare and execute SQL queries. - */ -@Slf4j -public record VertxJdbcClient(Map clients) implements JdbcClient { - @Override - public ResolvedQuery prepareQuery(SqlQuery query, Context context) { - var sqlClient = clients.get(query.getDatabase()); - if (sqlClient == null) { - throw new RuntimeException("Could not find database engine: " + query.getDatabase()); - } - - var preparedQuery = sqlClient.preparedQuery(query.getSql()); - - return new ResolvedSqlQuery(query, new PreparedSqrlQueryImpl(preparedQuery)); - } - - @Override - public ResolvedQuery unpreparedQuery(SqlQuery sqlQuery, Context context) { - return new ResolvedSqlQuery(sqlQuery, null); - } - - public Future> execute(PreparedQuery> query, Tuple tup) { - return query.execute(tup); - } - - public Future> execute(DatabaseType database, String query, Tuple tup) { - var sqlClient = clients.get(database); - - return execute(sqlClient.preparedQuery(query), tup); - } - - public record PreparedSqrlQueryImpl(PreparedQuery> preparedQuery) - implements PreparedSqrlQuery>> {} -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/JsonDeserializer.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/JsonDeserializer.java deleted file mode 100644 index 0cd44bff98..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/JsonDeserializer.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.kafka; - -import com.datasqrl.graphql.SqrlObjectMapper; -import com.fasterxml.jackson.core.JsonProcessingException; -import java.io.UnsupportedEncodingException; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Deserializer; - -public class JsonDeserializer implements Deserializer { - private String encoding; - - public JsonDeserializer() { - this.encoding = StandardCharsets.UTF_8.name(); - } - - @Override - public void configure(Map configs, boolean isKey) { - var propertyName = isKey ? "key.deserializer.encoding" : "value.deserializer.encoding"; - Object encodingValue = configs.get(propertyName); - if (encodingValue == null) { - encodingValue = configs.get("deserializer.encoding"); - } - - if (encodingValue instanceof String string) { - this.encoding = string; - } - } - - @Override - public Map deserialize(String topic, byte[] data) { - try { - return data == null - ? null - : SqrlObjectMapper.MAPPER.readValue(new String(data, this.encoding), Map.class); - } catch (UnsupportedEncodingException var4) { - throw new SerializationException( - "Error when deserializing byte[] to string due to unsupported encoding " + this.encoding); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/JsonSerializer.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/JsonSerializer.java deleted file mode 100644 index cb5b35f76f..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/JsonSerializer.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.kafka; - -import com.datasqrl.graphql.SqrlObjectMapper; -import com.fasterxml.jackson.core.JsonProcessingException; -import java.io.UnsupportedEncodingException; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Serializer; - -public class JsonSerializer implements Serializer { - private String encoding; - - public JsonSerializer() { - this.encoding = StandardCharsets.UTF_8.name(); - } - - @Override - public void configure(Map configs, boolean isKey) { - var propertyName = isKey ? "key.serializer.encoding" : "value.serializer.encoding"; - Object encodingValue = configs.get(propertyName); - if (encodingValue == null) { - encodingValue = configs.get("serializer.encoding"); - } - - if (encodingValue instanceof String string) { - this.encoding = string; - } - } - - @Override - public byte[] serialize(String topic, Map data) { - try { - return data == null - ? null - : SqrlObjectMapper.MAPPER.writeValueAsString(data).getBytes(this.encoding); - } catch (UnsupportedEncodingException var4) { - throw new SerializationException( - "Error when serializing string to byte[] due to unsupported encoding " + this.encoding); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/KafkaDataFetcherFactory.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/KafkaDataFetcherFactory.java deleted file mode 100644 index eeac906ebe..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/KafkaDataFetcherFactory.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.kafka; - -import com.datasqrl.graphql.exec.StandardExecutionContext; -import com.datasqrl.graphql.io.SinkConsumer; -import com.datasqrl.graphql.server.Context; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.RootGraphqlModel.KafkaSubscriptionCoords; -import graphql.schema.DataFetcher; -import graphql.schema.DataFetchingEnvironment; -import io.vertx.core.json.JsonObject; -import java.util.Map; -import java.util.stream.Collectors; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -public class KafkaDataFetcherFactory { - - public static DataFetcher create( - SinkConsumer consumer, KafkaSubscriptionCoords coords, Context context) { - var deferredFlux = - Flux.create(sink -> consumer.listen(sink::next, sink::error, (x) -> sink.complete())) - .share(); - - return new DataFetcher<>() { - @Override - public Publisher get(DataFetchingEnvironment env) throws Exception { - var execContext = - new StandardExecutionContext( - context, - env, - RootGraphqlModel.VariableArgument.convertArguments(env.getArguments())); - Map fieldNameToValue = - coords.getEqualityConditions().entrySet().stream() - .collect( - Collectors.toMap( - Map.Entry::getKey, - entry -> entry.getValue().accept(execContext, execContext))); - return deferredFlux.filter(entry -> filterSubscription(entry, fieldNameToValue)); - } - - private boolean filterSubscription(Object data, Map fieldNameToValue) { - for (Map.Entry filter : fieldNameToValue.entrySet()) { - var argValue = filter.getValue(); - if (argValue == null) { - return false; - } - - Map objectMap; - if (data instanceof Map map) { - objectMap = map; - } else if (data instanceof JsonObject object) { - objectMap = object.getMap(); - } else { - objectMap = Map.of(); - } - - var retrievedData = objectMap.get(filter.getKey()); - if (!argValue.equals(retrievedData)) { - return false; - } - } - - return true; - } - }; - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/KafkaSinkConsumer.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/KafkaSinkConsumer.java deleted file mode 100644 index 5da5417df8..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/KafkaSinkConsumer.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.kafka; - -import com.datasqrl.graphql.io.SinkConsumer; -import io.vertx.kafka.client.consumer.KafkaConsumer; -import java.util.function.Consumer; -import lombok.AllArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@AllArgsConstructor -public class KafkaSinkConsumer implements SinkConsumer { - - final KafkaConsumer consumer; - - @Override - public void listen( - Consumer listener, Consumer errorHandler, Consumer endOfStream) { - consumer - .handler( - kafkaRecord -> { - try { - Object result = kafkaRecord.value(); - listener.accept(result); - } catch (Exception e) { - errorHandler.accept(e); - } - }) - .exceptionHandler(errorHandler::accept) - .endHandler(endOfStream::accept); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/KafkaSinkProducer.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/KafkaSinkProducer.java deleted file mode 100644 index 3f482c1315..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/kafka/KafkaSinkProducer.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.kafka; - -import com.datasqrl.graphql.io.SinkProducer; -import com.datasqrl.graphql.io.SinkResult; -import io.vertx.core.Future; -import io.vertx.kafka.client.producer.KafkaProducer; -import io.vertx.kafka.client.producer.KafkaProducerRecord; -import io.vertx.kafka.client.producer.RecordMetadata; -import java.time.Instant; -import java.util.Map; -import lombok.AllArgsConstructor; - -@AllArgsConstructor -public class KafkaSinkProducer implements SinkProducer { - - private final String topic; - private final KafkaProducer kafkaProducer; - - @Override - public Future send(Map entry) { - final KafkaProducerRecord producerRecord; - - try { - producerRecord = KafkaProducerRecord.create(topic, entry); - } catch (Exception e) { - return Future.failedFuture(e); - } - // TODO: generate UUID server side - return kafkaProducer - .send(producerRecord) - .map( - result -> - new SinkResult(Instant.ofEpochMilli(((RecordMetadata) result).getTimestamp()))); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/swagger/SwaggerService.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/swagger/SwaggerService.java deleted file mode 100644 index f7a1565563..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/graphql/swagger/SwaggerService.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.swagger; - -import com.datasqrl.graphql.config.SwaggerConfig; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.operation.ApiOperation; -import com.datasqrl.graphql.server.operation.FunctionDefinition; -import com.datasqrl.graphql.server.operation.RestMethodType; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.swagger.v3.core.util.Json; -import io.swagger.v3.oas.models.OpenAPI; -import io.swagger.v3.oas.models.Operation; -import io.swagger.v3.oas.models.PathItem; -import io.swagger.v3.oas.models.Paths; -import io.swagger.v3.oas.models.info.Contact; -import io.swagger.v3.oas.models.info.Info; -import io.swagger.v3.oas.models.info.License; -import io.swagger.v3.oas.models.media.Content; -import io.swagger.v3.oas.models.media.MediaType; -import io.swagger.v3.oas.models.media.Schema; -import io.swagger.v3.oas.models.parameters.Parameter; -import io.swagger.v3.oas.models.parameters.RequestBody; -import io.swagger.v3.oas.models.responses.ApiResponse; -import io.swagger.v3.oas.models.responses.ApiResponses; -import io.swagger.v3.oas.models.servers.Server; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.regex.Pattern; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ObjectUtils; - -@RequiredArgsConstructor -@Slf4j -public class SwaggerService { - - private static final Pattern QUERY_PARAMS_PATTERN = Pattern.compile("\\{\\?([^}]+)\\}"); - private static final Pattern PATH_PARAMS_PATTERN = Pattern.compile("\\{([^}?]+)\\}"); - private static final ObjectMapper objectMapper = Json.mapper(); - - private final SwaggerConfig swaggerConfig; - private final RootGraphqlModel model; - private final String modelVersion; - private final String restEndpoint; - - public String generateSwaggerJson(String requestHost) { - try { - var openAPI = createOpenAPI(requestHost); - return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(openAPI); - } catch (JsonProcessingException e) { - log.error("Failed to generate Swagger JSON", e); - return "{}"; - } - } - - private OpenAPI createOpenAPI(String requestHost) { - var openAPI = new OpenAPI(); - - // Set API info - var info = - new Info() - .title(swaggerConfig.getTitle()) - .description(swaggerConfig.getDescription()) - .version(swaggerConfig.getVersion()); - - if (swaggerConfig.getContact() != null) { - var contact = - new Contact() - .name(swaggerConfig.getContact()) - .url(swaggerConfig.getContactUrl()) - .email(swaggerConfig.getContactEmail()); - info.contact(contact); - } - - if (swaggerConfig.getLicense() != null) { - var license = - new License().name(swaggerConfig.getLicense()).url(swaggerConfig.getLicenseUrl()); - info.license(license); - } - - openAPI.info(info); - - // Add server based on request host - var serverUrl = requestHost != null ? requestHost : "http://localhost:8888"; - var server = new Server().url(serverUrl).description("DataSQRL API Server"); - openAPI.addServersItem(server); - - // Generate paths from REST operations - var paths = new Paths(); - for (ApiOperation operation : model.getOperations()) { - if (operation.isRestEndpoint()) { - addOperationToPath(paths, operation); - } - } - openAPI.paths(paths); - - return openAPI; - } - - private void addOperationToPath(Paths paths, ApiOperation operation) { - var uriTemplate = operation.getUriTemplate(); - var httpMethod = operation.getRestMethod(); - - if (uriTemplate == null || httpMethod == null) { - return; - } - - // Convert URI template to OpenAPI path - var pathPattern = convertUriTemplateToOpenApiPath(uriTemplate); - - var pathItem = paths.get(pathPattern); - if (pathItem == null) { - pathItem = new PathItem(); - paths.put(pathPattern, pathItem); - } - - var swaggerOperation = createSwaggerOperation(operation, uriTemplate); - - switch (httpMethod) { - case GET -> pathItem.get(swaggerOperation); - case POST -> pathItem.post(swaggerOperation); - case NONE -> throw new UnsupportedOperationException("Should not be called"); - } - } - - private String convertUriTemplateToOpenApiPath(String uriTemplate) { - // Remove query parameters pattern {?param1,param2} - var path = QUERY_PARAMS_PATTERN.matcher(uriTemplate).replaceAll(""); - - // Convert path parameters {param} to {param} (same format) - // No change needed for OpenAPI format - - // Ensure path starts with / - if (!path.startsWith("/")) { - path = "/" + path; - } - - // Add REST endpoint prefix to match actual server routes - path = restEndpoint + path; - - return path; - } - - private Operation createSwaggerOperation(ApiOperation operation, String uriTemplate) { - var swaggerOperation = - new Operation() - .operationId(operation.getName()) - .summary(operation.getName()) - .description(operation.getFunction().getDescription()); - - // Add parameters - if (operation.getRestMethod() == RestMethodType.GET) { - var parameters = extractParameters(uriTemplate); - if (!parameters.isEmpty()) { - swaggerOperation.parameters(parameters); - } - } - - // Add request body - if (operation.getRestMethod() == RestMethodType.POST) { - var requestBody = buildRequestBody(operation); - requestBody.ifPresent(swaggerOperation::setRequestBody); - } - - // Add responses - var responses = new ApiResponses(); - - // Success response - var successResponse = - new ApiResponse() - .description("Successful operation") - .content( - new Content() - .addMediaType( - "application/json", - new MediaType() - .schema( - new Schema<>() - .type("object") - .addProperty( - "data", - new Schema<>() - .type("object") - .description("Response data"))))); - responses.addApiResponse("200", successResponse); - - // Error response - var errorResponse = - new ApiResponse() - .description("Error response") - .content( - new Content() - .addMediaType( - "application/json", - new MediaType() - .schema( - new Schema<>() - .type("object") - .addProperty( - "errors", - new Schema<>() - .type("array") - .items(new Schema<>().type("object")))))); - responses.addApiResponse("400", errorResponse); - - swaggerOperation.responses(responses); - - return swaggerOperation; - } - - private List extractParameters(String uriTemplate) { - List parameters = new ArrayList<>(); - - // Extract path parameters - var pathMatcher = PATH_PARAMS_PATTERN.matcher(uriTemplate); - while (pathMatcher.find()) { - var paramName = pathMatcher.group(1); - if (!paramName.contains("?")) { // Skip query parameter syntax - var parameter = - new Parameter() - .name(paramName) - .in("path") - .required(true) - .schema(new Schema<>().type("string")); - parameters.add(parameter); - } - } - - // Extract query parameters - var queryMatcher = QUERY_PARAMS_PATTERN.matcher(uriTemplate); - while (queryMatcher.find()) { - var queryParams = queryMatcher.group(1); - var paramNames = queryParams.split(","); - for (String paramName : paramNames) { - var parameter = - new Parameter() - .name(paramName.trim()) - .in("query") - .required(false) - .schema(new Schema<>().type("string")); - parameters.add(parameter); - } - } - - return parameters; - } - - private Optional buildRequestBody(ApiOperation operation) { - var fn = operation.getFunction(); - var params = fn.getParameters(); - var props = params.getProperties(); - - if (props.isEmpty()) { - return Optional.empty(); - } - - var requestBody = new RequestBody(); - requestBody.description(fn.getDescription()); - requestBody.content( - new Content() - .addMediaType("application/json", new MediaType().schema(paramsToSchema(params)))); - - return Optional.of(requestBody); - } - - private Schema paramsToSchema(FunctionDefinition.Parameters params) { - var schema = new Schema<>(); - schema.type(params.getType()); - - addPropsToSchema(schema, params.getProperties()); - - if (ObjectUtils.isNotEmpty(params.getRequired())) { - schema.required(params.getRequired()); - } - - return schema; - } - - private Schema argToSchema(FunctionDefinition.Argument arg) { - var schema = new Schema<>(); - schema.type(arg.getType()); - schema.description(arg.getDescription()); - - if (ObjectUtils.isNotEmpty(arg.getEnumValues())) { - schema._enum(new ArrayList<>(arg.getEnumValues())); - } - - if (arg.getItems() != null) { - schema.items(argToSchema(arg.getItems())); - } - - addPropsToSchema(schema, arg.getProperties()); - - if (ObjectUtils.isNotEmpty(arg.getRequired())) { - schema.required(arg.getRequired()); - } - - return schema; - } - - private void addPropsToSchema(Schema schema, Map props) { - if (ObjectUtils.isNotEmpty(props)) { - for (var entry : props.entrySet()) { - schema.addProperty(entry.getKey(), argToSchema(entry.getValue())); - } - } - } - - public String generateSwaggerUI() { - var swaggerUIHtml = - """ - - - - %s - - - - -
- - - - - - """; - - return String.format( - swaggerUIHtml, swaggerConfig.getTitle(), swaggerConfig.getEndpoint(modelVersion)); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/util/DuckDbExtensions.java b/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/util/DuckDbExtensions.java deleted file mode 100644 index baf3797326..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/main/java/com/datasqrl/util/DuckDbExtensions.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.util; - -import static com.datasqrl.env.EnvVariableNames.DUCKDB_EXTENSIONS_DIR; - -import com.datasqrl.graphql.config.JdbcConfig; -import java.util.Optional; -import java.util.StringJoiner; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; - -@RequiredArgsConstructor -@Slf4j -public final class DuckDbExtensions { - - private final StringJoiner joiner = new StringJoiner(";", "", ";"); - - private final JdbcConfig.DuckDbConfig config; - - public Optional buildInitSql() { - var extensionDir = System.getenv(DUCKDB_EXTENSIONS_DIR); - - if (extensionDir == null || extensionDir.trim().isEmpty()) { - log.warn("Environment variable {} is not set, extensions will not be loaded.", extensionDir); - return Optional.empty(); - } - - joiner.add("SET extension_directory='" + extensionDir + "'"); - joiner.add("LOAD iceberg"); - joiner.add("LOAD httpfs"); - - if (config.isUseDiskCache()) { - joiner.add("LOAD cache_httpfs"); - } - - if (config.isUseVersionGuessing()) { - joiner.add("SET unsafe_enable_version_guessing = true"); - } - - return Optional.of(joiner.toString()); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/GraphQLJwtHandlerIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/GraphQLJwtHandlerIT.java deleted file mode 100644 index 3eed95af99..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/GraphQLJwtHandlerIT.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import static org.apache.kafka.clients.admin.AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; - -import com.datasqrl.graphql.config.CorsHandlerOptions; -import com.datasqrl.graphql.config.KafkaConfig; -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.config.ServletConfig; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.RootGraphqlModel.KafkaSubscriptionCoords; -import com.datasqrl.graphql.server.RootGraphqlModel.StringSchema; -import io.vertx.core.AsyncResult; -import io.vertx.core.Handler; -import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.http.HttpServerOptions; -import io.vertx.core.http.WebSocketConnectOptions; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.auth.JWTOptions; -import io.vertx.ext.auth.PubSecKeyOptions; -import io.vertx.ext.auth.jwt.JWTAuth; -import io.vertx.ext.auth.jwt.JWTAuthOptions; -import io.vertx.ext.web.client.HttpResponse; -import io.vertx.ext.web.client.WebClient; -import io.vertx.junit5.VertxExtension; -import io.vertx.junit5.VertxTestContext; -import io.vertx.pgclient.PgConnectOptions; -import io.vertx.sqlclient.PoolOptions; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import lombok.SneakyThrows; -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.NewTopic; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.kafka.KafkaContainer; -import org.testcontainers.postgresql.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; - -@ExtendWith(VertxExtension.class) -@Testcontainers -class GraphQLJwtHandlerIT { - private static final int SERVER_PORT = 8889; // Use different port to avoid conflicts - - @Container - private final KafkaContainer kafkaContainer = - new KafkaContainer(DockerImageName.parse("apache/kafka-native:3.9.1")); - - @Container - private final PostgreSQLContainer postgresContainer = - new PostgreSQLContainer( - DockerImageName.parse("ankane/pgvector:v0.5.0").asCompatibleSubstituteFor("postgres")) - .withDatabaseName("datasqrl") - .withUsername("foo") - .withPassword("secret"); - - Vertx vertx; - GraphQLServerVerticle graphQLServerVerticle; - ServerConfig serverConfig; - RootGraphqlModel model; - - @SneakyThrows - @BeforeEach - void setup(VertxTestContext testContext) { - try (var admin = - AdminClient.create( - Map.of(BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()))) { - admin.createTopics(List.of(new NewTopic("mytopic", Optional.empty(), Optional.empty()))); - } - vertx = Vertx.vertx(); - - // Create the model with GraphQL schema and Kafka subscription - model = - RootGraphqlModel.builder() - .schema( - StringSchema.builder() - .schema( - "type Query { " - + " mock: String " - + "}" - + "type Subscription { " - + " mock: MySub " - + "}" - + "type MySub { val: String }") - .build()) - .subscription( - KafkaSubscriptionCoords.builder() - .topic("mytopic") - .fieldName("mock") - .equalityConditions(Map.of()) - .build()) - .build(); - - // Create server config with JWT auth and Kafka settings - serverConfig = new ServerConfig(); - serverConfig.setJwtAuth( - new JWTAuthOptions() - .addPubSecKey(new PubSecKeyOptions().setAlgorithm("HS256").setBuffer("dGVzdA=="))); - serverConfig.setPoolOptions(new PoolOptions()); - serverConfig.setServletConfig(new ServletConfig()); - serverConfig.setCorsHandlerOptions(new CorsHandlerOptions()); - serverConfig.setKafkaSubscriptionConfig( - new KafkaConfig.KafkaSubscriptionConfig( - Map.of( - BOOTSTRAP_SERVERS_CONFIG, - kafkaContainer.getBootstrapServers(), - KEY_DESERIALIZER_CLASS_CONFIG, - "com.datasqrl.graphql.kafka.JsonDeserializer", - VALUE_DESERIALIZER_CLASS_CONFIG, - "com.datasqrl.graphql.kafka.JsonDeserializer"))); - - // Configure PostgreSQL connection - PgConnectOptions pgOptions = new PgConnectOptions(); - pgOptions.setDatabase(postgresContainer.getDatabaseName()); - pgOptions.setHost(postgresContainer.getHost()); - pgOptions.setPort(postgresContainer.getMappedPort(PostgreSQLContainer.POSTGRESQL_PORT)); - pgOptions.setUser(postgresContainer.getUsername()); - pgOptions.setPassword(postgresContainer.getPassword()); - serverConfig.setPgConnectOptions(pgOptions); - HttpServerOptions httpServerOptions = - new HttpServerOptions().setPort(SERVER_PORT).setHost("0.0.0.0"); - serverConfig.setHttpServerOptions(httpServerOptions); - - // Create a minimal HTTP server with GraphQL verticle (following McpApiValidationIT pattern) - var httpServer = vertx.createHttpServer(httpServerOptions); - var router = io.vertx.ext.web.Router.router(vertx); - - // Add basic handlers - router.route().handler(io.vertx.ext.web.handler.BodyHandler.create()); - router.route().handler(io.vertx.ext.web.handler.LoggerHandler.create()); - - // Add CORS handler - var corsHandler = - io.vertx.ext.web.handler.CorsHandler.create() - .addOrigin("*") - .allowedMethod(io.vertx.core.http.HttpMethod.GET) - .allowedMethod(io.vertx.core.http.HttpMethod.POST) - .allowedMethod(io.vertx.core.http.HttpMethod.OPTIONS) - .allowedHeader("Access-Control-Request-Method") - .allowedHeader("Access-Control-Allow-Credentials") - .allowedHeader("Access-Control-Allow-Origin") - .allowedHeader("Access-Control-Allow-Headers") - .allowedHeader("Content-Type") - .allowedHeader("Authorization"); - router.route().handler(corsHandler); - - // Create the GraphQL server verticle directly (similar to McpBridgeVerticle) - var authProviders = - List.of( - (io.vertx.ext.auth.authentication.AuthenticationProvider) - io.vertx.ext.auth.jwt.JWTAuth.create(vertx, serverConfig.getJwtAuth())); - graphQLServerVerticle = - new GraphQLServerVerticle( - router, serverConfig, "v1", model, authProviders, Optional.empty()); - - vertx - .deployVerticle(graphQLServerVerticle) - .compose( - deploymentId -> { - httpServer.requestHandler(router); - return httpServer.listen(); - }) - .onSuccess( - server -> { - System.out.println("GraphQL server started on port " + SERVER_PORT); - testContext.completeNow(); - }) - .onFailure(testContext::failNow); - } - - @SneakyThrows - @AfterEach - void teardown(VertxTestContext testContext) { - if (vertx != null) { - vertx - .close() - .onComplete( - ar -> { - testContext.completeNow(); - }); - } else { - testContext.completeNow(); - } - } - - @Test - void jwtAuthentication(VertxTestContext testContext) { - var provider = JWTAuth.create(vertx, this.serverConfig.getJwtAuth()); - - // Generate token - var token = provider.generateToken(new JsonObject(), new JWTOptions().setExpiresInSeconds(60)); - - sendQuery( - token, - ar -> { - if (ar.succeeded()) { - if (ar.result().statusCode() != 200) { - testContext.failNow("Status code not 200: " + ar.result().statusCode()); - } else { - testContext.completeNow(); - } - } else { - testContext.failNow(ar.cause()); - } - }); - } - - @Test - void invalidJWTAuthentication(VertxTestContext testContext) { - sendQuery( - "Badtoken", - ar -> { - if (ar.succeeded()) { - if (ar.result().statusCode() != 401) { - testContext.failNow("Status code not 401: " + ar.result().statusCode()); - } else { - testContext.completeNow(); - } - } else { - testContext.failNow(ar.cause()); - } - }); - } - - @Test - void websocketBadAuth(VertxTestContext testContext) { - var options = - new WebSocketConnectOptions() - .setPort(SERVER_PORT) - .setHost("localhost") - .setURI("/v1/graphql") - .putHeader("Authorization", "Bearer badToken"); - - var client = vertx.createWebSocketClient(); - - client - .connect(options) - .onSuccess( - ws -> { - // Should not succeed - testContext.failNow("Should fail"); - }) - .onFailure( - err -> { - // Failure is expected - testContext.completeNow(); - }); - } - - @Disabled - @Test - void websocket(VertxTestContext testContext) { - // var provider = JWTAuth.create(vertx, this.serverConfig.getAuthOptions()); - // var token = provider.generateToken(new JsonObject(), - // new JWTOptions().setExpiresInSeconds(60)); - // - // var options = new WebSocketConnectOptions() - // .setPort(SERVER_PORT) - // .setHost("localhost") - // .setURI("/graphql") // Your actual WebSocket endpoint URI - // .addHeader("Authorization", "Bearer " + token); // Send the JWT as part of the initial - // request headers - // - // var initMessage = "{\"type\":\"connection_init\",\"payload\":{}}"; // connection - // initialization message - // - // var graphqlSubscription = - // "{\"type\":\"subscribe\",\"id\":\"1\",\"payload\":{\"query\":\"subscription { mock { val } - // }\"}}"; - // // Connect using the WebSocket client - // vertx.createHttpClient().webSocket(options, wsResult -> { - // if (wsResult.succeeded()) { - // var ws = wsResult.result(); - // // Send a GraphQL query as a text message - // ws.writeTextMessage(initMessage); - // ws.handler(message -> { - // if (message.toString().contains("next")) { - // if - // (message.toString().equals("{\"id\":\"1\",\"type\":\"next\",\"payload\":{\"data\":{\"mock\":{\"val\":\"x\"}}}}")) { - // testContext.completeNow(); - // } else { - // testContext.failNow("Unexpected message:" + message); - // } - // } else if (message.toString().contains("connection_ack")) { - // ws.writeTextMessage(graphqlSubscription); - // - // var props = new Properties(); - // props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()); - // props.put(GROUP_ID_CONFIG, UUID.randomUUID().toString()); - // props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - // StringSerializer.class.getName()); - // props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - // StringSerializer.class.getName()); - // - // try { - // Thread.sleep(1000); - // } catch (InterruptedException e) { - // throw new RuntimeException(e); - // } - // KafkaProducer producer = KafkaProducer.create(Vertx.vertx(), props); - // var jsonMessage = new JsonObject().put("val", "x"); - // producer.send(new KafkaProducerRecordImpl("mytopic", - // jsonMessage.toString()),(Handler>)(metadata)->{ - // System.out.println(metadata.result().getTopic()); - // System.out.println(metadata.result().getTimestamp()); - // }); - // } - // }); - // } else { - // testContext.failNow(wsResult.cause()); // Fail the test if the WebSocket connection - // could not be established - // } - // }); - } - - private void sendQuery(String token, Handler>> callback) { - var query = new JsonObject().put("query", "query { mock }"); - var webClient = WebClient.create(vertx); - webClient - .post(SERVER_PORT, "localhost", "/v1/graphql") - .putHeader("Authorization", "Bearer " + token) - .putHeader("Content-Type", "application/json") - .sendJson(query) - .onComplete(callback); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/JsonEnvVarDeserializerTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/JsonEnvVarDeserializerTest.java deleted file mode 100644 index 10e0b130a3..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/JsonEnvVarDeserializerTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Map; -import org.junit.jupiter.api.Test; - -class JsonEnvVarDeserializerTest { - - @Test - void deserialization() { - var jsonEnvVarDeserializer = new JsonEnvVarDeserializer(); - var s = - jsonEnvVarDeserializer.replaceWithEnv( - Map.of("PGPASSWORD", "G:eXB4-(70b~$afas%8#.riC3fs1H"), "${PGPASSWORD}"); - - assertThat(s).isEqualTo("G:eXB4-(70b~$afas%8#.riC3fs1H"); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/RestBridgeVerticleTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/RestBridgeVerticleTest.java deleted file mode 100644 index 54ca19d037..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/RestBridgeVerticleTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.config.ServletConfig; -import com.datasqrl.graphql.server.ModelContainer; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.operation.ApiOperation; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.io.Resources; -import graphql.ExecutionInput; -import graphql.ExecutionResult; -import graphql.GraphQL; -import io.vertx.core.MultiMap; -import io.vertx.core.Promise; -import io.vertx.core.Vertx; -import io.vertx.core.http.HttpServerRequest; -import io.vertx.core.http.HttpServerResponse; -import io.vertx.core.json.JsonObject; -import io.vertx.ext.web.RequestBody; -import io.vertx.ext.web.Route; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.RoutingContext; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -class RestBridgeVerticleTest { - - @Mock private Router router; - @Mock private ServerConfig serverConfig; - @Mock private ServletConfig servletConfig; - @Mock private GraphQLServerVerticle graphQLServerVerticle; - @Mock private GraphQL graphQL; - @Mock private RoutingContext routingContext; - @Mock private HttpServerRequest request; - @Mock private HttpServerResponse response; - @Mock private ExecutionResult executionResult; - @Mock private RequestBody requestBody; - @Mock private Route route; - - @Captor private ArgumentCaptor executionInputCaptor; - - private RootGraphqlModel model; - private RestBridgeVerticle restBridgeVerticle; - private Vertx vertx; - - @BeforeEach - @SneakyThrows - void given_setupMocks_when_initializingTest_then_preparesTestEnvironment() { - MockitoAnnotations.openMocks(this); - vertx = Vertx.vertx(); - - // Read RootGraphQLModel from resource - model = - new ObjectMapper() - .readValue(Resources.getResource("testdata/vertx-rest.json"), ModelContainer.class) - .models - .get("v1"); - - // Setup basic mocks - when(serverConfig.getServletConfig()).thenReturn(servletConfig); - when(servletConfig.getRestEndpoint("v1")).thenReturn("/v1/api/rest"); - when(graphQLServerVerticle.getGraphQLEngine()).thenReturn(graphQL); - when(routingContext.request()).thenReturn(request); - when(routingContext.response()).thenReturn(response); - when(response.setStatusCode(any(Integer.class))).thenReturn(response); - when(response.putHeader(any(String.class), any(String.class))).thenReturn(response); - when(request.params()).thenReturn(MultiMap.caseInsensitiveMultiMap()); - when(routingContext.pathParams()).thenReturn(new HashMap<>()); - - // Mock router methods to return mock Route - when(router.get(any(String.class))).thenReturn(route); - when(router.post(any(String.class))).thenReturn(route); - when(route.handler(any())).thenReturn(route); - - restBridgeVerticle = - new RestBridgeVerticle(router, serverConfig, "v1", model, List.of(), graphQLServerVerticle); - } - - @Test - void given_restEndpointWithGetMethod_when_setupRestEndpoints_then_createsGetRoute() { - // When - Promise promise = Promise.promise(); - restBridgeVerticle.start(promise); - - // Then - verify(router).get("/v1/api/rest/queries/HighTempAlert"); - assertThat(promise.future().succeeded()).isTrue(); - } - - @Test - void given_restEndpointWithPostMethod_when_setupRestEndpoints_then_createsPostRoute() { - // When - Promise promise = Promise.promise(); - restBridgeVerticle.start(promise); - - // Then - verify(router).post("/v1/api/rest/mutations/SensorReading"); - assertThat(promise.future().succeeded()).isTrue(); - } - - @Test - void given_postRequestWithEventList_when_handlerExecuted_then_executesGraphQLMutation() - throws Exception { - // Given - Setup the test data - var eventList = - List.of( - Map.of("sensorid", 1, "temperature", 25.5), Map.of("sensorid", 2, "temperature", 30.0)); - var jsonBody = new JsonObject().put("event", eventList); - - // Mock the request body - when(requestBody.asJsonObject()).thenReturn(jsonBody); - when(routingContext.body()).thenReturn(requestBody); - - // Mock the GraphQL execution result - var mutationResult = - Map.of( - "SensorReading", - List.of( - Map.of("sensorid", 1, "temperature", 25.5, "event_time", "2023-07-08T12:00:00Z"), - Map.of("sensorid", 2, "temperature", 30.0, "event_time", "2023-07-08T12:00:00Z"))); - when(executionResult.getData()).thenReturn(mutationResult); - when(executionResult.getErrors()).thenReturn(List.of()); - - // Mock the GraphQL engine to return a completed future - var completedFuture = CompletableFuture.completedFuture(executionResult); - when(graphQL.executeAsync(any(ExecutionInput.class))).thenReturn(completedFuture); - - // Setup response mocking for successful completion - when(response.end(any(String.class))).thenReturn(null); - - // When - Extract parameters and simulate the bridge call directly - var operation = addSensorReadingOperation(); - Map parameters = - RestBridgeVerticle.extractParameters(routingContext, operation); - - // Call the bridge method directly to trigger GraphQL execution - var future = restBridgeVerticle.bridgeRequestToGraphQL(routingContext, operation, parameters); - - // Wait for the future to complete (it should complete immediately with our mock) - var result = future.result(); - - // Then - Verify the parameters were extracted correctly - assertThat(parameters).containsKey("event"); - assertThat(parameters.get("event")).isInstanceOf(List.class); - - // Verify the GraphQL execution was called with correct parameters - verify(graphQL).executeAsync(executionInputCaptor.capture()); - - ExecutionInput capturedInput = executionInputCaptor.getValue(); - assertThat(capturedInput.getQuery()).contains("mutation SensorReading"); - assertThat(capturedInput.getOperationName()).isEqualTo("SensorReading"); - assertThat(capturedInput.getVariables()).containsKey("event"); - - // Verify the extracted event data matches what we sent - @SuppressWarnings("unchecked") - List capturedEventList = (List) capturedInput.getVariables().get("event"); - assertThat(capturedEventList).hasSize(2); - - // Verify the result data - assertThat(result).isEqualTo(executionResult); - assertThat((Object) result.getData()).isEqualTo(mutationResult); - assertThat(result.getErrors()).isEmpty(); - } - - @Test - void given_getRequestWithQueryParams_when_extractParameters_then_extractsCorrectParameters() { - // Given - var operation = getHighTempOperation(); - var queryParams = MultiMap.caseInsensitiveMultiMap(); - queryParams.add("offset", "10"); - queryParams.add("limit", "20"); - when(request.params()).thenReturn(queryParams); - - // When - Map parameters = - RestBridgeVerticle.extractParameters(routingContext, operation); - - // Then - assertThat(parameters).containsEntry("offset", 10L); - assertThat(parameters).containsEntry("limit", 20L); - } - - @Test - void given_getRequestWithPathParams_when_extractParameters_then_extractsCorrectParameters() { - // Given - var operation = getSensorMaxOperation(); - var pathParams = new HashMap(); - pathParams.put("sensorid", "123"); - when(routingContext.pathParams()).thenReturn(pathParams); - var queryParams = MultiMap.caseInsensitiveMultiMap(); - queryParams.add("offset", "10"); - queryParams.add("limit", "5"); - when(request.params()).thenReturn(queryParams); - - // When - Map parameters = - RestBridgeVerticle.extractParameters(routingContext, operation); - - // Then - assertThat(parameters).containsEntry("sensorid", 123L); - assertThat(parameters).containsEntry("offset", 10L); - assertThat(parameters).containsEntry("limit", 5L); - } - - @Test - void given_postRequestWithJsonBody_when_extractParameters_then_extractsCorrectParameters() { - // Given - var jsonBody = - new JsonObject() - .put("event", List.of(new JsonObject().put("sensorid", 1).put("temperature", 25.5))); - when(requestBody.asJsonObject()).thenReturn(jsonBody); - when(routingContext.body()).thenReturn(requestBody); - - // When - Map parameters = - RestBridgeVerticle.extractParameters(routingContext, addSensorReadingOperation()); - - // Then - assertThat(parameters).containsKey("event"); - assertThat(parameters.get("event")).isInstanceOf(List.class); - } - - @Test - void given_uriTemplateWithQueryParams_when_convertUriTemplateToRoute_then_removesQueryParams() { - // When - Promise promise = Promise.promise(); - restBridgeVerticle.start(promise); - - // Then - Should create route without query params - verify(router).get("/v1/api/rest/queries/HighTempAlert"); - } - - @Test - void given_uriTemplateWithPathParams_when_convertUriTemplateToRoute_then_convertsToVertxParams() { - // When - Promise promise = Promise.promise(); - restBridgeVerticle.start(promise); - - // Then - Should convert {sensorid} to :sensorid - verify(router).get("/v1/api/rest/queries/:sensorid/maxTemp"); - } - - private ApiOperation getOperationByName(String name) { - return model.getOperations().stream() - .filter(op -> op.getName().equalsIgnoreCase(name)) - .findFirst() - .orElseThrow(() -> new RuntimeException("Operation not found: " + name)); - } - - private ApiOperation getHighTempOperation() { - return getOperationByName("GetHighTempAlert"); - } - - private ApiOperation getSensorMaxOperation() { - return getOperationByName("GetSensorMaxTemp"); - } - - private ApiOperation addSensorReadingOperation() { - return getOperationByName("AddSensorReading"); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/WriteIT.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/WriteIT.java deleted file mode 100644 index 749e0049a5..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/WriteIT.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import static org.apache.kafka.clients.admin.AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_OFFSET_RESET_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.datasqrl.graphql.config.KafkaConfig; -import com.datasqrl.graphql.config.ServerConfig; -import com.datasqrl.graphql.jdbc.DatabaseType; -import com.datasqrl.graphql.jdbc.VertxJdbcClient; -import com.datasqrl.graphql.server.GraphQLEngineBuilder; -import com.datasqrl.graphql.server.PaginationType; -import com.datasqrl.graphql.server.RootGraphqlModel; -import com.datasqrl.graphql.server.RootGraphqlModel.ArgumentLookupQueryCoords; -import com.datasqrl.graphql.server.RootGraphqlModel.KafkaMutationCoords; -import com.datasqrl.graphql.server.RootGraphqlModel.QueryWithArguments; -import com.datasqrl.graphql.server.RootGraphqlModel.SqlQuery; -import com.datasqrl.graphql.server.RootGraphqlModel.StringSchema; -import graphql.ExecutionInput; -import graphql.ExecutionResult; -import graphql.GraphQL; -import io.vertx.core.Vertx; -import io.vertx.junit5.VertxExtension; -import io.vertx.pgclient.PgBuilder; -import io.vertx.pgclient.PgConnectOptions; -import io.vertx.sqlclient.PoolOptions; -import io.vertx.sqlclient.SqlClient; -import java.time.Duration; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import lombok.SneakyThrows; -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.errors.WakeupException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.kafka.KafkaContainer; -import org.testcontainers.postgresql.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; - -@ExtendWith(VertxExtension.class) -@Testcontainers -class WriteIT { - - @Container - private final KafkaContainer kafkaContainer = - new KafkaContainer(DockerImageName.parse("apache/kafka-native:3.9.1")); - - @Container - private final PostgreSQLContainer postgresContainer = - new PostgreSQLContainer( - DockerImageName.parse("ankane/pgvector:v0.5.0").asCompatibleSubstituteFor("postgres")) - .withDatabaseName("foo") - .withUsername("foo") - .withPassword("secret") - .withDatabaseName("datasqrl"); - - private SqlClient client; - - Vertx vertx; - RootGraphqlModel model; - String topicName = "topic-1"; - - ServerConfig config; - - @SneakyThrows - @BeforeEach - void init(Vertx vertx) { - PgConnectOptions options = new PgConnectOptions(); - options.setDatabase(postgresContainer.getDatabaseName()); - options.setHost(postgresContainer.getHost()); - options.setPort(postgresContainer.getMappedPort(PostgreSQLContainer.POSTGRESQL_PORT)); - options.setUser(postgresContainer.getUsername()); - options.setPassword(postgresContainer.getPassword()); - - options.setCachePreparedStatements(true); - options.setPipeliningLimit(100_000); - - config = mock(ServerConfig.class); - when(config.getPgConnectOptions()).thenReturn(options); - when(config.getKafkaMutationConfig()) - .thenReturn(new KafkaConfig.KafkaMutationConfig(getKafkaConfig())); - - var client = - PgBuilder.client().with(new PoolOptions()).connectingTo(options).using(vertx).build(); - this.client = client; - this.vertx = vertx; - this.model = getCustomerModel(); - } - - private Map getKafkaConfig() { - var props = new HashMap(); - props.put(BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); - props.put(GROUP_ID_CONFIG, "kafka-test-listener"); - props.put( - KEY_DESERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringDeserializer"); - props.put( - VALUE_DESERIALIZER_CLASS_CONFIG, - "org.apache.kafka.common.serialization.StringDeserializer"); - props.put(AUTO_OFFSET_RESET_CONFIG, "earliest"); - props.put(KEY_SERIALIZER_CLASS_CONFIG, "com.datasqrl.graphql.kafka.JsonSerializer"); - props.put(VALUE_SERIALIZER_CLASS_CONFIG, "com.datasqrl.graphql.kafka.JsonSerializer"); - - return props; - } - - private RootGraphqlModel getCustomerModel() { - return RootGraphqlModel.builder() - .schema( - StringSchema.builder() - .schema( - """ - scalar DateTime - type Query { \ - customer: Customer \ - } \ - type Mutation {\ - addCustomer(event: CreateCustomerEvent): Customer\ - } \ - input CreateCustomerEvent {\ - customerid: Int\ - ts: DateTime\ - } \ - type Customer {\ - customerid: Int \ - ts: DateTime\ - }\ - """) - .build()) - .query( - ArgumentLookupQueryCoords.builder() - .parentType("Query") - .fieldName("customer") - .exec( - QueryWithArguments.builder() - .query( - new SqlQuery( - "SELECT customerid FROM Customer", - List.of(), - PaginationType.NONE, - 0, - DatabaseType.POSTGRES)) - .build()) - .build()) - .mutation( - new KafkaMutationCoords("addCustomer", false, topicName, Map.of(), false, Map.of())) - .build(); - } - - @AfterEach - void after() { - client.close(); - } - - @SneakyThrows - @Test - void test() { - // Create topic using AdminClient - var adminProps = new Properties(); - adminProps.put(BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers()); - try (var adminClient = AdminClient.create(adminProps)) { - var newTopic = new NewTopic(topicName, 1, (short) 1); - adminClient.createTopics(Collections.singleton(newTopic)).all().get(); - } - - var props = new Properties(); - props.putAll(getKafkaConfig()); - var consumer = new KafkaConsumer(props); - consumer.subscribe(Collections.singletonList(topicName)); - - GraphQL graphQL = - model - .accept( - new GraphQLEngineBuilder.Builder() - .withMutationConfiguration(new MutationConfigurationImpl(vertx, config)) - .withSubscriptionConfiguration(new SubscriptionConfigurationImpl(vertx, config)) - .build(), - new VertxContext( - new VertxJdbcClient(Map.of(DatabaseType.POSTGRES, client)), null, null)) - .build(); - - ExecutionInput executionInput = - ExecutionInput.newExecutionInput() - .query( - "mutation ($event: CreateCustomerEvent!) { addCustomer(event: $event) { customerid," - + " ts } }") - .variables( - Map.of("event", Map.of("customerid", 123, "ts", "2001-01-01T10:00:00-05:00"))) - .build(); - - ExecutionResult executionResult = graphQL.execute(executionInput); - - Map data = executionResult.getData(); - assertThat(data).hasSize(1); - assertThat(data.get("addCustomer")) - .hasToString("{customerid=123, ts=2001-01-01T10:00:00.000-05:00}"); - - try { - ConsumerRecords records = consumer.poll(Duration.ofMillis(1000)); - assertThat(records.isEmpty()).isFalse(); - - for (ConsumerRecord record : records) { - assertThat(record.value()).startsWith("{\"customerid\":123"); - } - } catch (WakeupException e) { - // Ignore exception - } finally { - consumer.close(); - } - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/config/ServerConfigTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/config/ServerConfigTest.java deleted file mode 100644 index aa2d37641a..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/config/ServerConfigTest.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import static com.datasqrl.graphql.SqrlObjectMapper.MAPPER; -import static org.assertj.core.api.Assertions.*; - -import java.util.Map; -import org.junit.jupiter.api.Test; - -class ServerConfigTest { - - @Test - void given_emptyJson_when_constructorCalled_then_createsConfigWithDefaults() { - var json = MAPPER.createObjectNode(); - - var serverConfig = ServerConfigUtil.fromConfigMap(MAPPER.convertValue(json, Map.class)); - - assertThat(serverConfig.getServletConfig()).isNotNull(); - assertThat(serverConfig.getGraphQLHandlerOptions()).isNotNull(); - assertThat(serverConfig.getGraphiQLHandlerOptions()).isNull(); - assertThat(serverConfig.getHttpServerOptions()).isNotNull(); - assertThat(serverConfig.getPgConnectOptions()).isNotNull(); - assertThat(serverConfig.getPoolOptions()).isNotNull(); - assertThat(serverConfig.getCorsHandlerOptions()).isNotNull(); - assertThat(serverConfig.getJwtAuth()).isNull(); - assertThat(serverConfig.getSwaggerConfig()).isNotNull(); - assertThat(serverConfig.getKafkaMutationConfig()).isNull(); - assertThat(serverConfig.getKafkaSubscriptionConfig()).isNull(); - } - - @Test - void given_jsonWithAllFields_when_constructorCalled_then_createsConfigWithAllValues() { - var json = MAPPER.createObjectNode(); - json.set("servletConfig", MAPPER.createObjectNode().put("graphQLEndpoint", "/custom-graphql")); - json.set("graphQLHandlerOptions", MAPPER.createObjectNode()); - json.set("graphiQLHandlerOptions", MAPPER.createObjectNode()); - json.set("httpServerOptions", MAPPER.createObjectNode().put("port", 9999)); - json.set( - "pgConnectOptions", - MAPPER.createObjectNode().put("host", "custom-host").put("port", "1234")); - json.set("poolOptions", MAPPER.createObjectNode().put("maxSize", 20)); - json.set("corsHandlerOptions", MAPPER.createObjectNode().put("allowCredentials", true)); - json.set("jwtAuth", MAPPER.createObjectNode().put("algorithm", "HS256")); - json.set("swaggerConfig", MAPPER.createObjectNode().put("enabled", true)); - - var kafkaMutationConfig = MAPPER.createObjectNode(); - kafkaMutationConfig.put("bootstrap.servers", "localhost:9092"); - kafkaMutationConfig.put("topic", "mutations"); - json.set("kafkaMutationConfig", kafkaMutationConfig); - - var kafkaSubscriptionConfig = MAPPER.createObjectNode(); - kafkaSubscriptionConfig.put("bootstrap.servers", "localhost:9092"); - kafkaSubscriptionConfig.put("groupId", "test-group"); - json.set("kafkaSubscriptionConfig", kafkaSubscriptionConfig); - - var serverConfig = ServerConfigUtil.fromConfigMap(MAPPER.convertValue(json, Map.class)); - - assertThat(serverConfig.getServletConfig()).isNotNull(); - assertThat(serverConfig.getGraphQLHandlerOptions()).isNotNull(); - assertThat(serverConfig.getGraphiQLHandlerOptions()).isNotNull(); - assertThat(serverConfig.getHttpServerOptions()).isNotNull(); - assertThat(serverConfig.getPgConnectOptions()).isNotNull(); - assertThat(serverConfig.getPoolOptions()).isNotNull(); - assertThat(serverConfig.getCorsHandlerOptions()).isNotNull(); - assertThat(serverConfig.getJwtAuth()).isNotNull(); - assertThat(serverConfig.getSwaggerConfig()).isNotNull(); - assertThat(serverConfig.getKafkaMutationConfig()).isNotNull(); - assertThat(serverConfig.getKafkaSubscriptionConfig()).isNotNull(); - } - - @Test - void given_jsonWithNullFields_when_constructorCalled_then_handlesNullsCorrectly() { - var json = MAPPER.createObjectNode(); - json.putNull("servletConfig"); - json.putNull("graphQLHandlerOptions"); - json.putNull("httpServerOptions"); - json.putNull("pgConnectOptions"); - json.putNull("poolOptions"); - json.putNull("corsHandlerOptions"); - json.putNull("jwtAuth"); - json.putNull("swaggerConfig"); - json.putNull("kafkaMutationConfig"); - json.putNull("kafkaSubscriptionConfig"); - - var serverConfig = ServerConfigUtil.fromConfigMap(MAPPER.convertValue(json, Map.class)); - - // Fields with empty defaults should still be created - assertThat(serverConfig.getServletConfig()).isNotNull(); - assertThat(serverConfig.getGraphQLHandlerOptions()).isNotNull(); - assertThat(serverConfig.getHttpServerOptions()).isNotNull(); - assertThat(serverConfig.getPoolOptions()).isNotNull(); - assertThat(serverConfig.getCorsHandlerOptions()).isNotNull(); - assertThat(serverConfig.getSwaggerConfig()).isNotNull(); - - // PgConnectOptions uses empty default when null - assertThat(serverConfig.getPgConnectOptions()).isNotNull(); - - // Null default mappings - these should be null when explicitly null or invalid - assertThat(serverConfig.getJwtAuth()).isNull(); - assertThat(serverConfig.getKafkaMutationConfig()).isNull(); - assertThat(serverConfig.getKafkaSubscriptionConfig()).isNull(); - } - - @Test - void given_constructorWithJson_when_created_then_configurationIsApplied() { - var json = MAPPER.createObjectNode(); - json.set("servletConfig", MAPPER.createObjectNode().put("graphQLEndpoint", "/test")); - - var serverConfig = ServerConfigUtil.fromConfigMap(MAPPER.convertValue(json, Map.class)); - - assertThat(serverConfig.getServletConfig()).isNotNull(); - assertThat(serverConfig.getGraphQLHandlerOptions()).isNotNull(); - } - - @Test - void given_corsHandlerOptionsWithWildcardHeaders_when_constructorCalled_then_allowsAllHeaders() { - var json = MAPPER.createObjectNode(); - var corsOptions = MAPPER.createObjectNode(); - corsOptions.put("allowedOrigin", "*"); - corsOptions.set( - "allowedMethods", MAPPER.createArrayNode().add("POST").add("GET").add("OPTIONS")); - corsOptions.set("allowedHeaders", MAPPER.createArrayNode().add("*")); - json.set("corsHandlerOptions", corsOptions); - - var serverConfig = ServerConfigUtil.fromConfigMap(MAPPER.convertValue(json, Map.class)); - - assertThat(serverConfig.getCorsHandlerOptions()).isNotNull(); - assertThat(serverConfig.getCorsHandlerOptions().getAllowedOrigin()).isEqualTo("*"); - assertThat(serverConfig.getCorsHandlerOptions().getAllowedMethods()) - .containsExactlyInAnyOrder("POST", "GET", "OPTIONS"); - assertThat(serverConfig.getCorsHandlerOptions().getAllowedHeaders()).containsExactly("*"); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/config/ServerConfigUtilTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/config/ServerConfigUtilTest.java deleted file mode 100644 index 4da1f638fa..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/config/ServerConfigUtilTest.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.config; - -import static org.assertj.core.api.Assertions.assertThat; - -import io.vertx.ext.web.handler.graphql.GraphiQLHandlerOptions; -import org.junit.jupiter.api.Test; - -class ServerConfigUtilTest { - - @Test - void given_validVersionAndEndpoint_when_getVersionedEndpoint_then_returnsVersionedPath() { - // given - var version = "v1"; - var endpoint = "/graphql"; - - // when - var result = ServerConfigUtil.getVersionedEndpoint(version, endpoint); - - // then - assertThat(result).isEqualTo("/v1/graphql"); - } - - @Test - void given_nullEndpoint_when_getVersionedEndpoint_then_returnsNull() { - // given - var version = "v1"; - String endpoint = null; - - // when - var result = ServerConfigUtil.getVersionedEndpoint(version, endpoint); - - // then - assertThat(result).isNull(); - } - - @Test - void given_emptyVersion_when_getVersionedEndpoint_then_returnsVersionedPath() { - // given - var version = ""; - var endpoint = "/graphql"; - - // when - var result = ServerConfigUtil.getVersionedEndpoint(version, endpoint); - - // then - assertThat(result).isEqualTo("//graphql"); - } - - @Test - void given_endpointWithLeadingSlash_when_getVersionedEndpoint_then_returnsVersionedPath() { - // given - var version = "v2"; - var endpoint = "/api/graphql"; - - // when - var result = ServerConfigUtil.getVersionedEndpoint(version, endpoint); - - // then - assertThat(result).isEqualTo("/v2/api/graphql"); - } - - @Test - void given_endpointWithoutLeadingSlash_when_getVersionedEndpoint_then_returnsVersionedPath() { - // given - var version = "v1"; - var endpoint = "graphql"; - - // when - var result = ServerConfigUtil.getVersionedEndpoint(version, endpoint); - - // then - assertThat(result).isEqualTo("/v1graphql"); - } - - @Test - void given_nullOptions_when_createVersionedGraphiQLHandlerOptions_then_returnsNull() { - // given - var version = "v1"; - GraphiQLHandlerOptions options = null; - - // when - var result = ServerConfigUtil.createVersionedGraphiQLHandlerOptions(version, options); - - // then - assertThat(result).isNull(); - } - - @Test - void - given_optionsWithGraphQLUri_when_createVersionedGraphiQLHandlerOptions_then_updatesGraphQLUri() { - // given - var version = "v1"; - var options = new GraphiQLHandlerOptions(); - options.setGraphQLUri("/graphql"); - - // when - var result = ServerConfigUtil.createVersionedGraphiQLHandlerOptions(version, options); - - // then - assertThat(result).isNotNull(); - assertThat(result.getGraphQLUri()).isEqualTo("/v1/graphql"); - } - - @Test - void - given_optionsWithGraphQLWSUri_when_createVersionedGraphiQLHandlerOptions_then_updatesGraphQLWSUri() { - // given - var version = "v1"; - var options = new GraphiQLHandlerOptions(); - options.setGraphWSQLUri("/graphql-ws"); - - // when - var result = ServerConfigUtil.createVersionedGraphiQLHandlerOptions(version, options); - - // then - assertThat(result).isNotNull(); - assertThat(result.getGraphQLWSUri()).isEqualTo("/v1/graphql-ws"); - } - - @Test - void given_optionsWithBothUris_when_createVersionedGraphiQLHandlerOptions_then_updatesBothUris() { - // given - var version = "api"; - var options = new GraphiQLHandlerOptions(); - options.setGraphQLUri("/graphql"); - options.setGraphWSQLUri("/subscriptions"); - - // when - var result = ServerConfigUtil.createVersionedGraphiQLHandlerOptions(version, options); - - // then - assertThat(result).isNotNull(); - assertThat(result.getGraphQLUri()).isEqualTo("/api/graphql"); - assertThat(result.getGraphQLWSUri()).isEqualTo("/api/subscriptions"); - } - - @Test - void - given_emptyVersion_when_createVersionedGraphiQLHandlerOptions_then_updatesWithEmptyVersion() { - // given - var version = ""; - var options = new GraphiQLHandlerOptions(); - options.setGraphQLUri("/graphql"); - options.setGraphWSQLUri("/graphql-ws"); - - // when - var result = ServerConfigUtil.createVersionedGraphiQLHandlerOptions(version, options); - - // then - assertThat(result).isNotNull(); - assertThat(result.getGraphQLUri()).isEqualTo("//graphql"); - assertThat(result.getGraphQLWSUri()).isEqualTo("//graphql-ws"); - } - - @Test - void - given_complexEndpoints_when_createVersionedGraphiQLHandlerOptions_then_handlesComplexPaths() { - // given - var version = "v2.1"; - var options = new GraphiQLHandlerOptions(); - options.setGraphQLUri("/api/v1/graphql"); - options.setGraphWSQLUri("/api/v1/subscriptions"); - - // when - var result = ServerConfigUtil.createVersionedGraphiQLHandlerOptions(version, options); - - // then - assertThat(result).isNotNull(); - assertThat(result.getGraphQLUri()).isEqualTo("/v2.1/api/v1/graphql"); - assertThat(result.getGraphQLWSUri()).isEqualTo("/v2.1/api/v1/subscriptions"); - } - - @Test - void - given_originalOptions_when_createVersionedGraphiQLHandlerOptions_then_returnsNewInstanceWithoutModifyingOriginal() { - // given - var version = "v1"; - var options = new GraphiQLHandlerOptions(); - options.setGraphQLUri("/graphql"); - options.setGraphWSQLUri("/graphql-ws"); - var originalGraphQLUri = options.getGraphQLUri(); - var originalGraphQLWSUri = options.getGraphQLWSUri(); - - // when - var result = ServerConfigUtil.createVersionedGraphiQLHandlerOptions(version, options); - - // then - assertThat(result).isNotSameAs(options); - assertThat(result.getGraphQLUri()).isEqualTo("/v1/graphql"); - assertThat(result.getGraphQLWSUri()).isEqualTo("/v1/graphql-ws"); - - // Verify original options remain unchanged - assertThat(options.getGraphQLUri()).isEqualTo(originalGraphQLUri); - assertThat(options.getGraphQLWSUri()).isEqualTo(originalGraphQLWSUri); - } -} diff --git a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcherTest.java b/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcherTest.java deleted file mode 100644 index 4e482b2c85..0000000000 --- a/sqrl-server/sqrl-server-vertx-base/src/test/java/com/datasqrl/graphql/util/CaseInsensitiveJsonDataFetcherTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql.util; - -import static org.assertj.core.api.Assertions.assertThat; - -import io.vertx.core.json.JsonObject; -import org.junit.jupiter.api.Test; - -class CaseInsensitiveJsonDataFetcherTest { - - private final CaseInsensitiveJsonDataFetcher fetcher = - new CaseInsensitiveJsonDataFetcher("testkey"); - - @Test - void caseInsensitivePropertyFetcherNonNullValue() { - var jsonObject = new JsonObject(); - jsonObject.put("TestKey", "TestValue"); - - assertThat(fetcher.fetchJsonObject(jsonObject)).isEqualTo("TestValue"); - } - - @Test - void caseInsensitivePropertyFetcherNullValue() { - // Creating a JsonObject with a key but null value - var jsonObject = new JsonObject(); - jsonObject.put("TestKey", null); - - assertThat(fetcher.fetchJsonObject(jsonObject)).isNull(); - } - - @Test - void caseInsensitivePropertyFetcherNoMatch() { - // Creating a JsonObject without the matching key - var jsonObject = new JsonObject(); - jsonObject.put("AnotherKey", "SomeValue"); - - assertThat(fetcher.fetchJsonObject(jsonObject)).isNull(); - } -} diff --git a/sqrl-server/sqrl-server-vertx/Dockerfile b/sqrl-server/sqrl-server-vertx/Dockerfile deleted file mode 100644 index bb026ba659..0000000000 --- a/sqrl-server/sqrl-server-vertx/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright © 2021 DataSQRL (contact@datasqrl.com) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -ARG DUCKDB_EXTENSIONS_TAG=latest -FROM datasqrl/duckdb-extensions:${DUCKDB_EXTENSIONS_TAG} - -# Create directories for application, configuration, and logs -RUN mkdir -p /opt/sqrl/app /opt/sqrl/config /opt/sqrl/logs - -# Copy application jar to separate directory -COPY target/vertx-server.jar /opt/sqrl/app/vertx-server.jar -COPY src/main/resources/log4j2-debug.properties /opt/sqrl/app/log4j2-debug.properties -COPY entrypoint.sh /entrypoint.sh - -RUN chmod +x /entrypoint.sh - -# Set working directory to config directory by default -WORKDIR /opt/sqrl/config - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/sqrl-server/sqrl-server-vertx/entrypoint.sh b/sqrl-server/sqrl-server-vertx/entrypoint.sh deleted file mode 100755 index 51534f11fd..0000000000 --- a/sqrl-server/sqrl-server-vertx/entrypoint.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# -# Copyright © 2021 DataSQRL (contact@datasqrl.com) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -e - -# Enable debug mode if DEBUG environment variable is set -if [[ -n "${SQRL_DEBUG+x}" && -n "$SQRL_DEBUG" ]]; then - SQRL_JVM_ARGS="-Dlog4j2.configurationFile=/opt/sqrl/app/log4j2-debug.properties" - set -x -fi - -# Change to config directory (default WORKDIR) -cd /opt/sqrl/config - -# Run the application jar from the app directory using SqrlLauncher to enable metrics -exec java $SQRL_JVM_ARGS -cp /opt/sqrl/app/vertx-server.jar com.datasqrl.graphql.SqrlLauncher diff --git a/sqrl-server/sqrl-server-vertx/pom.xml b/sqrl-server/sqrl-server-vertx/pom.xml deleted file mode 100644 index 6ce823190e..0000000000 --- a/sqrl-server/sqrl-server-vertx/pom.xml +++ /dev/null @@ -1,117 +0,0 @@ - - - - 4.0.0 - - com.datasqrl - sqrl-server - 1.0-SNAPSHOT - - - sqrl-server-vertx - SQRL :: Server :: Vert.x - - - com.datasqrl.graphql.HttpServerVerticle - false - datasqrl/sqrl-server - - - - - - com.datasqrl - sqrl-server-vertx-base - - - - io.vertx - vertx-launcher-application - - - - - vertx-server - - - io.reactiverse - vertx-maven-plugin - - true - true - com.datasqrl.graphql.SqrlLauncher - ${skipVertexFatJar} - - - - package - - package - - - - - - - com.spotify - dockerfile-maven-plugin - - - build-docker-image - - build - - package - - - - - - - - - instrument - - - - - com.marvinformatics.jacoco - easy-jacoco-maven-plugin - ${easy-jacoco-maven-plugin.version} - - - instrument-uber-jar - - instrument-jar - - - ${project.build.directory}/vertx-server.jar - ${project.build.directory}/vertx-server.jar - - com/datasqrl/* - - - - - - - - - - diff --git a/sqrl-server/sqrl-server-vertx/src/main/java/com/datasqrl/graphql/SqrlLauncher.java b/sqrl-server/sqrl-server-vertx/src/main/java/com/datasqrl/graphql/SqrlLauncher.java deleted file mode 100644 index 2b3139d65c..0000000000 --- a/sqrl-server/sqrl-server-vertx/src/main/java/com/datasqrl/graphql/SqrlLauncher.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright © 2021 DataSQRL (contact@datasqrl.com) - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datasqrl.graphql; - -import com.datasqrl.env.GlobalEnvironmentStore; -import io.micrometer.prometheusmetrics.PrometheusConfig; -import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; -import io.vertx.launcher.application.HookContext; -import io.vertx.launcher.application.VertxApplication; -import io.vertx.launcher.application.VertxApplicationHooks; -import io.vertx.micrometer.MicrometerMetricsFactory; - -/** Main entry point for launching the Vert.x application with Prometheus metrics. */ -public class SqrlLauncher implements VertxApplicationHooks { - - public static void main(String[] args) { - GlobalEnvironmentStore.putAll(System.getenv()); - - if (args == null || args.length == 0) { - args = new String[] {HttpServerVerticle.class.getName()}; - } - VertxApplication vertxApplication = new VertxApplication(args, new SqrlLauncher()); - vertxApplication.launch(); - } - - @Override - public void beforeStartingVertx(HookContext context) { - var prometheusMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); - // Register the Prometheus registry with Micrometer's global registries - io.micrometer.core.instrument.Metrics.addRegistry(prometheusMeterRegistry); - var metricsOptions = - new MicrometerMetricsFactory(prometheusMeterRegistry).newOptions().setEnabled(true); - context.vertxOptions().setMetricsOptions(metricsOptions); - } -} diff --git a/sqrl-server/sqrl-server-vertx/src/main/resources/log4j2-debug.properties b/sqrl-server/sqrl-server-vertx/src/main/resources/log4j2-debug.properties deleted file mode 100644 index cf98a2569b..0000000000 --- a/sqrl-server/sqrl-server-vertx/src/main/resources/log4j2-debug.properties +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright © 2021 DataSQRL (contact@datasqrl.com) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Console appender -appender.console.type = Console -appender.console.name = Console -appender.console.target = SYSTEM_OUT -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n - -# RollingFile appender for DetailedRequestTracer -appender.requestTrace.type = RollingFile -appender.requestTrace.name = RequestTraceFile -appender.requestTrace.fileName = /opt/sqrl/logs/request-trace.log -appender.requestTrace.filePattern = /opt/sqrl/logs/request-trace.%i.log.gz -appender.requestTrace.layout.type = PatternLayout -appender.requestTrace.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n -appender.requestTrace.policies.type = Policies -appender.requestTrace.policies.size.type = SizeBasedTriggeringPolicy -appender.requestTrace.policies.size.size = 50MB -appender.requestTrace.strategy.type = DefaultRolloverStrategy -appender.requestTrace.strategy.max = 10 - -# Flink logger -logger.flink.name = org.apache.flink -logger.flink.level = WARN - -# DetailedRequestTracer logger -logger.requestTracer.name = com.datasqrl.graphql.DetailedRequestTracer -logger.requestTracer.level = DEBUG -logger.requestTracer.additivity = false -logger.requestTracer.appenderRef.file.ref = RequestTraceFile - -# Root logger -rootLogger.level = INFO -rootLogger.appenderRef.console.ref = Console diff --git a/sqrl-server/sqrl-server-vertx/src/main/resources/log4j2.properties b/sqrl-server/sqrl-server-vertx/src/main/resources/log4j2.properties deleted file mode 100644 index 52101fb168..0000000000 --- a/sqrl-server/sqrl-server-vertx/src/main/resources/log4j2.properties +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright © 2021 DataSQRL (contact@datasqrl.com) -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Console appender -appender.console.type = Console -appender.console.name = Console -appender.console.target = SYSTEM_OUT -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n - -# Flink logger -logger.flink.name = org.apache.flink -logger.flink.level = WARN - -# Root logger -rootLogger.level = INFO -rootLogger.appenderRef.console.ref = Console diff --git a/sqrl-testing/sqrl-testing-container/pom.xml b/sqrl-testing/sqrl-testing-container/pom.xml index a50889e945..172c9e3f7f 100644 --- a/sqrl-testing/sqrl-testing-container/pom.xml +++ b/sqrl-testing/sqrl-testing-container/pom.xml @@ -132,7 +132,7 @@ com.datasqrl - sqrl-server-vertx + sqrl-server-spring ${project.version} pom provided diff --git a/sqrl-testing/sqrl-testing-container/src/test/java/com/datasqrl/container/testing/KafkaStartupFailureContainerIT.java b/sqrl-testing/sqrl-testing-container/src/test/java/com/datasqrl/container/testing/KafkaStartupFailureContainerIT.java index e7ef404986..76988fb06e 100644 --- a/sqrl-testing/sqrl-testing-container/src/test/java/com/datasqrl/container/testing/KafkaStartupFailureContainerIT.java +++ b/sqrl-testing/sqrl-testing-container/src/test/java/com/datasqrl/container/testing/KafkaStartupFailureContainerIT.java @@ -56,7 +56,7 @@ void givenMissingKafkaEnvVar_whenServerStarts_thenServiceTerminatesCleanly() { .withLogConsumer(logConsumer); assertThatThrownBy(() -> serverContainer.start()) - .isExactlyInstanceOf(ContainerLaunchException.class) + .isInstanceOfAny(ContainerLaunchException.class, IllegalStateException.class) .hasMessageContaining("failed"); // Get container logs from the consumer @@ -92,7 +92,7 @@ void givenInvalidKafkaBootstrapServers_whenServerStarts_thenServiceTerminatesCle .withLogConsumer(logConsumer); assertThatThrownBy(() -> serverContainer.start()) - .isExactlyInstanceOf(ContainerLaunchException.class) + .isInstanceOfAny(ContainerLaunchException.class, IllegalStateException.class) .hasMessageContaining("failed"); // Get container logs from the consumer diff --git a/sqrl-testing/sqrl-testing-integration/pom.xml b/sqrl-testing/sqrl-testing-integration/pom.xml index 5bd33ceae6..a232ca7497 100644 --- a/sqrl-testing/sqrl-testing-integration/pom.xml +++ b/sqrl-testing/sqrl-testing-integration/pom.xml @@ -50,24 +50,6 @@ test-jar test - - io.vertx - vertx-junit5 - test - - - - junit - junit - - - - - io.vertx - vertx-web-client - test - - org.testcontainers testcontainers diff --git a/sqrl-testing/sqrl-testing-integration/src/test/java/com/datasqrl/engines/TestContainersForTestGoal.java b/sqrl-testing/sqrl-testing-integration/src/test/java/com/datasqrl/engines/TestContainersForTestGoal.java index e780755302..5876c05d60 100644 --- a/sqrl-testing/sqrl-testing-integration/src/test/java/com/datasqrl/engines/TestContainersForTestGoal.java +++ b/sqrl-testing/sqrl-testing-integration/src/test/java/com/datasqrl/engines/TestContainersForTestGoal.java @@ -127,8 +127,11 @@ public TestContainerHook visit(PostgresLogTestEngine engine, Void context) { @Override public TestContainerHook visit(KafkaTestEngine engine, Void context) { return new TestContainerHook() { + // Using Docker Hub image to avoid rate limiting from docker.redpanda.com registry final RedpandaContainer testKafka = - new RedpandaContainer("docker.redpanda.com/redpandadata/redpanda:v23.1.2"); + new RedpandaContainer( + DockerImageName.parse("redpandadata/redpanda:v23.1.2") + .asCompatibleSubstituteFor("docker.redpanda.com/redpandadata/redpanda")); @Override public void start() {