From 2e6a60c597e03e7c3a5c81e6e785ebfce81d2bdb Mon Sep 17 00:00:00 2001 From: Siddharth Agrawal Date: Wed, 23 Apr 2025 13:08:21 -0700 Subject: [PATCH] chore: add sample demonstrating sending a nested proto --- samples/install-without-bom/pom.xml | 30 +++++ samples/snapshot/pom.xml | 31 +++++ samples/snippets/pom.xml | 33 ++++++ .../bigquerystorage/WriteNestedProto.java | 73 ++++++++++++ samples/snippets/src/main/proto/nested.proto | 40 +++++++ .../snippets/src/main/proto/separate.proto | 27 +++++ .../bigquerystorage/WriteNestedProtoIT.java | 109 ++++++++++++++++++ 7 files changed, 343 insertions(+) create mode 100644 samples/snippets/src/main/java/com/example/bigquerystorage/WriteNestedProto.java create mode 100644 samples/snippets/src/main/proto/nested.proto create mode 100644 samples/snippets/src/main/proto/separate.proto create mode 100644 samples/snippets/src/test/java/com/example/bigquerystorage/WriteNestedProtoIT.java diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 205488efe8..c7717c1782 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -18,6 +18,7 @@ + 3.25.4 1.8 1.8 UTF-8 @@ -64,6 +65,11 @@ exporter-metrics 0.34.0 + + com.google.protobuf + protobuf-java-util + ${project.protobuf-java.version} + junit @@ -81,6 +87,13 @@ + + + kr.motd.maven + os-maven-plugin + 1.7.1 + + org.codehaus.mojo @@ -95,6 +108,7 @@ ../snippets/src/main/java + ../snippets/src/main/proto @@ -111,6 +125,22 @@ + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${project.protobuf-java.version}:exe:${os.detected.classifier} + ../snippets/src/main/proto + + + + + compile + + + + diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index bbbf756040..bbead4559d 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -18,6 +18,7 @@ + 3.25.4 1.8 1.8 UTF-8 @@ -66,6 +67,12 @@ 0.34.0 + + com.google.protobuf + protobuf-java-util + ${project.protobuf-java.version} + + junit junit @@ -82,6 +89,13 @@ + + + kr.motd.maven + os-maven-plugin + 1.7.1 + + org.codehaus.mojo @@ -96,6 +110,7 @@ ../snippets/src/main/java + ../snippets/src/main/proto @@ -112,6 +127,22 @@ + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${project.protobuf-java.version}:exe:${os.detected.classifier} + ../snippets/src/main/proto + + + + + compile + + + + diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index 9ee9018a0f..f403bcd5c6 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -18,6 +18,7 @@ + 3.25.4 1.8 1.8 UTF-8 @@ -81,6 +82,11 @@ exporter-metrics 0.34.0 + + com.google.protobuf + protobuf-java-util + ${project.protobuf-java.version} + @@ -97,4 +103,31 @@ test + + + + + kr.motd.maven + os-maven-plugin + 1.7.1 + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${project.protobuf-java.version}:exe:${os.detected.classifier} + + + + + compile + + + + + + diff --git a/samples/snippets/src/main/java/com/example/bigquerystorage/WriteNestedProto.java b/samples/snippets/src/main/java/com/example/bigquerystorage/WriteNestedProto.java new file mode 100644 index 0000000000..92ab52724c --- /dev/null +++ b/samples/snippets/src/main/java/com/example/bigquerystorage/WriteNestedProto.java @@ -0,0 +1,73 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquerystorage; + +// [START bigquerystorage_writenestedproto] +import com.google.api.core.ApiFuture; +import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.ProtoRows; +import com.google.cloud.bigquery.storage.v1.ProtoSchemaConverter; +import com.google.cloud.bigquery.storage.v1.StreamWriter; +import com.google.protobuf.Descriptors.DescriptorValidationException; +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public class WriteNestedProto { + + public static void runWriteNestedProto(String projectId, String datasetName, String tableName) + throws DescriptorValidationException, InterruptedException, IOException { + StreamWriter streamWriter = + StreamWriter.newBuilder( + "projects/" + + projectId + + "/datasets/" + + datasetName + + "/tables/" + + tableName + + "/_default") + .setWriterSchema(ProtoSchemaConverter.convert(HasNestedMessage.getDescriptor())) + .build(); + ProtoRows protoRows = + ProtoRows.newBuilder() + .addSerializedRows( + HasNestedMessage.newBuilder() + .setFoo("foo") + .setBar( + HasNestedMessage.InnerMessage.newBuilder() + .setMyInt(12345) + .setMyString("bar") + .build()) + .build() + .toByteString()) + .addSerializedRows( + HasSeparateNestedMessage.newBuilder() + .setFoo("foo2") + .setBar( + SeparateMessage.newBuilder().setMyInt(123456).setMyString("bar2").build()) + .build() + .toByteString()) + .build(); + ApiFuture future = streamWriter.append(protoRows); + try { + AppendRowsResponse response = future.get(); + System.out.println("Appended records successfully."); + } catch (ExecutionException e) { + System.out.println(e); + } + } +} +// [END bigquerystorage_writenestedproto] diff --git a/samples/snippets/src/main/proto/nested.proto b/samples/snippets/src/main/proto/nested.proto new file mode 100644 index 0000000000..63db8bd11d --- /dev/null +++ b/samples/snippets/src/main/proto/nested.proto @@ -0,0 +1,40 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +package nestedprotos; + +import "separate.proto"; + +option java_multiple_files = true; +option java_package = "com.example.bigquerystorage"; +option java_outer_classname = "NestedProtos"; + +message HasNestedMessage { + optional string foo = 1; + + message InnerMessage { + optional int64 my_int = 1; + optional string my_string = 2; + } + + optional InnerMessage bar = 2; +} + +message HasSeparateNestedMessage { + optional string foo = 1; + optional SeparateMessage bar = 2; +} diff --git a/samples/snippets/src/main/proto/separate.proto b/samples/snippets/src/main/proto/separate.proto new file mode 100644 index 0000000000..8aca5219ce --- /dev/null +++ b/samples/snippets/src/main/proto/separate.proto @@ -0,0 +1,27 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto3"; +package nestedprotos; + +option java_multiple_files = true; +option java_package = "com.example.bigquerystorage"; +option java_outer_classname = "SeparateProtos"; + +message SeparateMessage { + optional int64 my_int = 1; + optional string my_string = 2; +} diff --git a/samples/snippets/src/test/java/com/example/bigquerystorage/WriteNestedProtoIT.java b/samples/snippets/src/test/java/com/example/bigquerystorage/WriteNestedProtoIT.java new file mode 100644 index 0000000000..6293b301a7 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/bigquerystorage/WriteNestedProtoIT.java @@ -0,0 +1,109 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquerystorage; + +import static com.google.common.truth.Truth.assertThat; +import static junit.framework.TestCase.assertNotNull; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQuery.DatasetDeleteOption; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.DatasetId; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.UUID; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class WriteNestedProtoIT { + + private static final String GOOGLE_CLOUD_PROJECT = System.getenv("GOOGLE_CLOUD_PROJECT"); + + private ByteArrayOutputStream bout; + private PrintStream out; + private BigQuery bigquery; + private String datasetName; + private String tableName; + + private static void requireEnvVar(String varName) { + assertNotNull( + "Environment variable " + varName + " is required to perform these tests.", + System.getenv(varName)); + } + + @BeforeClass + public static void checkRequirements() { + requireEnvVar("GOOGLE_CLOUD_PROJECT"); + } + + @Before + public void setUp() { + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + + bigquery = BigQueryOptions.getDefaultInstance().getService(); + + // Create a new dataset and table for each test. + datasetName = "WRITE_STREAM_TEST" + UUID.randomUUID().toString().substring(0, 8); + tableName = "DEFAULT_STREAM_TEST" + UUID.randomUUID().toString().substring(0, 8); + Schema schema = + Schema.of( + com.google.cloud.bigquery.Field.newBuilder("foo", StandardSQLTypeName.STRING).build(), + com.google.cloud.bigquery.Field.newBuilder( + "bar", + StandardSQLTypeName.STRUCT, + FieldList.of( + com.google.cloud.bigquery.Field.newBuilder( + "my_int", StandardSQLTypeName.INT64) + .build(), + com.google.cloud.bigquery.Field.newBuilder( + "my_string", StandardSQLTypeName.STRING) + .build())) + .build()); + bigquery.create(DatasetInfo.newBuilder(datasetName).build()); + TableInfo tableInfo = + TableInfo.newBuilder(TableId.of(datasetName, tableName), StandardTableDefinition.of(schema)) + .build(); + bigquery.create(tableInfo); + } + + @After + public void tearDown() { + bigquery.delete( + DatasetId.of(GOOGLE_CLOUD_PROJECT, datasetName), DatasetDeleteOption.deleteContents()); + System.setOut(null); + } + + @Test + public void testWriteNestedProto() throws Exception { + WriteNestedProto.runWriteNestedProto(GOOGLE_CLOUD_PROJECT, datasetName, tableName); + assertThat(bout.toString()).contains("Appended records successfully."); + } +}