diff --git a/metadata/BigQueryToSpannerDelete_metadata b/metadata/BigQueryToSpannerDelete_metadata new file mode 100644 index 0000000..3f9745f --- /dev/null +++ b/metadata/BigQueryToSpannerDelete_metadata @@ -0,0 +1,61 @@ +{ + "name": "BigQueryToSpannerDelete", + "description": "Pipeline to insert records to Spanner table from BigQuery query results.", + "parameters": [ + { + "name": "query", + "label": "SQL Query text", + "help_text": "SQL Query text to read records from BigQuery." + }, + { + "name": "projectId", + "label": "Output Spanner Project ID (Google Cloud Project ID)", + "help_text": "Project ID Cloud Spanner to store is belong to." + }, + { + "name": "instanceId", + "label": "Output Spanner Instance ID", + "help_text": "Cloud Spanner instance ID to store." + }, + { + "name": "databaseId", + "label": "Output Database ID", + "help_text": "Cloud Spanner database ID to store." + }, + { + "name": "table", + "label": "Output Spanner Table Name", + "help_text": "Cloud Spanner target table name to store." + }, + { + "name": "outputError", + "label": "GCS path to output error record as avro files.", + "help_text": "Path and filename prefix for writing output avro files. ex: gs://MyBucket/xxx", + "regexes": ["^gs:\/\/[^\n\r]+$"] + }, + { + "name": "mutationOp", + "label": "Spanner insert policy. `INSERT` or `UPDATE` or `REPLACE` or `INSERT_OR_UPDATE`", + "help_text": "Detail: https://googleapis.github.io/google-cloud-java/google-cloud-clients/apidocs/com/google/cloud/spanner/Mutation.Op.html" + }, + { + "name": "primaryKeyFields", + "label": "Key field on destination Spanner table. (Required table auto generation)", + "help_text": "If destination Spanner table not exists, template will create the table using this field as key.", + "is_optional": true + }, + { + "name": "outputNotify", + "label": "OutputNotify", + "help_text": "GCS path to put notification file that contains output file paths.", + "regexes": ["^gs:\/\/[^\n\r]+$"], + "is_optional": true + }, + { + "name": "splitField", + "label": "SplitField", + "help_text": "Field name to split output destination by the value", + "is_optional": true + } + ] +} \ No newline at end of file diff --git a/src/main/java/com/mercari/solution/templates/BigQueryToSpannerDelete.java b/src/main/java/com/mercari/solution/templates/BigQueryToSpannerDelete.java new file mode 100644 index 0000000..b6d10db --- /dev/null +++ b/src/main/java/com/mercari/solution/templates/BigQueryToSpannerDelete.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) Mercari, Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +package com.mercari.solution.templates; + +import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Struct; +import com.mercari.solution.transforms.dofns.StructToMutationDoFn; +import com.mercari.solution.util.converter.RecordToStructConverter; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.coders.SerializableCoder; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; +import org.apache.beam.sdk.io.gcp.spanner.SpannerIO; +import org.apache.beam.sdk.options.Description; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.options.ValueProvider; +import org.apache.beam.sdk.transforms.ParDo; + +/** + *
BigQueryToSpannerDelete template inserts BigQuery query results to specified Cloud Spanner table.
+ *+ * Template parameters are as follows. + * + *
| Parameter | Type | Description |
|---|---|---|
| query | String | SQL query to read records from BigQuery |
| projectId | String | Project ID for Spanner you will recover |
| instanceId | String | Spanner instanceID you will recover. |
| databaseId | String | Spanner databaseID you will recover. |
| table | String | Spanner table name to insert records. |
| mutationOp | String | Spanner insert policy. INSERT or UPDATE or REPLACE or INSERT_OR_UPDATE |