From 01837d9b33e7c060caf1f82fe8f71d2966102d79 Mon Sep 17 00:00:00 2001 From: Eli Boyarski Date: Fri, 10 Apr 2026 14:53:01 +0300 Subject: [PATCH] Clarify that multiple projects and datasets are supported: 1. Make the --project flag only required when --data-from-yaml isn't specified, 2. Update README.md, 3. Update data.yaml to showcase multiple projects and datasets. --- README.md | 7 ++--- _examples/python/data.yaml | 4 +++ cmd/bigquery-emulator/main.go | 51 ++++++++++++++++++++++------------- server/testdata/data.yaml | 4 +++ 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index bf5c2c65c..06da00b37 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ For example, it has the following features. - Wildcard table - Templated Argument Function - JavaScript UDF +- Multi-project, multi-dataset support If you want to know the specific features supported, please see [here](https://github.com/goccy/go-zetasqlite#status) @@ -85,14 +86,14 @@ Usage: bigquery-emulator [OPTIONS] Application Options: - --project= specify the project name - --dataset= specify the dataset name + --project= specify the project name, unless --data-from-yaml is specified + --dataset= specify the dataset name, unless --data-from-yaml is specified --port= specify the http port number. this port used by bigquery api (default: 9050) --grpc-port= specify the grpc port number. this port used by bigquery storage api (default: 9060) --log-level= specify the log level (debug/info/warn/error) (default: error) --log-format= specify the log format (console/json) (default: console) --database= specify the database file if required. if not specified, it will be on memory - --data-from-yaml= specify the path to the YAML file that contains the initial data + --data-from-yaml= specify the path to a YAML file which defines the projects and datasets and contains the initial data -v, --version print version Help Options: diff --git a/_examples/python/data.yaml b/_examples/python/data.yaml index 81266bd63..d965c62df 100644 --- a/_examples/python/data.yaml +++ b/_examples/python/data.yaml @@ -18,3 +18,7 @@ projects: - id: 2 name: bob createdAt: "2022-10-21T00:00:00" +- id: test2 + datasets: + - id: dataset1 + - id: dataset2 \ No newline at end of file diff --git a/cmd/bigquery-emulator/main.go b/cmd/bigquery-emulator/main.go index 6fefc3c1f..ac56b653a 100644 --- a/cmd/bigquery-emulator/main.go +++ b/cmd/bigquery-emulator/main.go @@ -15,15 +15,15 @@ import ( ) type option struct { - Project string `description:"specify the project name" long:"project" env:"BIGQUERY_EMULATOR_PROJECT"` - Dataset string `description:"specify the dataset name" long:"dataset" env:"BIGQUERY_EMULATOR_DATASET"` + Project string `description:"specify the project name, unless --data-from-yaml is specified" long:"project" env:"BIGQUERY_EMULATOR_PROJECT"` + Dataset string `description:"specify the dataset name, unless --data-from-yaml is specified" long:"dataset" env:"BIGQUERY_EMULATOR_DATASET"` Host string `description:"specify the host" long:"host" default:"0.0.0.0"` HTTPPort uint16 `description:"specify the http port number. this port used by bigquery api" long:"port" default:"9050"` GRPCPort uint16 `description:"specify the grpc port number. this port used by bigquery storage api" long:"grpc-port" default:"9060"` LogLevel server.LogLevel `description:"specify the log level (debug/info/warn/error)" long:"log-level" default:"error"` LogFormat server.LogFormat `description:"specify the log format (console/json)" long:"log-format" default:"console"` Database string `description:"specify the database file if required. if not specified, it will be on memory" long:"database"` - DataFromYAML string `description:"specify the path to the YAML file that contains the initial data" long:"data-from-yaml"` + DataFromYAML string `description:"specify the path to a YAML file which defines the projects and datasets and contains the initial data" long:"data-from-yaml"` Version bool `description:"print version" long:"version" short:"v"` } @@ -75,41 +75,54 @@ func runServer(args []string, opt option) error { fmt.Fprintf(os.Stdout, "version: %s (%s)\n", version, revision) return nil } - if opt.Project == "" { - return fmt.Errorf("the required flag --project was not specified") + if opt.Project == "" && opt.DataFromYAML == "" { + return fmt.Errorf("specifying either the --project flag or --data-from-yaml flag is required") } + + if opt.Project == "" && opt.Dataset != "" { + return fmt.Errorf("specifying the --dataset flag requires specifying the --project flag") + } + var db server.Storage if opt.Database == "" { db = server.TempStorage } else { db = server.Storage(fmt.Sprintf("file:%s?cache=shared", opt.Database)) } - project := types.NewProject(opt.Project) - if opt.Dataset != "" { - project.Datasets = append(project.Datasets, types.NewDataset(opt.Dataset)) - } + bqServer, err := server.New(db) if err != nil { return err } - if err := bqServer.SetProject(project.ID); err != nil { - return err + + if opt.Project != "" { + project := types.NewProject(opt.Project) + if err := bqServer.SetProject(project.ID); err != nil { + return err + } + + if opt.Dataset != "" { + project.Datasets = append(project.Datasets, types.NewDataset(opt.Dataset)) + } + + if err := bqServer.Load(server.StructSource(project)); err != nil { + return err + } } - if err := bqServer.Load(server.StructSource(project)); err != nil { - return err + + if opt.DataFromYAML != "" { + if err := bqServer.Load(server.YAMLSource(opt.DataFromYAML)); err != nil { + return err + } } + if err := bqServer.SetLogLevel(opt.LogLevel); err != nil { return err } if err := bqServer.SetLogFormat(opt.LogFormat); err != nil { return err } - if opt.DataFromYAML != "" { - if err := bqServer.Load(server.YAMLSource(opt.DataFromYAML)); err != nil { - return err - } - } - + ctx := context.Background() interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM) diff --git a/server/testdata/data.yaml b/server/testdata/data.yaml index a13d3149d..5e751d44f 100644 --- a/server/testdata/data.yaml +++ b/server/testdata/data.yaml @@ -54,3 +54,7 @@ projects: - num: 1.2345 bignum: 1.234567891234 interval: 1-6 15 0:0:0 +- id: test2 + datasets: + - id: dataset1 + - id: dataset2 \ No newline at end of file