From 8f001d8083e480a88da85947830dca7966c937cd Mon Sep 17 00:00:00 2001 From: JhonCode Date: Thu, 20 Apr 2023 00:15:28 -0500 Subject: [PATCH] New Example with ZincSearch engine NoSQL and update Readme --- CONTRIBUTORS | 1 + README.md | 135 +++++++++++++++++++++++++++++++++++++++++++++++- example_test.go | 35 +++++++++++++ 3 files changed, 170 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 176cb1d..eaadd66 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,2 +1,3 @@ edhaight [@edhaight](https://github.com/edhaight) Manuel Laflamme [@mlaflamm](https://github.com/mlaflamm) +Jhon Arcila Castaño [@JhonCodeU](https://github.com/JhonCodeU) \ No newline at end of file diff --git a/README.md b/README.md index af869fe..c3e6053 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,92 @@ [![Build Status](https://github.com/olivere/ndjson/workflows/Test/badge.svg)](https://github.com/olivere/ndjson/actions) +## Ndjson The `ndjson` package implements reading and writing files according to the [ndjson specification](http://ndjson.org/). +## Documentation + +This Go library allows reading and writing files in the newline delimited JSON (NDJSON) format, which is a simple way to represent structured data that can be read and written efficiently. + +The package provides two main types, ndjson.Reader and ndjson.Writer, that allow reading and writing NDJSON data, respectively. It also provides a method for decoding NDJSON data into Go structs and encoding Go structs into NDJSON data. + +The example code provided demonstrates how to use the ndjson package to read and write NDJSON files, as well as encode and decode Go structs to and from NDJSON data. + +## Installation +To install this package, you can use the go get command: +```bash +go get github.com/olivere/ndjson +``` + +## Usage +```go +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/olivere/ndjson" +) + +type Location struct { + City string `json:"city"` +} + +func main() { + // Writing to an ndjson file + locations := []Location{ + {City: "Munich"}, + {City: "Berlin"}, + {City: "London"}, + } + + file, err := os.Create("locations.ndjson") + if err != nil { + panic(err) + } + defer file.Close() + + writer := ndjson.NewWriter(file) + for _, loc := range locations { + if err := writer.Encode(loc); err != nil { + panic(err) + } + } + writer.Flush() + + // Reading from an ndjson file + file, err = os.Open("locations.ndjson") + if err != nil { + panic(err) + } + defer file.Close() + + reader := ndjson.NewReader(file) + for reader.Next() { + var loc Location + if err := reader.Decode(&loc); err != nil { + panic(err) + } + fmt.Println(loc.City) + } + if err := reader.Err(); err != nil { + panic(err) + } +} + +``` +This example writes three `Location` objects to an ndjson file called `locations.ndjson`, and then reads them back from the file, printing the `City` field of each object to the console. + Example: ```go package ndjson_test import ( + "bytes" "fmt" "os" "strings" @@ -22,6 +99,12 @@ type Location struct { City string `json:"city"` } + +type collection struct { + Index map[string]interface{} `json:"index"` +} + +// ExampleReader demonstrates how to use ndjson.Reader to read NDJSON data from a string and decode it into a Go struct. func ExampleReader() { r := ndjson.NewReader(strings.NewReader(`{"city":"Munich"} {"city":"Berlin"} @@ -44,6 +127,7 @@ func ExampleReader() { // London } +// ExampleWriter demonstrates how to use ndjson.Writer to encode Go structs into NDJSON data and write it to an output stream. func ExampleWriter() { locations := []Location{ {City: "Munich"}, @@ -55,6 +139,7 @@ func ExampleWriter() { if err := r.Encode(loc); err != nil { fmt.Fprintf(os.Stderr, "Encode failed: %v", err) return + // If an error occurs while encoding, the function stops writing and returns an error message. } } // Output: @@ -62,8 +147,56 @@ func ExampleWriter() { // {"city":"Berlin"} // {"city":"London"} } + +// ExampleWriter_zincSearch demonstrates how to use ndjson.Writer to encode Go structs and write NDJSON data to an output stream +// for use with the ZincSearch engine. +func ExampleWriter_zincSearch() { + locations := []Location{ + {City: "Munich"}, + } + + index := map[string]interface{}{"_index": "States"} + collections := []collection{{Index: index}} + + var buf bytes.Buffer + r := ndjson.NewWriter(&buf) + + for _, c := range collections { + if err := r.Encode(c); err != nil { + fmt.Fprintf(os.Stderr, "Encode failed: %v", err) + return + // If an error occurs while encoding, the function stops writing and returns an error message. + } + } + + for _, loc := range locations { + if err := r.Encode(loc); err != nil { + fmt.Fprintf(os.Stderr, "Encode failed: %v", err) + return + // If an error occurs while encoding, the function stops writing and returns an error message. + } + } +} + +// Output: +// { "index" : { "_index" : "States" } } +// {"city":"Munich"} + ``` -# License +The ExampleWriter_zincSearch demonstrates how to use the ndjson package to write multiple JSON documents into a single NDJSON stream. In this example, we have a slice of Location structs and a slice of collection structs. + +Before writing the Location documents, we first write the collection documents that define Elasticsearch indices. Each collection struct includes metadata for the index, such as the name of the index ("_index") and its value ("States"). + +Once the collection documents are written, we iterate over the Location slice and encode each struct as an NDJSON document using the Encode method of the ndjson.Writer. The resulting NDJSON stream can be used to index the Location documents into an Elasticsearch index. + +The expected output of the ExampleWriter_zincSearch function is the NDJSON stream containing the collection and Location documents, formatted for Elasticsearch bulk indexing. + +## Contributing + +Pull requests are welcome. For major changes, please open an issue first +to discuss what you would like to change. + +## License MIT. See [LICENSE](https://github.com/olivere/ndjson/blob/master/LICENSE) file. diff --git a/example_test.go b/example_test.go index 44e8017..fb4d675 100644 --- a/example_test.go +++ b/example_test.go @@ -1,6 +1,7 @@ package ndjson_test import ( + "bytes" "fmt" "os" "strings" @@ -12,6 +13,10 @@ type Location struct { City string `json:"city"` } +type collection struct { + Index map[string]interface{} `json:"index"` +} + func ExampleReader() { r := ndjson.NewReader(strings.NewReader(`{"city":"Munich"} {"city":"Berlin"} @@ -73,3 +78,33 @@ func ExampleWriter() { // {"city":"Berlin"} // {"city":"London"} } + +func ExampleWriter_zincSearch() { + locations := []Location{ + {City: "Munich"}, + } + + index := map[string]interface{}{"_index": "States"} + collections := []collection{{Index: index}} + + var buf bytes.Buffer + r := ndjson.NewWriter(&buf) + + for _, c := range collections { + if err := r.Encode(c); err != nil { + fmt.Fprintf(os.Stderr, "Encode failed: %v", err) + return + } + } + + for _, loc := range locations { + if err := r.Encode(loc); err != nil { + fmt.Fprintf(os.Stderr, "Encode failed: %v", err) + return + } + } +} + +// Output: +// { "index" : { "_index" : "States" } } +// {"city":"Munich"}