From cca84b649b814a3936d2bfd1bfe5a49abc2e8ccc Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Thu, 16 Mar 2023 12:22:08 +0100 Subject: [PATCH 01/10] feature: post new config handler --- go.mod | 39 +++++++++++++ go.sum | 118 +++++++++++++++++++++++++++++++++++++++ main.go | 47 ++++++++++++++++ model/config.go | 7 +++ server/handlers.go | 52 +++++++++++++++++ server/helper.go | 18 ++++++ server/server.go | 19 +++++++ service/configService.go | 28 ++++++++++ store/configStore.go | 26 +++++++++ 9 files changed, 354 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 model/config.go create mode 100644 server/handlers.go create mode 100644 server/helper.go create mode 100644 server/server.go create mode 100644 service/configService.go create mode 100644 store/configStore.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f7a9ace --- /dev/null +++ b/go.mod @@ -0,0 +1,39 @@ +module kuiper + +go 1.18 + +require ( + github.com/gin-gonic/gin v1.9.0 + github.com/google/uuid v1.3.0 + github.com/opentracing/opentracing-go v1.2.0 + go.opentelemetry.io/otel/trace v1.14.0 +) + +require ( + github.com/bytedance/sonic v1.8.5 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.11.2 // indirect + github.com/goccy/go-json v0.10.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/leodido/go-urn v1.2.2 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.7 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + go.opentelemetry.io/otel v1.14.0 // indirect + golang.org/x/arch v0.3.0 // indirect + golang.org/x/crypto v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..67f78e0 --- /dev/null +++ b/go.sum @@ -0,0 +1,118 @@ +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.8.3 h1:pf6fGl5eqWYKkx1RcD4qpuX+BIUaduv/wTm5ekWJ80M= +github.com/bytedance/sonic v1.8.3/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.8.5 h1:kjX0/vo5acEQ/sinD/18SkA/lDDUk23F0RcaHvI7omc= +github.com/bytedance/sonic v1.8.5/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= +github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= +github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= +github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= +github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.1 h1:lEs5Ob+oOG/Ze199njvzHbhn6p9T+h64F5hRj69iTTo= +github.com/goccy/go-json v0.10.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/leodido/go-urn v1.2.2 h1:7z68G0FCGvDk646jz1AelTYNYWrTNm0bEcFAo147wt4= +github.com/leodido/go-urn v1.2.2/go.mod h1:kUaIbLZWttglzwNuG0pgsh5vuV6u2YcGBYz1hIPjtOQ= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= +github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rwtodd/Go.Sed v0.0.0-20210816025313-55464686f9ef/go.mod h1:8AEUvGVi2uQ5b24BIhcr0GCcpd/RNAFWaN2CJFrWIIQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.10 h1:eimT6Lsr+2lzmSZxPhLFoOWFmQqwk0fllJJ5hEbTXtQ= +github.com/ugorji/go/codec v1.2.10/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= +go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= +go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.2.0 h1:W1sUEHXiJTfjaFJ5SLo0N6lZn+0eO5gWD1MFeTGqQEY= +golang.org/x/arch v0.2.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/main.go b/main.go new file mode 100644 index 0000000..13c3df1 --- /dev/null +++ b/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "context" + "kuiper/server" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gin-gonic/gin" +) + +func main() { + router := gin.New() + handler := server.NewConfigHandler() + + router.POST("/api/config", handler.CreateConfig) + + // start server + srv := &http.Server{Addr: "0.0.0.0:8080", Handler: router} + go func() { + log.Println("server starting") + if err := srv.ListenAndServe(); err != nil { + if err != http.ErrServerClosed { + log.Fatal(err) + } + } + }() + + quit := make(chan os.Signal) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit + + log.Println("service shutting down ...") + + // gracefully stop server + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := srv.Shutdown(ctx); err != nil { + log.Fatal(err) + } + log.Println("server stopped") +} diff --git a/model/config.go b/model/config.go new file mode 100644 index 0000000..ecd42d9 --- /dev/null +++ b/model/config.go @@ -0,0 +1,7 @@ +package model + +type Config struct { + ID string `json:"id"` + Version string `json:"version"` + Entries map[string]string `json:"entries"` +} diff --git a/server/handlers.go b/server/handlers.go new file mode 100644 index 0000000..82121a8 --- /dev/null +++ b/server/handlers.go @@ -0,0 +1,52 @@ +package server + +import ( + "errors" + "mime" + "net/http" + + "github.com/gin-gonic/gin" +) + +func (ch configHandler) CreateConfig(c *gin.Context) { + ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.CreateConfig") + defer span.End() + + contentType := c.Request.Header.Get("Content-Type") + // idemKey := c.Request.Header.Get("x-idempotency-key") + + mediatype, _, err := mime.ParseMediaType(contentType) + if err != nil { + span.RecordError(err) + http.Error(c.Writer, err.Error(), http.StatusBadRequest) + return + } + + if mediatype != "application/json" { + err := errors.New("Expect application/json Content-Type") + span.RecordError(err) + http.Error(c.Writer, err.Error(), http.StatusUnsupportedMediaType) + return + } + + rt, err := decodeConfigBody(c.Request.Body) + if err != nil || rt.Version == "" || rt.Entries == nil { + span.RecordError(err) + http.Error(c.Writer, "Invalid JSON format", http.StatusBadRequest) + return + } + + // if ch.idempotencyService.FindRequestId(ctx, requestId) == true { + // http.Error(c.Writer, "Request has been already sent", http.StatusBadRequest) + // return + // } + + cid, err := ch.configService.CreateConfig(ctx, rt) + + // reqId := "" + // if err == nil { + // reqId = ts.idempotencyService.SaveRequestId(ctx) + // } + + c.JSON(http.StatusOK, gin.H{"id": cid}) +} diff --git a/server/helper.go b/server/helper.go new file mode 100644 index 0000000..1b618b9 --- /dev/null +++ b/server/helper.go @@ -0,0 +1,18 @@ +package server + +import ( + "encoding/json" + "io" + "kuiper/model" +) + +func decodeConfigBody(r io.Reader) (model.Config, error) { + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + + var config *model.Config + if err := dec.Decode(&config); err != nil { + return model.Config{}, err + } + return *config, nil +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..eed5346 --- /dev/null +++ b/server/server.go @@ -0,0 +1,19 @@ +package server + +import ( + "kuiper/service" + + "go.opentelemetry.io/otel/trace" +) + +type ConfigHandler interface { +} + +type configHandler struct { + tracer trace.Tracer + configService service.ConfigService +} + +func NewConfigHandler() configHandler { + return configHandler{} +} diff --git a/service/configService.go b/service/configService.go new file mode 100644 index 0000000..63e3124 --- /dev/null +++ b/service/configService.go @@ -0,0 +1,28 @@ +package service + +import ( + "context" + "kuiper/model" + "kuiper/store" +) + +type ConfigService interface { + CreateConfig(ctx context.Context, config model.Config) (string, error) + GetConfig(id string) (model.Config, error) +} + +func NewConfigService(cs store.ConfigStore) ConfigService { + return configService{store: cs} +} + +type configService struct { + store store.ConfigStore +} + +func (cs configService) CreateConfig(ctx context.Context, config model.Config) (string, error) { + return cs.store.CreateConfig(config, context.TODO()) +} + +func (cs configService) GetConfig(id string) (model.Config, error) { + return cs.store.GetConfig(id, context.TODO()) +} diff --git a/store/configStore.go b/store/configStore.go new file mode 100644 index 0000000..1521cee --- /dev/null +++ b/store/configStore.go @@ -0,0 +1,26 @@ +package store + +import ( + "context" + "kuiper/model" +) + +type ConfigStore interface { + CreateConfig(cfg model.Config, ctx context.Context) (string, error) + GetConfig(id string, ctx context.Context) (model.Config, error) +} + +func NewConfigStore() ConfigStore { + return configStore{} +} + +type configStore struct { +} + +func (cStore configStore) CreateConfig(cfg model.Config, ctx context.Context) (string, error) { + return "", nil +} + +func (cStore configStore) GetConfig(id string, ctx context.Context) (model.Config, error) { + return model.Config{}, nil +} From 76fa1f83403771a1646fd453fe44cc350a6b6866 Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Sun, 19 Mar 2023 12:30:46 +0100 Subject: [PATCH 02/10] feature: create and get a config --- server/handlers.go | 16 +++++++++ service/configService.go | 27 +++++++++----- store/configStore.go | 78 +++++++++++++++++++++++++++++++++++----- 3 files changed, 104 insertions(+), 17 deletions(-) diff --git a/server/handlers.go b/server/handlers.go index 82121a8..3a2c74e 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -50,3 +50,19 @@ func (ch configHandler) CreateConfig(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"id": cid}) } + +func (ch configHandler) GetConfig(c *gin.Context) { + ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.CreateConfig") + defer span.End() + + id := c.Param("id") + ver := c.Param("ver") + + cfg, err := ch.configService.GetConfig(ctx, id, ver) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error:": "No value under key"}) + return + } + + c.JSON(http.StatusOK, cfg) +} diff --git a/service/configService.go b/service/configService.go index 63e3124..06192c3 100644 --- a/service/configService.go +++ b/service/configService.go @@ -4,25 +4,34 @@ import ( "context" "kuiper/model" "kuiper/store" + "log" + + "go.opentelemetry.io/otel/trace" ) type ConfigService interface { - CreateConfig(ctx context.Context, config model.Config) (string, error) - GetConfig(id string) (model.Config, error) + CreateConfig(ctx context.Context, cfg model.Config) (string, error) + GetConfig(ctx context.Context, id, ver string) (model.Config, error) } -func NewConfigService(cs store.ConfigStore) ConfigService { - return configService{store: cs} +func NewConfigService(cs store.ConfigStore, logger log.Logger, trace trace.Tracer) ConfigService { + return configService{store: cs, logger: logger, trace: trace} } type configService struct { - store store.ConfigStore + store store.ConfigStore + logger log.Logger + trace trace.Tracer } -func (cs configService) CreateConfig(ctx context.Context, config model.Config) (string, error) { - return cs.store.CreateConfig(config, context.TODO()) +func (cs configService) CreateConfig(ctx context.Context, cfg model.Config) (string, error) { + nCtx, span := cs.trace.Start(ctx, "configService.CreateConfig") + defer span.End() + return cs.store.CreateConfig(nCtx, cfg) } -func (cs configService) GetConfig(id string) (model.Config, error) { - return cs.store.GetConfig(id, context.TODO()) +func (cs configService) GetConfig(ctx context.Context, id, ver string) (model.Config, error) { + nCtx, span := cs.trace.Start(ctx, "configService.GetConfig") + defer span.End() + return cs.store.GetConfig(nCtx, id, ver) } diff --git a/store/configStore.go b/store/configStore.go index 1521cee..ff780d8 100644 --- a/store/configStore.go +++ b/store/configStore.go @@ -2,25 +2,87 @@ package store import ( "context" + "encoding/json" + "fmt" "kuiper/model" + "log" + "time" + + "github.com/google/uuid" + clientv3 "go.etcd.io/etcd/client/v3" + "go.opentelemetry.io/otel/trace" ) +//ConfigStore is used for persistance of configurations. +//Configs are uniquely identified by ID and version. Each ID can have multiple versions. type ConfigStore interface { - CreateConfig(cfg model.Config, ctx context.Context) (string, error) - GetConfig(id string, ctx context.Context) (model.Config, error) + //CreateConfig() persists a config and returns it's ID as string. + CreateConfig(ctx context.Context, cfg model.Config) (string, error) + //GetConfig() finds a config by it's ID and version. + GetConfig(ctx context.Context, id, ver string) (model.Config, error) } -func NewConfigStore() ConfigStore { - return configStore{} +func NewConfigStore(cli clientv3.Client, logger log.Logger, trace trace.Tracer) ConfigStore { + return configStore{cli: cli, logger: logger, trace: trace} } type configStore struct { + logger log.Logger + cli clientv3.Client + trace trace.Tracer } -func (cStore configStore) CreateConfig(cfg model.Config, ctx context.Context) (string, error) { - return "", nil +func makeKey(id, ver string) string { + key := fmt.Sprintf("config/%s/%s/", id, ver) + return key } -func (cStore configStore) GetConfig(id string, ctx context.Context) (model.Config, error) { - return model.Config{}, nil +func (cStore configStore) CreateConfig(ctx context.Context, cfg model.Config) (string, error) { + _, span := cStore.trace.Start(ctx, "configStoreEtcd.CreateConfig") + defer span.End() + id := uuid.NewString() + + key := makeKey(id, cfg.Version) + + jsonB, err := json.Marshal(cfg) + if err != nil { + span.RecordError(err) + return "", err + } + + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) + _, err = cStore.cli.KV.Put(kvCtx, key, string(jsonB)) + cancel() + if err != nil { + span.RecordError(err) + return "", err + } + + return id, nil +} + +func (cStore configStore) GetConfig(ctx context.Context, id, ver string) (model.Config, error) { + _, span := cStore.trace.Start(ctx, "configStoreEtcd.GetConfig") + defer span.End() + + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) + res, err := cStore.cli.KV.Get(kvCtx, makeKey(id, ver)) + cancel() + if err != nil { + span.RecordError(err) + return model.Config{}, err + } + + kvs := res.Kvs + kv := kvs[0] + + data := kv.Value + var cfg model.Config + err = json.Unmarshal(data, &cfg) + if err != nil { + span.RecordError(err) + return model.Config{}, nil + } + + return cfg, nil } From 4dd49fdbf7bf16657550d90fb3a00e9af3d80b7a Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Sun, 19 Mar 2023 13:58:00 +0100 Subject: [PATCH 03/10] feature: tracing --- go.mod | 18 ++++++++-- go.sum | 89 ++++++++++++++++++++++++++++++++++++------------ main.go | 35 ++++++++++++++++++- server/server.go | 6 ++-- util/tracing.go | 37 ++++++++++++++++++++ 5 files changed, 158 insertions(+), 27 deletions(-) create mode 100644 util/tracing.go diff --git a/go.mod b/go.mod index f7a9ace..ab8268c 100644 --- a/go.mod +++ b/go.mod @@ -5,13 +5,19 @@ go 1.18 require ( github.com/gin-gonic/gin v1.9.0 github.com/google/uuid v1.3.0 - github.com/opentracing/opentracing-go v1.2.0 + go.etcd.io/etcd/client/v3 v3.5.7 + go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.40.0 + go.opentelemetry.io/otel v1.14.0 + go.opentelemetry.io/otel/exporters/jaeger v1.14.0 + go.opentelemetry.io/otel/sdk v1.14.0 go.opentelemetry.io/otel/trace v1.14.0 ) require ( github.com/bytedance/sonic v1.8.5 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -19,6 +25,8 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.11.2 // indirect github.com/goccy/go-json v0.10.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/leodido/go-urn v1.2.2 // indirect @@ -28,12 +36,18 @@ require ( github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect - go.opentelemetry.io/otel v1.14.0 // indirect + go.etcd.io/etcd/api/v3 v3.5.7 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.17.0 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/crypto v0.7.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/grpc v1.53.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 67f78e0..b47bc42 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,13 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.8.3 h1:pf6fGl5eqWYKkx1RcD4qpuX+BIUaduv/wTm5ekWJ80M= -github.com/bytedance/sonic v1.8.3/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/bytedance/sonic v1.8.5 h1:kjX0/vo5acEQ/sinD/18SkA/lDDUk23F0RcaHvI7omc= github.com/bytedance/sonic v1.8.5/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -25,12 +27,14 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= -github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= -github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.1 h1:lEs5Ob+oOG/Ze199njvzHbhn6p9T+h64F5hRj69iTTo= github.com/goccy/go-json v0.10.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -38,6 +42,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -52,16 +58,17 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= -github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rwtodd/Go.Sed v0.0.0-20210816025313-55464686f9ef/go.mod h1:8AEUvGVi2uQ5b24BIhcr0GCcpd/RNAFWaN2CJFrWIIQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -72,47 +79,85 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.10 h1:eimT6Lsr+2lzmSZxPhLFoOWFmQqwk0fllJJ5hEbTXtQ= -github.com/ugorji/go/codec v1.2.10/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= +go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= +go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= +go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= +go.etcd.io/etcd/client/v3 v3.5.7 h1:u/OhpiuCgYY8awOHlhIhmGIGpxfBU/GZBUP3m/3/Iz4= +go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw= +go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.40.0 h1:E4MMXDxufRnIHXhoTNOlNsdkWpC5HdLhfj84WNRKPkc= +go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.40.0/go.mod h1:A8+gHkpqTfMKxdKWq1pp360nAs096K26CH5Sm2YHDdA= +go.opentelemetry.io/contrib/propagators/b3 v1.15.0 h1:bMaonPyFcAvZ4EVzkUNkfnUHP5Zi63CIDlA3dRsEg8Q= go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= +go.opentelemetry.io/otel/exporters/jaeger v1.14.0 h1:CjbUNd4iN2hHmWekmOqZ+zSCU+dzZppG8XsV+A3oc8Q= +go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA= +go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= +go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.2.0 h1:W1sUEHXiJTfjaFJ5SLo0N6lZn+0eO5gWD1MFeTGqQEY= -golang.org/x/arch v0.2.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/main.go b/main.go index 13c3df1..542d7c4 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,9 @@ package main import ( "context" "kuiper/server" + "kuiper/service" + "kuiper/store" + "kuiper/util" "log" "net/http" "os" @@ -11,13 +14,43 @@ import ( "time" "github.com/gin-gonic/gin" + clientv3 "go.etcd.io/etcd/client/v3" + "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" ) func main() { + logger := log.Default() + + ctx := context.Background() + //init exporter + exporter, err := util.NewJaegerExporter("http://127.0.0.1:14268/api/traces") + if err != nil { + logger.Fatalf(err.Error()) + } + //init traceprovider + tp := util.NewTraceProvider(exporter) + defer func() { _ = tp.Shutdown(ctx) }() + otel.SetTracerProvider(tp) + tracer := tp.Tracer("kuiper") + otel.SetTextMapPropagator(propagation.TraceContext{}) + router := gin.New() - handler := server.NewConfigHandler() + router.Use(gin.Recovery()) + router.Use(otelgin.Middleware("kuiper")) + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: []string{"http://127.0.0.1:2379"}, + DialTimeout: 10 * time.Second, + }) + + cfgStore := store.NewConfigStore(*cli, *logger, tracer) + cfgService := service.NewConfigService(cfgStore, *logger, tracer) + handler := server.NewConfigHandler(tracer, *logger, cfgService) router.POST("/api/config", handler.CreateConfig) + router.GET("/api/config/:id/:ver", handler.GetConfig) // start server srv := &http.Server{Addr: "0.0.0.0:8080", Handler: router} diff --git a/server/server.go b/server/server.go index eed5346..dca29ad 100644 --- a/server/server.go +++ b/server/server.go @@ -2,6 +2,7 @@ package server import ( "kuiper/service" + "log" "go.opentelemetry.io/otel/trace" ) @@ -11,9 +12,10 @@ type ConfigHandler interface { type configHandler struct { tracer trace.Tracer + logger log.Logger configService service.ConfigService } -func NewConfigHandler() configHandler { - return configHandler{} +func NewConfigHandler(tracer trace.Tracer, logger log.Logger, configService service.ConfigService) configHandler { + return configHandler{tracer: tracer, logger: logger, configService: configService} } diff --git a/util/tracing.go b/util/tracing.go new file mode 100644 index 0000000..e1fe9d6 --- /dev/null +++ b/util/tracing.go @@ -0,0 +1,37 @@ +package util + +import ( + "go.opentelemetry.io/otel/exporters/jaeger" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" +) + +func NewTraceProvider(exp sdktrace.SpanExporter) *sdktrace.TracerProvider { + r, err := resource.Merge( + resource.Default(), + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("kuiper"), + ), + ) + + if err != nil { + panic(err.Error()) + } + + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exp), + sdktrace.WithResource(r), + ) +} + +func NewJaegerExporter(address string) (*jaeger.Exporter, error) { + exp, err := jaeger.New( + jaeger.WithCollectorEndpoint( + jaeger.WithEndpoint(address))) + if err != nil { + return nil, err + } + return exp, nil +} From a2ded66cb207463ea66c3e30670878ce40d087dc Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Mon, 20 Mar 2023 12:21:41 +0100 Subject: [PATCH 04/10] add: create new version of cfg --- go.mod | 4 +++ go.sum | 16 +++++++++++ main.go | 3 +- model/config.go | 1 - server/handlers.go | 60 ++++++++++++++++++++++++++++++++++++++-- server/server.go | 3 ++ service/configService.go | 31 +++++++++++++++++++-- store/configStore.go | 52 ++++++++++++++++++++++++++-------- 8 files changed, 150 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index ab8268c..32724c8 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/gin-gonic/gin v1.9.0 github.com/google/uuid v1.3.0 + github.com/nats-io/nats.go v1.24.0 go.etcd.io/etcd/client/v3 v3.5.7 go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.40.0 go.opentelemetry.io/otel v1.14.0 @@ -33,6 +34,9 @@ require ( github.com/mattn/go-isatty v0.0.17 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nats-io/nats-server/v2 v2.9.15 // indirect + github.com/nats-io/nkeys v0.3.0 // indirect + github.com/nats-io/nuid v1.0.1 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect diff --git a/go.sum b/go.sum index b47bc42..d373daa 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -53,11 +54,21 @@ github.com/leodido/go-urn v1.2.2 h1:7z68G0FCGvDk646jz1AelTYNYWrTNm0bEcFAo147wt4= github.com/leodido/go-urn v1.2.2/go.mod h1:kUaIbLZWttglzwNuG0pgsh5vuV6u2YcGBYz1hIPjtOQ= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= +github.com/nats-io/nats-server/v2 v2.9.15 h1:MuwEJheIwpvFgqvbs20W8Ish2azcygjf4Z0liVu2I4c= +github.com/nats-io/nats-server/v2 v2.9.15/go.mod h1:QlCTy115fqpx4KSOPFIxSV7DdI6OxtZsGOL1JLdeRlE= +github.com/nats-io/nats.go v1.24.0 h1:CRiD8L5GOQu/DcfkmgBcTTIQORMwizF+rPk6T0RaHVQ= +github.com/nats-io/nats.go v1.24.0/go.mod h1:dVQF+BK3SzUZpwyzHedXsvH3EO38aVKuOPkkHlv5hXA= +github.com/nats-io/nkeys v0.3.0 h1:cgM5tL53EvYRU+2YLXIK0G2mJtK12Ft9oeooSZMA2G8= +github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= @@ -112,6 +123,7 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -120,6 +132,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -128,14 +141,17 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= diff --git a/main.go b/main.go index 542d7c4..dadbc13 100644 --- a/main.go +++ b/main.go @@ -49,8 +49,9 @@ func main() { cfgService := service.NewConfigService(cfgStore, *logger, tracer) handler := server.NewConfigHandler(tracer, *logger, cfgService) - router.POST("/api/config", handler.CreateConfig) + router.POST("/api/config", handler.SaveConfig) router.GET("/api/config/:id/:ver", handler.GetConfig) + router.POST("/api/config/:id/", handler.CreateNewVersion) // start server srv := &http.Server{Addr: "0.0.0.0:8080", Handler: router} diff --git a/model/config.go b/model/config.go index ecd42d9..f0d372f 100644 --- a/model/config.go +++ b/model/config.go @@ -1,7 +1,6 @@ package model type Config struct { - ID string `json:"id"` Version string `json:"version"` Entries map[string]string `json:"entries"` } diff --git a/server/handlers.go b/server/handlers.go index 3a2c74e..f755c35 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -2,13 +2,14 @@ package server import ( "errors" + "kuiper/service" "mime" "net/http" "github.com/gin-gonic/gin" ) -func (ch configHandler) CreateConfig(c *gin.Context) { +func (ch configHandler) SaveConfig(c *gin.Context) { ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.CreateConfig") defer span.End() @@ -26,13 +27,14 @@ func (ch configHandler) CreateConfig(c *gin.Context) { err := errors.New("Expect application/json Content-Type") span.RecordError(err) http.Error(c.Writer, err.Error(), http.StatusUnsupportedMediaType) + c.JSON(http.StatusUnsupportedMediaType, gin.H{"error:": "Only application/json is accepted"}) return } rt, err := decodeConfigBody(c.Request.Body) - if err != nil || rt.Version == "" || rt.Entries == nil { + if err != nil || rt.Entries == nil { span.RecordError(err) - http.Error(c.Writer, "Invalid JSON format", http.StatusBadRequest) + c.JSON(http.StatusInternalServerError, gin.H{"error:": "Invalid JSON"}) return } @@ -42,6 +44,13 @@ func (ch configHandler) CreateConfig(c *gin.Context) { // } cid, err := ch.configService.CreateConfig(ctx, rt) + if err == service.NoVersionError { + c.JSON(http.StatusBadRequest, gin.H{"error:": "No version supplied"}) + return + } else if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error:": "Error when saving config"}) + return + } // reqId := "" // if err == nil { @@ -66,3 +75,48 @@ func (ch configHandler) GetConfig(c *gin.Context) { c.JSON(http.StatusOK, cfg) } + +func (ch configHandler) CreateNewVersion(c *gin.Context) { + ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.CreateNewVersion") + defer span.End() + + id := c.Param("id") + + contentType := c.Request.Header.Get("Content-Type") + // idemKey := c.Request.Header.Get("x-idempotency-key") + + mediatype, _, err := mime.ParseMediaType(contentType) + if err != nil { + span.RecordError(err) + http.Error(c.Writer, err.Error(), http.StatusBadRequest) + return + } + + if mediatype != "application/json" { + err := errors.New("Expect application/json Content-Type") + span.RecordError(err) + http.Error(c.Writer, err.Error(), http.StatusUnsupportedMediaType) + c.JSON(http.StatusUnsupportedMediaType, gin.H{"error:": "Only application/json is accepted"}) + return + } + + rt, err := decodeConfigBody(c.Request.Body) + if err != nil || rt.Entries == nil { + span.RecordError(err) + c.JSON(http.StatusInternalServerError, gin.H{"error:": "Invalid JSON"}) + return + } + + err = ch.configService.CreateNewVersion(ctx, rt, id) + if err == service.NoVersionError { + c.JSON(http.StatusBadRequest, gin.H{"error:": "No version supplied"}) + return + } + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error:": "Invalid JSON"}) + return + } + + c.JSON(http.StatusCreated, gin.H{"id": id}) + return +} diff --git a/server/server.go b/server/server.go index dca29ad..fbf9112 100644 --- a/server/server.go +++ b/server/server.go @@ -4,10 +4,13 @@ import ( "kuiper/service" "log" + "github.com/gin-gonic/gin" "go.opentelemetry.io/otel/trace" ) type ConfigHandler interface { + SaveConfig(c *gin.Context) + GetConfig(c *gin.Context) } type configHandler struct { diff --git a/service/configService.go b/service/configService.go index 06192c3..e7c2766 100644 --- a/service/configService.go +++ b/service/configService.go @@ -2,6 +2,7 @@ package service import ( "context" + "errors" "kuiper/model" "kuiper/store" "log" @@ -9,9 +10,15 @@ import ( "go.opentelemetry.io/otel/trace" ) +var NoVersionError = errors.New("Must supply version name when creating a new config") + type ConfigService interface { + //Checks if cfg is a valid config and tries to persist it. CreateConfig(ctx context.Context, cfg model.Config) (string, error) - GetConfig(ctx context.Context, id, ver string) (model.Config, error) + //Finds a config by id and version + GetConfig(ctx context.Context, id, ver string) (map[string]string, error) + //Creates a new version of already existing config + CreateNewVersion(ctx context.Context, cfg model.Config, id string) error } func NewConfigService(cs store.ConfigStore, logger log.Logger, trace trace.Tracer) ConfigService { @@ -27,11 +34,29 @@ type configService struct { func (cs configService) CreateConfig(ctx context.Context, cfg model.Config) (string, error) { nCtx, span := cs.trace.Start(ctx, "configService.CreateConfig") defer span.End() - return cs.store.CreateConfig(nCtx, cfg) + + if len(cfg.Version) == 0 { + span.RecordError(NoVersionError) + return "", NoVersionError + } + + return cs.store.SaveConfig(nCtx, cfg) } -func (cs configService) GetConfig(ctx context.Context, id, ver string) (model.Config, error) { +func (cs configService) GetConfig(ctx context.Context, id, ver string) (map[string]string, error) { nCtx, span := cs.trace.Start(ctx, "configService.GetConfig") defer span.End() return cs.store.GetConfig(nCtx, id, ver) } + +func (cs configService) CreateNewVersion(ctx context.Context, cfg model.Config, id string) error { + nCtx, span := cs.trace.Start(ctx, "configService.CreateNewVersion") + defer span.End() + + if len(cfg.Version) == 0 { + span.RecordError(NoVersionError) + return NoVersionError + } + + return cs.store.SaveVersion(nCtx, cfg, id) +} diff --git a/store/configStore.go b/store/configStore.go index ff780d8..5316003 100644 --- a/store/configStore.go +++ b/store/configStore.go @@ -16,10 +16,12 @@ import ( //ConfigStore is used for persistance of configurations. //Configs are uniquely identified by ID and version. Each ID can have multiple versions. type ConfigStore interface { - //CreateConfig() persists a config and returns it's ID as string. - CreateConfig(ctx context.Context, cfg model.Config) (string, error) + //SaveConfig() persists a config and returns it's ID as string. + SaveConfig(ctx context.Context, cfg model.Config) (string, error) //GetConfig() finds a config by it's ID and version. - GetConfig(ctx context.Context, id, ver string) (model.Config, error) + GetConfig(ctx context.Context, id, ver string) (map[string]string, error) + //Creates a new version for an already existing id + SaveVersion(ctx context.Context, cfg model.Config, id string) error } func NewConfigStore(cli clientv3.Client, logger log.Logger, trace trace.Tracer) ConfigStore { @@ -37,14 +39,16 @@ func makeKey(id, ver string) string { return key } -func (cStore configStore) CreateConfig(ctx context.Context, cfg model.Config) (string, error) { +func (cStore configStore) SaveConfig(ctx context.Context, cfg model.Config) (string, error) { _, span := cStore.trace.Start(ctx, "configStoreEtcd.CreateConfig") defer span.End() - id := uuid.NewString() + + var id string + id = uuid.NewString() key := makeKey(id, cfg.Version) - jsonB, err := json.Marshal(cfg) + jsonB, err := json.Marshal(cfg.Entries) if err != nil { span.RecordError(err) return "", err @@ -61,28 +65,52 @@ func (cStore configStore) CreateConfig(ctx context.Context, cfg model.Config) (s return id, nil } -func (cStore configStore) GetConfig(ctx context.Context, id, ver string) (model.Config, error) { +func (cStore configStore) GetConfig(ctx context.Context, id, ver string) (map[string]string, error) { _, span := cStore.trace.Start(ctx, "configStoreEtcd.GetConfig") defer span.End() + entries := make(map[string]string) + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) res, err := cStore.cli.KV.Get(kvCtx, makeKey(id, ver)) cancel() if err != nil { span.RecordError(err) - return model.Config{}, err + return entries, err } kvs := res.Kvs kv := kvs[0] data := kv.Value - var cfg model.Config - err = json.Unmarshal(data, &cfg) + err = json.Unmarshal(data, &entries) + if err != nil { + span.RecordError(err) + return entries, err + } + + return entries, nil +} + +func (cStore configStore) SaveVersion(ctx context.Context, cfg model.Config, id string) error { + _, span := cStore.trace.Start(ctx, "configStoreEtcd.SaveVersion") + defer span.End() + + key := makeKey(id, cfg.Version) + + jsonB, err := json.Marshal(cfg.Entries) + if err != nil { + span.RecordError(err) + return err + } + + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) + _, err = cStore.cli.KV.Put(kvCtx, key, string(jsonB)) + cancel() if err != nil { span.RecordError(err) - return model.Config{}, nil + return err } - return cfg, nil + return nil } From 5848297bf95a2071baf6ee0bed58d239aa335323 Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Wed, 22 Mar 2023 15:38:27 +0100 Subject: [PATCH 05/10] add: delete by prefix, new version can't be added if id doesn't exist --- main.go | 2 + server/handlers.go | 44 ++++++++++++- server/server.go | 3 + service/configService.go | 17 +++++ store/configStore.go | 130 ++++++++++++++++++++++++++++++++++----- 5 files changed, 180 insertions(+), 16 deletions(-) diff --git a/main.go b/main.go index dadbc13..11bf7ca 100644 --- a/main.go +++ b/main.go @@ -52,6 +52,8 @@ func main() { router.POST("/api/config", handler.SaveConfig) router.GET("/api/config/:id/:ver", handler.GetConfig) router.POST("/api/config/:id/", handler.CreateNewVersion) + router.DELETE("/api/config/:id/:ver", handler.DeleteConfig) + router.DELETE("/api/config/:id", handler.DeleteConfigsWithPrefix) // start server srv := &http.Server{Addr: "0.0.0.0:8080", Handler: router} diff --git a/server/handlers.go b/server/handlers.go index f755c35..cc562c1 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -3,6 +3,7 @@ package server import ( "errors" "kuiper/service" + "kuiper/store" "mime" "net/http" @@ -61,7 +62,7 @@ func (ch configHandler) SaveConfig(c *gin.Context) { } func (ch configHandler) GetConfig(c *gin.Context) { - ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.CreateConfig") + ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.GetConfig") defer span.End() id := c.Param("id") @@ -108,9 +109,16 @@ func (ch configHandler) CreateNewVersion(c *gin.Context) { } err = ch.configService.CreateNewVersion(ctx, rt, id) - if err == service.NoVersionError { + switch err { + case service.NoVersionError: c.JSON(http.StatusBadRequest, gin.H{"error:": "No version supplied"}) return + case store.KeyAlreadyExistsError: + c.JSON(http.StatusConflict, gin.H{"error:": "Version already exists"}) + return + case store.ErrorNotFound: + c.JSON(http.StatusNotFound, gin.H{"error:": "Configuration with given ID doesn't exist"}) + return } if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error:": "Invalid JSON"}) @@ -120,3 +128,35 @@ func (ch configHandler) CreateNewVersion(c *gin.Context) { c.JSON(http.StatusCreated, gin.H{"id": id}) return } + +func (ch configHandler) DeleteConfig(c *gin.Context) { + ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.DeleteConfig") + defer span.End() + + id := c.Param("id") + ver := c.Param("ver") + + cfg, err := ch.configService.DeleteConfig(ctx, id, ver) + if err == store.ErrorNotFound { + c.JSON(http.StatusNotFound, gin.H{"error:": "Configuration with the given ID and version doesn't exist"}) + return + } else if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error:": "Failure when connecting to database"}) + } + + c.JSON(http.StatusOK, cfg) +} + +func (ch configHandler) DeleteConfigsWithPrefix(c *gin.Context) { + ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.DeleteConfigsWithPrefix") + defer span.End() + + id := c.Param("id") + + cfg, err := ch.configService.DeleteConfigsWithPrefix(ctx, id) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error:": "Failure when connecting to database"}) + } + + c.JSON(http.StatusOK, cfg) +} diff --git a/server/server.go b/server/server.go index fbf9112..1a36967 100644 --- a/server/server.go +++ b/server/server.go @@ -11,6 +11,9 @@ import ( type ConfigHandler interface { SaveConfig(c *gin.Context) GetConfig(c *gin.Context) + DeleteConfig(c *gin.Context) + CreateNewVersion(c *gin.Context) + DeleteConfigsWithPrefix(c *gin.Context) } type configHandler struct { diff --git a/service/configService.go b/service/configService.go index e7c2766..78357b6 100644 --- a/service/configService.go +++ b/service/configService.go @@ -11,6 +11,7 @@ import ( ) var NoVersionError = errors.New("Must supply version name when creating a new config") +var DbError = errors.New("Error happened while connecting to database") type ConfigService interface { //Checks if cfg is a valid config and tries to persist it. @@ -19,6 +20,10 @@ type ConfigService interface { GetConfig(ctx context.Context, id, ver string) (map[string]string, error) //Creates a new version of already existing config CreateNewVersion(ctx context.Context, cfg model.Config, id string) error + //Deletes config by id and version, returns error when config wasn't foun + DeleteConfig(ctx context.Context, id, ver string) (cfg map[string]string, err error) + //Deletes all configs with the given ID + DeleteConfigsWithPrefix(ctx context.Context, id string) (deleted map[string]map[string]string, err error) } func NewConfigService(cs store.ConfigStore, logger log.Logger, trace trace.Tracer) ConfigService { @@ -60,3 +65,15 @@ func (cs configService) CreateNewVersion(ctx context.Context, cfg model.Config, return cs.store.SaveVersion(nCtx, cfg, id) } + +func (cs configService) DeleteConfig(ctx context.Context, id, ver string) (cfg map[string]string, err error) { + nCtx, span := cs.trace.Start(ctx, "configService.DeleteConfig") + defer span.End() + return cs.store.DeleteConfig(nCtx, id, ver) +} + +func (cs configService) DeleteConfigsWithPrefix(ctx context.Context, id string) (deleted map[string]map[string]string, err error) { + nCtx, span := cs.trace.Start(ctx, "configService.DeleteConfig") + defer span.End() + return cs.store.DeleteConfigsWithPrefix(nCtx, id) +} diff --git a/store/configStore.go b/store/configStore.go index 5316003..a207ec0 100644 --- a/store/configStore.go +++ b/store/configStore.go @@ -3,6 +3,7 @@ package store import ( "context" "encoding/json" + "errors" "fmt" "kuiper/model" "log" @@ -13,6 +14,9 @@ import ( "go.opentelemetry.io/otel/trace" ) +var ErrorNotFound = errors.New("Config not found") +var KeyAlreadyExistsError = errors.New("Version already exists for the given ID") + //ConfigStore is used for persistance of configurations. //Configs are uniquely identified by ID and version. Each ID can have multiple versions. type ConfigStore interface { @@ -22,13 +26,17 @@ type ConfigStore interface { GetConfig(ctx context.Context, id, ver string) (map[string]string, error) //Creates a new version for an already existing id SaveVersion(ctx context.Context, cfg model.Config, id string) error + //Deletes a config and returns the config that was deleted + DeleteConfig(ctx context.Context, id, ver string) (map[string]string, error) + //Deletes all the configs with the given ID and returns them + DeleteConfigsWithPrefix(ctx context.Context, id string) (map[string]map[string]string, error) } func NewConfigStore(cli clientv3.Client, logger log.Logger, trace trace.Tracer) ConfigStore { - return configStore{cli: cli, logger: logger, trace: trace} + return configStoreEtcd{cli: cli, logger: logger, trace: trace} } -type configStore struct { +type configStoreEtcd struct { logger log.Logger cli clientv3.Client trace trace.Tracer @@ -38,8 +46,12 @@ func makeKey(id, ver string) string { key := fmt.Sprintf("config/%s/%s/", id, ver) return key } +func makeIdPrefix(id string) string { + key := fmt.Sprintf("config/%s/", id) + return key +} -func (cStore configStore) SaveConfig(ctx context.Context, cfg model.Config) (string, error) { +func (cStore configStoreEtcd) SaveConfig(ctx context.Context, cfg model.Config) (string, error) { _, span := cStore.trace.Start(ctx, "configStoreEtcd.CreateConfig") defer span.End() @@ -65,7 +77,7 @@ func (cStore configStore) SaveConfig(ctx context.Context, cfg model.Config) (str return id, nil } -func (cStore configStore) GetConfig(ctx context.Context, id, ver string) (map[string]string, error) { +func (cStore configStoreEtcd) GetConfig(ctx context.Context, id, ver string) (map[string]string, error) { _, span := cStore.trace.Start(ctx, "configStoreEtcd.GetConfig") defer span.End() @@ -80,20 +92,36 @@ func (cStore configStore) GetConfig(ctx context.Context, id, ver string) (map[st } kvs := res.Kvs - kv := kvs[0] + if len(kvs) > 0 { + kv := kvs[0] + data := kv.Value + err = json.Unmarshal(data, &entries) + if err != nil { + span.RecordError(err) + return entries, err + } + return entries, nil + } - data := kv.Value - err = json.Unmarshal(data, &entries) + return entries, ErrorNotFound +} + +func (cStore configStoreEtcd) getPrefixCount(ctx context.Context, id string) (int64, error) { + _, span := cStore.trace.Start(ctx, "configStoreEtcd.getPrefixCount") + defer span.End() + + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) + key := makeIdPrefix(id) + res, err := cStore.cli.Get(kvCtx, key, clientv3.WithPrefix(), clientv3.WithCountOnly()) + cancel() if err != nil { - span.RecordError(err) - return entries, err + return 0, err } - - return entries, nil + return res.Count, nil } -func (cStore configStore) SaveVersion(ctx context.Context, cfg model.Config, id string) error { - _, span := cStore.trace.Start(ctx, "configStoreEtcd.SaveVersion") +func (cStore configStoreEtcd) SaveVersion(ctx context.Context, cfg model.Config, id string) error { + ctx, span := cStore.trace.Start(ctx, "configStoreEtcd.SaveVersion") defer span.End() key := makeKey(id, cfg.Version) @@ -104,13 +132,87 @@ func (cStore configStore) SaveVersion(ctx context.Context, cfg model.Config, id return err } + c, err := cStore.getPrefixCount(ctx, id) + if err != nil { + return err + } + if c == 0 { + return ErrorNotFound + } + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) - _, err = cStore.cli.KV.Put(kvCtx, key, string(jsonB)) + op := clientv3.OpPut(key, string(jsonB)) + res, err := cStore.cli.Txn(kvCtx).If(clientv3.Compare(clientv3.Version(key), "=", 0)).Then(op).Commit() cancel() if err != nil { span.RecordError(err) return err } + if !res.Succeeded { + return KeyAlreadyExistsError + } + return nil } + +func (cStore configStoreEtcd) DeleteConfig(ctx context.Context, id, ver string) (map[string]string, error) { + _, span := cStore.trace.Start(ctx, "configStoreEtcd.DeleteConfig") + defer span.End() + + entries := make(map[string]string) + key := makeKey(id, ver) + + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) + res, err := cStore.cli.KV.Delete(kvCtx, key, clientv3.WithPrevKV()) + cancel() + if err != nil { + span.RecordError(err) + return entries, err + } + + kvs := res.PrevKvs + if len(kvs) > 0 { + kv := kvs[0] + data := kv.Value + err = json.Unmarshal(data, &entries) + if err != nil { + span.RecordError(err) + return entries, err + } + return entries, nil + } + + return entries, ErrorNotFound +} + +func (cStore configStoreEtcd) DeleteConfigsWithPrefix(ctx context.Context, id string) (map[string]map[string]string, error) { + _, span := cStore.trace.Start(ctx, "configStoreEtcd.DeleteConfig") + defer span.End() + + cfgs := make(map[string]map[string]string) + key := makeIdPrefix(id) + + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) + res, err := cStore.cli.KV.Delete(kvCtx, key, clientv3.WithPrevKV(), clientv3.WithPrefix()) + cancel() + if err != nil { + span.RecordError(err) + return cfgs, err + } + + kvs := res.PrevKvs + for _, kv := range kvs { + cfg := make(map[string]string) + data := kv.Value + err = json.Unmarshal(data, &cfg) + if err != nil { + span.RecordError(err) + return cfgs, err + } + + cfgs[string(kv.Key)] = cfg + } + + return cfgs, nil +} From 7b1531185758b675ead36c3021f58417c0796324 Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Fri, 24 Mar 2023 13:32:20 +0100 Subject: [PATCH 06/10] add: publish to nats when new config is created --- main.go | 4 +++- model/config.go | 1 + server/handlers.go | 26 +++++++++++++------------- server/helper.go | 11 +++++++++++ server/server.go | 6 ++++-- util/nats.go | 7 +++++++ 6 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 util/nats.go diff --git a/main.go b/main.go index 11bf7ca..f80f15f 100644 --- a/main.go +++ b/main.go @@ -45,9 +45,11 @@ func main() { DialTimeout: 10 * time.Second, }) + natsCon, err := util.Conn("http://127.0.0.1:4222") + cfgStore := store.NewConfigStore(*cli, *logger, tracer) cfgService := service.NewConfigService(cfgStore, *logger, tracer) - handler := server.NewConfigHandler(tracer, *logger, cfgService) + handler := server.NewConfigHandler(tracer, *logger, cfgService, *natsCon) router.POST("/api/config", handler.SaveConfig) router.GET("/api/config/:id/:ver", handler.GetConfig) diff --git a/model/config.go b/model/config.go index f0d372f..812e1ec 100644 --- a/model/config.go +++ b/model/config.go @@ -1,6 +1,7 @@ package model type Config struct { + Service string `json:"service"` Version string `json:"version"` Entries map[string]string `json:"entries"` } diff --git a/server/handlers.go b/server/handlers.go index cc562c1..3343cf8 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -1,7 +1,10 @@ package server import ( + "encoding/json" "errors" + "fmt" + "kuiper/model" "kuiper/service" "kuiper/store" "mime" @@ -10,6 +13,10 @@ import ( "github.com/gin-gonic/gin" ) +func natsKey(serviceName string) string { + return fmt.Sprintf("config.%s", serviceName) +} + func (ch configHandler) SaveConfig(c *gin.Context) { ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.CreateConfig") defer span.End() @@ -32,19 +39,15 @@ func (ch configHandler) SaveConfig(c *gin.Context) { return } - rt, err := decodeConfigBody(c.Request.Body) - if err != nil || rt.Entries == nil { + newCfg, err := decodeNewConfigBody(c.Request.Body) + if err != nil || newCfg.Entries == nil { span.RecordError(err) c.JSON(http.StatusInternalServerError, gin.H{"error:": "Invalid JSON"}) return } - // if ch.idempotencyService.FindRequestId(ctx, requestId) == true { - // http.Error(c.Writer, "Request has been already sent", http.StatusBadRequest) - // return - // } - - cid, err := ch.configService.CreateConfig(ctx, rt) + config := model.Config{Version: newCfg.Version, Entries: newCfg.Entries} + cid, err := ch.configService.CreateConfig(ctx, config) if err == service.NoVersionError { c.JSON(http.StatusBadRequest, gin.H{"error:": "No version supplied"}) return @@ -53,11 +56,8 @@ func (ch configHandler) SaveConfig(c *gin.Context) { return } - // reqId := "" - // if err == nil { - // reqId = ts.idempotencyService.SaveRequestId(ctx) - // } - + cfgJson, _ := json.Marshal(config.Entries) + ch.nats.Publish(natsKey(newCfg.ServiceName), cfgJson) c.JSON(http.StatusOK, gin.H{"id": cid}) } diff --git a/server/helper.go b/server/helper.go index 1b618b9..d8ca2ea 100644 --- a/server/helper.go +++ b/server/helper.go @@ -16,3 +16,14 @@ func decodeConfigBody(r io.Reader) (model.Config, error) { } return *config, nil } + +func decodeNewConfigBody(r io.Reader) (model.NewConfigDto, error) { + dec := json.NewDecoder(r) + dec.DisallowUnknownFields() + + var config *model.NewConfigDto + if err := dec.Decode(&config); err != nil { + return model.NewConfigDto{}, err + } + return *config, nil +} diff --git a/server/server.go b/server/server.go index 1a36967..caa23f7 100644 --- a/server/server.go +++ b/server/server.go @@ -5,6 +5,7 @@ import ( "log" "github.com/gin-gonic/gin" + "github.com/nats-io/nats.go" "go.opentelemetry.io/otel/trace" ) @@ -20,8 +21,9 @@ type configHandler struct { tracer trace.Tracer logger log.Logger configService service.ConfigService + nats nats.Conn } -func NewConfigHandler(tracer trace.Tracer, logger log.Logger, configService service.ConfigService) configHandler { - return configHandler{tracer: tracer, logger: logger, configService: configService} +func NewConfigHandler(tracer trace.Tracer, logger log.Logger, configService service.ConfigService, conn nats.Conn) configHandler { + return configHandler{tracer: tracer, logger: logger, configService: configService, nats: conn} } diff --git a/util/nats.go b/util/nats.go new file mode 100644 index 0000000..4b25365 --- /dev/null +++ b/util/nats.go @@ -0,0 +1,7 @@ +package util + +import "github.com/nats-io/nats.go" + +func Conn(natsAddr string) (*nats.Conn, error) { + return nats.Connect(natsAddr) +} From 54b79b2c261b1f85d5610a25d3eb6402f8052978 Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Tue, 28 Mar 2023 22:39:56 +0200 Subject: [PATCH 07/10] fix: DELETE requests now return 404 if nothing was deleted --- server/handlers.go | 51 +++++++++++++++++++++++----------------- server/helper.go | 11 --------- service/configService.go | 5 ++++ store/configStore.go | 22 +++++++++++------ 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/server/handlers.go b/server/handlers.go index 3343cf8..1d613a9 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "kuiper/model" "kuiper/service" "kuiper/store" "mime" @@ -35,29 +34,34 @@ func (ch configHandler) SaveConfig(c *gin.Context) { err := errors.New("Expect application/json Content-Type") span.RecordError(err) http.Error(c.Writer, err.Error(), http.StatusUnsupportedMediaType) - c.JSON(http.StatusUnsupportedMediaType, gin.H{"error:": "Only application/json is accepted"}) + c.JSON(http.StatusUnsupportedMediaType, gin.H{"error": "Only application/json is accepted"}) return } - newCfg, err := decodeNewConfigBody(c.Request.Body) + newCfg, err := decodeConfigBody(c.Request.Body) if err != nil || newCfg.Entries == nil { span.RecordError(err) - c.JSON(http.StatusInternalServerError, gin.H{"error:": "Invalid JSON"}) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Invalid JSON"}) return } - config := model.Config{Version: newCfg.Version, Entries: newCfg.Entries} - cid, err := ch.configService.CreateConfig(ctx, config) + cid, err := ch.configService.CreateConfig(ctx, newCfg) if err == service.NoVersionError { - c.JSON(http.StatusBadRequest, gin.H{"error:": "No version supplied"}) + c.JSON(http.StatusBadRequest, gin.H{"error": "No version supplied"}) + return + } else if err == store.KeyAlreadyExistsError { + c.JSON(http.StatusConflict, gin.H{"error": "Version already exists for the service"}) + return + } else if err == service.NoServiceNameError { + c.JSON(http.StatusBadRequest, gin.H{"error": "No service name supplied"}) return } else if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error:": "Error when saving config"}) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Error when saving config"}) return } - cfgJson, _ := json.Marshal(config.Entries) - ch.nats.Publish(natsKey(newCfg.ServiceName), cfgJson) + cfgJson, _ := json.Marshal(newCfg.Entries) + ch.nats.Publish(natsKey(newCfg.Service), cfgJson) c.JSON(http.StatusOK, gin.H{"id": cid}) } @@ -70,7 +74,7 @@ func (ch configHandler) GetConfig(c *gin.Context) { cfg, err := ch.configService.GetConfig(ctx, id, ver) if err != nil { - c.JSON(http.StatusNotFound, gin.H{"error:": "No value under key"}) + c.JSON(http.StatusNotFound, gin.H{"error": "No value under key"}) return } @@ -97,31 +101,31 @@ func (ch configHandler) CreateNewVersion(c *gin.Context) { err := errors.New("Expect application/json Content-Type") span.RecordError(err) http.Error(c.Writer, err.Error(), http.StatusUnsupportedMediaType) - c.JSON(http.StatusUnsupportedMediaType, gin.H{"error:": "Only application/json is accepted"}) + c.JSON(http.StatusUnsupportedMediaType, gin.H{"error": "Only application/json is accepted"}) return } rt, err := decodeConfigBody(c.Request.Body) if err != nil || rt.Entries == nil { span.RecordError(err) - c.JSON(http.StatusInternalServerError, gin.H{"error:": "Invalid JSON"}) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Invalid JSON"}) return } err = ch.configService.CreateNewVersion(ctx, rt, id) switch err { case service.NoVersionError: - c.JSON(http.StatusBadRequest, gin.H{"error:": "No version supplied"}) + c.JSON(http.StatusBadRequest, gin.H{"error": "No version supplied"}) return case store.KeyAlreadyExistsError: - c.JSON(http.StatusConflict, gin.H{"error:": "Version already exists"}) + c.JSON(http.StatusConflict, gin.H{"error": "Version already exists"}) return case store.ErrorNotFound: - c.JSON(http.StatusNotFound, gin.H{"error:": "Configuration with given ID doesn't exist"}) + c.JSON(http.StatusNotFound, gin.H{"error": "Configuration with given ID doesn't exist"}) return } if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error:": "Invalid JSON"}) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Invalid JSON"}) return } @@ -138,10 +142,10 @@ func (ch configHandler) DeleteConfig(c *gin.Context) { cfg, err := ch.configService.DeleteConfig(ctx, id, ver) if err == store.ErrorNotFound { - c.JSON(http.StatusNotFound, gin.H{"error:": "Configuration with the given ID and version doesn't exist"}) + c.JSON(http.StatusNotFound, gin.H{"error": "Configuration with the given ID and version doesn't exist"}) return } else if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error:": "Failure when connecting to database"}) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failure when connecting to database"}) } c.JSON(http.StatusOK, cfg) @@ -154,8 +158,13 @@ func (ch configHandler) DeleteConfigsWithPrefix(c *gin.Context) { id := c.Param("id") cfg, err := ch.configService.DeleteConfigsWithPrefix(ctx, id) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error:": "Failure when connecting to database"}) + if err == store.ErrorNotFound { + errorMsg := fmt.Sprintf("No configurations found for %s", id) + c.JSON(http.StatusNotFound, gin.H{"error": errorMsg}) + return + } else if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failure when connecting to database"}) + return } c.JSON(http.StatusOK, cfg) diff --git a/server/helper.go b/server/helper.go index d8ca2ea..1b618b9 100644 --- a/server/helper.go +++ b/server/helper.go @@ -16,14 +16,3 @@ func decodeConfigBody(r io.Reader) (model.Config, error) { } return *config, nil } - -func decodeNewConfigBody(r io.Reader) (model.NewConfigDto, error) { - dec := json.NewDecoder(r) - dec.DisallowUnknownFields() - - var config *model.NewConfigDto - if err := dec.Decode(&config); err != nil { - return model.NewConfigDto{}, err - } - return *config, nil -} diff --git a/service/configService.go b/service/configService.go index 78357b6..4c652b7 100644 --- a/service/configService.go +++ b/service/configService.go @@ -11,6 +11,7 @@ import ( ) var NoVersionError = errors.New("Must supply version name when creating a new config") +var NoServiceNameError = errors.New("Must supply service name when creating a new config") var DbError = errors.New("Error happened while connecting to database") type ConfigService interface { @@ -44,6 +45,10 @@ func (cs configService) CreateConfig(ctx context.Context, cfg model.Config) (str span.RecordError(NoVersionError) return "", NoVersionError } + if len(cfg.Service) == 0 { + span.RecordError(NoServiceNameError) + return "", NoServiceNameError + } return cs.store.SaveConfig(nCtx, cfg) } diff --git a/store/configStore.go b/store/configStore.go index a207ec0..c3e0e53 100644 --- a/store/configStore.go +++ b/store/configStore.go @@ -9,7 +9,6 @@ import ( "log" "time" - "github.com/google/uuid" clientv3 "go.etcd.io/etcd/client/v3" "go.opentelemetry.io/otel/trace" ) @@ -17,8 +16,8 @@ import ( var ErrorNotFound = errors.New("Config not found") var KeyAlreadyExistsError = errors.New("Version already exists for the given ID") -//ConfigStore is used for persistance of configurations. -//Configs are uniquely identified by ID and version. Each ID can have multiple versions. +// ConfigStore is used for persistance of configurations. +// Configs are uniquely identified by ID and version. Each ID can have multiple versions. type ConfigStore interface { //SaveConfig() persists a config and returns it's ID as string. SaveConfig(ctx context.Context, cfg model.Config) (string, error) @@ -56,7 +55,7 @@ func (cStore configStoreEtcd) SaveConfig(ctx context.Context, cfg model.Config) defer span.End() var id string - id = uuid.NewString() + id = cfg.Service key := makeKey(id, cfg.Version) @@ -67,12 +66,18 @@ func (cStore configStoreEtcd) SaveConfig(ctx context.Context, cfg model.Config) } kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) - _, err = cStore.cli.KV.Put(kvCtx, key, string(jsonB)) + op := clientv3.OpPut(key, string(jsonB)) + res, err := cStore.cli.Txn(kvCtx).If(clientv3.Compare(clientv3.Version(key), "=", 0)).Then(op).Commit() cancel() if err != nil { span.RecordError(err) return "", err } + if !res.Succeeded { + err = KeyAlreadyExistsError + span.RecordError(KeyAlreadyExistsError) + return "", err + } return id, nil } @@ -172,7 +177,7 @@ func (cStore configStoreEtcd) DeleteConfig(ctx context.Context, id, ver string) } kvs := res.PrevKvs - if len(kvs) > 0 { + if res.Deleted > 0 { kv := kvs[0] data := kv.Value err = json.Unmarshal(data, &entries) @@ -187,7 +192,7 @@ func (cStore configStoreEtcd) DeleteConfig(ctx context.Context, id, ver string) } func (cStore configStoreEtcd) DeleteConfigsWithPrefix(ctx context.Context, id string) (map[string]map[string]string, error) { - _, span := cStore.trace.Start(ctx, "configStoreEtcd.DeleteConfig") + _, span := cStore.trace.Start(ctx, "configStoreEtcd.DeleteConfigWithPrefix") defer span.End() cfgs := make(map[string]map[string]string) @@ -200,6 +205,9 @@ func (cStore configStoreEtcd) DeleteConfigsWithPrefix(ctx context.Context, id st span.RecordError(err) return cfgs, err } + if res.Deleted == 0 { + return cfgs, ErrorNotFound + } kvs := res.PrevKvs for _, kv := range kvs { From 90eca445f2e4a8544c8d9f844af8a0e284906813 Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Sun, 2 Apr 2023 12:05:28 +0200 Subject: [PATCH 08/10] feature: get all versions for service --- main.go | 23 +++++++--- model/config.go | 8 ++-- server/handlers.go | 23 ++++++++++ server/server.go | 1 + service/configService.go | 36 +++++++++++++--- store/configStore.go | 92 ++++++++++++++++++++++++++++------------ store/keyUtils.go | 20 +++++++++ 7 files changed, 162 insertions(+), 41 deletions(-) create mode 100644 store/keyUtils.go diff --git a/main.go b/main.go index f80f15f..f657bd8 100644 --- a/main.go +++ b/main.go @@ -20,12 +20,18 @@ import ( "go.opentelemetry.io/otel/propagation" ) +const kuiperName = "kuiper" + func main() { logger := log.Default() + config, err := server.NewConfig() + if err != nil { + logger.Fatalf("Error: %s", err.Error()) + } ctx := context.Background() //init exporter - exporter, err := util.NewJaegerExporter("http://127.0.0.1:14268/api/traces") + exporter, err := util.NewJaegerExporter(config.JaegerAddress) if err != nil { logger.Fatalf(err.Error()) } @@ -33,19 +39,25 @@ func main() { tp := util.NewTraceProvider(exporter) defer func() { _ = tp.Shutdown(ctx) }() otel.SetTracerProvider(tp) - tracer := tp.Tracer("kuiper") + tracer := tp.Tracer(kuiperName) otel.SetTextMapPropagator(propagation.TraceContext{}) router := gin.New() router.Use(gin.Recovery()) - router.Use(otelgin.Middleware("kuiper")) + router.Use(otelgin.Middleware(kuiperName)) cli, err := clientv3.New(clientv3.Config{ - Endpoints: []string{"http://127.0.0.1:2379"}, + Endpoints: []string{config.EtcdAddress}, DialTimeout: 10 * time.Second, }) + if err != nil { + logger.Fatalf(err.Error()) + } - natsCon, err := util.Conn("http://127.0.0.1:4222") + natsCon, err := util.Conn(config.NatsAddress) + if err != nil { + logger.Fatalf(err.Error()) + } cfgStore := store.NewConfigStore(*cli, *logger, tracer) cfgService := service.NewConfigService(cfgStore, *logger, tracer) @@ -53,6 +65,7 @@ func main() { router.POST("/api/config", handler.SaveConfig) router.GET("/api/config/:id/:ver", handler.GetConfig) + router.GET("/api/config/:id", handler.GetConfigsByService) router.POST("/api/config/:id/", handler.CreateNewVersion) router.DELETE("/api/config/:id/:ver", handler.DeleteConfig) router.DELETE("/api/config/:id", handler.DeleteConfigsWithPrefix) diff --git a/model/config.go b/model/config.go index 812e1ec..620dafa 100644 --- a/model/config.go +++ b/model/config.go @@ -1,7 +1,9 @@ package model +type Entries = map[string]string + type Config struct { - Service string `json:"service"` - Version string `json:"version"` - Entries map[string]string `json:"entries"` + Service string `json:"service"` + Version string `json:"version"` + Entries Entries `json:"entries"` } diff --git a/server/handlers.go b/server/handlers.go index 1d613a9..6ba88df 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -81,6 +81,29 @@ func (ch configHandler) GetConfig(c *gin.Context) { c.JSON(http.StatusOK, cfg) } +func (ch configHandler) GetConfigsByService(c *gin.Context) { + ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.GetConfigsByService") + defer span.End() + + id := c.Param("id") + if c.Query("latest") == "true" { + cfg, err := ch.configService.GetLatestConfigByService(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "No value under key"}) + return + } + c.JSON(http.StatusOK, cfg) + return + } + cfgs, err := ch.configService.GetConfigsByService(ctx, id) + if err != nil { + c.JSON(http.StatusNotFound, gin.H{"error": "No value under key"}) + return + } + c.JSON(http.StatusOK, cfgs) + +} + func (ch configHandler) CreateNewVersion(c *gin.Context) { ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.CreateNewVersion") defer span.End() diff --git a/server/server.go b/server/server.go index caa23f7..d991928 100644 --- a/server/server.go +++ b/server/server.go @@ -15,6 +15,7 @@ type ConfigHandler interface { DeleteConfig(c *gin.Context) CreateNewVersion(c *gin.Context) DeleteConfigsWithPrefix(c *gin.Context) + GetConfigsByService(c *gin.Context) } type configHandler struct { diff --git a/service/configService.go b/service/configService.go index 4c652b7..7eb1050 100644 --- a/service/configService.go +++ b/service/configService.go @@ -18,13 +18,17 @@ type ConfigService interface { //Checks if cfg is a valid config and tries to persist it. CreateConfig(ctx context.Context, cfg model.Config) (string, error) //Finds a config by id and version - GetConfig(ctx context.Context, id, ver string) (map[string]string, error) + GetConfig(ctx context.Context, id, ver string) (model.Entries, error) //Creates a new version of already existing config CreateNewVersion(ctx context.Context, cfg model.Config, id string) error //Deletes config by id and version, returns error when config wasn't foun - DeleteConfig(ctx context.Context, id, ver string) (cfg map[string]string, err error) + DeleteConfig(ctx context.Context, id, ver string) (cfg model.Entries, err error) //Deletes all configs with the given ID - DeleteConfigsWithPrefix(ctx context.Context, id string) (deleted map[string]map[string]string, err error) + DeleteConfigsWithPrefix(ctx context.Context, id string) (deleted map[string]model.Entries, err error) + //Gets the latest config of a service + GetLatestConfigByService(ctx context.Context, id string) (map[string]model.Entries, error) + //Gets all of the service's configs + GetConfigsByService(ctx context.Context, id string) (map[string]model.Entries, error) } func NewConfigService(cs store.ConfigStore, logger log.Logger, trace trace.Tracer) ConfigService { @@ -53,12 +57,32 @@ func (cs configService) CreateConfig(ctx context.Context, cfg model.Config) (str return cs.store.SaveConfig(nCtx, cfg) } -func (cs configService) GetConfig(ctx context.Context, id, ver string) (map[string]string, error) { +func (cs configService) GetConfig(ctx context.Context, id, ver string) (model.Entries, error) { nCtx, span := cs.trace.Start(ctx, "configService.GetConfig") defer span.End() return cs.store.GetConfig(nCtx, id, ver) } +var IdNotProvidedError = errors.New("Id not provided") + +func (cs configService) GetConfigsByService(ctx context.Context, id string) (map[string]model.Entries, error) { + nCtx, span := cs.trace.Start(ctx, "configService.GetConfigsByService") + defer span.End() + if id == "" { + return make(map[string]model.Entries), IdNotProvidedError + } + return cs.store.GetConfigsByService(nCtx, id) +} + +func (cs configService) GetLatestConfigByService(ctx context.Context, id string) (map[string]model.Entries, error) { + nCtx, span := cs.trace.Start(ctx, "configService.GetConfigsByService") + defer span.End() + if id == "" { + return make(map[string]model.Entries), IdNotProvidedError + } + return cs.store.GetLatestConfigByService(nCtx, id) +} + func (cs configService) CreateNewVersion(ctx context.Context, cfg model.Config, id string) error { nCtx, span := cs.trace.Start(ctx, "configService.CreateNewVersion") defer span.End() @@ -71,13 +95,13 @@ func (cs configService) CreateNewVersion(ctx context.Context, cfg model.Config, return cs.store.SaveVersion(nCtx, cfg, id) } -func (cs configService) DeleteConfig(ctx context.Context, id, ver string) (cfg map[string]string, err error) { +func (cs configService) DeleteConfig(ctx context.Context, id, ver string) (cfg model.Entries, err error) { nCtx, span := cs.trace.Start(ctx, "configService.DeleteConfig") defer span.End() return cs.store.DeleteConfig(nCtx, id, ver) } -func (cs configService) DeleteConfigsWithPrefix(ctx context.Context, id string) (deleted map[string]map[string]string, err error) { +func (cs configService) DeleteConfigsWithPrefix(ctx context.Context, id string) (deleted map[string]model.Entries, err error) { nCtx, span := cs.trace.Start(ctx, "configService.DeleteConfig") defer span.End() return cs.store.DeleteConfigsWithPrefix(nCtx, id) diff --git a/store/configStore.go b/store/configStore.go index c3e0e53..8f234e6 100644 --- a/store/configStore.go +++ b/store/configStore.go @@ -4,11 +4,11 @@ import ( "context" "encoding/json" "errors" - "fmt" "kuiper/model" "log" "time" + "go.etcd.io/etcd/api/v3/mvccpb" clientv3 "go.etcd.io/etcd/client/v3" "go.opentelemetry.io/otel/trace" ) @@ -28,7 +28,11 @@ type ConfigStore interface { //Deletes a config and returns the config that was deleted DeleteConfig(ctx context.Context, id, ver string) (map[string]string, error) //Deletes all the configs with the given ID and returns them - DeleteConfigsWithPrefix(ctx context.Context, id string) (map[string]map[string]string, error) + DeleteConfigsWithPrefix(ctx context.Context, id string) (map[string]model.Entries, error) + //Gets all of the service's configs + GetConfigsByService(ctx context.Context, id string) (map[string]model.Entries, error) + //Gets only the latest config of a service + GetLatestConfigByService(ctx context.Context, id string) (map[string]model.Entries, error) } func NewConfigStore(cli clientv3.Client, logger log.Logger, trace trace.Tracer) ConfigStore { @@ -41,15 +45,6 @@ type configStoreEtcd struct { trace trace.Tracer } -func makeKey(id, ver string) string { - key := fmt.Sprintf("config/%s/%s/", id, ver) - return key -} -func makeIdPrefix(id string) string { - key := fmt.Sprintf("config/%s/", id) - return key -} - func (cStore configStoreEtcd) SaveConfig(ctx context.Context, cfg model.Config) (string, error) { _, span := cStore.trace.Start(ctx, "configStoreEtcd.CreateConfig") defer span.End() @@ -111,6 +106,62 @@ func (cStore configStoreEtcd) GetConfig(ctx context.Context, id, ver string) (ma return entries, ErrorNotFound } +func (cStore configStoreEtcd) GetConfigsByService(ctx context.Context, id string) (map[string]model.Entries, error) { + ctx, span := cStore.trace.Start(ctx, "configStoreEtcd.GetConfigsByService") + defer span.End() + + cfgs := make(map[string]model.Entries) + + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) + res, err := cStore.cli.KV.Get(kvCtx, makeIdPrefix(id), clientv3.WithPrefix()) + cancel() + if err != nil { + span.RecordError(err) + return cfgs, err + } + + return cStore.decodeConfigsFromKvs(res.Kvs, ctx) +} + +func (cStore configStoreEtcd) GetLatestConfigByService(ctx context.Context, id string) (map[string]model.Entries, error) { + ctx, span := cStore.trace.Start(ctx, "configStoreEtcd.GetLatestConfigByService") + defer span.End() + + cfgs := make(map[string]model.Entries) + + kvCtx, cancel := context.WithTimeout(ctx, time.Second*10) + opts := clientv3.WithLastRev() + opts = append(opts, clientv3.WithPrefix()) + res, err := cStore.cli.KV.Get(kvCtx, makeIdPrefix(id), opts...) + cancel() + if err != nil { + span.RecordError(err) + return cfgs, err + } + + return cStore.decodeConfigsFromKvs(res.Kvs, ctx) +} + +func (cStore configStoreEtcd) decodeConfigsFromKvs(kvs []*mvccpb.KeyValue, ctx context.Context) (map[string]model.Entries, error) { + ctx, span := cStore.trace.Start(ctx, "configStoreEtcd.decodeConfigsFromKvs") + defer span.End() + + cfgs := make(map[string]model.Entries) + for _, kv := range kvs { + cfg := make(map[string]string) + data := kv.Value + err := json.Unmarshal(data, &cfg) + if err != nil { + span.RecordError(err) + return cfgs, err + } + + ver := getVersionFromKey(string(kv.Key)) + cfgs[string(ver)] = cfg + } + return cfgs, nil +} + func (cStore configStoreEtcd) getPrefixCount(ctx context.Context, id string) (int64, error) { _, span := cStore.trace.Start(ctx, "configStoreEtcd.getPrefixCount") defer span.End() @@ -191,8 +242,8 @@ func (cStore configStoreEtcd) DeleteConfig(ctx context.Context, id, ver string) return entries, ErrorNotFound } -func (cStore configStoreEtcd) DeleteConfigsWithPrefix(ctx context.Context, id string) (map[string]map[string]string, error) { - _, span := cStore.trace.Start(ctx, "configStoreEtcd.DeleteConfigWithPrefix") +func (cStore configStoreEtcd) DeleteConfigsWithPrefix(ctx context.Context, id string) (map[string]model.Entries, error) { + ctx, span := cStore.trace.Start(ctx, "configStoreEtcd.DeleteConfigWithPrefix") defer span.End() cfgs := make(map[string]map[string]string) @@ -209,18 +260,5 @@ func (cStore configStoreEtcd) DeleteConfigsWithPrefix(ctx context.Context, id st return cfgs, ErrorNotFound } - kvs := res.PrevKvs - for _, kv := range kvs { - cfg := make(map[string]string) - data := kv.Value - err = json.Unmarshal(data, &cfg) - if err != nil { - span.RecordError(err) - return cfgs, err - } - - cfgs[string(kv.Key)] = cfg - } - - return cfgs, nil + return cStore.decodeConfigsFromKvs(res.PrevKvs, ctx) } diff --git a/store/keyUtils.go b/store/keyUtils.go new file mode 100644 index 0000000..e2e02ee --- /dev/null +++ b/store/keyUtils.go @@ -0,0 +1,20 @@ +package store + +import ( + "fmt" + "strings" +) + +func makeKey(id, ver string) string { + key := fmt.Sprintf("config/%s/%s/", id, ver) + return key +} +func makeIdPrefix(id string) string { + key := fmt.Sprintf("config/%s/", id) + return key +} + +func getVersionFromKey(key string) string { + split := strings.Split(string(key), "/") + return split[len(split)-2] +} From 799bbd275571f3cf282200c1c829abba3da5f40b Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Sun, 2 Apr 2023 12:05:44 +0200 Subject: [PATCH 09/10] feature: dockerfile --- Dockerfile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..97b28e6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM golang:alpine as build_container +WORKDIR /app +COPY ./go.mod ./go.sum ./ +RUN go mod download +COPY ./ . +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o configServer . + + +FROM alpine +WORKDIR /root/ +COPY --from=build_container /app/configServer . + +EXPOSE 8080 + +ENTRYPOINT ["./configServer"] From 7a6b6927ff0efb4e8300b912ab0233bb1bb90831 Mon Sep 17 00:00:00 2001 From: Filip Schenk Date: Tue, 25 Jul 2023 12:54:06 +0200 Subject: [PATCH 10/10] izmenice --- server/config.go | 35 +++++++++++++++++++++++++++++++++++ server/handlers.go | 4 ---- 2 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 server/config.go diff --git a/server/config.go b/server/config.go new file mode 100644 index 0000000..8dad5fd --- /dev/null +++ b/server/config.go @@ -0,0 +1,35 @@ +package server + +import ( + "errors" + "os" +) + +type Config struct { + JaegerAddress string + EtcdAddress string + NatsAddress string +} + +const ( + jaegerAddressEnv = "JAEGER_ADDRESS" + etcdAddressEnv = "ETCD_ADDRESS" + natsAddressEnv = "NATS_ADDRESS" +) + +func NewConfig() (Config, error) { + jagAddr, found := os.LookupEnv(jaegerAddressEnv) + if !found { + return Config{}, errors.New(jaegerAddressEnv + " environment variable not set") + } + etcdAddr, found := os.LookupEnv(etcdAddressEnv) + if !found { + return Config{}, errors.New(etcdAddressEnv + " environment variable not set") + } + natsAddr, found := os.LookupEnv(natsAddressEnv) + if !found { + return Config{}, errors.New(natsAddressEnv + " environment variable not set") + } + + return Config{JaegerAddress: jagAddr, EtcdAddress: etcdAddr, NatsAddress: natsAddr}, nil +} diff --git a/server/handlers.go b/server/handlers.go index 6ba88df..ab5fbcc 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -12,10 +12,6 @@ import ( "github.com/gin-gonic/gin" ) -func natsKey(serviceName string) string { - return fmt.Sprintf("config.%s", serviceName) -} - func (ch configHandler) SaveConfig(c *gin.Context) { ctx, span := ch.tracer.Start(c.Request.Context(), "configServer.CreateConfig") defer span.End()