diff --git a/go.mod b/go.mod index fc40c40..683a11b 100644 --- a/go.mod +++ b/go.mod @@ -6,16 +6,18 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/davecgh/go-spew v1.1.1 github.com/go-chi/chi v1.5.1 + github.com/go-redis/redis/v8 v8.11.0 // indirect github.com/imdario/mergo v0.3.9 + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible github.com/machinebox/progress v0.2.0 github.com/pelletier/go-toml v1.7.0 github.com/pkg/errors v0.9.1 - github.com/seatgeek/logrus-gelf-formatter v0.0.0-20180829220724-ce23ecb3f367 - github.com/sirupsen/logrus v1.6.0 + github.com/seatgeek/logrus-gelf-formatter v0.0.0-20210414080842-5b05eb8ff761 + github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.5.1 golang.org/x/net v0.0.0-20210119194325-5f4716e94777 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 gopkg.in/h2non/gock.v1 v1.0.15 - gopkg.in/yaml.v2 v2.2.8 + gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 7761d9c..5cf0cbd 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,32 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-chi/chi v1.5.1 h1:kfTK3Cxd/dkMu/rKs5ZceWYp+t5CtiE7vmaTv3LjC6w= github.com/go-chi/chi v1.5.1/go.mod h1:REp24E+25iKvxgeTfHmdUoL5x15kBiDBlnIl5bCwe2k= +github.com/go-redis/redis/v8 v8.11.0 h1:O1Td0mQ8UFChQ3N9zFQqo6kTU2cJ+/it88gDB+zg0wo= +github.com/go-redis/redis/v8 v8.11.0/go.mod h1:DLomh7y2e3ggQXQLd1YgmvIfecPJoFl7WU5SOQ/r06M= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= @@ -17,6 +37,13 @@ github.com/machinebox/progress v0.2.0 h1:7z8+w32Gy1v8S6VvDoOPPBah3nLqdKjr3GUly18 github.com/machinebox/progress v0.2.0/go.mod h1:hl4FywxSjfmkmCrersGhmJH7KwuKl+Ueq9BXkOny+iE= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -25,27 +52,75 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/seatgeek/logrus-gelf-formatter v0.0.0-20180829220724-ce23ecb3f367 h1:kjJVIlsZ+29dcgTtHVJDtK7ykrf7e1tW/63UvYSCEJw= github.com/seatgeek/logrus-gelf-formatter v0.0.0-20180829220724-ce23ecb3f367/go.mod h1:/THDZYi7F/BsVEcYzYPqdcWFQ+1C2InkawTKfLOAnzg= +github.com/seatgeek/logrus-gelf-formatter v0.0.0-20210414080842-5b05eb8ff761 h1:0b8DF5kR0PhRoRXDiEEdzrgBc8UqVY4JWLkQJCRsLME= +github.com/seatgeek/logrus-gelf-formatter v0.0.0-20210414080842-5b05eb8ff761/go.mod h1:/THDZYi7F/BsVEcYzYPqdcWFQ+1C2InkawTKfLOAnzg= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091 h1:DMyOG0U+gKfu8JZzg2UQe9MeaC1X+xQWlAKcRnjxjCw= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/gock.v1 v1.0.15 h1:SzLqcIlb/fDfg7UvukMpNcWsu7sI5tWwL+KCATZqks0= gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/xfile/copy.go b/xfile/copy.go new file mode 100644 index 0000000..c83171e --- /dev/null +++ b/xfile/copy.go @@ -0,0 +1,50 @@ +package fileutil + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// Copy will copy the file from src to dst, the paths have to be absolute to +// ensure consistent behavior. +func Copy(src string, dst string) (err error) { + if !filepath.IsAbs(src) || !filepath.IsAbs(dst) { + return fmt.Errorf("can't copy src to dst paths not abosulte between src: %s and dst: %s", src, dst) + } + + srcStat, err := os.Stat(src) + if err != nil { + return errors.Wrap(err, "failed to copy file") + } + + if !srcStat.Mode().IsRegular() { + return fmt.Errorf("failed to copy file %s not a regular file", src) + } + + srcFile, err := os.Open(src) + if err != nil { + return errors.Wrap(err, "failed to open file to copy") + } + defer func() { + err = srcFile.Close() + }() + + dstFile, err := os.Create(dst) + if err != nil { + return errors.Wrapf(err, "failed to create file to copy to for %s", src) + } + defer func() { + err = dstFile.Close() + }() + + _, err = io.Copy(dstFile, srcFile) + if err != nil { + return errors.Wrapf(err, "failed to copy file src: %s dst: %s", src, dstFile.Name()) + } + + return err +} diff --git a/xfile/create.go b/xfile/create.go new file mode 100644 index 0000000..31dffd4 --- /dev/null +++ b/xfile/create.go @@ -0,0 +1,103 @@ +package fileutil + +import ( + "crypto/rand" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// Specify the number of bytes you want to be different when create partial +// matching files. +const numOfDiffBytes = 4 + +// Create will create a file in a specific dir, for the specific size. The +// data inside of file is completely random. +func Create(dir string, size int) (fileName string, err error) { + tmpFile, err := ioutil.TempFile(dir, "") + if err != nil { + return "", errors.Wrapf(err, "failed to create tmp file inside of %s", dir) + } + defer func() { + err = tmpFile.Close() + }() + + var fileContent = make([]byte, size) + _, err = rand.Read(fileContent) + if err != nil { + return "", errors.Wrap(err, "failed to create random data for file") + } + + _, err = tmpFile.Write(fileContent) + if err != nil { + return "", errors.Wrapf(err, "failed to write random data to file %s", tmpFile.Name()) + } + + return tmpFile.Name(), err +} + +// useful for fuzzing + +// CreatePartialMatch will create two files that have the same size and the +// first few bytes but the final 4 bytes of the file are different. The 4 extra +// bytes are included into the specified size. +func CreatePartialMatch(dir string, size int) ([]string, error) { + if !filepath.IsAbs(dir) { + return nil, fmt.Errorf("cannot append to file, path %s is not absolute", dir) + } + + // Create the identical files. + originalFile, err := Create(dir, size-numOfDiffBytes) + if err != nil { + return nil, errors.Wrapf(err, "failed to create file inside %s", dir) + } + + var cpFile = fmt.Sprintf("%s_partial", originalFile) + + err = Copy(originalFile, cpFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to copy file to %s", dir) + } + + err = appendToFile(originalFile, numOfDiffBytes) + if err != nil { + return nil, errors.Wrapf(err, "failed to append random data to %s", originalFile) + } + + err = appendToFile(cpFile, numOfDiffBytes) + if err != nil { + return nil, errors.Wrapf(err, "failed to append random data to %s", originalFile) + } + + return []string{originalFile, cpFile}, nil +} + +func appendToFile(path string, size int) (err error) { + if !filepath.IsAbs(path) { + return fmt.Errorf("cannot append to file, path is not absolute: %s", path) + } + + f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return errors.Wrapf(err, "failed to open file %s", path) + } + defer func() { + err = f.Close() + }() + + var appendData = make([]byte, size) + + _, err = rand.Read(appendData) + if err != nil { + return errors.Wrapf(err, "failed to create random data for file") + } + + if _, err := f.Write(appendData); err != nil { + return errors.Wrapf(err, "failed to append data to file %s", path) + } + + return err +} diff --git a/xfile/create_test.go b/xfile/create_test.go new file mode 100644 index 0000000..81a17c8 --- /dev/null +++ b/xfile/create_test.go @@ -0,0 +1,124 @@ +package fileutil + +import ( + "io" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestCreate_Successful(t *testing.T) { + tmpDir, cleanUpFn := createTmpDir(t) + defer cleanUpFn() + + wantSize := 10 + + gotName, err := Create(tmpDir, wantSize) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + + f, err := os.Stat(gotName) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + + if int64(wantSize) != f.Size() { + t.Fatalf("Create file size does not match want size, want: %d got: %d", wantSize, f.Size()) + } +} + +func TestCreate_PathShouldBeAbs(t *testing.T) { + tmpDir, cleanUpFn := createTmpDir(t) + defer cleanUpFn() + + _, err := Create(filepath.Base(tmpDir), 10) + + if err == nil { + t.Fatal("Expected error got nil") + } +} + +func TestCreatePartialMatch_Success(t *testing.T) { + tmpDir, cleanUpFn := createTmpDir(t) + defer cleanUpFn() + + wantSize := 10 + + files, err := CreatePartialMatch(tmpDir, 10) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + + if len(files) != 2 { + t.Fatalf("Got unexpected amount of files want: 2 got: %d", len(files)) + } + + // Compare the first bytes + f1, err := os.Open(files[0]) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + defer f1.Close() + f1FirstBytes := make([]byte, wantSize-numOfDiffBytes) + if _, err := io.ReadFull(f1, f1FirstBytes); err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + + f2, err := os.Open(files[1]) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + defer f2.Close() + f2FirstBytes := make([]byte, wantSize-numOfDiffBytes) + if _, err := io.ReadFull(f2, f2FirstBytes); err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + + if !reflect.DeepEqual(f1FirstBytes, f2FirstBytes) { + t.Fatalf("First bytes are not equal") + } + + // Compare the last bytes are different. + f1LastBytes := make([]byte, numOfDiffBytes) + _, err = f1.ReadAt(f1LastBytes, int64(wantSize-numOfDiffBytes)) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + + f2LastBytes := make([]byte, numOfDiffBytes) + _, err = f2.ReadAt(f2LastBytes, int64(wantSize-numOfDiffBytes)) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + + if reflect.DeepEqual(f1LastBytes, f2LastBytes) { + t.Fatalf("Last bytes should be different") + } +} + +func TestCreatePartialMatch_PathShouldBeAbs(t *testing.T) { + tmpDir, cleanUpFn := createTmpDir(t) + defer cleanUpFn() + + _, err := CreatePartialMatch(filepath.Base(tmpDir), 10) + if err == nil { + t.Fatalf("Expected error got nil") + } +} + +func createTmpDir(t *testing.T) (string, func()) { + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Failed to create tmp dir for testdata: %v", err) + } + + return tmpDir, func() { + err := os.RemoveAll(tmpDir) + if err != nil { + t.Logf("Failed to clean up test %s: %v", t.Name(), err) + } + } +} diff --git a/xredis/client.go b/xredis/client.go new file mode 100644 index 0000000..63adc32 --- /dev/null +++ b/xredis/client.go @@ -0,0 +1,108 @@ +package xredis + +import ( + "context" + "crypto/tls" + "time" + + "github.com/go-redis/redis/v8" + "github.com/logrusorgru/aurora" + "github.com/thisisdevelopment/go-dockly/xlogger" +) + +// ICache defines and exposes the caching layer +type ICache interface { + // Set the value to key with TTL overwrite or TTL taken from config + Set(ctx context.Context, key string, value interface{}, t time.Duration) error + // Get will return value of key under cancellable context and try unmarshal the result into expected + Get(ctx context.Context, key string, expected interface{}) error + // GetCacheKeys returns all the keys in the oartial match pattern + GetCacheKeys(ctx context.Context, keyPattern string) ([]string, error) + // GetTTL returns the remaining time to live duration for key or 0 if expired + GetTTL(ctx context.Context, key string) (time.Duration, error) +} + +// Redis implements the ICache interface based on redis +type Redis struct { + redis *redis.Client + config *Config + log *xlogger.Logger +} + +type Config struct { + Host string + Pass string + DB int + Expiration int + PoolSize int `yaml:"pool_size"` + MaxRetries int `yaml:"max_retries"` + ConnTimeOut time.Duration `yaml:"conn_timeout"` + PollInterval time.Duration `yaml:"poll_interval"` + TLS bool +} + +// New constructs a cache class +func New(config *Config, log *xlogger.Logger) (ICache, error) { + + var opts = defaultOpts(config) + var client = redis.NewClient(opts) + + ctx, cancel := context.WithTimeout(context.Background(), config.ConnTimeOut) + defer cancel() + + err := client.Ping(ctx).Err() + if err != nil { + return nil, err + } + + log.Printf("connected to redis %s\n", aurora.Cyan(config.Host)) + + var o = &Redis{ + config: config, + log: log, + redis: client, + } + + go o.checkConnection() + + return o, nil +} + +func defaultOpts(config *Config) *redis.Options { + var opts = &redis.Options{ + Addr: config.Host, + Password: config.Pass, + DB: config.DB, + PoolSize: config.PoolSize, + MaxRetries: config.MaxRetries, + } + + if config.TLS { + opts.TLSConfig = new(tls.Config) + } + + return opts +} + +// pings the connection at tick interval and tries to continously reconnect on error +func (r *Redis) checkConnection() { + + for range time.Tick(r.config.PollInterval) { + ctx, cancel := context.WithTimeout(context.Background(), r.config.PollInterval) + + err := r.redis.Ping(ctx).Err() + if err != nil { + // redis disconnected + opts := defaultOpts(r.config) + client := redis.NewClient(opts) + r.redis = client + + if err := r.redis.Ping(ctx).Err(); err != nil { + r.log.Warningln("redis connnection failed, failed to set new one") + cancel() + continue + } + } + cancel() + } +} diff --git a/xredis/gzip.go b/xredis/gzip.go new file mode 100644 index 0000000..d86236d --- /dev/null +++ b/xredis/gzip.go @@ -0,0 +1,46 @@ +package xredis + +import ( + "bytes" + "compress/gzip" + + "github.com/pkg/errors" +) + +func (c *Redis) gzip(val []byte) ([]byte, error) { + var buf bytes.Buffer + var w = gzip.NewWriter(&buf) + + pos, err := w.Write(val) + if err != nil { + return nil, errors.Wrapf(err, "gzip content for pos %d", pos) + } + + err = w.Close() + if err != nil { + return nil, errors.Wrap(err, "close gzip content writer") + } + + return buf.Bytes(), nil +} + +func (c *Redis) gunzip(val []byte) ([]byte, error) { + var b = bytes.NewBuffer(val) + r, err := gzip.NewReader(b) + if err != nil { + return nil, errors.Wrap(err, "open gzip content reader") + } + + var res bytes.Buffer + pos, err := res.ReadFrom(r) + if err != nil { + return nil, errors.Wrapf(err, "read gzip content pos %d", pos) + } + + err = r.Close() + if err != nil { + return nil, errors.Wrap(err, "close gzip content reader") + } + + return res.Bytes(), nil +} diff --git a/xredis/methods.go b/xredis/methods.go new file mode 100644 index 0000000..e155b08 --- /dev/null +++ b/xredis/methods.go @@ -0,0 +1,119 @@ +package xredis + +import ( + "context" + "encoding/json" + "reflect" + "sort" + "time" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" +) + +// Set the value to key with TTL overwrite or TTL taken from config +func (c *Redis) Set(ctx context.Context, key string, value interface{}, t time.Duration) (err error) { + var g []byte + // 0 will default to config controlled cache ttl + if t == 0 { + t = time.Duration(c.config.Expiration) * time.Minute + } + + switch value.(type) { + // if set value is byte slice it is assumed you know what you do + // as in the value is already marshalled from some format eg yaml + case []byte: + // gzipped storing in redis yields x10 size reduction + g, err = c.gzip(value.([]byte)) + if err != nil { + return errors.Wrapf(err, "gzip byte %s", aurora.Yellow(key)) + } + + default: + // if set value is interface it will be json marshalled + b, err := json.Marshal(value) + if err != nil { + return errors.Wrapf(err, "marshal %s", aurora.Yellow(key)) + } + + g, err = c.gzip(b) + if err != nil { + return errors.Wrapf(err, "gzip interface %s", aurora.Yellow(key)) + } + } + + return c.redis.Set(ctx, key, g, t).Err() +} + +// Get will return value of key under cancellable context and try unmarshal the result into expected +func (c *Redis) Get(ctx context.Context, key string, expected interface{}) error { + + val, err := c.redis.Get(ctx, key).Bytes() + if err != nil { + return errors.Wrapf(err, "get %s", aurora.Yellow(key)) + } + + // each value this application controls is assumed to be gzipped compressed + b, err := c.gunzip(val) + if err != nil { + // or an error will be thrown in case it is eg json marshalled byte slice + return errors.Wrapf(err, "gunzip %s", aurora.Yellow(key)) + } + + switch expected.(type) { + // the special case bypassing an unmarshal indicating a different format otherwise json is assumed + case *[]byte: + // assign the raw byte slice to expected interface as is (caller handles payload) + reflect.ValueOf(expected).Elem().Set(reflect.ValueOf(b)) + default: + // we handle the payload and unmarshal into the expected interface directly + if err = json.Unmarshal(b, expected); err != nil { + return errors.Wrapf(err, "unmarshal %s ", aurora.Yellow(key)) + } + } + + return nil +} + +// GetCacheKeys returns all the keys in the oartial match pattern +func (c *Redis) GetCacheKeys(ctx context.Context, keyPattern string) (ks []string, err error) { + + var cursor uint64 + + for { + var keys []string + var err error + // only string keys are returned no payloads + keys, cursor, err = c.redis.Scan(ctx, cursor, keyPattern, 512).Result() + if err != nil { + return nil, err + } + + ks = append(ks, keys...) + + if cursor == 0 { + break + } + } + + sort.Slice(ks, func(i, j int) bool { + return ks[i] < ks[j] + }) + + return ks, err +} + +// GetTTL returns the remaining time to live duration for key or 0 if expired +func (c *Redis) GetTTL(ctx context.Context, key string) (time.Duration, error) { + + ttl, err := c.redis.TTL(ctx, key).Result() + if err != nil { + return 0, err + } + + if ttl < 0 { + return 0, nil + } + + return ttl, err +} diff --git a/xredis/suite_test.go b/xredis/suite_test.go new file mode 100644 index 0000000..c7574b1 --- /dev/null +++ b/xredis/suite_test.go @@ -0,0 +1,25 @@ +package xredis_test + +import ( + "testing" + + "github.com/thisisdevelopment/go-dockly/xhelper" + "github.com/thisisdevelopment/go-dockly/xlogger" + + "github.com/stretchr/testify/suite" +) + +type TestSuite struct { + suite.Suite + logger *xlogger.Logger + helper *xhelper.Helper +} + +func (s *TestSuite) SetupSuite() { + s.logger = xlogger.DefaultTestLogger(&s.Suite) + s.helper = xhelper.NewHelper(&s.Suite, s.logger) +} + +func TestRunner(t *testing.T) { + suite.Run(t, new(TestSuite)) +} diff --git a/xslice/integration_test.go b/xslice/integration_test.go index 7c72bb2..99242fb 100644 --- a/xslice/integration_test.go +++ b/xslice/integration_test.go @@ -10,13 +10,13 @@ import ( var syncslice = xslice.NewSyncSlice([]interface{}{"one", "two"}...) -func TestShift(t *testing.T) { +func TestIntegrationShift(t *testing.T) { var res = <-syncslice.Shift() spew.Dump(res) assert.Equal(t, res.Val, "one") } -func TestInsert(t *testing.T) { +func TestIntegrationInsert(t *testing.T) { syncslice.Append("three", "four") assert.GreaterOrEqual(t, syncslice.Len(), 3) } diff --git a/xslice/sync_test.go b/xslice/sync_test.go index 88135a1..f01eb7e 100644 --- a/xslice/sync_test.go +++ b/xslice/sync_test.go @@ -8,7 +8,7 @@ import ( "github.com/thisisdevelopment/go-dockly/xslice" ) -func TestUniq(t *testing.T) { +func TestSyncUniq(t *testing.T) { data := []struct{ in, out []interface{} }{ {[]interface{}{}, []interface{}{}}, @@ -34,7 +34,7 @@ func TestUniq(t *testing.T) { } } -func TestCut(t *testing.T) { +func TestSyncCut(t *testing.T) { data := []struct { in, out []interface{} @@ -64,7 +64,7 @@ func TestCut(t *testing.T) { } } -func TestStrip(t *testing.T) { +func TestSyncStrip(t *testing.T) { data := []struct { in, out []interface{} @@ -90,7 +90,7 @@ func TestStrip(t *testing.T) { } } -func TestDel(t *testing.T) { +func TestSyncDel(t *testing.T) { data := []struct { in, out []interface{} @@ -116,7 +116,7 @@ func TestDel(t *testing.T) { } } -func TestPop(t *testing.T) { +func TestSyncPop(t *testing.T) { data := []struct { in, out []interface{} @@ -143,7 +143,7 @@ func TestPop(t *testing.T) { } } -func TestShift(t *testing.T) { +func TestSyncShift(t *testing.T) { data := []struct { in, out []interface{} @@ -171,7 +171,7 @@ func TestShift(t *testing.T) { } } -func TestUnShift(t *testing.T) { +func TestSyncUnShift(t *testing.T) { data := []struct { in, out []interface{} @@ -195,7 +195,7 @@ func TestUnShift(t *testing.T) { } } -func TestFilter(t *testing.T) { +func TestSyncFilter(t *testing.T) { data := []struct { in, out []interface{} @@ -220,7 +220,7 @@ func TestFilter(t *testing.T) { } } -func TestContains(t *testing.T) { +func TestSyncContains(t *testing.T) { var ok bool s := xslice.NewSyncSlice([]interface{}{"abc", "def"}...) @@ -228,7 +228,7 @@ func TestContains(t *testing.T) { assert.Equal(t, ok, true, "did not match") } -func TestReverse(t *testing.T) { +func TestSyncReverse(t *testing.T) { data := []struct{ in, out []interface{} }{ {[]interface{}{}, []interface{}{}}, @@ -250,7 +250,7 @@ func TestReverse(t *testing.T) { } } -func TestContainsAny(t *testing.T) { +func TestSyncContainsAny(t *testing.T) { data := []struct{ src, tgt, out []interface{} }{ {[]interface{}{}, []interface{}{}, []interface{}{}}, @@ -270,7 +270,7 @@ func TestContainsAny(t *testing.T) { } } -func TestInsert(t *testing.T) { +func TestSyncInsert(t *testing.T) { data := []struct { in, ins, out []interface{}