diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ae03357..e8f933fd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: jobs: build: + permissions: write-all runs-on: ubuntu-latest steps: - name: Checkout diff --git a/cmd/redis-shake/main.go b/cmd/redis-shake/main.go index 70f5fad7..b26cb6f6 100644 --- a/cmd/redis-shake/main.go +++ b/cmd/redis-shake/main.go @@ -82,11 +82,11 @@ func main() { var theReader reader.Reader if config.Config.Type == "sync" { theReader = reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS, source.ElastiCachePSync) - } else if config.Config.Type == "restore" { // TODO: new aof reader + } else if config.Config.Type == "restore" { if source.RDBFilePath != "" { theReader = reader.NewRDBReader(source.RDBFilePath) } else { - theReader = reader.NewAOFReader(source.AOFFilePath) // 如果是mp-aof 用户传入 manifest文件的地址 ，其他的传递aof的地址 + theReader = reader.NewAOFReader(source.AOFFilePath) } } else if config.Config.Type == "scan" { diff --git a/go.mod b/go.mod index 4dc27cf2..d0c58329 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,16 @@ module github.com/alibaba/RedisShake go 1.17 require ( + github.com/go-redis/redis v6.15.9+incompatible github.com/pelletier/go-toml/v2 v2.0.0-beta.3 github.com/rs/zerolog v1.28.0 github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.27.10 // indirect + golang.org/x/sys v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index 55959bc7..9ea34504 100644 --- a/go.sum +++ b/go.sum @@ -5,11 +5,80 @@ github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= +github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= +github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= +github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pelletier/go-toml/v2 v2.0.0-beta.3 h1:PNCTU4naEJ8mKal97P3A2qDU74QRQGlv4FXiL1XDqi4= github.com/pelletier/go-toml/v2 v2.0.0-beta.3/go.mod h1:aNseLYu/uKskg0zpr/kbr2z8yGuWtotWf/0BpGIAL2Y= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -19,15 +88,143 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942 h1:t0lM6y/M5IiUZyvbBTcngso8SZEZICH7is9B6g/obVU= github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/aof/aof.go b/internal/aof/aof.go index 0f1eb09f..05f53995 100644 --- a/internal/aof/aof.go +++ b/internal/aof/aof.go @@ -1,37 +1,53 @@ package aof import ( + "bufio" "bytes" - "encoding/binary" - "log" + "fmt" + "io" + "os" + "path/filepath" "strconv" + "strings" + "time" "unicode" "github.com/alibaba/RedisShake/internal/entry" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/rdb" + "github.com/alibaba/RedisShake/internal/statistics" ) const ( - AofManifestFileTypeBase = "b" /* Base file */ - AofManifestTypeHist = "h" /* History file */ - AofManifestTypeIncr = "i" /* INCR file */ + AOFManifestFileTypeBase = "b" /* Base File */ + AOFManifestTypeHist = "h" /* History File */ + AOFManifestTypeIncr = "i" /* INCR File */ + RDBFormatSuffix = ".RDB" + AOFFormatSuffix = ".AOF" + BaseFileSuffix = ".Base" + IncrFileSuffix = ".incr" + TempFileNamePrefix = "temp-" + COK = 1 + CERR = -1 + EINTR = 4 + ManifestNameSuffix = ".manifest" + AOFNotExist = 1 + AOFOpenErr = 3 + AOFOK = 0 + AOFEmpty = 2 + AOFFailed = 4 + AOFTruncated = 5 + SizeMax = 128 + RDBFlagsAOFPreamble = 1 << 0 ) -/* AOF manifest definition */ -type aofInfo struct { - fileName string - fileSeq int64 - aofFileType string -} +var RDBFileBeingLoaded string = "" -func IntToBytes(n int) []byte { - data := int64(n) - bytebuf := bytes.NewBuffer([]byte{}) - binary.Write(bytebuf, binary.BigEndian, data) - return bytebuf.Bytes() -} +func Ustime() int64 { + tv := time.Now() + ust := int64(tv.UnixNano()) / 1000 + return ust -func aofInfoCreate() *aofInfo { - return new(aofInfo) } func StringNeedsRepr(s string) int { @@ -49,37 +65,30 @@ func StringNeedsRepr(s string) int { return 0 } -/*如果字符串中包含要转义的字符，则返回一 - *通过sdscatrer（），否则为零。 - * - *通常，这应该用于以某种方式帮助保护聚合字符串 - *与sdsspliargs（）兼容。因此，空间也将 - *被视为需要逃跑。 - */ -/*func stringTrim(s string,cset string)string{ - slen:=len(s) - sp:=0 - ep:=slen-1 - - for sp<=ep&&strings.ContainsRune(cset,rune(s[sp])){ - sp++ - } - - for ep>sp&&strings.ContainsRune(cset,rune(s[sp])){ - ep-- - } - trimmed:=s[sp:ep+1] - trimmedlen:=len(trimmed) - if sp<0||ep= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F') +} + +func HexDigitToInt(c byte) int { switch c { case '0': return 0 @@ -118,10 +127,9 @@ func hexDigitToInt(c byte) int { } } -func splitArgs(line string) ([]string, int) { - var p string - p = line - var current string +func SplitArgs(line string) ([]string, int) { + var p string = line + var Current string var vector []string argc := 0 i := 0 @@ -131,58 +139,50 @@ func splitArgs(line string) ([]string, int) { i++ } if i < lens { - // Get a token inq := false // Set to true if we are in "quotes" insq := false // Set to true if we are in 'single quotes' done := false - if current == "" { - current = "" - } for !done { if inq { - _, err1 := strconv.ParseInt(string(p[i+2]), 16, 64) - _, err2 := strconv.ParseInt(string(p[i+3]), 16, 64) - if p[i] == '\\' && (p[i+1]) == 'x' && err1 == nil && err2 == nil { - int16 := (hexDigitToInt((p[i+2])) * 16) + hexDigitToInt(p[i+3]) - //var bytes []byte - //bytes=IntToBytes(int16) - //current = stringcatlen(current,bytes,1) - current += string(int16) - i += 3 + + if p[i] == '\\' && (p[i+1]) == 'x' && IsHexDigit(p[i+2]) && IsHexDigit(p[i+3]) { + _, err1 := strconv.ParseInt(string(p[i+2]), 16, 64) + _, err2 := strconv.ParseInt(string(p[i+3]), 16, 64) + if err1 == nil && err2 == nil { + int16 := (HexDigitToInt((p[i+2])) * 16) + HexDigitToInt(p[i+3]) + Current = Current + fmt.Sprint(int16) + i += 3 + } } else if p[i] == '\\' && i+1 < lens { - c := p[i] + var c byte i++ switch p[i] { case 'n': c = '\n' - break case 'r': c = 'r' - break case 'a': c = '\a' - break default: c = p[i] - break } - current += string(c) + Current += string(c) } else if p[i] == '"' { - if i+1 < lens && unicode.IsSpace((rune(p[i+1]))) { + if i+1 < lens && !unicode.IsSpace((rune(p[i+1]))) { return nil, 0 } done = true } else if i >= lens { return nil, 0 } else { - current += string(p[i]) + Current += string(p[i]) } } else if insq { if p[i] == '\\' && p[i+1] == '\'' { i++ - current += "'" + Current += "'" } else if p[i] == '\'' { if i+1 < lens && !unicode.IsSpace((rune(p[i+1]))) { return nil, 0 @@ -191,23 +191,19 @@ func splitArgs(line string) ([]string, int) { } else if i >= lens { return nil, 0 } else { - current += string(p[i]) + Current += string(p[i]) } } else { switch p[i] { - case ' ', '\n', '\r', '\t': + case ' ', '\n', '\r', '\t', '\000': done = true - break case '"': inq = true - break case '\'': insq = true - break default: - current += string(p[i]) - break + Current += string(p[i]) } } if i < lens { @@ -218,9 +214,9 @@ func splitArgs(line string) ([]string, int) { } } - vector = append(vector, current) + vector = append(vector, Current) argc++ - current = "" + Current = "" } else { return vector, argc @@ -229,7 +225,7 @@ func splitArgs(line string) ([]string, int) { } } -func stringcatlen(s string, t []byte, lent int) string { +func Stringcatlen(s string, t []byte, lent int) string { curlen := len(s) if curlen == 0 { @@ -240,21 +236,165 @@ func stringcatlen(s string, t []byte, lent int) string { copy(buf[:curlen], []byte(s)) copy(buf[curlen:], t) - //buf[curlen+lent] = 0 ///0 return string(buf) } -func aofInfoDup(orig *aofInfo) *aofInfo { +func Stringcatprintf(s string, fmtStr string, args ...interface{}) string { + result := fmt.Sprintf(fmtStr, args...) + if s == "" { + return result + } else { + return s + result + } +} + +func Stringcatrepr(s string, p string, length int) string { + s = s + string("\"") + for i := 0; i < length; i++ { + switch p[i] { + case '\\', '"': + s = Stringcatprintf(s, "\\%c", p[i]) + case '\n': + s = s + "\\n" + case '\r': + s = s + "\\r" + case '\t': + s = s + "\\t" + case '\a': + s = s + "\\a" + case '\b': + s = s + "\\b" + default: + if strconv.IsPrint(rune(p[i])) { + s = s + string(p[i]) + } else { + s = s + "\\x%02x" + } + } + } + return s + "\"" +} + +func UpdateLoadingFileName(FileName string) { + RDBFileBeingLoaded = FileName +} + +/* AOF manifest definition */ +type AOFInfo struct { + FileName string + FileSeq int64 + AOFFileType string +} + +func AOFInfoCreate() *AOFInfo { + return new(AOFInfo) +} + +var AOF_Info AOFInfo = *AOFInfoCreate() + +func (a *AOFInfo) GetAOFInfoName() string { + return a.FileName +} + +func AOFInfoDup(orig *AOFInfo) *AOFInfo { if orig == nil { - log.Fatal("Assertion failed: orig != nil") + log.Panicf("Assertion failed: orig != nil") } - ai := aofInfoCreate() - ai.fileName = orig.fileName - ai.fileSeq = orig.fileSeq - ai.aofFileType = orig.aofFileType + ai := AOFInfoCreate() + ai.FileName = orig.FileName + ai.FileSeq = orig.FileSeq + ai.AOFFileType = orig.AOFFileType return ai } +func AOFInfoFormat(buf string, ai *AOFInfo) string { + var AOFManifestcreate string + if StringNeedsRepr(ai.FileName) == 1 { + AOFManifestcreate = Stringcatrepr("", ai.FileName, len(ai.FileName)) + } + var ret string + if AOFManifestcreate != "" { + ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, AOFManifestcreate, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) + } else { + ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, ai.FileName, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) + } + return ret +} + +type INFO struct { + AOFDirName string + AOFUseRDBPreamble int + AOFManifest *AOFManifest + AOFFileName string + AOFCurrentSize int64 + AOFRewriteBaseSize int64 +} + +var AOFFileInfo INFO + +func (a *INFO) GetAOFDirName() string { + return a.AOFDirName +} + +func NewAOFFileInfo(aofFilePath string) *INFO { + return &INFO{ + AOFDirName: filepath.Dir(aofFilePath), + AOFUseRDBPreamble: 0, + AOFManifest: nil, + AOFFileName: filepath.Base(aofFilePath), + AOFCurrentSize: 0, + AOFRewriteBaseSize: 0, + } +} + +func (a *INFO) SetAOFDirName(DirName string) { + a.AOFDirName = DirName +} + +func (a *INFO) GetAOFUseRDBPreamble() int { + return a.AOFUseRDBPreamble +} + +func (a *INFO) SetAOFUseRDBPreamble(useRDBPreamble int) { + a.AOFUseRDBPreamble = useRDBPreamble +} + +func (a *INFO) GetAOFManifest() *AOFManifest { + return a.AOFManifest +} +func (a *INFO) SetAOFManifest(manifest *AOFManifest) { + a.AOFManifest = manifest +} + +func (a *INFO) GetAOFFileName() string { + return a.AOFFileName +} + +func (a *INFO) SetAOFFileName(FileName string) { + a.AOFFileName = FileName +} + +func (a *INFO) GetAOFCurrentSize() int64 { + return a.AOFCurrentSize +} + +func (a *INFO) SetAOFCurrentSize(size int64) { + a.AOFCurrentSize = size +} + +func (a *INFO) GetAOFRewriteBaseSize() int64 { + return a.AOFRewriteBaseSize +} + +func (a *INFO) SetAOFRewriteBaseSize(size int64) { + a.AOFRewriteBaseSize = size +} + +type listIter struct { + next *listNode + Direction int +} + type lists struct { head, tail *listNode len uint64 @@ -266,40 +406,32 @@ type listNode struct { value interface{} } -func listCreate() *lists { +func ListCreate() *lists { lists := &lists{} lists.head = nil lists.tail = nil lists.len = 0 return lists } +func ListNext(iter *listIter) *listNode { + Current := iter.next -type aofManifest struct { - baseAofInfo *aofInfo - incrAofList *lists - historyList *lists - currBaseFileSeq int64 - currIncrFIleSeq int64 - dirty int64 -} - -// TODO: 待填充完整loader -type Loader struct { - filPath string - ch chan *entry.Entry + if Current != nil { + if iter.Direction == 0 { + iter.next = Current.next + } else { + iter.next = Current.prev + } + } + return Current } -func listAddNodeTail(lists *lists, value interface{}) *lists { - node := &listNode{ - value: value, - prev: nil, - next: nil, - } - listLinkNodeTail(lists, node) - return lists +func (list *lists) ListsRewind(li *listIter) { + li.next = list.head + li.Direction = 0 } -func listLinkNodeTail(lists *lists, node *listNode) { +func ListLinkNodeTail(lists *lists, node *listNode) { if lists.len == 0 { lists.head = node lists.tail = node @@ -313,6 +445,111 @@ func listLinkNodeTail(lists *lists, node *listNode) { } lists.len++ } + +func ListAddNodeTail(lists *lists, value interface{}) *lists { + node := &listNode{ + value: value, + prev: nil, + next: nil, + } + ListLinkNodeTail(lists, node) + return lists +} + +func ListsRewindTail(list *lists, li *listIter) { + li.next = list.tail + li.Direction = 1 +} + +func ListDup(orig *lists) *lists { + var copy *lists + var iter listIter + var node *listNode + copy = ListCreate() + if copy == nil { + return nil + } + copy.ListsRewind(&iter) + node = ListNext(&iter) + var value interface{} + for node != nil { + value = node.value + } + + if ListAddNodeTail(copy, value) == nil { + return nil + } + return copy +} + +func ListIndex(list *lists, index int64) *listNode { + var n *listNode + + if index < 0 { + index = (-index) - 1 + n = list.tail + for ; index > 0 && n != nil; index-- { + n = n.prev + } + } else { + n = list.head + for ; index > 0 && n != nil; index-- { + n = n.next + } + } + return n +} + +func ListLinkNodeHead(list *lists, node *listNode) { + if list.len == 0 { + list.head = node + list.tail = node + node.prev = nil + node.next = nil + } else { + node.prev = nil + node.next = list.head + list.head.prev = node + list.head = node + } + list.len++ +} + +func ListAddNodeHead(list *lists, value interface{}) *lists { + node := &listNode{ + value: value, + } + ListLinkNodeHead(list, node) + + return list +} + +func ListUnlinkNode(list *lists, node *listNode) { + if node.prev != nil { + node.prev.next = node.next + } else { + list.head = node.next + } + if node.next != nil { + node.next.prev = node.prev + } else { + list.tail = node.prev + } + node.next = nil + node.prev = nil + + list.len-- +} +func ListDelNode(list *lists, node *listNode) { + ListUnlinkNode(list, node) + +} + +type Loader struct { + filPath string + ch chan *entry.Entry +} + func NewLoader(filPath string, ch chan *entry.Entry) *Loader { ld := new(Loader) ld.ch = ch @@ -320,19 +557,503 @@ func NewLoader(filPath string, ch chan *entry.Entry) *Loader { return ld } -// TODO：完成checAofMain后写单测进行测试 -func (ld *Loader) ParseRDB() int { - // 加载aof目录 - // 进行check_aof， aof - CheckAofMain(ld.filPath) - // TODO：执行加载 - return 0 +type AOFManifest struct { + BaseAOFInfo *AOFInfo + incrAOFList *lists + HistoryList *lists + CurrBaseFileSeq int64 + CurrIncrFileSeq int64 + Dirty int64 +} + +func AOFManifestcreate() *AOFManifest { + am := &AOFManifest{ + incrAOFList: ListCreate(), + HistoryList: ListCreate(), + } + return am } -func aofManifestcreate() *aofManifest { - am := &aofManifest{ - incrAofList: listCreate(), - historyList: listCreate(), +func AOFManifestDup(orig *AOFManifest) *AOFManifest { + if orig == nil { + panic("orig is nil") + } + + am := &AOFManifest{ + CurrBaseFileSeq: orig.CurrBaseFileSeq, + CurrIncrFileSeq: orig.CurrIncrFileSeq, + Dirty: orig.Dirty, + } + + if orig.BaseAOFInfo != nil { + am.BaseAOFInfo = AOFInfoDup(orig.BaseAOFInfo) + } + + am.incrAOFList = ListDup(orig.incrAOFList) + am.HistoryList = ListDup(orig.HistoryList) + + if am.incrAOFList == nil || am.HistoryList == nil { + log.Panicf("IncrAOFlist or HistoryAOFlist is nil") } return am } + +func GetAOFManifestAsString(am *AOFManifest) string { + if am == nil { + panic("am is nil") + } + var buf string + var ln *listNode + var li listIter + + if am.BaseAOFInfo != nil { + buf = AOFInfoFormat(buf, am.BaseAOFInfo) + } + am.HistoryList.ListsRewind(&li) + ln = ListNext(&li) + for ln != nil { + ai, ok := ln.value.(*AOFInfo) + if ok { + buf = AOFInfoFormat(buf, ai) + } + ln = ListNext(&li) + } + + am.incrAOFList.ListsRewind(&li) + ln = ListNext(&li) + for ln != nil { + ai, ok := ln.value.(*AOFInfo) + if ok { + buf = AOFInfoFormat(buf, ai) + } + ln = ListNext(&li) + } + + return buf + +} + +func GetNewBaseFileNameAndMarkPreAsHistory(am *AOFManifest) string { + if am == nil { + log.Panicf("AOFManifest is nil") + } + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType != AOFManifestFileTypeBase { + log.Panicf("Base_AOF_info has invalid File_type") + } + am.BaseAOFInfo.AOFFileType = AOFManifestTypeHist + } + var formatSuffix string + if AOFFileInfo.AOFUseRDBPreamble == 1 { + formatSuffix = RDBFormatSuffix + } else { + formatSuffix = AOFFormatSuffix + } + ai := AOFInfoCreate() + ai.FileName = Stringcatprintf("%s.%d%s%d", AOF_Info.GetAOFInfoName(), am.CurrBaseFileSeq+1, BaseFileSuffix, formatSuffix) + ai.FileSeq = am.CurrBaseFileSeq + 1 + ai.AOFFileType = AOFManifestFileTypeBase + am.BaseAOFInfo = ai + am.Dirty = 1 + return am.BaseAOFInfo.FileName +} + +func AOFLoadManifestFromDisk() { + AOFFileInfo.AOFManifest = AOFManifestcreate() + if DirExists(AOFFileInfo.AOFDirName) == 0 { + log.Infof("The AOF Directory %v doesn't exist\n", AOFFileInfo.AOFDirName) + return + } + + am_Name := GetAOFManifestFileName() + am_Filepath := MakePath(AOFFileInfo.AOFDirName, am_Name) + if FileExist(am_Filepath) == 0 { + log.Infof("The AOF Directory %v doesn't exist\n", AOFFileInfo.AOFDirName) + return + } + + am := AOFLoadManifestFromFile(am_Filepath) + //if am != nil { + AOFFileInfo.AOFManifest = am + //} +} + +func GetNewIncrAOFName(am *AOFManifest) string { + ai := AOFInfoCreate() + ai.AOFFileType = AOFManifestTypeIncr + ai.FileName = Stringcatprintf("", "%s.%d%s%s", AOFFileInfo.AOFFileName, am.CurrIncrFileSeq+1, IncrFileSuffix, AOFFormatSuffix) + ai.FileSeq = am.CurrIncrFileSeq + 1 + ListAddNodeTail(am.incrAOFList, ai) + am.Dirty = 1 + return ai.FileName +} + +func GetTempIncrAOFNanme() string { + return Stringcatprintf("", "%s%s%s", TempFileNamePrefix, AOFFileInfo.AOFFileName, IncrFileSuffix) +} + +func GetLastIncrAOFName(am *AOFManifest) string { + if am == nil { + log.Panicf(("AOFManifest is nil")) + } + + if am.incrAOFList.len == 0 { + return GetNewIncrAOFName(am) + } + + lastnode := ListIndex(am.incrAOFList, -1) + + ai, ok := lastnode.value.(AOFInfo) + if !ok { + log.Panicf("Failed to convert lastnode.value to AOFInfo") + } + return ai.FileName +} + +func GetAOFManifestFileName() string { + return AOFFileInfo.AOFFileName +} + +func GetTempAOFManifestFileName() string { + return Stringcatprintf("", "%s%s", TempFileNamePrefix, AOFFileInfo.AOFFileName) +} + +func StartLoading(size int64, RDBflags int, async int) { + /* Load the DB */ + statistics.Metrics.Loading = true + if async == 1 { + statistics.Metrics.AsyncLoading = true + } + statistics.Metrics.LoadingStartTime = time.Now().Unix() + statistics.Metrics.LoadingLoadedBytes = 0 + statistics.Metrics.LoadingTotalBytes = size + log.Infof("The AOF File starts loading.\n") +} + +func StopLoading(ret int) { + statistics.Metrics.Loading = false + statistics.Metrics.AsyncLoading = false + if ret == AOFOK || ret == AOFTruncated { + log.Infof("The AOF File was successfully loaded\n") + } else { + log.Infof("There was an error opening the AOF File.\n") + } +} + +func AOFFileExist(FileName string) int { + Filepath := MakePath(AOFFileInfo.AOFDirName, FileName) + ret := FileExist(Filepath) + return ret +} + +func GetAppendOnlyFileSize(FileName string, status *int) int64 { + var size int64 + + AOFFilePath := MakePath(AOFFileInfo.AOFDirName, FileName) + + stat, err := os.Stat(AOFFilePath) + if err != nil { + if status != nil { + if os.IsNotExist(err) { + *status = AOFNotExist + } else { + *status = AOFOpenErr + } + } + log.Panicf("Unable to obtain the AOF File %v length. stat: %v", FileName, err.Error()) + size = 0 + } else { + if status != nil { + *status = AOFOK + } + size = stat.Size() + } + return size +} + +func GetBaseAndIncrAppendOnlyFilesSize(am *AOFManifest, status *int) int64 { + var size int64 + var ln *listNode = new(listNode) + var li *listIter = new(listIter) + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType != AOFManifestFileTypeBase { + log.Panicf("File type must be Base.") + } + size += GetAppendOnlyFileSize(am.BaseAOFInfo.FileName, status) + if *status != AOFOK { + return 0 + } + } + + am.incrAOFList.ListsRewind(li) + ln = ListNext(li) + for ln != nil { + ai := ln.value.(*AOFInfo) + if ai.AOFFileType != AOFManifestTypeIncr { + log.Panicf("File type must be Incr") + } + size += GetAppendOnlyFileSize(ai.FileName, status) + if *status != AOFOK { + return 0 + } + ln = ListNext(li) + } + return size +} + +func GetBaseAndIncrAppendOnlyFilesNum(am *AOFManifest) int { + num := 0 + if am.BaseAOFInfo != nil { + num++ + } + if am.incrAOFList != nil { + num += int(am.incrAOFList.len) + } + return num +} + +func (ld *Loader) LoadSingleAppendOnlyFile(FileName string, ch chan *entry.Entry, LastFile bool) int { + ret := AOFOK + AOFFilepath := MakePath(AOFFileInfo.AOFDirName, FileName) + var sizes int64 = 0 + fp, err := os.Open(AOFFilepath) + if err != nil { + if os.IsNotExist(err) { + if _, err := os.Stat(AOFFilepath); err == nil || !os.IsNotExist(err) { + log.Infof("Fatal error: can't open the append log File %v for reading: %v", FileName, err.Error()) + return AOFOpenErr + } else { + log.Infof("The append log File %v doesn't exist: %v", FileName, err.Error()) + return AOFNotExist + } + + } + defer fp.Close() + + stat, _ := fp.Stat() + if stat.Size() == 0 { + return AOFEmpty + } + } + sig := make([]byte, 5) + if n, err := fp.Read(sig); err != nil || n != 5 || !bytes.Equal(sig, []byte("REDIS")) { + if _, err := fp.Seek(0, 0); err != nil { + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed + return ret + } + } else { + sizes += 5 + log.Infof("Reading RDB Base File on AOF loading...") + ldRDB := rdb.NewLoader(AOFFilepath, ch) + ldRDB.ParseRDB() + return AOFOK + //Skipped RDB checksum and has not been processed yet. + } + reader := bufio.NewReader(fp) + for { + + line, err := reader.ReadBytes('\n') + { + if err != nil { + if err == io.EOF { + break + } + } else { + _, errs := fp.Seek(0, io.SeekCurrent) + if errs != nil { + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed + return ret + } + } + sizes += int64(len(line)) + + if line[0] == '#' { + continue + } + if line[0] != '*' { + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) + } + argc, _ := strconv.ParseInt(string(line[1:len(line)-2]), 10, 64) + if argc < 1 { + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) + } + if argc > int64(SizeMax) { + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) + } + e := entry.NewEntry() + argv := []string{} + + for j := 0; j < int(argc); j++ { + line, err := reader.ReadString('\n') + if err != nil || line[0] != '$' { + if err == io.EOF { + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed + return ret + } else { + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) + } + } + sizes += int64(len(line)) + len, _ := strconv.ParseInt(string(line[1:len(line)-2]), 10, 64) + argstring := make([]byte, len+2) + argstring, err = reader.ReadBytes('\n') + if err != nil || argstring[len+1] != '\n' { + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed + return ret + } + /*if ConsumeNewline(argstring[len-2:]) == 0 { + return 0 + }*/ + argstring = argstring[:len] + argv = append(argv, string(argstring)) + + sizes += len + 2 + } + for _, value := range argv { + e.Argv = append(e.Argv, value) + } + ld.ch <- e + if sizes >= CheckAOFInfof.pos && LastFile { + break + } + } + + } + statistics.Metrics.LoadingLoadedBytes = sizes + return ret +} + +func (ld *Loader) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Entry) int { + if am == nil { + log.Panicf("AOFManifest is null") + } + status := AOFOK + ret := AOFOK + var start int64 + var totalSize int64 = 0 + var BaseSize int64 = 0 + var AOFName string + var totalNum, AOFNum, lastFile int + + if AOFFileExist(AOFFileInfo.AOFFileName) == 1 { + if DirExists(AOFFileInfo.AOFDirName) == 0 || + (am.BaseAOFInfo == nil && am.incrAOFList.len == 0) || + (am.BaseAOFInfo != nil && am.incrAOFList.len == 0 && + strings.Compare(am.BaseAOFInfo.FileName, AOFFileInfo.AOFFileName) == 0 && AOFFileExist(AOFFileInfo.AOFFileName) == 0) { + log.Panicf("This is an old version of the AOF File") + } + } + + if am.BaseAOFInfo == nil && am.incrAOFList == nil { + return AOFNotExist + } + + totalNum = GetBaseAndIncrAppendOnlyFilesNum(am) + if totalNum <= 0 { + log.Panicf("Assertion failed: IncrAppendOnlyFilestotalNum > 0") + } + + totalSize = GetBaseAndIncrAppendOnlyFilesSize(am, &status) + if status != AOFOK { + if status == AOFNotExist { + status = AOFFailed + } + return status + } else if totalSize == 0 { + return AOFEmpty + } + + StartLoading(totalSize, RDBFlagsAOFPreamble, 0) + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType == AOFManifestFileTypeBase { + AOFName = string(am.BaseAOFInfo.FileName) + UpdateLoadingFileName(AOFName) + BaseSize = GetAppendOnlyFileSize(AOFName, nil) + lastFile = totalNum + start = Ustime() + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch, false) + if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) { + log.Infof("DB loaded from Base File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) + } + + if ret == AOFEmpty { + ret = AOFOK + } + + if ret == AOFTruncated && lastFile == 0 { + ret = AOFFailed + log.Infof("Fatal error: the truncated File is not the last File") + } + + if ret == AOFOpenErr || ret == AOFFailed { + if ret == AOFOK || ret == AOFTruncated { + log.Infof("The AOF File was successfully loaded\n") + } else { + if ret == AOFOpenErr { + log.Panicf("There was an error opening the AOF File.\n") + } else { + log.Panicf("Failed to open AOF File.\n") + } + } + return ret + } + } + totalNum-- + } + + if am.incrAOFList.len > 0 { + var ln *listNode = new(listNode) + var li listIter + + am.incrAOFList.ListsRewind(&li) + ln = ListNext(&li) + for ln != nil { + ai := ln.value.(*AOFInfo) + if ai.AOFFileType != AOFManifestTypeIncr { + log.Panicf("The manifestType must be Incr") + } + AOFName = ai.FileName + UpdateLoadingFileName(AOFName) + lastFile = totalNum + AOFNum++ + start = Ustime() + if lastFile == 1 { + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch, true) + } else { + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch, false) + } + if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) { + log.Infof("DB loaded from incr File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) + } + + if ret == AOFEmpty { + ret = AOFOK + } + + if ret == AOFTruncated && lastFile == 0 { + ret = AOFFailed + log.Infof("Fatal error: the truncated File is not the last File\n") + } + + if ret == AOFOpenErr || ret == AOFFailed { + if ret == AOFOpenErr { + log.Infof("There was an error opening the AOF File.\n") + } else { + log.Infof("Failed to open AOF File.\n") + } + return ret + } + ln = ListNext(&li) + } + totalNum-- + } + + AOFFileInfo.AOFCurrentSize = totalSize + AOFFileInfo.AOFRewriteBaseSize = BaseSize + return ret + +} diff --git a/internal/aof/aof_check.go b/internal/aof/aof_check.go index 2a86be5e..210b741d 100644 --- a/internal/aof/aof_check.go +++ b/internal/aof/aof_check.go @@ -4,80 +4,94 @@ import ( "bufio" "fmt" "io" - "log" + "math" "os" "path" + "path/filepath" "strconv" "strings" - //"time" -) -type AofFileType string + "github.com/alibaba/RedisShake/internal/config" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/rdb" +) -var errors [1044]byte -var line int64 = 1 -var epos int64 +type AOFFileType string const ( - aofResp AofFileType = "AOF_RESP" - aofRdbPreamble AofFileType = "AOF_RDB_PREAMBLE" - aofMultiPart AofFileType = "AOF_MULTI_PART" - rdbCheckMode = 1 - MANIFEST_MAX_LINE = 1024 - AOF_CHECK_OK = 0 - AOF_CHECK_EMPTY = 1 - AOF_CHECK_TRUNCATED = 2 - AOF_CHECK_TIMESTAMP_TRUNCATED = 3 - toTimestamp = 0 - AOF_MANIFEST_KEY_FILE_NAME = "file" - AOF_MANIFEST_KEY_FILE_SEQ = "seq" - AOF_MANIFEST_KEY_FILE_TYPE = "type" - AOF_ANNOTATION_LINE_MAX_LEN = 1024 + AOFResp AOFFileType = "AOF_RESP" + AOFRdbPreamble AOFFileType = "AOF_RDB_PREAMBLE" + AOFMultiPart AOFFileType = "AOF_MULTI_PART" + ManifestMaxLine = 1024 + AOFCheckOk = 0 + AOFCheckEmpty = 1 + AOFCheckTruncated = 2 + AOFCheckTimeStampTruncated = 3 + AOFManifestKeyFileName = "File" + AOFManifestKeyFileSeq = "seq" + AOFManifestKeyFileType = "type" + AOFAnnoTationLineMaxLen = 1024 ) +type CheckAOFINFOF struct { + line int64 + fp *os.File + pos int64 + toTimestamp int64 +} + +func NewCheckAOFINFOF() *CheckAOFINFOF { + return &CheckAOFINFOF{ + line: 0, + fp: nil, + pos: 0, + toTimestamp: 0, + } +} + +var CheckAOFInfof = NewCheckAOFINFOF() + // check 里面的主函数 -func CheckAofMain(aofFilePath string) (checkResult bool, fileType AofFileType, err error) { - /* - getAofType := getInputAofFileTye(filePath) // 获取aof文件的类型 - switch getAOFType { - case AOF_MULTI_PART: - checkResult, err = checkMultiPartAof(dirpath, filepath, fix) - return checkResult, aofMultiPart, nil - case AOF_RESP: - checkResult, err := checkOldStyleAof(filepath) - return checkResult, aofResp, nil - case AOF_RDB_PREAMBLE: - checkResult, err := checkOldStyleAof(filepath) - return checkResult, aofRdbPreamble, nil - } - return result, err - */ - //TODO: mock result - return true, aofMultiPart, nil +func CheckAOFMain(AOFFilePath string) (checkResult bool, FileType AOFFileType, err error) { + var Filepaths string + var dirpath string + fix := 1 + Filepaths = AOFFilePath + dirpath = filepath.Dir(string(Filepaths)) + + FileType = GetInputAOFFileType(Filepaths) + switch FileType { + case "AOF_MULTI_PART": + CheckMultipartAOF(dirpath, Filepaths, fix) + case "AOF_RESP": + CheckOldStyleAOF(Filepaths, fix, false) + case "AOF_RDB_PREAMBLE": + CheckOldStyleAOF(Filepaths, fix, true) + } + return true, AOFMultiPart, nil } -func getInputAofFileType(aofFilepath string) AofFileType { - if filelsManifest(aofFilepath) { +func GetInputAOFFileType(AOFFilePath string) AOFFileType { + if FilelsManifest(AOFFilePath) { return "AOF_MULTI_PART" - } else if fileIsRDB(aofFilepath) { + } else if FileIsRDB(AOFFilePath) { return "AOF_RDB_PREAMBLE" } else { return "AOF_RESP" } } -func filelsManifest(aofFilepath string) bool { +func FilelsManifest(AOFFilePath string) bool { var is_manifest bool = false - fp, err := os.Open(aofFilepath) + log.Infof("FIleLsMainifest:%v", AOFFilePath) + fp, err := os.Open(AOFFilePath) if err != nil { - fmt.Printf("Cannot open file %s:%s\n", aofFilepath, err.Error()) - os.Exit(1) + log.Panicf("Cannot open File %v:%v\n", AOFFilePath, err.Error()) } - sb, err := os.Stat(aofFilepath) + sb, err := os.Stat(AOFFilePath) if err != nil { - fmt.Printf("cannot stat file: %s\n", aofFilepath) - os.Exit(1) + log.Panicf("cannot stat File: %v\n", AOFFilePath) } size := sb.Size() if size == 0 { @@ -91,11 +105,10 @@ func filelsManifest(aofFilepath string) bool { if err == io.EOF { break } else { - fmt.Printf("cannot read file: %s\n", aofFilepath) - os.Exit(1) + log.Panicf("cannot read File: %v\n", AOFFilePath) } } - if lines[0] == '#' { + if lines[0] == '#' || len(lines) < 4 { continue } else if lines[:4] == "file" { is_manifest = true @@ -105,148 +118,146 @@ func filelsManifest(aofFilepath string) bool { return is_manifest } -func fileIsRDB(aofFilepath string) bool { - fp, err := os.Open(aofFilepath) +func FileIsRDB(AOFFilePath string) bool { + fp, err := os.Open(AOFFilePath) + if err != nil { - fmt.Printf("Cannot open file %s:%s\n", aofFilepath, err.Error()) - os.Exit(1) + log.Panicf("Cannot open File %v:%v\n", AOFFilePath, err.Error()) } - sb, err := os.Stat(aofFilepath) + + defer fp.Close() + sb, err := os.Stat(AOFFilePath) if err != nil { - fmt.Printf("cannot stat file: %s\n", aofFilepath) - os.Exit(1) + log.Panicf("cannot stat File: %v\n", AOFFilePath) } size := sb.Size() if size == 0 { - fp.Close() return false } if size >= 8 { sig := make([]byte, 5) _, err := fp.Read(sig) if err == nil && string(sig) == "REDIS" { - fp.Close() return true } } - fp.Close() return false } -func printAofStyle(ret int, aofFileName string, aofType string) { +func OutPutAOFStyle(ret int, AOFFileName string, AOFType string) { switch ret { - case AOF_CHECK_OK: - fmt.Printf("%s %s is valid\n", aofType, aofFileName) - case AOF_CHECK_EMPTY: - fmt.Printf("%s %s is empty\n", aofType, aofFileName) - case AOF_CHECK_TIMESTAMP_TRUNCATED: - fmt.Printf("Successfully truncated AOF %s to timestamp %d\n", aofFileName, toTimestamp) - case AOF_CHECK_TRUNCATED: - fmt.Printf("Successfully truncated AOF %s\n", aofFileName) + case AOFCheckOk: + log.Infof("%v %v is valid\n", AOFType, AOFFileName) + case AOFCheckEmpty: + log.Infof("%v %v is empty\n", AOFType, AOFFileName) + case AOFCheckTimeStampTruncated: + log.Infof("Successfully truncated AOF %v to timestamp %d\n", AOFFileName, CheckAOFInfof.toTimestamp) + case AOFCheckTruncated: + log.Infof("Successfully truncated AOF %v\n", AOFFileName) } } -func makePath(paths string, filename string) string { - return path.Join(paths, filename) +func MakePath(Paths string, FileName string) string { + return path.Join(Paths, FileName) } -func pathIsBaseName(path string) bool { - return strings.IndexByte(path, '/') == -1 && strings.IndexByte(path, '\\') == -1 +func PathIsBaseName(Path string) bool { + return strings.IndexByte(Path, '/') == -1 && strings.IndexByte(Path, '\\') == -1 } -func readArgc(fp *os.File, target *int64) int { - return readLong(fp, '*', target) +func ReadArgc(rd *bufio.Reader, target *int64) int { + return ReadLong(rd, ' ', target) } -func readString(fp *os.File, target *string) int { +func ReadString(rd *bufio.Reader, target *string) int { var len int64 *target = "" - if readLong(fp, '$', &len) == 0 { + if ReadLong(rd, '$', &len) == 0 { return 0 } if len < 0 || len > math.MaxInt64-2 { - fmt.Printf("Expected to read string of %d bytes, which is not in the suitable range\n", len) + log.Infof("Expected to read string of %d bytes, which is not in the suitable range\n", len) return 0 } - - // Increase length to also consume \r\n len += 2 + // Increase length to also consume \r\n data := make([]byte, len) - if readBytes(fp, &data, len) == 0 { + if ReadBytes(rd, &data, len) == 0 { return 0 } - if consumeNewline(data[len-2:]) == 0 { + if ConsumeNewline(data[len-2:]) == 0 { return 0 } - - *target = string(data[:len-2]) + data = data[:len-2] //\r\n + *target = string(data) return 1 } -func readBytes(fp *os.File, target *[]byte, length int64) int { - var real int64 - epos, _ = fp.Seek(0, io.SeekCurrent) - n, err := fp.Read(*target) - real = int64(n) - if err != nil || real != length { - fmt.Printf("Expected to read %d bytes, got %d bytes\n", length, real) +func ReadBytes(rd *bufio.Reader, target *[]byte, length int64) int { + var err error + *target, err = rd.ReadBytes('\n') + if err != nil || (*target)[length-1] != '\n' { + log.Infof("AOF format error:%s", *target) return 0 } + CheckAOFInfof.pos += length return 1 } -func consumeNewline(buf []byte) int { +func ConsumeNewline(buf []byte) int { if buf[0] != '\r' || buf[1] != '\n' { - fmt.Printf("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) + log.Infof("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) return 0 } - line += 1 + CheckAOFInfof.line += 1 return 1 } -func readLong(fp *os.File, prefix byte, target *int64) int { - buf := make([]byte, 128) +func ReadLong(rd *bufio.Reader, prefix byte, target *int64) int { + var err error - epos, err = fp.Seek(0, io.SeekCurrent) - if err != nil { - fmt.Printf("Failed to get current position, aborting...\n") - os.Exit(1) - } - reader := bufio.NewReader(fp) - if _, err := reader.ReadBytes('\n'); err != nil { - return 0 - } - buf, err = reader.ReadBytes('\n') + var value int64 + buf, err := rd.ReadBytes('\n') if err != nil { - fmt.Println("Failed to read line from file") - return 0 - } - if buf[0] != prefix { - fmt.Printf("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) + log.Infof("Failed to read line from File") return 0 } - value, err := strconv.ParseInt(string(buf[1:len(buf)-1]), 10, 64) //去除了换行符*8* - if err != nil { - fmt.Println("Failed to parse long value") - return 0 + CheckAOFInfof.pos += int64(len(buf)) + if prefix != ' ' { + if buf[0] != prefix { + log.Infof("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) + return 0 + } + value, err = strconv.ParseInt(string(buf[1:len(buf)-2]), 10, 64) //Removed line breaks\r\n + if err != nil { + log.Infof("Failed to parse long value") + return 0 + } + } else { + value, err = strconv.ParseInt(string(buf[0:len(buf)-2]), 10, 64) //Removed line breaks\r\n + if err != nil { + log.Infof("Failed to parse long value") + return 0 + } } *target = value - line += 1 + CheckAOFInfof.line += 1 return 1 } -func aofLoadManifestFromFile(am_filepath string) *aofManifest { + +func AOFLoadManifestFromFile(am_Filepath string) *AOFManifest { var maxseq int64 - am := aofManifestcreate() - fp, err := os.Open(am_filepath) + am := AOFManifestcreate() + fp, err := os.Open(am_Filepath) if err != nil { - log.Fatalf("Fatal error:can't open the AOF manifest %s for reading: %s", am_filepath, err) + log.Panicf("Fatal error:can't open the AOF manifest %v for reading: %v", am_Filepath, err) } var argv []string - var ai *aofInfo + var ai *AOFInfo var line string linenum := 0 reader := bufio.NewReader(fp) @@ -255,66 +266,75 @@ func aofLoadManifestFromFile(am_filepath string) *aofManifest { if err != nil { if err == io.EOF { if linenum == 0 { - log.Fatalf("Found an empty AOF manifest") + log.Infof("Found an empty AOF manifest") + am = nil + return am } else { break } } else { - log.Fatalf("Read AOF manifest failed") + log.Infof("Read AOF manifest failed") + am = nil + return am + } } + linenum++ if buf[0] == '#' { continue } if !strings.Contains(buf, "\n") { - log.Fatalf("The AOF manifest file contains too long line") + log.Infof("The AOF manifest File contains too long line") + return nil } line = strings.Trim(buf, " \t\r\n") if len(line) == 0 { - log.Fatalf("Invalid AOF manifest file format") + log.Infof("Invalid AOF manifest File format") + return nil } argc := 0 - argv, argc = splitArgs(line) + argv, argc = SplitArgs(line) if argc < 6 || argc%2 != 0 { - log.Fatalf("Invalid AOF manifest file format") + log.Infof("Invalid AOF manifest File format") + am = nil + return am } - ai = aofInfoCreate() + ai = AOFInfoCreate() for i := 0; i < argc; i += 2 { - if strings.EqualFold(argv[i], AOF_MANIFEST_KEY_FILE_NAME) { - ai.fileName = string(argv[i+1]) - if !pathIsBaseName(string(ai.fileName)) { - log.Fatalf("File can't be a path, just a filename") + if strings.EqualFold(argv[i], AOFManifestKeyFileName) { + ai.FileName = string(argv[i+1]) + if !PathIsBaseName(string(ai.FileName)) { + log.Panicf("File can't be a path, just a Filename") } - } else if strings.EqualFold(argv[i], AOF_MANIFEST_KEY_FILE_SEQ) { - ai.fileSeq, _ = strconv.ParseInt(argv[i+1], 10, 64) - } else if strings.EqualFold(argv[i], AOF_MANIFEST_KEY_FILE_TYPE) { - ai.aofFileType = string(argv[i+1][0]) + } else if strings.EqualFold(argv[i], AOFManifestKeyFileSeq) { + ai.FileSeq, _ = strconv.ParseInt(argv[i+1], 10, 64) + } else if strings.EqualFold(argv[i], AOFManifestKeyFileType) { + ai.AOFFileType = string(argv[i+1][0]) } } - if ai.fileName == "" || ai.fileSeq == 0 || ai.aofFileType == "" { - log.Fatalf("Invalid AOF manifest file format") + if ai.FileName == "" || ai.FileSeq == 0 || ai.AOFFileType == "" { + log.Panicf("Invalid AOF manifest File format") } - //==nil - if ai.aofFileType == AofManifestFileTypeBase { - if am.baseAofInfo != nil { - log.Fatalf("Found duplicate base file information") + if ai.AOFFileType == AOFManifestFileTypeBase { + if am.BaseAOFInfo != nil { + log.Panicf("Found duplicate Base File information") } - am.baseAofInfo = ai - am.currBaseFileSeq = ai.fileSeq - } else if ai.aofFileType == AofManifestTypeHist { - am.historyList = listAddNodeTail(am.historyList, ai) - } else if ai.aofFileType == AofManifestTypeIncr { - if ai.fileSeq <= maxseq { - log.Fatalf("Found a non-monotonic sequence number") + am.BaseAOFInfo = ai + am.CurrBaseFileSeq = ai.FileSeq + } else if ai.AOFFileType == AOFManifestTypeHist { + am.HistoryList = ListAddNodeTail(am.HistoryList, ai) + } else if ai.AOFFileType == AOFManifestTypeIncr { + if ai.FileSeq <= maxseq { + log.Panicf("Found a non-monotonic sequence number") } - am.incrAofList = listAddNodeTail(am.historyList, ai) - am.currIncrFIleSeq = ai.fileSeq - maxseq = ai.fileSeq + am.incrAOFList = ListAddNodeTail(am.HistoryList, ai) + am.CurrIncrFileSeq = ai.FileSeq + maxseq = ai.FileSeq } else { - log.Fatalf("Unknown AOF file type") + log.Panicf("Unknown AOF File type") } line = " " ai = nil @@ -323,31 +343,29 @@ func aofLoadManifestFromFile(am_filepath string) *aofManifest { return am } -func processRESP(fp *os.File, filename string, outMulti *int) int { +func ProcessRESP(rd *bufio.Reader, Filename string, outMulti *int) int { var argc int64 var str string - if readArgc(fp, &argc) == 0 { + if ReadArgc(rd, &argc) == 0 { return 0 } for i := int64(0); i < argc; i++ { - if readString(fp, &str) == 0 { + if ReadString(rd, &str) == 0 { return 0 } if i == 0 { if strings.EqualFold(str, "multi") { - (*outMulti)++ - if (*outMulti) > 0 { - err := fmt.Errorf("Unexpected MULTI in AOF %s", filename) - fmt.Println(err.Error()) + if (*outMulti) != 0 { + log.Infof("Unexpected MULTI in AOF %v", Filename) return 0 } + (*outMulti)++ } else if strings.EqualFold(str, "exec") { (*outMulti)-- if (*outMulti) != 0 { - err := fmt.Errorf("Unexpected EXEC in AOF %s", filename) - fmt.Println(err.Error()) + log.Infof("Unexpected EXEC in AOF %v", Filename) return 0 } } @@ -357,243 +375,191 @@ func processRESP(fp *os.File, filename string, outMulti *int) int { return 1 } -// 截断可能有问题 -func processAnnotations(fp *os.File, filename string, lastFile bool) int { - - epos, err := fp.Seek(0, io.SeekCurrent) - if err != nil { - fmt.Printf("Failed to get current position, aborting...\n") - os.Exit(1) - } - reader := bufio.NewReader(fp) - buf, _, err := reader.ReadLine() +func ProcessAnnotations(rd *bufio.Reader, Filename string, lastFile bool) int { + buf, _, err := rd.ReadLine() if err != nil { - fmt.Printf("Failed to read annotations from AOF %s, aborting...\n", filename) - os.Exit(1) + log.Panicf("Failed to read annotations from AOF %v, aborting...\n", Filename) } - - if toTimestamp != 0 && strings.HasPrefix(string(buf), "#TS:") { + if CheckAOFInfof.toTimestamp != 0 && strings.HasPrefix(string(buf), "TS:") { var ts int64 - ts, err = strconv.ParseInt(strings.TrimPrefix(string(buf), "#TS:"), 10, 64) + ts, err = strconv.ParseInt(strings.TrimPrefix(string(buf), "TS:"), 10, 64) if err != nil { - fmt.Println("Invalid timestamp annotation") - os.Exit(1) + log.Panicf("Invalid timestamp annotation") } - if ts <= toTimestamp { + + if ts <= CheckAOFInfof.toTimestamp { + CheckAOFInfof.pos += int64(len(buf)) + 2 return 1 } - if epos == 0 { - fmt.Printf("AOF %s has nothing before timestamp %ld, aborting...\n", filename, toTimestamp) - os.Exit(1) + + if CheckAOFInfof.pos == 0 { + log.Panicf("AOF %v has nothing before timestamp %d, aborting...\n", Filename, CheckAOFInfof.toTimestamp) } - if lastFile == false { - fmt.Printf("Failed to truncate AOF %s to timestamp %ld to offset %ld because it is not the last file.\n", filename, toTimestamp, epos) - fmt.Println("If you insist, please delete all files after this file according to the manifest file and delete the corresponding records in manifest file manually. Then re-run redis-check-aof.") - os.Exit(1) + + if !lastFile { + log.Infof("Failed to truncate AOF %v to timestamp %d to offset %d because it is not the last File.\n", Filename, CheckAOFInfof.toTimestamp, CheckAOFInfof.pos) + log.Panicf("If you insist, please delete all Files after this File according to the manifest File and delete the corresponding records in manifest File manually. Then re-run redis-check-AOF.") } + // Truncate remaining AOF if exceeding 'toTimestamp' - if err := fp.Truncate(epos); err != nil { - fmt.Printf("Failed to truncate AOF %s to timestamp %ld\n", filename, toTimestamp) - os.Exit(1) - } else { - return 0 - } + /*if err := CheckAOFInfof.fp.Truncate(CheckAOFInfof.pos); err != nil { + log.Panicf("Failed to truncate AOF %v to timestamp %d\n", Filename, CheckAOFInfof.toTimestamp) + } else {*/ + //CheckAOFInfof.pos += int64(len(buf)) + 2 + return 0 + //} } + CheckAOFInfof.pos += int64(len(buf)) + 2 return 1 } -/* -func checkRdbMain(argc int, argv *[]string, fp *os.File) int { - - if argc != 2 && fp == nil { - fmt.Fprintf(os.Stderr, "Usage: %s \n", argv[0]) - os.Exit(1) - } else if argv[1] == "-v" || argv[1] == "--version" { - version := checkRdbVersion() - fmt.Printf("redis-check-rdb %s\n", version) - os.Exit(0) - } - - tv := time.Now() - sec := int64(tv.Unix()) - usec := int64(tv.Nanosecond()) / 1000 - pid := int64(os.Getpid()) - seed := ((sec * 1000000) + usec) ^ pid - init_genrand64(seed) - rdbCheckMode = 1 - rdbCheckInfo("Checking RDB file %s", args[1]) - rdbCheckSetupSignals() - retval := redis_Check_RDB(args[1], fp) - if retval == 0 { - rdbCheckInfo("\\o/ RDB looks OK! \\o/") - rdbShowGenericInfo() - } - if fp != nil { - if retval == 0 { - return 0 - } else { - return 1 - } - } - os.Exit(retval) - } -*/ -func checkMultipartAof(dirpath string, manifestFilepath string, fix int) { +func CheckMultipartAOF(DirPath string, ManifestFilePath string, fix int) { totalNum := 0 - aofNum := 0 + AOFNum := 0 var ret int - am := aofLoadManifestFromFile(manifestFilepath) - if am.baseAofInfo != nil { + am := AOFLoadManifestFromFile(ManifestFilePath) + if am.BaseAOFInfo != nil { totalNum++ } - if am.incrAofList != nil { - totalNum += int(am.incrAofList.len) - } - if am.baseAofInfo != nil { - aofFilename := am.baseAofInfo.fileName - aofFilepath := makePath(dirpath, aofFilename) - lastFile := (aofNum + 1) == totalNum - aofPreable := fileIsRDB(aofFilepath) - if aofPreable { - fmt.Printf("Start to check BASE AOF (RDB format).\n") + if am.incrAOFList != nil { + totalNum += int(am.incrAOFList.len) + } + if am.BaseAOFInfo != nil { + AOFFileName := am.BaseAOFInfo.FileName + AOFFilePath := MakePath(DirPath, AOFFileName) + AOFNum++ + lastFile := AOFNum == totalNum + AOFPreable := FileIsRDB(AOFFilePath) + if AOFPreable { + log.Infof("Start to check Base AOF (RDB format).\n") } else { - fmt.Printf("Start to check BASE AOF (AOF format).\n") + log.Infof("Start to check Base AOF (AOF format).\n") } - ret = checkSingleAof(aofFilename, aofFilepath, lastFile, fix, aofPreable) - printAofStyle(ret, aofFilename, "BASE AOF") + ret = CheckSingleAOF(AOFFileName, AOFFilePath, lastFile, fix, AOFPreable) + OutPutAOFStyle(ret, AOFFileName, "Base AOF") } - if am.incrAofList.len != 0 { - fmt.Printf("start to check INCR INCR files.") + if am.incrAOFList.len != 0 { + log.Infof("start to check INCR INCR Files.\n") var ln *listNode - ln = am.incrAofList.head + ln = am.incrAOFList.head for ln != nil { - ai := ln.value.(*aofInfo) - aofFilename := ai.fileName - aofFilepath := makePath(dirpath, aofFilename) - lastFile := (aofNum + 1) == totalNum - ret = checkSingleAof(aofFilename, aofFilepath, lastFile, fix, false) - printAofStyle(ret, aofFilename, "INCR AOF") - //stringfree(aofFilepath) + ai := ln.value.(*AOFInfo) + AOFFileName := ai.FileName + AOFFilePath := MakePath(DirPath, AOFFileName) + AOFNum++ + lastFile := AOFNum == totalNum + ret = CheckSingleAOF(AOFFileName, AOFFilePath, lastFile, fix, false) + OutPutAOFStyle(ret, AOFFileName, "INCR AOF") ln = ln.next } } - //aofManifestFree(am) - fmt.Println("All AOF files and manifest are vaild") + log.Infof("All AOF Files and manifest are vaild") } -func checkOldStyleAof(aofFilepath string, fix int, preamble bool) { - fmt.Printf("Start checking Old-Style AOF\n") - var ret = checkSingleAof(aofFilepath, aofFilepath, true, fix, preamble) - printAofStyle(ret, aofFilepath, "AOF") +func CheckOldStyleAOF(AOFFilePath string, fix int, preamble bool) { + log.Infof("Start checking Old-Style AOF\n") + var ret = CheckSingleAOF(AOFFilePath, AOFFilePath, true, fix, preamble) + OutPutAOFStyle(ret, AOFFilePath, "AOF") } - -func checkSingleAof(aofFilename, aofFilepath string, lastFile bool, fix int, preamble bool) int { - var pos, diff int64 +func CheckSingleAOF(AOFFileName, AOFFilePath string, lastFile bool, fix int, preamble bool) int { + var rdbpos int64 = 0 + CheckAOFInfof.toTimestamp = config.Config.Source.AOFTruncateToTimestamp multi := 0 - buf := make([]byte, 2) - - fp, err := os.OpenFile(aofFilepath, os.O_RDWR, 0666) + CheckAOFInfof.pos = 0 + buf := make([]byte, 1) + var err error + CheckAOFInfof.fp, err = os.OpenFile(AOFFilePath, os.O_RDWR, 0666) if err != nil { - log.Fatalf("Cannot open file %s:%s,aborting...\n", aofFilepath, err) + log.Panicf("Cannot open File %v:%v,aborting...\n", AOFFilePath, err) } - sb, err := fp.Stat() + sb, err := CheckAOFInfof.fp.Stat() if err != nil { - log.Fatalf("Cannot stat file: %s,aborting...\n", aofFilename) + log.Panicf("Cannot stat File: %v,aborting...\n", AOFFileName) } size := sb.Size() if size == 0 { - return AOF_CHECK_EMPTY + return AOFCheckEmpty } - if preamble != false { - argv := []string{aofFilepath} - err := checkRdbMain(2, argv, fp) - if err == -1 { - log.Fatal("RDB preamble of AOF file is not sane, aborting.") + rd := bufio.NewReader(CheckAOFInfof.fp) + if preamble { + + rdbpos = rdb.RedisCheckRDBMain(AOFFilePath, CheckAOFInfof.fp) + rdbpos += 8 //The RDB checksum has not been processed yet. + if rdbpos == -1 { + log.Panicf("RDB preamble of AOF File is not sane, aborting.\n") } else { - fmt.Println("RDB preamble is OK, proceeding with AOF tail...") - } - } - for { - if multi != 0 { - var err error - pos, err = fp.Seek(0, io.SeekCurrent) + log.Infof("RDB preamble is OK, proceeding with AOF tail...\n") + _, err = CheckAOFInfof.fp.Seek(rdbpos, io.SeekStart) if err != nil { - log.Fatalf(("Failed to seek in AOF %s: %s"), aofFilename, err) + + log.Panicf(("Failed to seek in AOF %v: %v"), AOFFileName, err) } + CheckAOFInfof.pos = rdbpos } - if _, err := fp.Read(buf); err != nil { + } + + for { + + if _, err := rd.Read(buf); err != nil { + if err == io.EOF { + break } - log.Fatalf("Failed to read from AOF %s, aborting...\n", aofFilename) + log.Panicf("Failed to read from AOF %v, aborting...\n", AOFFileName) } - if _, err := fp.Seek(-1, io.SeekCurrent); err != nil { - log.Fatalf("Failed to fseek in AOF %s: %s", aofFilename, err) - } - - switch buf[0] { - case '#': - if processAnnotations(fp, aofFilepath, lastFile) == 0 { - fp.Close() - return AOF_CHECK_TIMESTAMP_TRUNCATED + CheckAOFInfof.pos += int64(len(buf)) + if buf[0] == '#' { + if ProcessAnnotations(rd, AOFFilePath, lastFile) == 0 { + CheckAOFInfof.fp.Close() + return AOFCheckTimeStampTruncated } - case '*': - if processRESP(fp, aofFilepath, &multi) == 0 { + } else if buf[0] == '*' { + if ProcessRESP(rd, AOFFilePath, &multi) == 0 { break } - default: - fmt.Printf("AOF %s format error\n", aofFilename) + } else { + log.Infof("AOF %v format error\n", AOFFileName) break } } - if _, err := fp.Stat(); err == nil && multi == 1 && len(errors) == 0 { - if _, err := fp.Seek(0, io.SeekEnd); err != nil { - if _, err := fp.Seek(0, io.SeekCurrent); err == io.EOF { - fmt.Println("Reached EOF before reading EXEC for MULTI") - } - } - } - - if len(errors) > 0 { - fmt.Println(errors) - } - diff = size - pos - if diff == 0 && toTimestamp == 1 { - fmt.Printf("Truncate nothing in AOF %s to timestamp %d\n", aofFilename, toTimestamp) - return AOF_CHECK_OK + diff := size - CheckAOFInfof.pos + if diff == 0 && CheckAOFInfof.toTimestamp == 1 { + log.Infof("Truncate nothing in AOF %v to timestamp %d\n", AOFFileName, CheckAOFInfof.toTimestamp) + return AOFCheckOk } - fmt.Printf("AOF analyzed: filename=%s, size=%d, ok_up_to=%d, ok_up_to_line=%d, diff=%d\n", aofFilename, size, pos, line, diff) + log.Infof("AOF analyzed: Filename=%v, size=%d, ok_up_to=%d, ok_up_to_line=%d, diff=%d\n", AOFFileName, size, CheckAOFInfof.pos, CheckAOFInfof.line, diff) if diff > 0 { if fix == 1 { - if lastFile == false { - fmt.Printf("Failed to truncate AOF %s because it is not the last file\n", aofFilename) + if !lastFile { + log.Panicf("Failed to truncate AOF %v because it is not the last File\n", AOFFileName) os.Exit(1) } - fmt.Printf("this will shrink the AOF %s from %d bytes,with %d bytes,to %d bytes\n", aofFilename, size, diff, pos) + fmt.Printf("this will shrink the AOF %v from %d bytes,with %d bytes,to %d bytes\n", AOFFileName, size, diff, CheckAOFInfof.pos) fmt.Print("Continue? [y/N]: ") reader := bufio.NewReader(os.Stdin) input, err := reader.ReadString('\n') if err != nil || strings.ToLower(string(input[0])) != "y" { - fmt.Println("Aborting...") - os.Exit(1) + log.Panicf("Aborting...") + } - if err := fp.Truncate(pos); err != nil { - fmt.Printf("Failed to truncate AOF %s\n", aofFilename) - os.Exit(1) + if err := CheckAOFInfof.fp.Truncate(CheckAOFInfof.pos); err != nil { + log.Panicf("Failed to truncate AOF %v\n", AOFFileName) + } else { - return AOF_CHECK_TRUNCATED + return AOFCheckTruncated } } else { - fmt.Printf("AOF %s is not valid.Use the --fix potion to try fixing it.\n", aofFilename) - os.Exit(1) + log.Panicf("AOF %v is not valid.Use the --fix potion to try fixing it.\n", AOFFileName) } } - fp.Close() - return AOF_CHECK_OK + CheckAOFInfof.fp.Close() + + return AOFCheckOk } diff --git a/internal/commands/keys.go b/internal/commands/keys.go index 0899e055..1094821c 100644 --- a/internal/commands/keys.go +++ b/internal/commands/keys.go @@ -2,11 +2,12 @@ package commands import ( "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/utils" "math" "strconv" "strings" + + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/utils" ) // CalcKeys https://redis.io/docs/reference/key-specs/ diff --git a/internal/config/config.go b/internal/config/config.go index e4b4fbea..25c786f5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,8 +19,9 @@ type tomlSource struct { ElastiCachePSync string `toml:"elasticache_psync"` // restore mode - RDBFilePath string `toml:"rdb_file_path"` - AOFFilePath string `toml:"aof_file_path"` // add the aof path + RDBFilePath string `toml:"rdb_file_path"` + AOFFilePath string `toml:"aof_file_path"` + AOFTruncateToTimestamp int64 `toml:"aof_truncate_to_timestamp"` } type tomlTarget struct { @@ -76,6 +77,7 @@ func init() { // restore Config.Source.RDBFilePath = "" Config.Source.AOFFilePath = "" + Config.Source.AOFTruncateToTimestamp = 0 // target Config.Target.Type = "standalone" diff --git a/internal/rdb/rdb_check.go b/internal/rdb/rdb_check.go new file mode 100644 index 00000000..4d724d23 --- /dev/null +++ b/internal/rdb/rdb_check.go @@ -0,0 +1,201 @@ +package rdb + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "strconv" + "time" + + "github.com/alibaba/RedisShake/internal/config" + "github.com/alibaba/RedisShake/internal/entry" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/rdb/structure" + "github.com/alibaba/RedisShake/internal/rdb/types" +) + +func RedisCheckRDBMain(filepath string, fp *os.File) int64 { + if fp == nil { + fmt.Fprintf(os.Stderr, "The file: %s fp is nil", filepath) + os.Exit(1) + } + log.Infof("Checking RDB file %s", filepath) + ld := NewLoader(filepath, nil) + + RDBPos := ld.CheckParseRDB() + + if RDBPos > 0 { + log.Infof("\\o/ RDB looks OK! \\o/") + } else { + return -1 + } + return RDBPos +} +func (ld *Loader) CheckParseRDB() int64 { + var err error + ld.fp, err = os.OpenFile(ld.filPath, os.O_RDONLY, 0666) + if err != nil { + log.Panicf("open file failed. filepath=[%s], error=[%s]", ld.filPath, err) + } + defer func() { + err = ld.fp.Close() + if err != nil { + log.Panicf("close file failed. filepath=[%s], error=[%s]", ld.filPath, err) + } + }() + rd := bufio.NewReader(ld.fp) + //magic + version + buf := make([]byte, 9) + _, err = io.ReadFull(rd, buf) + if err != nil { + log.PanicError(err) + } + if !bytes.Equal(buf[:5], []byte("REDIS")) { + log.Panicf("verify magic string, invalid file format. bytes=[%v]", buf[:5]) + } + version, err := strconv.Atoi(string(buf[5:])) + if err != nil { + log.PanicError(err) + } + log.Infof("RDB version: %d", version) + + // read entries + rdbpos := ld.CheckParseRDBEntry(rd) + + return rdbpos +} + +func (ld *Loader) CheckParseRDBEntry(rd *bufio.Reader) int64 { + // for stat + var RDBPos int64 + var rdbsize int64 = 9 + UpdateRDBSize := func() { + var err error + RDBPos, err = ld.fp.Seek(0, io.SeekCurrent) + if err != nil { + log.PanicError(err) + } + + } + defer UpdateRDBSize() + + // read one entry + tick := time.Tick(time.Second * 1) + for { + typeByte := structure.ReadByte(rd) + rdbsize += 1 + switch typeByte { + case kFlagIdle: + tempIdle, tempOffset := structure.ReadLengthWithOffset(rd) + ld.idle = int64(tempIdle) + rdbsize += tempOffset + case kFlagFreq: + ld.freq = int64(structure.ReadByte(rd)) + rdbsize += 1 + case kFlagAUX: + key, tempOffset := structure.ReadStringWithOffset(rd) + rdbsize += tempOffset + value, tempOffset := structure.ReadStringWithOffset(rd) + rdbsize += tempOffset + + if key == "repl-stream-db" { + var err error + ld.replStreamDbId, err = strconv.Atoi(value) + if err != nil { + log.PanicError(err) + } + log.Infof("RDB repl-stream-db: %d position: %d", ld.replStreamDbId, RDBPos) + } else if key == "lua" { + e := entry.NewEntry() + e.Argv = []string{"script", "load", value} + e.IsBase = true + + log.Infof("LUA script: [%s]", value) + } else { + log.Infof("RDB AUX fields. key=[%s], value=[%s]", key, value) + } + case kFlagResizeDB: + dbSize, dbSizeOffset := structure.ReadLengthWithOffset(rd) + expireSize, expireSizeOffset := structure.ReadLengthWithOffset(rd) + rdbsize += dbSizeOffset + expireSizeOffset + log.Infof("RDB resize db. dbsize=[%d], expiresize=[%d]", dbSize, expireSize) + case kFlagExpireMs: + ld.expireMs = int64(structure.ReadUint64(rd)) - time.Now().UnixMilli() + rdbsize += 8 + if ld.expireMs < 0 { + ld.expireMs = 1 + } + case kFlagExpire: + ld.expireMs = int64(structure.ReadUint32(rd))*1000 - time.Now().UnixMilli() + rdbsize += 4 + if ld.expireMs < 0 { + ld.expireMs = 1 + } + case kFlagSelect: + DBId, DBIDOffset := structure.ReadLengthWithOffset(rd) + ld.nowDBId = int(DBId) + rdbsize += DBIDOffset + case kEOF: + UpdateRDBSize() + return rdbsize + default: + key, tempOffset := structure.ReadStringWithOffset(rd) + rdbsize += tempOffset + var value bytes.Buffer + anotherReader := io.TeeReader(rd, &value) + o, tempOffsets := types.ParseObjectWithOffset(anotherReader, typeByte, key) + + rdbsize += tempOffsets + if uint64(value.Len()) > config.Config.Advanced.TargetRedisProtoMaxBulkLen { + cmds := o.Rewrite() + for _, cmd := range cmds { + e := entry.NewEntry() + e.IsBase = true + e.DbId = ld.nowDBId + e.Argv = cmd + + } + if ld.expireMs != 0 { + e := entry.NewEntry() + e.IsBase = true + e.DbId = ld.nowDBId + e.Argv = []string{"PEXPIRE", key, strconv.FormatInt(ld.expireMs, 10)} + } + } else { + e := entry.NewEntry() + e.IsBase = true + e.DbId = ld.nowDBId + + v := ld.createValueDump(typeByte, value.Bytes()) + + //value + e.Argv = []string{"restore", key, strconv.FormatInt(ld.expireMs, 10), v} + if config.Config.Advanced.RDBRestoreCommandBehavior == "rewrite" { + if config.Config.Target.Version < 3.0 { + log.Panicf("RDB restore command behavior is rewrite, but target redis version is %f, not support REPLACE modifier,position: %d", config.Config.Target.Version, RDBPos) + } + e.Argv = append(e.Argv, "replace") + } + if ld.idle != 0 && config.Config.Target.Version >= 5.0 { + e.Argv = append(e.Argv, "idletime", strconv.FormatInt(ld.idle, 10)) + } + if ld.freq != 0 && config.Config.Target.Version >= 5.0 { + e.Argv = append(e.Argv, "freq", strconv.FormatInt(ld.freq, 10)) + } + + } + ld.expireMs = 0 + ld.idle = 0 + ld.freq = 0 + } + select { + case <-tick: + UpdateRDBSize() + default: + } + UpdateRDBSize() + } + return RDBPos +} diff --git a/internal/rdb/structure/float.go b/internal/rdb/structure/float.go index 8351d96a..580af530 100644 --- a/internal/rdb/structure/float.go +++ b/internal/rdb/structure/float.go @@ -2,10 +2,11 @@ package structure import ( "encoding/binary" - "github.com/alibaba/RedisShake/internal/log" "io" "math" "strconv" + + "github.com/alibaba/RedisShake/internal/log" ) func ReadFloat(rd io.Reader) float64 { @@ -33,6 +34,30 @@ func ReadFloat(rd io.Reader) float64 { } } +func ReadFloatWithOffset(rd io.Reader) (float64, int64) { + u := ReadUint8(rd) + switch u { + case 253: + return math.NaN(), 1 + case 254: + return math.Inf(0), 1 + case 255: + return math.Inf(-1), 1 + default: + buf := make([]byte, u) + _, err := io.ReadFull(rd, buf) + if err != nil { + return 0, 2 + } + + v, err := strconv.ParseFloat(string(buf), 64) + if err != nil { + log.PanicError(err) + } + return v, 2 + } +} + func ReadDouble(rd io.Reader) float64 { var buf = make([]byte, 8) _, err := io.ReadFull(rd, buf) diff --git a/internal/rdb/structure/intset.go b/internal/rdb/structure/intset.go index a829cd60..e440287e 100644 --- a/internal/rdb/structure/intset.go +++ b/internal/rdb/structure/intset.go @@ -30,3 +30,27 @@ func ReadIntset(rd io.Reader) []string { } return elements } + +func ReadIntsetWithOffset(rd io.Reader) ([]string, int64) { + tempString, offset := ReadStringWithOffset(rd) + rd = bufio.NewReader(strings.NewReader(tempString)) + + encodingType := int(ReadUint32(rd)) + size := int(ReadUint32(rd)) + elements := make([]string, size) + + for i := 0; i < size; i++ { + intBytes := ReadBytes(rd, encodingType) + var intString string + switch encodingType { + case 2: + intString = strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10) + case 4: + intString = strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10) + case 8: + intString = strconv.FormatInt(int64(int64(binary.LittleEndian.Uint64(intBytes))), 10) + } + elements[i] = intString + } + return elements, offset +} diff --git a/internal/rdb/structure/length.go b/internal/rdb/structure/length.go index e62543fe..bd82c903 100644 --- a/internal/rdb/structure/length.go +++ b/internal/rdb/structure/length.go @@ -3,8 +3,9 @@ package structure import ( "encoding/binary" "fmt" - "github.com/alibaba/RedisShake/internal/log" "io" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -29,8 +30,8 @@ func ReadLength(rd io.Reader) uint64 { func readEncodedLength(rd io.Reader) (length uint64, special bool, err error) { var lengthBuffer = make([]byte, 8) - firstByte := ReadByte(rd) + var offset int64 = 1 first2bits := (firstByte & 0xc0) >> 6 // first 2 bits of encoding switch first2bits { case RDB6ByteLen: @@ -38,6 +39,7 @@ func readEncodedLength(rd io.Reader) (length uint64, special bool, err error) { case RDB14ByteLen: nextByte := ReadByte(rd) length = (uint64(firstByte)&0x3f)<<8 | uint64(nextByte) + offset += 1 case len32or64Bit: if firstByte == RDB32ByteLen { _, err = io.ReadFull(rd, lengthBuffer[0:4]) @@ -45,12 +47,14 @@ func readEncodedLength(rd io.Reader) (length uint64, special bool, err error) { return 0, false, fmt.Errorf("read len32Bit failed: %s", err.Error()) } length = uint64(binary.BigEndian.Uint32(lengthBuffer)) + offset += 4 } else if firstByte == RDB64ByteLen { _, err = io.ReadFull(rd, lengthBuffer) if err != nil { return 0, false, fmt.Errorf("read len64Bit failed: %s", err.Error()) } length = binary.BigEndian.Uint64(lengthBuffer) + offset += 8 } else { return 0, false, fmt.Errorf("illegal length encoding: %x", firstByte) } @@ -60,3 +64,51 @@ func readEncodedLength(rd io.Reader) (length uint64, special bool, err error) { } return length, special, nil } + +func ReadLengthWithOffset(rd io.Reader) (uint64, int64) { + length, special, offset, err := readEncodedLengthWithOffset(rd) + if special { + log.Panicf("illegal length special=true, encoding: %d", length) + } + if err != nil { + log.PanicError(err) + } + return length, offset +} +func readEncodedLengthWithOffset(rd io.Reader) (length uint64, special bool, offsets int64, err error) { + var lengthBuffer = make([]byte, 8) + firstByte := ReadByte(rd) + var offset int64 = 1 + first2bits := (firstByte & 0xc0) >> 6 // first 2 bits of encoding + switch first2bits { + case RDB6ByteLen: + length = uint64(firstByte) & 0x3f + case RDB14ByteLen: + nextByte := ReadByte(rd) + length = (uint64(firstByte)&0x3f)<<8 | uint64(nextByte) + offset += 1 + case len32or64Bit: + if firstByte == RDB32ByteLen { + _, err = io.ReadFull(rd, lengthBuffer[0:4]) + offset += 4 + if err != nil { + return 0, false, offset, fmt.Errorf("read len32Bit failed: %s", err.Error()) + } + length = uint64(binary.BigEndian.Uint32(lengthBuffer)) + + } else if firstByte == RDB64ByteLen { + _, err = io.ReadFull(rd, lengthBuffer) + offset += 8 + if err != nil { + return 0, false, offset, fmt.Errorf("read len64Bit failed: %s", err.Error()) + } + length = binary.BigEndian.Uint64(lengthBuffer) + } else { + return 0, false, offset, fmt.Errorf("illegal length encoding: %x", firstByte) + } + case lenSpecial: + special = true + length = uint64(firstByte) & 0x3f + } + return length, special, offset, nil +} diff --git a/internal/rdb/structure/listpack.go b/internal/rdb/structure/listpack.go index 2d30374a..65e93d94 100644 --- a/internal/rdb/structure/listpack.go +++ b/internal/rdb/structure/listpack.go @@ -2,11 +2,12 @@ package structure import ( "bufio" - "github.com/alibaba/RedisShake/internal/log" "io" "math" "strconv" "strings" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -54,6 +55,23 @@ func ReadListpack(rd io.Reader) []string { } return elements } +func ReadListpackWithOffset(rd io.Reader) ([]string, int64) { + tempstring, offset := ReadStringWithOffset(rd) + rd = bufio.NewReader(strings.NewReader(tempstring)) + + _ = ReadUint32(rd) // bytes + size := int(ReadUint16(rd)) + var elements []string + for i := 0; i < size; i++ { + ele := readListpackEntry(rd) + elements = append(elements, ele) + } + lastByte := ReadByte(rd) + if lastByte != 0xFF { + log.Panicf("ReadListpack: last byte is not 0xFF, but [%d]", lastByte) + } + return elements, offset +} // redis/src/Listpack.c lpGet() func readListpackEntry(rd io.Reader) string { diff --git a/internal/rdb/structure/string.go b/internal/rdb/structure/string.go index f53559e3..2b0ae7e4 100644 --- a/internal/rdb/structure/string.go +++ b/internal/rdb/structure/string.go @@ -1,9 +1,10 @@ package structure import ( - "github.com/alibaba/RedisShake/internal/log" "io" "strconv" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -42,6 +43,41 @@ func ReadString(rd io.Reader) string { return string(ReadBytes(rd, int(length))) } +func ReadStringWithOffset(rd io.Reader) (string, int64) { + length, special, offset, err := readEncodedLengthWithOffset(rd) + if err != nil { + log.PanicError(err) + } + if special { + switch length { + case RDBEncInt8: + b := ReadInt8(rd) + offset += 1 + return strconv.Itoa(int(b)), offset + case RDBEncInt16: + b := ReadInt16(rd) + offset += 2 + return strconv.Itoa(int(b)), offset + case RDBEncInt32: + b := ReadInt32(rd) + offset += 4 + return strconv.Itoa(int(b)), offset + case RDBEncLZF: + inLen, tempoffset := ReadLengthWithOffset(rd) + offset += tempoffset + outLen, tempoffset := ReadLengthWithOffset(rd) + offset += tempoffset + in := ReadBytes(rd, int(inLen)) + offset += int64(inLen) + return lzfDecompress(in, int(outLen)), offset + default: + log.Panicf("Unknown string encode type %d", length) + } + } + offset += int64(length) + return string(ReadBytes(rd, int(length))), offset +} + func lzfDecompress(in []byte, outLen int) string { out := make([]byte, outLen) diff --git a/internal/rdb/structure/ziplist.go b/internal/rdb/structure/ziplist.go index 141ea026..3a7b370e 100644 --- a/internal/rdb/structure/ziplist.go +++ b/internal/rdb/structure/ziplist.go @@ -3,10 +3,11 @@ package structure import ( "bufio" "encoding/binary" - "github.com/alibaba/RedisShake/internal/log" "io" "strconv" "strings" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -24,6 +25,7 @@ const ( ) func ReadZipList(rd io.Reader) []string { + rd = bufio.NewReader(strings.NewReader(ReadString(rd))) // The general layout of the ziplist is as follows: @@ -51,6 +53,36 @@ func ReadZipList(rd io.Reader) []string { return elements } +func ReadZipListWithOffset(rd io.Reader) ([]string, int64) { + + Strings, offset := ReadStringWithOffset(rd) + rd = bufio.NewReader(strings.NewReader(Strings)) + + // The general layout of the ziplist is as follows: + // ... + _ = ReadUint32(rd) // zlbytes + _ = ReadUint32(rd) // zltail + + size := int(ReadUint16(rd)) + var elements []string + if size == 65535 { // 2^16-1, we need to traverse the entire list to know how many items it holds. + for firstByte := ReadByte(rd); firstByte != 0xFE; firstByte = ReadByte(rd) { + ele := readZipListEntry(rd, firstByte) + elements = append(elements, ele) + } + } else { + for i := 0; i < size; i++ { + firstByte := ReadByte(rd) + ele := readZipListEntry(rd, firstByte) + elements = append(elements, ele) + } + if lastByte := ReadByte(rd); lastByte != 0xFF { + log.Panicf("invalid zipList lastByte encoding: %d", lastByte) + } + } + return elements, offset +} + /* * So practically an entry is encoded in the following way: * diff --git a/internal/rdb/types/hash.go b/internal/rdb/types/hash.go index e27b67e8..5a9785e9 100644 --- a/internal/rdb/types/hash.go +++ b/internal/rdb/types/hash.go @@ -1,9 +1,10 @@ package types import ( + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) type HashObject struct { @@ -28,6 +29,27 @@ func (o *HashObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { } } +func (o *HashObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + o.key = key + o.value = make(map[string]string) + switch typeByte { + case rdbTypeHash: + offset := o.readHashWithOffset(rd) + return offset + case rdbTypeHashZipmap: + o.readHashZipmap(rd) + case rdbTypeHashZiplist: + offset := o.readHashZiplistWithOffset(rd) + return offset + case rdbTypeHashListpack: + offset := o.readHashListpackWithOffset(rd) + return offset + default: + log.Panicf("unknown hash type. typeByte=[%d]", typeByte) + return 0 + } + return 0 +} func (o *HashObject) readHash(rd io.Reader) { size := int(structure.ReadLength(rd)) for i := 0; i < size; i++ { @@ -37,6 +59,18 @@ func (o *HashObject) readHash(rd io.Reader) { } } +func (o *HashObject) readHashWithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + for i := 0; i < int(size); i++ { + key, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + value, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + o.value[key] = value + } + return offset +} + func (o *HashObject) readHashZipmap(rd io.Reader) { log.Panicf("not implemented rdbTypeZipmap") } @@ -51,6 +85,17 @@ func (o *HashObject) readHashZiplist(rd io.Reader) { } } +func (o *HashObject) readHashZiplistWithOffset(rd io.Reader) int64 { + list, offset := structure.ReadZipListWithOffset(rd) + size := len(list) + for i := 0; i < size; i += 2 { + key := list[i] + value := list[i+1] + o.value[key] = value + } + return offset +} + func (o *HashObject) readHashListpack(rd io.Reader) { list := structure.ReadListpack(rd) size := len(list) @@ -59,6 +104,18 @@ func (o *HashObject) readHashListpack(rd io.Reader) { value := list[i+1] o.value[key] = value } + +} + +func (o *HashObject) readHashListpackWithOffset(rd io.Reader) int64 { + list, offset := structure.ReadListpackWithOffset(rd) + size := len(list) + for i := 0; i < size; i += 2 { + key := list[i] + value := list[i+1] + o.value[key] = value + } + return offset } func (o *HashObject) Rewrite() []RedisCmd { diff --git a/internal/rdb/types/interface.go b/internal/rdb/types/interface.go index 7d9fb3d5..b72d331a 100644 --- a/internal/rdb/types/interface.go +++ b/internal/rdb/types/interface.go @@ -1,8 +1,9 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" "io" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -99,6 +100,41 @@ func ParseObject(rd io.Reader, typeByte byte, key string) RedisObject { return nil } +func ParseObjectWithOffset(rd io.Reader, typeByte byte, key string) (RedisObject, int64) { + switch typeByte { + case rdbTypeString: // string + o := new(StringObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeList, rdbTypeListZiplist, rdbTypeListQuicklist, rdbTypeListQuicklist2: // list + o := new(ListObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeSet, rdbTypeSetIntset: // set + o := new(SetObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeZSet, rdbTypeZSet2, rdbTypeZSetZiplist, rdbTypeZSetListpack: // zset + o := new(ZsetObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeHash, rdbTypeHashZipmap, rdbTypeHashZiplist, rdbTypeHashListpack: // hash + o := new(HashObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeStreamListpacks, rdbTypeStreamListpacks2: // stream + o := new(StreamObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeModule, rdbTypeModule2: // module + o := new(ModuleObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + } + log.Panicf("unknown type byte: %d", typeByte) + return nil, 0 +} + func moduleTypeNameByID(moduleId uint64) string { nameList := make([]byte, 9) moduleId >>= 10 diff --git a/internal/rdb/types/list.go b/internal/rdb/types/list.go index 57e23c08..73fae7d5 100644 --- a/internal/rdb/types/list.go +++ b/internal/rdb/types/list.go @@ -1,9 +1,10 @@ package types import ( + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) // quicklist node container formats @@ -33,6 +34,27 @@ func (o *ListObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { log.Panicf("unknown list type %d", typeByte) } } +func (o *ListObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + var offset int64 = 0 + o.key = key + switch typeByte { + case rdbTypeList: + offset = o.readListWithOffset(rd) + return offset + case rdbTypeListZiplist: + o.elements, offset = structure.ReadZipListWithOffset(rd) + return offset + case rdbTypeListQuicklist: + offset = o.readQuickListWithOffset(rd) + return offset + case rdbTypeListQuicklist2: + offset = o.readQuickList2WithOffset(rd) + return offset + default: + log.Panicf("unknown list type %d", typeByte) + } + return offset +} func (o *ListObject) Rewrite() []RedisCmd { cmds := make([]RedisCmd, len(o.elements)) @@ -50,6 +72,16 @@ func (o *ListObject) readList(rd io.Reader) { o.elements = append(o.elements, ele) } } +func (o *ListObject) readListWithOffset(rd io.Reader) int64 { + sizes, offset := structure.ReadLengthWithOffset(rd) + size := int(sizes) + for i := 0; i < size; i++ { + ele, offsets := structure.ReadStringWithOffset(rd) + offset += offsets + o.elements = append(o.elements, ele) + } + return offset +} func (o *ListObject) readQuickList(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -58,6 +90,15 @@ func (o *ListObject) readQuickList(rd io.Reader) { o.elements = append(o.elements, ziplistElements...) } } +func (o *ListObject) readQuickListWithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + for i := 0; i < int(size); i++ { + ziplistElements, offsets := structure.ReadZipListWithOffset(rd) + offset += offsets + o.elements = append(o.elements, ziplistElements...) + } + return offset +} func (o *ListObject) readQuickList2(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -74,3 +115,23 @@ func (o *ListObject) readQuickList2(rd io.Reader) { } } } + +func (o *ListObject) readQuickList2WithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + for i := 0; i < int(size); i++ { + container, offsets := structure.ReadLengthWithOffset(rd) + offset += offsets + if container == quicklistNodeContainerPlain { + ele, offsets := structure.ReadStringWithOffset(rd) + offset += offsets + o.elements = append(o.elements, ele) + } else if container == quicklistNodeContainerPacked { + listpackElements, TempOffsets := structure.ReadListpackWithOffset(rd) + offset += TempOffsets + o.elements = append(o.elements, listpackElements...) + } else { + log.Panicf("unknown quicklist container %d", container) + } + } + return offset +} diff --git a/internal/rdb/types/module2.go b/internal/rdb/types/module2.go index 26ab46e0..a6c4492a 100644 --- a/internal/rdb/types/module2.go +++ b/internal/rdb/types/module2.go @@ -1,9 +1,10 @@ package types import ( + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) type ModuleObject struct { @@ -34,6 +35,38 @@ func (o *ModuleObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { } } +func (o *ModuleObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + if typeByte == rdbTypeModule { + log.Panicf("module type with version 1 is not supported, key=[%s]", key) + } + moduleId, offset := structure.ReadLengthWithOffset(rd) + moduleName := moduleTypeNameByID(moduleId) + opcode := structure.ReadByte(rd) + offset += 1 + for opcode != rdbModuleOpcodeEOF { + switch opcode { + case rdbModuleOpcodeSINT: + case rdbModuleOpcodeUINT: + _, tempOffset := structure.ReadLengthWithOffset(rd) + offset += tempOffset + case rdbModuleOpcodeFLOAT: + _, tempOffset := structure.ReadFloatWithOffset(rd) + offset += tempOffset + case rdbModuleOpcodeDOUBLE: + structure.ReadDouble(rd) + offset += 8 + case rdbModuleOpcodeSTRING: + _, tempOffset := structure.ReadStringWithOffset(rd) + offset += tempOffset + default: + log.Panicf("unknown module opcode=[%d], module name=[%s]", opcode, moduleName) + } + opcode = structure.ReadByte(rd) + offset += 1 + } + return offset +} + func (o *ModuleObject) Rewrite() []RedisCmd { log.Panicf("module Rewrite not implemented") return nil diff --git a/internal/rdb/types/set.go b/internal/rdb/types/set.go index ac02f378..807cd173 100644 --- a/internal/rdb/types/set.go +++ b/internal/rdb/types/set.go @@ -1,9 +1,10 @@ package types import ( + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) type SetObject struct { @@ -22,6 +23,21 @@ func (o *SetObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { log.Panicf("unknown set type. typeByte=[%d]", typeByte) } } +func (o *SetObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + o.key = key + switch typeByte { + case rdbTypeSet: + tempOffset := o.readSetWithOffset(rd) + return tempOffset + case rdbTypeSetIntset: + var tempOffsets int64 + o.elements, tempOffsets = structure.ReadIntsetWithOffset(rd) + return tempOffsets + default: + log.Panicf("unknown set type. typeByte=[%d]", typeByte) + } + return 0 +} func (o *SetObject) readSet(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -31,6 +47,16 @@ func (o *SetObject) readSet(rd io.Reader) { o.elements[i] = val } } +func (o *SetObject) readSetWithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + o.elements = make([]string, size) + for i := 0; i < int(size); i++ { + val, TempOffsets := structure.ReadStringWithOffset(rd) + offset += TempOffsets + o.elements[i] = val + } + return offset +} func (o *SetObject) Rewrite() []RedisCmd { cmds := make([]RedisCmd, len(o.elements)) diff --git a/internal/rdb/types/stream.go b/internal/rdb/types/stream.go index 9b0a6b03..d7df7016 100644 --- a/internal/rdb/types/stream.go +++ b/internal/rdb/types/stream.go @@ -3,10 +3,11 @@ package types import ( "encoding/binary" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" "io" "strconv" + + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/rdb/structure" ) /* @@ -56,6 +57,22 @@ func (o *StreamObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { } } +func (o *StreamObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + o.key = key + switch typeByte { + case rdbTypeStreamListpacks: + offset := o.readStreamWithOffset(rd, key, typeByte) + return offset + case rdbTypeStreamListpacks2: + offset := o.readStreamWithOffset(rd, key, typeByte) + return offset + default: + log.Panicf("unknown hash type. typeByte=[%d]", typeByte) + return 0 + } + +} + // see redis rewriteStreamObject() func (o *StreamObject) readStream(rd io.Reader, masterKey string, typeByte byte) { @@ -227,6 +244,192 @@ func (o *StreamObject) readStream(rd io.Reader, masterKey string, typeByte byte) } } +func (o *StreamObject) readStreamWithOffset(rd io.Reader, masterKey string, typeByte byte) int64 { + // 1. length(number of listpack), k1, v1, k2, v2, ..., number, ms, seq + + /* Load the number of Listpack. */ + nListpack, offset := structure.ReadLengthWithOffset(rd) + for i := 0; i < int(nListpack); i++ { + /* Load key */ + key, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + /* key is streamId, like: 1612181627287-0 */ + masterMs := int64(binary.BigEndian.Uint64([]byte(key[:8]))) + masterSeq := int64(binary.BigEndian.Uint64([]byte(key[8:]))) + + /* value is a listpack */ + elements, tempOffsets := structure.ReadListpackWithOffset(rd) + offset += tempOffsets + inx := 0 + + /* The front of stream listpack is master entry */ + /* Parse the master entry */ + count := nextInteger(&inx, elements) // count + deleted := nextInteger(&inx, elements) // deleted + numFields := int(nextInteger(&inx, elements)) // num-fields + + fields := elements[3 : 3+numFields] // fields + inx = 3 + numFields + + // master entry end by zero + lastEntry := nextString(&inx, elements) + if lastEntry != "0" { + log.Panicf("master entry not ends by zero. lastEntry=[%s]", lastEntry) + } + + /* Parse entries */ + for count != 0 || deleted != 0 { + flags := nextInteger(&inx, elements) // [is_same_fields|is_deleted] + entryMs := nextInteger(&inx, elements) + entrySeq := nextInteger(&inx, elements) + + args := []string{"xadd", masterKey, fmt.Sprintf("%v-%v", entryMs+masterMs, entrySeq+masterSeq)} + + if flags&2 == 2 { // same fields, get field from master entry. + for j := 0; j < numFields; j++ { + args = append(args, fields[j], nextString(&inx, elements)) + } + } else { // get field by lp.Next() + num := int(nextInteger(&inx, elements)) + args = append(args, elements[inx:inx+num*2]...) + inx += num * 2 + } + + _ = nextString(&inx, elements) // lp_count + + if flags&1 == 1 { // is_deleted + deleted -= 1 + } else { + count -= 1 + o.cmds = append(o.cmds, args) + } + } + } + + /* Load total number of items inside the stream. */ + _, tempOffsets := structure.ReadLengthWithOffset(rd) // number + offset += tempOffsets + /* Load the last entry ID. */ + lastMs, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + lastSeq, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + lastid := fmt.Sprintf("%v-%v", lastMs, lastSeq) + if nListpack == 0 { + /* Use the XADD MAXLEN 0 trick to generate an empty stream if + * the key we are serializing is an empty string, which is possible + * for the Stream type. */ + args := []string{"xadd", masterKey, "MAXLEN", "0", lastid, "x", "y"} + o.cmds = append(o.cmds, args) + } + + /* Append XSETID after XADD, make sure lastid is correct, + * in case of XDEL lastid. */ + o.cmds = append(o.cmds, []string{"xsetid", masterKey, lastid}) + + if typeByte == rdbTypeStreamListpacks2 { + /* Load the first entry ID. */ + _, tempOffsets = structure.ReadLengthWithOffset(rd) // first_ms + offset += tempOffsets + _, tempOffsets = structure.ReadLengthWithOffset(rd) // first_seq + offset += tempOffsets + /* Load the maximal deleted entry ID. */ + _, tempOffsets = structure.ReadLengthWithOffset(rd) // max_deleted_ms + offset += tempOffsets + _, tempOffsets = structure.ReadLengthWithOffset(rd) // max_deleted_seq + offset += tempOffsets + + /* Load the offset. */ + _, tempOffsets = structure.ReadLengthWithOffset(rd) // offset + offset += tempOffsets + } + + /* 2. nConsumerGroup, groupName, ms, seq, PEL, Consumers */ + + /* Load the number of groups. */ + nConsumerGroup, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + for i := 0; i < int(nConsumerGroup); i++ { + /* Load groupName */ + groupName, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + /* Load the last ID */ + lastMs, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + lastSeq, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + lastid := fmt.Sprintf("%v-%v", lastMs, lastSeq) + + /* Create Group */ + o.cmds = append(o.cmds, []string{"CREATE", masterKey, groupName, lastid}) + + /* Load group offset. */ + if typeByte == rdbTypeStreamListpacks2 { + _, tempOffsets = structure.ReadLengthWithOffset(rd) // offset + offset += tempOffsets + } + + /* Load the global PEL */ + nPel, tempOffsets := structure.ReadLengthWithOffset(rd) + mapId2Time := make(map[string]uint64) + mapId2Count := make(map[string]uint64) + + for j := 0; j < int(nPel); j++ { + /* Load streamId */ + tmpBytes := structure.ReadBytes(rd, 16) + offset += 16 + ms := binary.BigEndian.Uint64(tmpBytes[:8]) + seq := binary.BigEndian.Uint64(tmpBytes[8:]) + streamId := fmt.Sprintf("%v-%v", ms, seq) + + /* Load deliveryTime */ + deliveryTime := structure.ReadUint64(rd) + offset += 8 + /* Load deliveryCount */ + deliveryCount, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + /* Save deliveryTime and deliveryCount */ + mapId2Time[streamId] = deliveryTime + mapId2Count[streamId] = deliveryCount + } + + /* Generate XCLAIMs for each consumer that happens to + * have pending entries. Empty consumers are discarded. */ + nConsumer, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + for j := 0; j < int(nConsumer); j++ { + /* Load consumerName */ + consumerName, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + + /* Load lastSeenTime */ + _ = structure.ReadUint64(rd) + offset += 8 + /* Consumer PEL */ + nPEL, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + for i := 0; i < int(nPEL); i++ { + + /* Load streamId */ + tmpBytes := structure.ReadBytes(rd, 16) + offset += 16 + ms := binary.BigEndian.Uint64(tmpBytes[:8]) + seq := binary.BigEndian.Uint64(tmpBytes[8:]) + streamId := fmt.Sprintf("%v-%v", ms, seq) + + /* Send */ + args := []string{ + "xclaim", masterKey, groupName, consumerName, "0", streamId, + "TIME", strconv.FormatUint(mapId2Time[streamId], 10), + "RETRYCOUNT", strconv.FormatUint(mapId2Count[streamId], 10), + "JUSTID", "FORCE"} + o.cmds = append(o.cmds, args) + } + } + } + return offset +} + func nextInteger(inx *int, elements []string) int64 { ele := elements[*inx] *inx++ diff --git a/internal/rdb/types/string.go b/internal/rdb/types/string.go index 2adf51c6..6cb07b06 100644 --- a/internal/rdb/types/string.go +++ b/internal/rdb/types/string.go @@ -1,8 +1,9 @@ package types import ( - "github.com/alibaba/RedisShake/internal/rdb/structure" "io" + + "github.com/alibaba/RedisShake/internal/rdb/structure" ) type StringObject struct { @@ -14,7 +15,12 @@ func (o *StringObject) LoadFromBuffer(rd io.Reader, key string, _ byte) { o.key = key o.value = structure.ReadString(rd) } - +func (o *StringObject) LoadFromBufferWithOffset(rd io.Reader, key string, _ byte) int64 { + var offsets int64 + o.key = key + o.value, offsets = structure.ReadStringWithOffset(rd) + return offsets +} func (o *StringObject) Rewrite() []RedisCmd { cmd := RedisCmd{} cmd = append(cmd, "set", o.key, o.value) diff --git a/internal/rdb/types/zset.go b/internal/rdb/types/zset.go index 34f99b37..fc2b0611 100644 --- a/internal/rdb/types/zset.go +++ b/internal/rdb/types/zset.go @@ -2,9 +2,10 @@ package types import ( "fmt" + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) type ZSetEntry struct { @@ -32,6 +33,26 @@ func (o *ZsetObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { log.Panicf("unknown zset type. typeByte=[%d]", typeByte) } } +func (o *ZsetObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + o.key = key + switch typeByte { + case rdbTypeZSet: + offset := o.readZsetWithOffset(rd) + return offset + case rdbTypeZSet2: + offset := o.readZset2WithOffset(rd) + return offset + case rdbTypeZSetZiplist: + offset := o.readZsetZiplistWithOffset(rd) + return offset + case rdbTypeZSetListpack: + offset := o.readZsetListpackWithOffset(rd) + return offset + default: + log.Panicf("unknown zset type. typeByte=[%d]", typeByte) + } + return 0 +} func (o *ZsetObject) readZset(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -42,6 +63,19 @@ func (o *ZsetObject) readZset(rd io.Reader) { o.elements[i].Score = fmt.Sprintf("%f", score) } } +func (o *ZsetObject) readZsetWithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + o.elements = make([]ZSetEntry, size) + var tempOffsets int64 = 0 + for i := 0; i < int(size); i++ { + o.elements[i].Member, tempOffsets = structure.ReadStringWithOffset(rd) + offset += tempOffsets + score, tempOffsets := structure.ReadFloatWithOffset(rd) + offset += tempOffsets + o.elements[i].Score = fmt.Sprintf("%f", score) + } + return offset +} func (o *ZsetObject) readZset2(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -52,6 +86,19 @@ func (o *ZsetObject) readZset2(rd io.Reader) { o.elements[i].Score = fmt.Sprintf("%f", score) } } +func (o *ZsetObject) readZset2WithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + o.elements = make([]ZSetEntry, size) + var tempOffsets int64 + for i := 0; i < int(size); i++ { + o.elements[i].Member, tempOffsets = structure.ReadStringWithOffset(rd) + offset += tempOffsets + score := structure.ReadDouble(rd) + offset += 8 + o.elements[i].Score = fmt.Sprintf("%f", score) + } + return offset +} func (o *ZsetObject) readZsetZiplist(rd io.Reader) { list := structure.ReadZipList(rd) @@ -66,6 +113,19 @@ func (o *ZsetObject) readZsetZiplist(rd io.Reader) { } } +func (o *ZsetObject) readZsetZiplistWithOffset(rd io.Reader) int64 { + list, offset := structure.ReadZipListWithOffset(rd) + size := len(list) + if size%2 != 0 { + log.Panicf("zset listpack size is not even. size=[%d]", size) + } + o.elements = make([]ZSetEntry, size/2) + for i := 0; i < size; i += 2 { + o.elements[i/2].Member = list[i] + o.elements[i/2].Score = list[i+1] + } + return offset +} func (o *ZsetObject) readZsetListpack(rd io.Reader) { list := structure.ReadListpack(rd) size := len(list) @@ -79,6 +139,20 @@ func (o *ZsetObject) readZsetListpack(rd io.Reader) { } } +func (o *ZsetObject) readZsetListpackWithOffset(rd io.Reader) int64 { + list, offset := structure.ReadListpackWithOffset(rd) + size := len(list) + if size%2 != 0 { + log.Panicf("zset listpack size is not even. size=[%d]", size) + } + o.elements = make([]ZSetEntry, size/2) + for i := 0; i < size; i += 2 { + o.elements[i/2].Member = list[i] + o.elements[i/2].Score = list[i+1] + } + return offset +} + func (o *ZsetObject) Rewrite() []RedisCmd { cmds := make([]RedisCmd, len(o.elements)) for inx, ele := range o.elements { diff --git a/internal/reader/aof_reader.go b/internal/reader/aof_reader.go index c4fedcf4..b0e7969f 100644 --- a/internal/reader/aof_reader.go +++ b/internal/reader/aof_reader.go @@ -1,9 +1,14 @@ package reader -// this file references rdb_reader.go - import ( + "os" + "path" + "path/filepath" + + "github.com/alibaba/RedisShake/internal/aof" "github.com/alibaba/RedisShake/internal/entry" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/statistics" ) type aofReader struct { @@ -11,15 +16,60 @@ type aofReader struct { ch chan *entry.Entry } -// TODO:待完善参考rdb reader func NewAOFReader(path string) Reader { - - return nil + log.Infof("NewAOFReader: path=[%s]", path) + absolutePath, err := filepath.Abs(path) + if err != nil { + log.Panicf("NewAOFReader: filepath.Abs error: %s", err.Error()) + } + log.Infof("NewAOFReader: absolute path=[%s]", absolutePath) + r := &aofReader{ + path: absolutePath, + ch: make(chan *entry.Entry), + } + return r } func (r *aofReader) StartRead() chan *entry.Entry { - //调用 aof中的函数 - // aof.NewLoader() - // aof.ParseRDB() - return nil + r.ch = make(chan *entry.Entry, 1024) + + go func() { + aof.AOFFileInfo = *(aof.NewAOFFileInfo(r.path)) + + aof.AOFLoadManifestFromDisk() + am := aof.AOFFileInfo.GetAOFManifest() + + if am == nil { + paths := path.Join(aof.AOFFileInfo.GetAOFDirName(), aof.AOFFileInfo.GetAOFFileName()) + aof.CheckAOFMain(r.path) + log.Infof("start send AOF path=[%s]", r.path) + fi, err := os.Stat(r.path) + if err != nil { + log.Panicf("NewAOFReader: os.Stat error：%s", err.Error()) + } + statistics.Metrics.AofFileSize = uint64(fi.Size()) + statistics.Metrics.AofReceivedSize = uint64(fi.Size()) + aofLoader := aof.NewLoader(r.path, r.ch) + _ = aofLoader.LoadSingleAppendOnlyFile(paths, r.ch, true) + log.Infof("Send AOF finished. path=[%s]", r.path) + close(r.ch) + } else { + paths := path.Join(aof.AOFFileInfo.GetAOFDirName(), aof.GetAOFManifestFileName()) + aof.CheckAOFMain(paths) + log.Infof("start send AOF。path=[%s]", r.path) + fi, err := os.Stat(r.path) + if err != nil { + log.Panicf("NewAOFReader: os.Stat error：%s", err.Error()) + } + statistics.Metrics.AofFileSize = uint64(fi.Size()) + statistics.Metrics.AofReceivedSize = uint64(fi.Size()) + aofLoader := aof.NewLoader(r.path, r.ch) + _ = aofLoader.LoadAppendOnlyFile(aof.AOFFileInfo.GetAOFManifest(), r.ch) + log.Infof("Send AOF finished. path=[%s]", r.path) + close(r.ch) + } + + }() + + return r.ch } diff --git a/internal/statistics/statistics.go b/internal/statistics/statistics.go index 9af801d5..d3d52abb 100644 --- a/internal/statistics/statistics.go +++ b/internal/statistics/statistics.go @@ -3,12 +3,13 @@ package statistics import ( "encoding/json" "fmt" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/log" "math/bits" "net/http" "strings" "time" + + "github.com/alibaba/RedisShake/internal/config" + "github.com/alibaba/RedisShake/internal/log" ) type metrics struct { @@ -26,9 +27,18 @@ type metrics struct { RdbReceivedSize uint64 `json:"rdb_received_size"` RdbSendSize uint64 `json:"rdb_send_size"` + //loading + Loading bool `json:"loading"` + AsyncLoading bool `json:"async_loading"` + LoadingStartTime int64 `json:"loading_start_time"` + LoadingLoadedBytes int64 `json:"loading_loaded_bytes"` + LoadingTotalBytes int64 `json:"loading_total_bytes"` + // aof AofReceivedOffset uint64 `json:"aof_received_offset"` AofAppliedOffset uint64 `json:"aof_applied_offset"` + AofFileSize uint64 `json:"aof_file_size"` + AofReceivedSize uint64 `json:"aof_received_size"` // for performance debug InQueueEntriesCount uint64 `json:"in_queue_entries_count"` diff --git a/restore.toml b/restore.toml index 6ed1d1ae..a510e51a 100644 --- a/restore.toml +++ b/restore.toml @@ -5,7 +5,8 @@ version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... # Path to the dump.rdb file. Absolute path or relative path. Note # that relative paths are relative to the dir directory. rdb_file_path = "dump.rdb" - +aof_file_path = "" +aof_truncate_to_timestamp=0 [target] type = "standalone" # standalone or cluster # When the target is a cluster, write the address of one of the nodes. diff --git a/test/aof_test/aof_test.go b/test/aof_test/aof_test.go new file mode 100644 index 00000000..dae3d59e --- /dev/null +++ b/test/aof_test/aof_test.go @@ -0,0 +1,207 @@ +package aof_test + +import ( + "fmt" + "net/http" + "os" + "runtime" + "testing" + + "github.com/alibaba/RedisShake/internal/commands" + "github.com/alibaba/RedisShake/internal/config" + "github.com/alibaba/RedisShake/internal/filter" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/reader" + "github.com/alibaba/RedisShake/internal/statistics" + "github.com/alibaba/RedisShake/internal/writer" + "github.com/go-redis/redis" +) + +const ( + AOFRestorePath = "/test_aof_restore.toml" + AppendOnlyAoFPath = "/appendonlydir/appendonly.aof.manifest" +) + +func AOFMain(configFile string, aofFilePath string) { + + // load aof config file + config.LoadFromFile(configFile) + + log.Init() + log.Infof("GOOS: %s, GOARCH: %s", runtime.GOOS, runtime.GOARCH) + log.Infof("Ncpu: %d, GOMAXPROCS: %d", config.Config.Advanced.Ncpu, runtime.GOMAXPROCS(0)) + log.Infof("pid: %d", os.Getpid()) + log.Infof("pprof_port: %d", config.Config.Advanced.PprofPort) + + // start pprof + if config.Config.Advanced.PprofPort != 0 { + go func() { + err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Config.Advanced.PprofPort), nil) + if err != nil { + log.PanicError(err) + } + }() + } + + // start statistics + if config.Config.Advanced.MetricsPort != 0 { + statistics.Metrics.Address = config.Config.Source.Address + go func() { + log.Infof("metrics url: http://localhost:%d", config.Config.Advanced.MetricsPort) + mux := http.NewServeMux() + mux.HandleFunc("/", statistics.Handler) + err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Config.Advanced.MetricsPort), mux) + if err != nil { + log.PanicError(err) + } + }() + } + + // create writer + var theWriter writer.Writer + target := &config.Config.Target + switch config.Config.Target.Type { + case "standalone": + theWriter = writer.NewRedisWriter(target.Address, target.Username, target.Password, target.IsTLS) + case "cluster": + theWriter = writer.NewRedisClusterWriter(target.Address, target.Username, target.Password, target.IsTLS) + default: + log.Panicf("unknown target type: %s", target.Type) + } + + var theReader reader.Reader + + theReader = reader.NewAOFReader(aofFilePath) + + fmt.Printf("the aof path:%v\n", aofFilePath) + ch := theReader.StartRead() + + // start sync + statistics.Init() + id := uint64(0) + for e := range ch { + statistics.UpdateInQueueEntriesCount(uint64(len(ch))) + // calc arguments + e.Id = id + id++ + e.CmdName, e.Group, e.Keys = commands.CalcKeys(e.Argv) + e.Slots = commands.CalcSlots(e.Keys) + + // filter + code := filter.Filter(e) + statistics.UpdateEntryId(e.Id) + if code == filter.Allow { + theWriter.Write(e) + statistics.AddAllowEntriesCount() + } else if code == filter.Disallow { + // do something + statistics.AddDisallowEntriesCount() + } else { + log.Panicf("error when run lua filter. entry: %s", e.ToString()) + } + } + theWriter.Close() + log.Infof("finished.") +} + +// if you use this test you need start redis in port 6379s +func TestMainFunction(t *testing.T) { + + // os.Args = []string{"redis-shake", "/home/hwy/kaiyuan/restore.toml"} + wdPath, err := os.Getwd() + if err != nil { + panic(err) + } + + configPath := wdPath + AOFRestorePath + aofFilePath := wdPath + AppendOnlyAoFPath + fmt.Printf("configPath:%v, aofFilepath:%v\n", configPath, aofFilePath) + + AOFMain(configPath, aofFilePath) //restore aof + + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", + DB: 0, + }) + + pong, err := client.Ping().Result() + if err != nil { + t.Fatalf("Failed to connect to Redis: %v", err) + } + fmt.Println("Connected to Redis:", pong) + /* + for i := 11; i <= 10000; i++ { + value := strconv.Itoa(i) + score := float64(i) + z := redis.Z{Score: score, Member: value} + err := client.ZAdd("myzset", z).Err() + fmt.Println(value) + if err != nil { + fmt.Println("Failed to write data to Redis:", err) + return + } + } + */ + // 读取整个有序集合 + zsetValues, err := client.ZRangeWithScores("myzset", 0, -1).Result() + if err != nil { + fmt.Println("Failed to read data from Redis:", err) + return + } + + // 遍历有序集合中的元素和分数 + for _, z := range zsetValues { + member := z.Member.(string) + score := z.Score + fmt.Printf("Member: %s, Score: %f\n", member, score) + } + /* + expected := map[string]string{ + "kl": "kl", + "key0": "2022-03-29 17:25:54.592593", + "key1": "2022-03-29 17:25:54.876326", + "key2": "2022-03-29 17:25:52.871918", + "key3": "2022-03-29 17:25:53.034060", + "key4": "2022-03-29 17:25:53.196913", + "key5": "2022-03-29 17:25:53.356234", + "key6": "2022-03-29 17:25:53.513544", + "key7": "2022-03-29 17:25:53.671556", + "key8": "2022-03-29 17:25:53.861237", + "key9": "2022-03-29 17:25:54.020518", + "key10": "2022-03-29 17:25:54.177881", + "key11": "2022-03-29 17:25:54.337640", + } + */ + /*for key, value := range expected { + result, err := client.Get(key).Result() + if err != nil { + t.Fatalf("Failed to read key %s from Redis: %v", key, err) + } + + if result != value { + t.Errorf("Value for key %s is incorrect. Expected: %s, Got: %s", key, value, result) + } + } + + for key := 11; key <= 10000; key++ { + result, err := client.Get(strconv.Itoa(key)).Result() + if err != nil { + t.Fatalf("Failed to read key %v from Redis: %v", key, err) + } + + if result != strconv.Itoa(key) { + t.Errorf("Value for key %v is incorrect. Expected: %v, Got: %v", key, key, result) + } + }*/ + + /*result, err := client.SMembers("superpowers").Result() + if err != nil { + t.Fatalf("Failed to read set from Redis: %v", err) + } + strings := result[0] + if strings != "reflexes" { + t.Errorf("read set wrong") + }*/ + +} diff --git a/test/aof_test/appendonlydir/appendonly.aof.manifest b/test/aof_test/appendonlydir/appendonly.aof.manifest new file mode 100644 index 00000000..eca380fc --- /dev/null +++ b/test/aof_test/appendonlydir/appendonly.aof.manifest @@ -0,0 +1,2 @@ +file appendonly.aof.2.base.rdb seq 2 type b +file appendonly.aof.2.incr.aof seq 2 type i diff --git a/test/aof_test/test_aof_restore.toml b/test/aof_test/test_aof_restore.toml new file mode 100644 index 00000000..e5d2f762 --- /dev/null +++ b/test/aof_test/test_aof_restore.toml @@ -0,0 +1,55 @@ +type = "restore" + +[source] +version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... +# Path to the dump.rdb file. Absolute path or relative path. Note +# that relative paths are relative to the dir directory. +rdb_file_path = "" +aof_file_path="/appendonly" #Please use an absolute path. +aof_truncate_to_timestamp=0 +[target] +type = "standalone" # standalone or cluster +# When the target is a cluster, write the address of one of the nodes. +# redis-shake will obtain other nodes through the `cluster nodes` command. +version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... +address = "127.0.0.1:6379" +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false + +[advanced] +dir = "data" + +# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores +ncpu = 3 + +# pprof port, 0 means disable +pprof_port = 0 + +# metric port, 0 means disable +metrics_port = 0 + +# log +log_file = "redis-shake.log" +log_level = "info" # debug, info or warn +log_interval = 5 # in seconds + +# redis-shake gets key and value from rdb file, and uses RESTORE command to +# create the key in target redis. Redis RESTORE will return a "Target key name +# is busy" error when key already exists. You can use this configuration item +# to change the default behavior of restore: +# panic: redis-shake will stop when meet "Target key name is busy" error. +# rewrite: redis-shake will replace the key with new value. +# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. +rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip + +# pipeline +pipeline_count_limit = 1024 + +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default. This amount is normally 1gb. +target_redis_client_max_querybuf_len = 1024_000_000 + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. +target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file