diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..84c22496 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,12 @@ +--- +name: Feature request +about: 功能性建议（Suggest an idea for this project） +title: '' +labels: 'type: feature request' +assignees: suxb201 + +--- + +请描述你的功能性建议。清晰地说明你希望添加或改进的功能，并尽可能提供详细的信息。 + +Please describe your feature request. Clearly state the functionality you would like to add or improve and provide as much detail as possible. diff --git a/.github/ISSUE_TEMPLATE/issue.md b/.github/ISSUE_TEMPLATE/issue.md index 93ba91cf..4ce5cd21 100644 --- a/.github/ISSUE_TEMPLATE/issue.md +++ b/.github/ISSUE_TEMPLATE/issue.md @@ -1,40 +1,34 @@ --- -name: issue -about: 大多数问题请使用这个模板 +name: Issue +about: 使用中遇到问题（Encountering problems while using） title: '' -labels: '' +labels: 'type: question' assignees: '' --- -- [ ] 请确保已经看过 wiki：https://github.com/alibaba/RedisShake/wiki -- [ ] 请确保已经学习过 Markdown 语法，良好的排版有助于维护人员了解你的问题 -- [ ] 请在此提供足够的信息供社区维护人员排查问题 -- [ ] 请在提交 issue 前删除此模板中多余的文字，包括这几句话 +### 问题描述（Issue Description） -**问题描述** +请在这里简要描述你遇到的问题。 -这里描述你的问题。 +Please provide a brief description of the issue you encountered. -redis-shake 的日志： -``` -shake 的日志贴在这里 -``` +### 环境信息（Environment） ---- +- RedisShake 版本（RedisShake Version）： +- Redis 源端版本（Redis Source Version）： +- Redis 目的端版本（Redis Destination Version）： +- Redis 部署方式（standalone/cluster/sentinel）： +- 是否在云服务商实例上部署（Deployed on Cloud Provider）： -**源端 Redis** -版本：版本号，自建还是云厂商？ -日志： -``` -日志贴在这里 -``` +### 日志信息（Logs） ---- +如果有错误日志或其他相关日志，请在这里提供。 + +If there are any error logs or other relevant logs, please provide them here. + +### 其他信息（Additional Information） + +请提供任何其他相关的信息，如配置文件、错误信息或截图等。 -**目的端 Redis** -版本：版本号，是否是集群？自建还是云厂商？ -日志： -``` -日志贴在这里 -``` +Please provide any additional information, such as configuration files, error messages, or screenshots. diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index 17aab4f4..d0af1e3e 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -1,3 +1,4 @@ +# https://github.com/release-drafter/release-drafter#example name-template: 'redis-shake-v$RESOLVED_VERSION' tag-template: 'v$RESOLVED_VERSION' template: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aa44ff4..fe6755ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,21 @@ name: CI -on: [ pull_request ] +on: + push: + branches: [ v4 ] + pull_request: + branches: [ v4 ] + workflow_dispatch: jobs: black-box-test: runs-on: ubuntu-latest strategy: + max-parallel: 1 matrix: - redis-version: [ 5, 6, 7 ] + redis-version: [ "2.8", "3.0", "4.0", "5.0", "6.0", "7.0" ] + fail-fast: false + container: ubuntu:latest steps: - name: Git checkout uses: actions/checkout@v2 @@ -19,19 +27,59 @@ jobs: - name: clone and make redis run: | - sudo apt-get install git + apt-get update + apt-get install -y --no-install-recommends git build-essential ca-certificates git clone https://github.com/redis/redis cd redis - git checkout ${{ matrix.redis-version }}.0 + git checkout ${{ matrix.redis-version }} make -j mkdir bin cp src/redis-server bin/redis-server echo "$GITHUB_WORKSPACE/redis/bin" >> $GITHUB_PATH + - name: clone and make TairString Module + if: contains( '5.0, 6.0, 7.0', matrix.redis-version) + run: | + cd $GITHUB_WORKSPACE + apt-get install -y cmake + git clone https://github.com/tair-opensource/TairString.git + cd TairString + mkdir build + cd build + cmake ../ && make -j + cp $GITHUB_WORKSPACE/TairString/lib/tairstring_module.so /lib + + + + - name: clone and make TairHash Module + if: contains( '5.0, 6.0, 7.0', matrix.redis-version) + run: | + cd $GITHUB_WORKSPACE + git clone https://github.com/tair-opensource/TairHash.git + cd TairHash + mkdir build + cd build + cmake ../ && make -j + cp $GITHUB_WORKSPACE/TairHash/lib/tairhash_module.so /lib + + + + - name: clone and make TairZset Module + if: contains( '5.0, 6.0, 7.0', matrix.redis-version) + run: | + cd $GITHUB_WORKSPACE + git clone https://github.com/tair-opensource/TairZset.git + cd TairZset + mkdir build + cd build + cmake ../ && make -j + cp $GITHUB_WORKSPACE/TairZset/lib/tairzset_module.so /lib + + - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.11' - name: make redis-shake run: | @@ -39,6 +87,5 @@ jobs: - name: test run: | - cd test - pip3 install -r requirements.txt - python3 main.py \ No newline at end of file + python -m pip install -r tests/requirements.txt + sh test.sh \ No newline at end of file diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 00000000..8cf21168 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,63 @@ +# Sample workflow for building and deploying a VitePress site to GitHub Pages +# +name: Pages + +on: + # Runs on pushes targeting the `main` branch. Change this to `master` if you're + # using the `master` branch as the default branch. + push: + branches: [ v4 ] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: pages + cancel-in-progress: false + +jobs: + # Build job + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: npm + cache-dependency-path: docs/package-lock.json + - name: Setup Pages + uses: actions/configure-pages@v3 + - name: Install dependencies + run: npm ci # or pnpm install / yarn install + working-directory: docs + - name: Build with VitePress + run: npm run docs:build # or pnpm docs:build / yarn docs:build + working-directory: docs + - name: Upload artifact + uses: actions/upload-pages-artifact@v2 + with: + path: docs/.vitepress/dist + + # Deployment job + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + needs: build + runs-on: ubuntu-latest + name: Deploy + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v2 \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ae03357..a0a37ed5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: jobs: build: + permissions: write-all runs-on: ubuntu-latest steps: - name: Checkout @@ -65,3 +66,23 @@ jobs: asset_content_type: application/gzip env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "Upload release windows-amd64" + uses: actions/upload-release-asset@v1 + with: + upload_url: ${{ steps.release.outputs.upload_url }} + asset_path: ./bin/redis-shake-windows-amd64.tar.gz + asset_name: redis-shake-windows-amd64.tar.gz + asset_content_type: application/gzip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: "Upload release windows-arm64" + uses: actions/upload-release-asset@v1 + with: + upload_url: ${{ steps.release.outputs.upload_url }} + asset_path: ./bin/redis-shake-windows-arm64.tar.gz + asset_name: redis-shake-windows-arm64.tar.gz + asset_content_type: application/gzip + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 61d3c65e..d864178d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ -.idea -data -__pycache__ -bin -.DS_Store +# system +.idea/ +__pycache__/ +.DS_Store/ + +# compiled output or test output +bin/ +dist/ +tmp/ +data/ *.log -*.rdb -*.aof diff --git a/README.md b/README.md index ee713ad4..e66793ab 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,45 @@ -# redis-shake +# RedisShake 4.x: Redis Data Processing & Migration Tool -[![CI](https://github.com/alibaba/RedisShake/actions/workflows/ci.yml/badge.svg?branch=v3)](https://github.com/alibaba/RedisShake/actions/workflows/ci.yml) +[![CI](https://github.com/tair-opensource/RedisShake/actions/workflows/ci.yml/badge.svg?event=push&branch=v4)](https://github.com/tair-opensource/RedisShake/actions/workflows/ci.yml) +[![CI](https://github.com/tair-opensource/RedisShake/actions/workflows/pages.yml/badge.svg?branch=v4)](https://github.com/tair-opensource/RedisShake/actions/workflows/pages.yml) +[![CI](https://github.com/tair-opensource/RedisShake/actions/workflows/release.yml/badge.svg?branch=v4)](https://github.com/tair-opensource/RedisShake/actions/workflows/release.yml) -- [中文文档](https://github.com/alibaba/RedisShake/wiki) +- [中文文档](https://tair-opensource.github.io/RedisShake/) +- [English Documentation](https://tair-opensource.github.io/RedisShake/en/) -redis-shake is a tool for Redis data migration and data filtering. +## Overview -## Feature +RedisShake is a tool designed for processing and migrating Redis data. It offers the following features: -* 🚄 High performance -* ✅ Tested on Redis 5.0, Redis 6.0 and Redis 7.0 -* 🤗 Support custom filtering rules -* 💪 Support large instance migration -* 💖 Support `restore` mode, `sync` mode and `scan` mode -* ☁️ Support Aliyun Redis and ElastiCache +1. **Redis Compatibility**: RedisShake is compatible with Redis versions ranging from 2.8 to 7.2, and supports various + deployment methods including standalone, master-slave, sentinel, and cluster. -For older versions of redis-shake (support codis, twemproxy) please -visit [here](https://github.com/alibaba/RedisShake/tree/develop). +2. **Cloud Service Compatibility**: RedisShake works seamlessly with popular Redis-like databases provided by leading + cloud service providers, including but not limited to: + - [Alibaba Cloud - ApsaraDB for Redis](https://www.alibabacloud.com/product/apsaradb-for-redis) + - [Alibaba Cloud - Tair](https://www.alibabacloud.com/product/tair) + - [AWS - ElastiCache](https://aws.amazon.com/elasticache/) + - [AWS - MemoryDB](https://aws.amazon.com/memorydb/) -![redis-shake2.PNG](https://s2.loli.net/2022/07/10/OZrSGutknlI8XNp.png) +3. **Module Compatibility**: RedisShake is compatible + with [TairString](https://github.com/tair-opensource/TairString), [TairZSet](https://github.com/tair-opensource/TairZset), + and [TairHash](https://github.com/tair-opensource/TairHash) modules. -![image.png](https://s2.loli.net/2022/06/30/vU346lVBrNofKzu.png) +4. **Multiple Export Modes**: RedisShake supports PSync, RDB, and Scan export modes. -# Document +5. **Data Processing**: RedisShake enables data filtering and transformation through custom scripts. -## Install +## Getting Started -### Binary package +### Installation -Download from Release: [https://github.com/alibaba/RedisShake/releases](https://github.com/alibaba/RedisShake/releases) +#### Download the Binary Package -### Compile from source +Download the binary package directly from the [Releases](https://github.com/tair-opensource/RedisShake/releases) page. -After downloading the source code, run the `sh build.sh` command to compile. +#### Compile from Source + +To compile from source, ensure that you have a Golang environment set up on your local machine: ```shell git clone https://github.com/alibaba/RedisShake @@ -40,84 +47,59 @@ cd RedisShake sh build.sh ``` -## Usage - -1. Edit `sync.toml` or `restore.toml`. -2. Start redis-shake. +### Usage -```shell -./bin/redis-shake redis-shake.toml -# or -./bin/redis-shake restore.toml -``` +Assume you have two Redis instances: -3. Check data synchronization status. +* Instance A: 127.0.0.1:6379 +* Instance B: 127.0.0.1:6380 -## Configure +Create a new configuration file `shake.toml`: -The redis-shake configuration file refers to `sync.toml` or `restore.toml`. +```toml +[sync_reader] +address = "127.0.0.1:6379" -## Data filtering +[redis_writer] +address = "127.0.0.1:6380" +``` -redis-shake supports custom filtering rules using lua scripts. redis-shake can be started with -the following command: +To start RedisShake, run the following command: ```shell -./bin/redis-shake sync.toml filter/xxx.lua +./redis-shake shake.toml ``` -Some following filter templates are provided in `filter` directory: - -1. `filter/print.lua`：print all commands -2. `filter/swap_db.lua`：swap the data of db0 and db1 - -### Custom filter rules - -Refer to `filter/print.lua` to create a new lua script, and implement the filter function in the lua script. The -arguments of the function are: - -- id: command id -- is_base: is the command read from the dump.rdb file -- group: command group, see the description file - under [redis/src/commands](https://github.com/redis/redis/tree/unstable/src/commands) -- cmd_name: command name -- keys: keys in command -- slots: slots in command -- db_id: database id -- timestamp_ms: timestamp of the command in milliseconds. The current version does not support it. - -The return value is: - -- code - - 0: allow this command to pass - - 1: this command is not allowed to pass - - 2: this command should not appear, and let redis-shake exit with an error -- db_id: redirected db_id +For more detailed information, please refer to the documentation: -# Contribution +- [中文文档](https://tair-opensource.github.io/RedisShake/) +- [English Documentation](https://tair-opensource.github.io/RedisShake/en/) -## Lua script +## Contributing -Welcome to share more creative lua scripts. +We welcome contributions from the community. For significant changes, please open an issue first to discuss what you +would like to change. We are particularly interested in: -1. Add lua scripts under `filters/`. -2. Add description to `README.md`. -3. Submit a pull request. +1. Adding support for more modules +2. Enhancing support for Readers and Writers +3. Sharing your Lua scripts and best practices -## Redis Module support +## History -1. Add code under `internal/rdb/types`. -2. Add a command file under `scripts/commands`, and use the script to generate a `table.go` file and move it to - the `internal/commands` directory. -3. Add test cases under `test/cases`. -4. Submit a pull request. +RedisShake is a project actively maintained by the [Tair team](https://github.com/tair-opensource) at Alibaba Cloud. Its +evolution can be traced back to its initial version, which was forked +from [redis-port](https://github.com/CodisLabs/redis-port). -# 感谢 +During its evolution: -redis-shake 旧版是阿里云基于豌豆荚开源的 redis-port 进行二次开发的一个支持 Redis 异构集群实时同步的工具。 -redis-shake v3 在 redis-shake 旧版的基础上重新组织代码结构，使其更具可维护性的版本。 +- The [RedisShake 2.x](https://github.com/tair-opensource/RedisShake/tree/v2) version brought a series of improvements + and updates, enhancing its overall stability and performance. +- The [RedisShake 3.x](https://github.com/tair-opensource/RedisShake/tree/v3) version represented a significant + milestone where the entire codebase was completely rewritten and optimized, leading to better efficiency and + usability. +- The current version, [RedisShake 4.x](https://github.com/tair-opensource/RedisShake/tree/v4), has further enhanced + features related to readers, configuration, observability, and functions. -redis-shake v3 参考借鉴了以下项目： +## License -- https://github.com/HDT3213/rdb -- https://github.com/sripathikrishnan/redis-rdb-tools \ No newline at end of file +RedisShake is open-sourced under the [MIT license](https://github.com/tair-opensource/RedisShake/blob/v2/license.txt). diff --git a/build.sh b/build.sh index c4e1378e..80d09c28 100755 --- a/build.sh +++ b/build.sh @@ -7,11 +7,7 @@ BIN_DIR=$(pwd)/bin/ rm -rf "$BIN_DIR" mkdir -p "$BIN_DIR" -cp sync.toml "$BIN_DIR" -cp scan.toml "$BIN_DIR" -cp restore.toml "$BIN_DIR" -cp -r filters "$BIN_DIR" -cp -r scripts/cluster_helper "$BIN_DIR" +cp shake.toml "$BIN_DIR" dist() { echo "try build GOOS=$1 GOARCH=$2" @@ -24,13 +20,13 @@ dist() { echo "build success GOOS=$1 GOARCH=$2" cd "$BIN_DIR" - tar -czvf ./redis-shake-"$1"-"$2".tar.gz ./sync.toml ./scan.toml ./restore.toml ./redis-shake ./filters ./cluster_helper + tar -czvf ./redis-shake-"$1"-"$2".tar.gz ./redis-shake ./shake.toml cd .. } if [ "$1" == "dist" ]; then echo "[ DIST ]" - for g in "linux" "darwin"; do + for g in "linux" "darwin" "windows"; do for a in "amd64" "arm64"; do dist "$g" "$a" done diff --git a/cmd/redis-shake/main.go b/cmd/redis-shake/main.go index 1f16b5a1..ed5700c5 100644 --- a/cmd/redis-shake/main.go +++ b/cmd/redis-shake/main.go @@ -1,120 +1,122 @@ package main import ( - "fmt" - "github.com/alibaba/RedisShake/internal/commands" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/filter" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/reader" - "github.com/alibaba/RedisShake/internal/statistics" - "github.com/alibaba/RedisShake/internal/writer" - "net/http" + "RedisShake/internal/config" + "RedisShake/internal/function" + "RedisShake/internal/log" + "RedisShake/internal/reader" + "RedisShake/internal/status" + "RedisShake/internal/utils" + "RedisShake/internal/writer" + "github.com/mcuadros/go-defaults" _ "net/http/pprof" - "os" - "runtime" ) func main() { - if len(os.Args) < 2 || len(os.Args) > 3 { - fmt.Println("Usage: redis-shake ") - fmt.Println("Example: redis-shake config.toml filter.lua") - os.Exit(1) - } - - // load filter file - if len(os.Args) == 3 { - luaFile := os.Args[2] - filter.LoadFromFile(luaFile) - } - - // load config - configFile := os.Args[1] - config.LoadFromFile(configFile) + v := config.LoadConfig() - log.Init() - log.Infof("GOOS: %s, GOARCH: %s", runtime.GOOS, runtime.GOARCH) - log.Infof("Ncpu: %d, GOMAXPROCS: %d", config.Config.Advanced.Ncpu, runtime.GOMAXPROCS(0)) - log.Infof("pid: %d", os.Getpid()) - log.Infof("pprof_port: %d", config.Config.Advanced.PprofPort) - if len(os.Args) == 2 { - log.Infof("No lua file specified, will not filter any cmd.") - } - - // start pprof - if config.Config.Advanced.PprofPort != 0 { - go func() { - err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Config.Advanced.PprofPort), nil) - if err != nil { - log.PanicError(err) - } - }() - } + log.Init(config.Opt.Advanced.LogLevel, config.Opt.Advanced.LogFile, config.Opt.Advanced.Dir) + utils.ChdirAndAcquireFileLock() + utils.SetNcpu() + utils.SetPprofPort() + function.Init() - // start statistics - if config.Config.Advanced.MetricsPort != 0 { - statistics.Metrics.Address = config.Config.Source.Address - go func() { - log.Infof("metrics url: http://localhost:%d", config.Config.Advanced.MetricsPort) - mux := http.NewServeMux() - mux.HandleFunc("/", statistics.Handler) - err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Config.Advanced.MetricsPort), mux) - if err != nil { - log.PanicError(err) - } - }() + // create reader + var theReader reader.Reader + if v.IsSet("sync_reader") { + opts := new(reader.SyncReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("sync_reader", opts) + if err != nil { + log.Panicf("failed to read the SyncReader config entry. err: %v", err) + } + if opts.Cluster { + theReader = reader.NewSyncClusterReader(opts) + log.Infof("create SyncClusterReader: %v", opts.Address) + } else { + theReader = reader.NewSyncStandaloneReader(opts) + log.Infof("create SyncStandaloneReader: %v", opts.Address) + } + } else if v.IsSet("scan_reader") { + opts := new(reader.ScanReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("scan_reader", opts) + if err != nil { + log.Panicf("failed to read the ScanReader config entry. err: %v", err) + } + if opts.Cluster { + theReader = reader.NewScanClusterReader(opts) + log.Infof("create ScanClusterReader: %v", opts.Address) + } else { + theReader = reader.NewScanStandaloneReader(opts) + log.Infof("create ScanStandaloneReader: %v", opts.Address) + } + } else if v.IsSet("rdb_reader") { + opts := new(reader.RdbReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("rdb_reader", opts) + if err != nil { + log.Panicf("failed to read the RdbReader config entry. err: %v", err) + } + theReader = reader.NewRDBReader(opts) + log.Infof("create RdbReader: %v", opts.Filepath) + } else if v.IsSet("aof_reader") { + opts := new(reader.AOFReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("aof_reader", opts) + if err != nil { + log.Panicf("failed to read the AOFReader config entry. err: %v", err) + } + theReader = reader.NewAOFReader(opts) + log.Infof("create AOFReader: %v", opts.Filepath) + } else { + log.Panicf("no reader config entry found") } // create writer var theWriter writer.Writer - target := &config.Config.Target - switch config.Config.Target.Type { - case "standalone": - theWriter = writer.NewRedisWriter(target.Address, target.Username, target.Password, target.IsTLS) - case "cluster": - theWriter = writer.NewRedisClusterWriter(target.Address, target.Username, target.Password, target.IsTLS) - default: - log.Panicf("unknown target type: %s", target.Type) - } - - // create reader - source := &config.Config.Source - var theReader reader.Reader - if config.Config.Type == "sync" { - theReader = reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS, source.ElastiCachePSync) - } else if config.Config.Type == "restore" { - theReader = reader.NewRDBReader(source.RDBFilePath) - } else if config.Config.Type == "scan" { - theReader = reader.NewScanReader(source.Address, source.Username, source.Password, source.IsTLS) + if v.IsSet("redis_writer") { + opts := new(writer.RedisWriterOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("redis_writer", opts) + if err != nil { + log.Panicf("failed to read the RedisStandaloneWriter config entry. err: %v", err) + } + if opts.Cluster { + theWriter = writer.NewRedisClusterWriter(opts) + log.Infof("create RedisClusterWriter: %v", opts.Address) + } else { + theWriter = writer.NewRedisStandaloneWriter(opts) + log.Infof("create RedisStandaloneWriter: %v", opts.Address) + } } else { - log.Panicf("unknown source type: %s", config.Config.Type) + log.Panicf("no writer config entry found") } - ch := theReader.StartRead() - // start sync - statistics.Init() - id := uint64(0) + // create status + status.Init(theReader, theWriter) + + log.Infof("start syncing...") + + ch := theReader.StartRead() for e := range ch { - statistics.UpdateInQueueEntriesCount(uint64(len(ch))) // calc arguments - e.Id = id - id++ - e.CmdName, e.Group, e.Keys = commands.CalcKeys(e.Argv) - e.Slots = commands.CalcSlots(e.Keys) + e.Parse() + status.AddReadCount(e.CmdName) // filter - code := filter.Filter(e) - statistics.UpdateEntryId(e.Id) - if code == filter.Allow { - theWriter.Write(e) - statistics.AddAllowEntriesCount() - } else if code == filter.Disallow { - // do something - statistics.AddDisallowEntriesCount() - } else { - log.Panicf("error when run lua filter. entry: %s", e.ToString()) + log.Debugf("function before: %v", e) + entries := function.RunFunction(e) + log.Debugf("function after: %v", entries) + + for _, entry := range entries { + entry.Parse() + theWriter.Write(entry) + status.AddWriteCount(entry.CmdName) } } - theWriter.Close() - log.Infof("finished.") + + theWriter.Close() // Wait for all writing operations to complete + utils.ReleaseFileLock() // Release file lock + log.Infof("all done") } diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..ff359c63 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +cache/ \ No newline at end of file diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts new file mode 100644 index 00000000..cea18636 --- /dev/null +++ b/docs/.vitepress/config.ts @@ -0,0 +1,126 @@ +import { defineConfig } from 'vitepress' + +// https://vitepress.dev/reference/site-config +export default defineConfig({ + head: [['link', { rel: 'icon', href: '/RedisShake/favicon.ico' }]], + base: "/RedisShake/", + title: "RedisShake", + description: "RedisShake is a tool for processing and migrating Redis data.", + srcDir: './src', + locales: { + root: { + label: '中文', + lang: 'zh', // optional, will be added as `lang` attribute on `html` tag + themeConfig: { + // https://vitepress.dev/reference/default-theme-config + nav: [ + { text: '主页', link: '/' }, + { text: '使用文档', link: '/zh/guide/getting-started' }, + { text: '云原生内存数据库 Tair', link: 'https://www.aliyun.com/product/apsaradb/kvstore/tair' } + ], + sidebar: [ + { + text: '介绍', + items: [ + { text: '什么是 RedisShake', link: '/zh/guide/introduction' }, + { text: '快速上手', link: '/zh/guide/getting-started' }, + { text: '配置', link: '/zh/guide/config' }, + { text: '迁移模式选择', link: '/zh/guide/mode' }, + ] + }, + { + text: 'Reader', + items: [ + { text: 'Sync Reader', link: '/zh/reader/sync_reader' }, + { text: 'Scan Reader', link: '/zh/reader/scan_reader' }, + { text: 'RDB Reader', link: '/zh/reader/rdb_reader' }, + ] + }, + { + text: 'Writer', + items: [ + { text: 'Redis Writer', link: '/zh/writer/redis_writer' }, + ] + }, + { + text: 'Function', + items: [ + { text: '什么是 function', link: '/zh/function/introduction' }, + { text: '最佳实践', link: '/zh/function/best_practices' } + ] + }, + { + text: 'Others', + items: [ + { text: 'Redis Modules', link: '/zh/others/modules' }, + ] + }, + ], + footer: { + message: 'Released under the MIT License.', + copyright: 'Copyright © 2019-present Tair' + } + } + }, + en: { + label: 'English', + lang: 'en', // optional, will be added as `lang` attribute on `html` tag + themeConfig: { + // https://vitepress.dev/reference/default-theme-config + nav: [ + { text: 'Home', link: '/en/' }, + { text: 'User Guide', link: '/en/guide/getting-started' }, + { text: 'Tair', link: 'https://www.alibabacloud.com/product/tair' } + ], + sidebar: [ + { + text: 'Introduction', + items: [ + { text: 'What is RedisShake', link: '/en/guide/introduction' }, + { text: 'Getting Started', link: '/en/guide/getting-started' }, + { text: 'Configuration', link: '/en/guide/config' }, + { text: 'Migration Mode Selection', link: '/en/guide/mode' }, + ] + }, + { + text: 'Reader', + items: [ + { text: 'Sync Reader', link: '/en/reader/sync_reader' }, + { text: 'Scan Reader', link: '/en/reader/scan_reader' }, + { text: 'RDB Reader', link: '/en/reader/rdb_reader' }, + ] + }, + { + text: 'Writer', + items: [ + { text: 'Redis Writer', link: '/en/writer/redis_writer' }, + ] + }, + { + text: 'Function', + items: [ + { text: 'What is function', link: '/en/function/introduction' }, + { text: 'Best Practices', link: '/en/function/best_practices' } + ] + }, + { + text: 'Others', + items: [ + { text: 'Redis Modules', link: '/en/others/modules' }, + ] + }, + ], + footer: { + message: 'Released under the MIT License.', + copyright: 'Copyright © 2019-present Tair' + } + } + }, + + }, + themeConfig: { + socialLinks: [ + { icon: 'github', link: 'https://github.com/tair-opensource/RedisShake' } + ], + } +}) diff --git a/docs/package-lock.json b/docs/package-lock.json new file mode 100644 index 00000000..017ce21d --- /dev/null +++ b/docs/package-lock.json @@ -0,0 +1,1272 @@ +{ + "name": "docs", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "devDependencies": { + "vitepress": "^1.0.0-rc.4" + } + }, + "node_modules/@algolia/autocomplete-core": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-core/-/autocomplete-core-1.9.3.tgz", + "integrity": "sha512-009HdfugtGCdC4JdXUbVJClA0q0zh24yyePn+KUGk3rP7j8FEe/m5Yo/z65gn6nP/cM39PxpzqKrL7A6fP6PPw==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-plugin-algolia-insights": "1.9.3", + "@algolia/autocomplete-shared": "1.9.3" + } + }, + "node_modules/@algolia/autocomplete-plugin-algolia-insights": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-plugin-algolia-insights/-/autocomplete-plugin-algolia-insights-1.9.3.tgz", + "integrity": "sha512-a/yTUkcO/Vyy+JffmAnTWbr4/90cLzw+CC3bRbhnULr/EM0fGNvM13oQQ14f2moLMcVDyAx/leczLlAOovhSZg==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-shared": "1.9.3" + }, + "peerDependencies": { + "search-insights": ">= 1 < 3" + } + }, + "node_modules/@algolia/autocomplete-preset-algolia": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-preset-algolia/-/autocomplete-preset-algolia-1.9.3.tgz", + "integrity": "sha512-d4qlt6YmrLMYy95n5TB52wtNDr6EgAIPH81dvvvW8UmuWRgxEtY0NJiPwl/h95JtG2vmRM804M0DSwMCNZlzRA==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-shared": "1.9.3" + }, + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/autocomplete-shared": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/@algolia/autocomplete-shared/-/autocomplete-shared-1.9.3.tgz", + "integrity": "sha512-Wnm9E4Ye6Rl6sTTqjoymD+l8DjSTHsHboVRYrKgEt8Q7UHm9nYbqhN/i0fhUYA3OAEH7WA8x3jfpnmJm3rKvaQ==", + "dev": true, + "peerDependencies": { + "@algolia/client-search": ">= 4.9.1 < 6", + "algoliasearch": ">= 4.9.1 < 6" + } + }, + "node_modules/@algolia/cache-browser-local-storage": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-browser-local-storage/-/cache-browser-local-storage-4.19.1.tgz", + "integrity": "sha512-FYAZWcGsFTTaSAwj9Std8UML3Bu8dyWDncM7Ls8g+58UOe4XYdlgzXWbrIgjaguP63pCCbMoExKr61B+ztK3tw==", + "dev": true, + "dependencies": { + "@algolia/cache-common": "4.19.1" + } + }, + "node_modules/@algolia/cache-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-common/-/cache-common-4.19.1.tgz", + "integrity": "sha512-XGghi3l0qA38HiqdoUY+wvGyBsGvKZ6U3vTiMBT4hArhP3fOGLXpIINgMiiGjTe4FVlTa5a/7Zf2bwlIHfRqqg==", + "dev": true + }, + "node_modules/@algolia/cache-in-memory": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/cache-in-memory/-/cache-in-memory-4.19.1.tgz", + "integrity": "sha512-+PDWL+XALGvIginigzu8oU6eWw+o76Z8zHbBovWYcrtWOEtinbl7a7UTt3x3lthv+wNuFr/YD1Gf+B+A9V8n5w==", + "dev": true, + "dependencies": { + "@algolia/cache-common": "4.19.1" + } + }, + "node_modules/@algolia/client-account": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-account/-/client-account-4.19.1.tgz", + "integrity": "sha512-Oy0ritA2k7AMxQ2JwNpfaEcgXEDgeyKu0V7E7xt/ZJRdXfEpZcwp9TOg4TJHC7Ia62gIeT2Y/ynzsxccPw92GA==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-analytics": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-analytics/-/client-analytics-4.19.1.tgz", + "integrity": "sha512-5QCq2zmgdZLIQhHqwl55ZvKVpLM3DNWjFI4T+bHr3rGu23ew2bLO4YtyxaZeChmDb85jUdPDouDlCumGfk6wOg==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-common/-/client-common-4.19.1.tgz", + "integrity": "sha512-3kAIVqTcPrjfS389KQvKzliC559x+BDRxtWamVJt8IVp7LGnjq+aVAXg4Xogkur1MUrScTZ59/AaUd5EdpyXgA==", + "dev": true, + "dependencies": { + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-personalization": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.19.1.tgz", + "integrity": "sha512-8CWz4/H5FA+krm9HMw2HUQenizC/DxUtsI5oYC0Jxxyce1vsr8cb1aEiSJArQT6IzMynrERif1RVWLac1m36xw==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/client-search": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.19.1.tgz", + "integrity": "sha512-mBecfMFS4N+yK/p0ZbK53vrZbL6OtWMk8YmnOv1i0LXx4pelY8TFhqKoTit3NPVPwoSNN0vdSN9dTu1xr1XOVw==", + "dev": true, + "dependencies": { + "@algolia/client-common": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/@algolia/logger-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/logger-common/-/logger-common-4.19.1.tgz", + "integrity": "sha512-i6pLPZW/+/YXKis8gpmSiNk1lOmYCmRI6+x6d2Qk1OdfvX051nRVdalRbEcVTpSQX6FQAoyeaui0cUfLYW5Elw==", + "dev": true + }, + "node_modules/@algolia/logger-console": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/logger-console/-/logger-console-4.19.1.tgz", + "integrity": "sha512-jj72k9GKb9W0c7TyC3cuZtTr0CngLBLmc8trzZlXdfvQiigpUdvTi1KoWIb2ZMcRBG7Tl8hSb81zEY3zI2RlXg==", + "dev": true, + "dependencies": { + "@algolia/logger-common": "4.19.1" + } + }, + "node_modules/@algolia/requester-browser-xhr": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-4.19.1.tgz", + "integrity": "sha512-09K/+t7lptsweRTueHnSnmPqIxbHMowejAkn9XIcJMLdseS3zl8ObnS5GWea86mu3vy4+8H+ZBKkUN82Zsq/zg==", + "dev": true, + "dependencies": { + "@algolia/requester-common": "4.19.1" + } + }, + "node_modules/@algolia/requester-common": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.19.1.tgz", + "integrity": "sha512-BisRkcWVxrDzF1YPhAckmi2CFYK+jdMT60q10d7z3PX+w6fPPukxHRnZwooiTUrzFe50UBmLItGizWHP5bDzVQ==", + "dev": true + }, + "node_modules/@algolia/requester-node-http": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.19.1.tgz", + "integrity": "sha512-6DK52DHviBHTG2BK/Vv2GIlEw7i+vxm7ypZW0Z7vybGCNDeWzADx+/TmxjkES2h15+FZOqVf/Ja677gePsVItA==", + "dev": true, + "dependencies": { + "@algolia/requester-common": "4.19.1" + } + }, + "node_modules/@algolia/transporter": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.19.1.tgz", + "integrity": "sha512-nkpvPWbpuzxo1flEYqNIbGz7xhfhGOKGAZS7tzC+TELgEmi7z99qRyTfNSUlW7LZmB3ACdnqAo+9A9KFBENviQ==", + "dev": true, + "dependencies": { + "@algolia/cache-common": "4.19.1", + "@algolia/logger-common": "4.19.1", + "@algolia/requester-common": "4.19.1" + } + }, + "node_modules/@babel/parser": { + "version": "7.22.10", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.10.tgz", + "integrity": "sha512-lNbdGsQb9ekfsnjFGhEiF4hfFqGgfOP3H3d27re3n+CGhNuTSUEQdfWk556sTLNTloczcdM5TYF2LhzmDQKyvQ==", + "dev": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@docsearch/css": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@docsearch/css/-/css-3.5.1.tgz", + "integrity": "sha512-2Pu9HDg/uP/IT10rbQ+4OrTQuxIWdKVUEdcw9/w7kZJv9NeHS6skJx1xuRiFyoGKwAzcHXnLp7csE99sj+O1YA==", + "dev": true + }, + "node_modules/@docsearch/js": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@docsearch/js/-/js-3.5.1.tgz", + "integrity": "sha512-EXi8de5njxgP6TV3N9ytnGRLG9zmBNTEZjR4VzwPcpPLbZxxTLG2gaFyJyKiFVQxHW/DPlMrDJA3qoRRGEkgZw==", + "dev": true, + "dependencies": { + "@docsearch/react": "3.5.1", + "preact": "^10.0.0" + } + }, + "node_modules/@docsearch/react": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@docsearch/react/-/react-3.5.1.tgz", + "integrity": "sha512-t5mEODdLzZq4PTFAm/dvqcvZFdPDMdfPE5rJS5SC8OUq9mPzxEy6b+9THIqNM9P0ocCb4UC5jqBrxKclnuIbzQ==", + "dev": true, + "dependencies": { + "@algolia/autocomplete-core": "1.9.3", + "@algolia/autocomplete-preset-algolia": "1.9.3", + "@docsearch/css": "3.5.1", + "algoliasearch": "^4.0.0" + }, + "peerDependencies": { + "@types/react": ">= 16.8.0 < 19.0.0", + "react": ">= 16.8.0 < 19.0.0", + "react-dom": ">= 16.8.0 < 19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.19.tgz", + "integrity": "sha512-1uOoDurJYh5MNqPqpj3l/TQCI1V25BXgChEldCB7D6iryBYqYKrbZIhYO5AI9fulf66sM8UJpc3UcCly2Tv28w==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.18.19.tgz", + "integrity": "sha512-4+jkUFQxZkQfQOOxfGVZB38YUWHMJX2ihZwF+2nh8m7bHdWXpixiurgGRN3c/KMSwlltbYI0/i929jwBRMFzbA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.18.19.tgz", + "integrity": "sha512-ae5sHYiP/Ogj2YNrLZbWkBmyHIDOhPgpkGvFnke7XFGQldBDWvc/AyYwSLpNuKw9UNkgnLlB/jPpnBmlF3G9Bg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.18.19.tgz", + "integrity": "sha512-HIpQvNQWFYROmWDANMRL+jZvvTQGOiTuwWBIuAsMaQrnStedM+nEKJBzKQ6bfT9RFKH2wZ+ej+DY7+9xHBTFPg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.18.19.tgz", + "integrity": "sha512-m6JdvXJQt0thNLIcWOeG079h2ivhYH4B5sVCgqb/B29zTcFd7EE8/J1nIUHhdtwGeItdUeqKaqqb4towwxvglQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.19.tgz", + "integrity": "sha512-G0p4EFMPZhGn/xVNspUyMQbORH3nlKTV0bFNHPIwLraBuAkTeMyxNviTe0ZXUbIXQrR1lrwniFjNFU4s+x7veQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.18.19.tgz", + "integrity": "sha512-hBxgRlG42+W+j/1/cvlnSa+3+OBKeDCyO7OG2ICya1YJaSCYfSpuG30KfOnQHI7Ytgu4bRqCgrYXxQEzy0zM5Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.18.19.tgz", + "integrity": "sha512-qtWyoQskfJlb9MD45mvzCEKeO4uCnDZ7lPFeNqbfaaJHqBiH9qA5Vu2EuckqYZuFMJWy1l4dxTf9NOulCVfUjg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.18.19.tgz", + "integrity": "sha512-X8g33tczY0GsJq3lhyBrjnFtaKjWVpp1gMq5IlF9BQJ3TUfSK74nQnz9mRIEejmcV+OIYn6bkOJeUaU1Knrljg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.18.19.tgz", + "integrity": "sha512-SAkRWJgb+KN+gOhmbiE6/wu23D6HRcGQi15cB13IVtBZZgXxygTV5GJlUAKLQ5Gcx0gtlmt+XIxEmSqA6sZTOw==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.18.19.tgz", + "integrity": "sha512-YLAslaO8NsB9UOxBchos82AOMRDbIAWChwDKfjlGrHSzS3v1kxce7dGlSTsrb0PJwo1KYccypN3VNjQVLtz7LA==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.18.19.tgz", + "integrity": "sha512-vSYFtlYds/oTI8aflEP65xo3MXChMwBOG1eWPGGKs/ev9zkTeXVvciU+nifq8J1JYMz+eQ4J9JDN0O2RKF8+1Q==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.18.19.tgz", + "integrity": "sha512-tgG41lRVwlzqO9tv9l7aXYVw35BxKXLtPam1qALScwSqPivI8hjkZLNH0deaaSCYCFT9cBIdB+hUjWFlFFLL9A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.18.19.tgz", + "integrity": "sha512-EgBZFLoN1S5RuB4cCJI31pBPsjE1nZ+3+fHRjguq9Ibrzo29bOLSBcH1KZJvRNh5qtd+fcYIGiIUia8Jw5r1lQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.18.19.tgz", + "integrity": "sha512-q1V1rtHRojAzjSigZEqrcLkpfh5K09ShCoIsdTakozVBnM5rgV58PLFticqDp5UJ9uE0HScov9QNbbl8HBo6QQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.18.19.tgz", + "integrity": "sha512-D0IiYjpZRXxGZLQfsydeAD7ZWqdGyFLBj5f2UshJpy09WPs3qizDCsEr8zyzcym6Woj/UI9ZzMIXwvoXVtyt0A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.18.19.tgz", + "integrity": "sha512-3tt3SOS8L3D54R8oER41UdDshlBIAjYhdWRPiZCTZ1E41+shIZBpTjaW5UaN/jD1ENE/Ok5lkeqhoNMbxstyxw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.18.19.tgz", + "integrity": "sha512-MxbhcuAYQPlfln1EMc4T26OUoeg/YQc6wNoEV8xvktDKZhLtBxjkoeESSo9BbPaGKhAPzusXYj5n8n5A8iZSrA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.18.19.tgz", + "integrity": "sha512-m0/UOq1wj25JpWqOJxoWBRM9VWc3c32xiNzd+ERlYstUZ6uwx5SZsQUtkiFHaYmcaoj+f6+Tfcl7atuAz3idwQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.18.19.tgz", + "integrity": "sha512-L4vb6pcoB1cEcXUHU6EPnUhUc4+/tcz4OqlXTWPcSQWxegfmcOprhmIleKKwmMNQVc4wrx/+jB7tGkjjDmiupg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.18.19.tgz", + "integrity": "sha512-rQng7LXSKdrDlNDb7/v0fujob6X0GAazoK/IPd9C3oShr642ri8uIBkgM37/l8B3Rd5sBQcqUXoDdEy75XC/jg==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.18.19.tgz", + "integrity": "sha512-z69jhyG20Gq4QL5JKPLqUT+eREuqnDAFItLbza4JCmpvUnIlY73YNjd5djlO7kBiiZnvTnJuAbOjIoZIOa1GjA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "node_modules/@types/web-bluetooth": { + "version": "0.0.17", + "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.17.tgz", + "integrity": "sha512-4p9vcSmxAayx72yn70joFoL44c9MO/0+iVEBIQXe3v2h2SiAsEIo/G5v6ObFWvNKRFjbrVadNf9LqEEZeQPzdA==", + "dev": true + }, + "node_modules/@vitejs/plugin-vue": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-vue/-/plugin-vue-4.2.3.tgz", + "integrity": "sha512-R6JDUfiZbJA9cMiguQ7jxALsgiprjBeHL5ikpXfJCH62pPHtI+JdJ5xWj6Ev73yXSlYl86+blXn1kZHQ7uElxw==", + "dev": true, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.0.0", + "vue": "^3.2.25" + } + }, + "node_modules/@vue/compiler-core": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.3.4.tgz", + "integrity": "sha512-cquyDNvZ6jTbf/+x+AgM2Arrp6G4Dzbb0R64jiG804HRMfRiFXWI6kqUVqZ6ZR0bQhIoQjB4+2bhNtVwndW15g==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.21.3", + "@vue/shared": "3.3.4", + "estree-walker": "^2.0.2", + "source-map-js": "^1.0.2" + } + }, + "node_modules/@vue/compiler-dom": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.3.4.tgz", + "integrity": "sha512-wyM+OjOVpuUukIq6p5+nwHYtj9cFroz9cwkfmP9O1nzH68BenTTv0u7/ndggT8cIQlnBeOo6sUT/gvHcIkLA5w==", + "dev": true, + "dependencies": { + "@vue/compiler-core": "3.3.4", + "@vue/shared": "3.3.4" + } + }, + "node_modules/@vue/compiler-sfc": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.3.4.tgz", + "integrity": "sha512-6y/d8uw+5TkCuzBkgLS0v3lSM3hJDntFEiUORM11pQ/hKvkhSKZrXW6i69UyXlJQisJxuUEJKAWEqWbWsLeNKQ==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.15", + "@vue/compiler-core": "3.3.4", + "@vue/compiler-dom": "3.3.4", + "@vue/compiler-ssr": "3.3.4", + "@vue/reactivity-transform": "3.3.4", + "@vue/shared": "3.3.4", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.0", + "postcss": "^8.1.10", + "source-map-js": "^1.0.2" + } + }, + "node_modules/@vue/compiler-ssr": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.3.4.tgz", + "integrity": "sha512-m0v6oKpup2nMSehwA6Uuu+j+wEwcy7QmwMkVNVfrV9P2qE5KshC6RwOCq8fjGS/Eak/uNb8AaWekfiXxbBB6gQ==", + "dev": true, + "dependencies": { + "@vue/compiler-dom": "3.3.4", + "@vue/shared": "3.3.4" + } + }, + "node_modules/@vue/devtools-api": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/@vue/devtools-api/-/devtools-api-6.5.0.tgz", + "integrity": "sha512-o9KfBeaBmCKl10usN4crU53fYtC1r7jJwdGKjPT24t348rHxgfpZ0xL3Xm/gLUYnc0oTp8LAmrxOeLyu6tbk2Q==", + "dev": true + }, + "node_modules/@vue/reactivity": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.3.4.tgz", + "integrity": "sha512-kLTDLwd0B1jG08NBF3R5rqULtv/f8x3rOFByTDz4J53ttIQEDmALqKqXY0J+XQeN0aV2FBxY8nJDf88yvOPAqQ==", + "dev": true, + "dependencies": { + "@vue/shared": "3.3.4" + } + }, + "node_modules/@vue/reactivity-transform": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/reactivity-transform/-/reactivity-transform-3.3.4.tgz", + "integrity": "sha512-MXgwjako4nu5WFLAjpBnCj/ieqcjE2aJBINUNQzkZQfzIZA4xn+0fV1tIYBJvvva3N3OvKGofRLvQIwEQPpaXw==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.20.15", + "@vue/compiler-core": "3.3.4", + "@vue/shared": "3.3.4", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.0" + } + }, + "node_modules/@vue/runtime-core": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.3.4.tgz", + "integrity": "sha512-R+bqxMN6pWO7zGI4OMlmvePOdP2c93GsHFM/siJI7O2nxFRzj55pLwkpCedEY+bTMgp5miZ8CxfIZo3S+gFqvA==", + "dev": true, + "dependencies": { + "@vue/reactivity": "3.3.4", + "@vue/shared": "3.3.4" + } + }, + "node_modules/@vue/runtime-dom": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.3.4.tgz", + "integrity": "sha512-Aj5bTJ3u5sFsUckRghsNjVTtxZQ1OyMWCr5dZRAPijF/0Vy4xEoRCwLyHXcj4D0UFbJ4lbx3gPTgg06K/GnPnQ==", + "dev": true, + "dependencies": { + "@vue/runtime-core": "3.3.4", + "@vue/shared": "3.3.4", + "csstype": "^3.1.1" + } + }, + "node_modules/@vue/server-renderer": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.3.4.tgz", + "integrity": "sha512-Q6jDDzR23ViIb67v+vM1Dqntu+HUexQcsWKhhQa4ARVzxOY2HbC7QRW/ggkDBd5BU+uM1sV6XOAP0b216o34JQ==", + "dev": true, + "dependencies": { + "@vue/compiler-ssr": "3.3.4", + "@vue/shared": "3.3.4" + }, + "peerDependencies": { + "vue": "3.3.4" + } + }, + "node_modules/@vue/shared": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.3.4.tgz", + "integrity": "sha512-7OjdcV8vQ74eiz1TZLzZP4JwqM5fA94K6yntPS5Z25r9HDuGNzaGdgvwKYq6S+MxwF0TFRwe50fIR/MYnakdkQ==", + "dev": true + }, + "node_modules/@vueuse/core": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@vueuse/core/-/core-10.3.0.tgz", + "integrity": "sha512-BEM5yxcFKb5btFjTSAFjTu5jmwoW66fyV9uJIP4wUXXU8aR5Hl44gndaaXp7dC5HSObmgbnR2RN+Un1p68Mf5Q==", + "dev": true, + "dependencies": { + "@types/web-bluetooth": "^0.0.17", + "@vueuse/metadata": "10.3.0", + "@vueuse/shared": "10.3.0", + "vue-demi": ">=0.14.5" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/core/node_modules/vue-demi": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.5.tgz", + "integrity": "sha512-o9NUVpl/YlsGJ7t+xuqJKx8EBGf1quRhCiT6D/J0pfwmk9zUwYkC7yrF4SZCe6fETvSM3UNL2edcbYrSyc4QHA==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/@vueuse/integrations": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@vueuse/integrations/-/integrations-10.3.0.tgz", + "integrity": "sha512-Jgiv7oFyIgC6BxmDtiyG/fxyGysIds00YaY7sefwbhCZ2/tjEx1W/1WcsISSJPNI30in28+HC2J4uuU8184ekg==", + "dev": true, + "dependencies": { + "@vueuse/core": "10.3.0", + "@vueuse/shared": "10.3.0", + "vue-demi": ">=0.14.5" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "async-validator": "*", + "axios": "*", + "change-case": "*", + "drauu": "*", + "focus-trap": "*", + "fuse.js": "*", + "idb-keyval": "*", + "jwt-decode": "*", + "nprogress": "*", + "qrcode": "*", + "sortablejs": "*", + "universal-cookie": "*" + }, + "peerDependenciesMeta": { + "async-validator": { + "optional": true + }, + "axios": { + "optional": true + }, + "change-case": { + "optional": true + }, + "drauu": { + "optional": true + }, + "focus-trap": { + "optional": true + }, + "fuse.js": { + "optional": true + }, + "idb-keyval": { + "optional": true + }, + "jwt-decode": { + "optional": true + }, + "nprogress": { + "optional": true + }, + "qrcode": { + "optional": true + }, + "sortablejs": { + "optional": true + }, + "universal-cookie": { + "optional": true + } + } + }, + "node_modules/@vueuse/integrations/node_modules/vue-demi": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.5.tgz", + "integrity": "sha512-o9NUVpl/YlsGJ7t+xuqJKx8EBGf1quRhCiT6D/J0pfwmk9zUwYkC7yrF4SZCe6fETvSM3UNL2edcbYrSyc4QHA==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/@vueuse/metadata": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@vueuse/metadata/-/metadata-10.3.0.tgz", + "integrity": "sha512-Ema3YhNOa4swDsV0V7CEY5JXvK19JI/o1szFO1iWxdFg3vhdFtCtSTP26PCvbUpnUtNHBY2wx5y3WDXND5Pvnw==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@vueuse/shared/-/shared-10.3.0.tgz", + "integrity": "sha512-kGqCTEuFPMK4+fNWy6dUOiYmxGcUbtznMwBZLC1PubidF4VZY05B+Oht7Jh7/6x4VOWGpvu3R37WHi81cKpiqg==", + "dev": true, + "dependencies": { + "vue-demi": ">=0.14.5" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@vueuse/shared/node_modules/vue-demi": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/vue-demi/-/vue-demi-0.14.5.tgz", + "integrity": "sha512-o9NUVpl/YlsGJ7t+xuqJKx8EBGf1quRhCiT6D/J0pfwmk9zUwYkC7yrF4SZCe6fETvSM3UNL2edcbYrSyc4QHA==", + "dev": true, + "hasInstallScript": true, + "bin": { + "vue-demi-fix": "bin/vue-demi-fix.js", + "vue-demi-switch": "bin/vue-demi-switch.js" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + }, + "peerDependencies": { + "@vue/composition-api": "^1.0.0-rc.1", + "vue": "^3.0.0-0 || ^2.6.0" + }, + "peerDependenciesMeta": { + "@vue/composition-api": { + "optional": true + } + } + }, + "node_modules/algoliasearch": { + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.19.1.tgz", + "integrity": "sha512-IJF5b93b2MgAzcE/tuzW0yOPnuUyRgGAtaPv5UUywXM8kzqfdwZTO4sPJBzoGz1eOy6H9uEchsJsBFTELZSu+g==", + "dev": true, + "dependencies": { + "@algolia/cache-browser-local-storage": "4.19.1", + "@algolia/cache-common": "4.19.1", + "@algolia/cache-in-memory": "4.19.1", + "@algolia/client-account": "4.19.1", + "@algolia/client-analytics": "4.19.1", + "@algolia/client-common": "4.19.1", + "@algolia/client-personalization": "4.19.1", + "@algolia/client-search": "4.19.1", + "@algolia/logger-common": "4.19.1", + "@algolia/logger-console": "4.19.1", + "@algolia/requester-browser-xhr": "4.19.1", + "@algolia/requester-common": "4.19.1", + "@algolia/requester-node-http": "4.19.1", + "@algolia/transporter": "4.19.1" + } + }, + "node_modules/ansi-sequence-parser": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ansi-sequence-parser/-/ansi-sequence-parser-1.1.1.tgz", + "integrity": "sha512-vJXt3yiaUL4UU546s3rPXlsry/RnM730G1+HkpKE012AN0sx1eOrxSu95oKDIonskeLTijMgqWZ3uDEe3NFvyg==", + "dev": true + }, + "node_modules/body-scroll-lock": { + "version": "4.0.0-beta.0", + "resolved": "https://registry.npmjs.org/body-scroll-lock/-/body-scroll-lock-4.0.0-beta.0.tgz", + "integrity": "sha512-a7tP5+0Mw3YlUJcGAKUqIBkYYGlYxk2fnCasq/FUph1hadxlTRjF+gAcZksxANnaMnALjxEddmSi/H3OR8ugcQ==", + "dev": true + }, + "node_modules/csstype": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", + "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==", + "dev": true + }, + "node_modules/esbuild": { + "version": "0.18.19", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.18.19.tgz", + "integrity": "sha512-ra3CaIKCzJp5bU5BDfrCc0FRqKj71fQi+gbld0aj6lN0ifuX2fWJYPgLVLGwPfA+ruKna+OWwOvf/yHj6n+i0g==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.18.19", + "@esbuild/android-arm64": "0.18.19", + "@esbuild/android-x64": "0.18.19", + "@esbuild/darwin-arm64": "0.18.19", + "@esbuild/darwin-x64": "0.18.19", + "@esbuild/freebsd-arm64": "0.18.19", + "@esbuild/freebsd-x64": "0.18.19", + "@esbuild/linux-arm": "0.18.19", + "@esbuild/linux-arm64": "0.18.19", + "@esbuild/linux-ia32": "0.18.19", + "@esbuild/linux-loong64": "0.18.19", + "@esbuild/linux-mips64el": "0.18.19", + "@esbuild/linux-ppc64": "0.18.19", + "@esbuild/linux-riscv64": "0.18.19", + "@esbuild/linux-s390x": "0.18.19", + "@esbuild/linux-x64": "0.18.19", + "@esbuild/netbsd-x64": "0.18.19", + "@esbuild/openbsd-x64": "0.18.19", + "@esbuild/sunos-x64": "0.18.19", + "@esbuild/win32-arm64": "0.18.19", + "@esbuild/win32-ia32": "0.18.19", + "@esbuild/win32-x64": "0.18.19" + } + }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "dev": true + }, + "node_modules/focus-trap": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-7.5.2.tgz", + "integrity": "sha512-p6vGNNWLDGwJCiEjkSK6oERj/hEyI9ITsSwIUICBoKLlWiTWXJRfQibCwcoi50rTZdbi87qDtUlMCmQwsGSgPw==", + "dev": true, + "dependencies": { + "tabbable": "^6.2.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/jsonc-parser": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.2.0.tgz", + "integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w==", + "dev": true + }, + "node_modules/magic-string": { + "version": "0.30.2", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.2.tgz", + "integrity": "sha512-lNZdu7pewtq/ZvWUp9Wpf/x7WzMTsR26TWV03BRZrXFsv+BI6dy8RAiKgm1uM/kyR0rCfUcqvOlXKG66KhIGug==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/mark.js": { + "version": "8.11.1", + "resolved": "https://registry.npmjs.org/mark.js/-/mark.js-8.11.1.tgz", + "integrity": "sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==", + "dev": true + }, + "node_modules/minisearch": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-6.1.0.tgz", + "integrity": "sha512-PNxA/X8pWk+TiqPbsoIYH0GQ5Di7m6326/lwU/S4mlo4wGQddIcf/V//1f9TB0V4j59b57b+HZxt8h3iMROGvg==", + "dev": true + }, + "node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "node_modules/postcss": { + "version": "8.4.27", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.27.tgz", + "integrity": "sha512-gY/ACJtJPSmUFPDCHtX78+01fHa64FaU4zaaWfuh1MhGJISufJAH4cun6k/8fwsHYeK4UQmENQK+tRLCFJE8JQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/preact": { + "version": "10.16.0", + "resolved": "https://registry.npmjs.org/preact/-/preact-10.16.0.tgz", + "integrity": "sha512-XTSj3dJ4roKIC93pald6rWuB2qQJO9gO2iLLyTe87MrjQN+HklueLsmskbywEWqCHlclgz3/M4YLL2iBr9UmMA==", + "dev": true, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/preact" + } + }, + "node_modules/rollup": { + "version": "3.27.2", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.27.2.tgz", + "integrity": "sha512-YGwmHf7h2oUHkVBT248x0yt6vZkYQ3/rvE5iQuVBh3WO8GcJ6BNeOkpoX1yMHIiBm18EMLjBPIoUDkhgnyxGOQ==", + "dev": true, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=14.18.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/search-insights": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/search-insights/-/search-insights-2.7.0.tgz", + "integrity": "sha512-GLbVaGgzYEKMvuJbHRhLi1qoBFnjXZGZ6l4LxOYPCp4lI2jDRB3jPU9/XNhMwv6kvnA9slTreq6pvK+b3o3aqg==", + "dev": true, + "peer": true, + "engines": { + "node": ">=8.16.0" + } + }, + "node_modules/shiki": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/shiki/-/shiki-0.14.3.tgz", + "integrity": "sha512-U3S/a+b0KS+UkTyMjoNojvTgrBHjgp7L6ovhFVZsXmBGnVdQ4K4U9oK0z63w538S91ATngv1vXigHCSWOwnr+g==", + "dev": true, + "dependencies": { + "ansi-sequence-parser": "^1.1.0", + "jsonc-parser": "^3.2.0", + "vscode-oniguruma": "^1.7.0", + "vscode-textmate": "^8.0.0" + } + }, + "node_modules/source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tabbable": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz", + "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew==", + "dev": true + }, + "node_modules/vite": { + "version": "4.4.9", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz", + "integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==", + "dev": true, + "dependencies": { + "esbuild": "^0.18.10", + "postcss": "^8.4.27", + "rollup": "^3.27.1" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + }, + "peerDependencies": { + "@types/node": ">= 14", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/vitepress": { + "version": "1.0.0-rc.4", + "resolved": "https://registry.npmjs.org/vitepress/-/vitepress-1.0.0-rc.4.tgz", + "integrity": "sha512-JCQ89Bm6ECUTnyzyas3JENo00UDJeK8q1SUQyJYou+4Yz5BKEc/F3O21cu++DnUT2zXc0kvQ2Aj4BZCc/nioXQ==", + "dev": true, + "dependencies": { + "@docsearch/css": "^3.5.1", + "@docsearch/js": "^3.5.1", + "@vitejs/plugin-vue": "^4.2.3", + "@vue/devtools-api": "^6.5.0", + "@vueuse/core": "^10.3.0", + "@vueuse/integrations": "^10.3.0", + "body-scroll-lock": "4.0.0-beta.0", + "focus-trap": "^7.5.2", + "mark.js": "8.11.1", + "minisearch": "^6.1.0", + "shiki": "^0.14.3", + "vite": "^4.4.9", + "vue": "^3.3.4" + }, + "bin": { + "vitepress": "bin/vitepress.js" + } + }, + "node_modules/vscode-oniguruma": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz", + "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==", + "dev": true + }, + "node_modules/vscode-textmate": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-8.0.0.tgz", + "integrity": "sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==", + "dev": true + }, + "node_modules/vue": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.3.4.tgz", + "integrity": "sha512-VTyEYn3yvIeY1Py0WaYGZsXnz3y5UnGi62GjVEqvEGPl6nxbOrCXbVOTQWBEJUqAyTUk2uJ5JLVnYJ6ZzGbrSw==", + "dev": true, + "dependencies": { + "@vue/compiler-dom": "3.3.4", + "@vue/compiler-sfc": "3.3.4", + "@vue/runtime-dom": "3.3.4", + "@vue/server-renderer": "3.3.4", + "@vue/shared": "3.3.4" + } + } + } +} diff --git a/docs/package.json b/docs/package.json new file mode 100644 index 00000000..444d549f --- /dev/null +++ b/docs/package.json @@ -0,0 +1,11 @@ +{ + "type": "module", + "devDependencies": { + "vitepress": "^1.0.0-rc.4" + }, + "scripts": { + "docs:dev": "vitepress dev", + "docs:build": "vitepress build", + "docs:preview": "vitepress preview" + } +} diff --git a/docs/src/en/function/best_practices.md b/docs/src/en/function/best_practices.md new file mode 100644 index 00000000..f663956a --- /dev/null +++ b/docs/src/en/function/best_practices.md @@ -0,0 +1,99 @@ +--- +outline: deep +--- + +# 最佳实践 + +## 过滤 + +### 过滤 Key + +```lua +local prefix = "user:" +local prefix_len = #prefix + +if string.sub(KEYS[1], 1, prefix_len) ~= prefix then + return +end + +shake.call(DB, ARGV) +``` + +效果是只将 key 以 `user:` 开头的源数据写入到目标端。没有考虑 `mset` 等多 key 命令的情况。 + +### 过滤 DB + +```lua +shake.log(DB) +if DB == 0 +then + return +end +shake.call(DB, ARGV) +``` + +效果是丢弃源端 `db` 0 的数据，将其他 `db` 的数据写入到目标端。 + + +### 过滤某类数据结构 + +可以通过 `GROUP` 变量来判断数据结构类型，支持的数据结构类型有：`STRING`、`LIST`、`SET`、`ZSET`、`HASH`、`SCRIPTING` 等。 + +#### 过滤 Hash 类型数据 +```lua +if GROUP == "HASH" then + return +end +shake.call(DB, ARGV) +``` + +效果是丢弃源端的 `hash` 类型数据，将其他数据写入到目标端。 + +#### 过滤 [LUA 脚本](https://redis.io/docs/interact/programmability/eval-intro/) + +```lua +if GROUP == "SCRIPTING" then + return +end +shake.call(DB, ARGV) +``` + +效果是丢弃源端的 `lua` 脚本，将其他数据写入到目标端。常见于主从同步至集群时，存在集群不支持的 LUA 脚本。 + +## 修改 + +### 修改 Key 的前缀 + +```lua +local prefix_old = "prefix_old_" +local prefix_new = "prefix_new_" + +shake.log("old=" .. table.concat(ARGV, " ")) + +for i, index in ipairs(KEY_INDEXES) do + local key = ARGV[index] + if string.sub(key, 1, #prefix_old) == prefix_old then + ARGV[index] = prefix_new .. string.sub(key, #prefix_old + 1) + end +end + +shake.log("new=" .. table.concat(ARGV, " ")) +shake.call(DB, ARGV) +``` +效果是将源端的 key `prefix_old_key` 写入到目标端的 key `prefix_new_key`。 + +### 交换 DB + +```lua +local db1 = 1 +local db2 = 2 + +if DB == db1 then + DB = db2 +elseif DB == db2 then + DB = db1 +end +shake.call(DB, ARGV) +``` + +效果是将源端的 `db 1` 写入到目标端的 `db 2`，将源端的 `db 2` 写入到目标端的 `db 1`, 其他 `db` 不变。 \ No newline at end of file diff --git a/docs/src/en/function/introduction.md b/docs/src/en/function/introduction.md new file mode 100644 index 00000000..579d0eec --- /dev/null +++ b/docs/src/en/function/introduction.md @@ -0,0 +1,56 @@ +--- +outline: deep +--- + +# 什么是 function + +RedisShake 通过提供 function 功能，实现了的 [ETL(提取-转换-加载)](https://en.wikipedia.org/wiki/Extract,_transform,_load) 中的 `transform` 能力。通过利用 function 可以实现类似功能： +* 更改数据所属的 `db`，比如将源端的 `db 0` 写入到目的端的 `db 1`。 +* 对数据进行筛选，例如，只将 key 以 `user:` 开头的源数据写入到目标端。 +* 改变 Key 的前缀，例如，将源端的 key `prefix_old_key` 写入到目标端的 key `prefix_new_key`。 +* ... + +要使用 function 功能，只需编写一份 lua 脚本。RedisShake 在从源端获取数据后，会将数据转换为 Redis 命令。然后，它会处理这些命令，从中解析出 `KEYS`、`ARGV`、`SLOTS`、`GROUP` 等信息，并将这些信息传递给 lua 脚本。lua 脚本会处理这些数据，并返回处理后的命令。最后，RedisShake 会将处理后的数据写入到目标端。 + +以下是一个具体的例子： +```toml +function = """ +shake.log(DB) +if DB == 0 +then + return +end +shake.call(DB, ARGV) +""" + +[sync_reader] +address = "127.0.0.1:6379" + +[redis_writer] +address = "127.0.0.1:6380" +``` +`DB` 是 RedisShake 提供的信息，表示当前数据所属的 db。`shake.log` 用于打印日志，`shake.call` 用于调用 Redis 命令。上述脚本的目的是丢弃源端 `db` 0 的数据，将其他 `db` 的数据写入到目标端。 + +除了 `DB`，还有其他信息如 `KEYS`、`ARGV`、`SLOTS`、`GROUP` 等，可供调用的函数有 `shake.log` 和 `shake.call`，具体请参考 [function API](#function-api)。 + +关于更多的示例，可以参考 [最佳实践](./best_practices.md)。 + +## function API + +### 变量 + +因为有些命令中含有多个 key，比如 `mset` 等命令。所以，`KEYS`、`KEY_INDEXES`、`SLOTS` 这三个变量都是数组类型。如果确认命令只有一个 key，可以直接使用 `KEYS[1]`、`KEY_INDEXES[1]`、`SLOTS[1]`。 + +| 变量 | 类型 | 示例 | 描述 | +|-|-|-|-----| +| DB | number | 1 | 命令所属的 `db` | +| GROUP | string | "LIST" | 命令所属的 `group`，符合 [Command key specifications](https://redis.io/docs/reference/key-specs/)，可以在 [commands](https://github.com/tair-opensource/RedisShake/tree/v4/scripts/commands) 中查询每个命令的 `group` 字段 | +| CMD | string | "XGROUP-DELCONSUMER" | 命令的名称 | +| KEYS | table | \{"key1", "key2"\} | 命令的所有 Key | +| KEY_INDEXES | table | \{2, 4\} | 命令的所有 Key 在 `ARGV` 中的索引 | +| SLOTS | table | \{9189, 4998\} | 当前命令的所有 Key 所属的 [slot](https://redis.io/docs/reference/cluster-spec/#key-distribution-model) | +| ARGV | table | \{"mset", "key1", "value1", "key2", "value2"\} | 命令的所有参数 | + +### 函数 +* `shake.call(DB, ARGV)`：返回一个 Redis 命令，RedisShake 会将该命令写入目标端。 +* `shake.log(msg)`：打印日志。 diff --git a/docs/src/en/guide/config.md b/docs/src/en/guide/config.md new file mode 100644 index 00000000..ffa19e1f --- /dev/null +++ b/docs/src/en/guide/config.md @@ -0,0 +1,82 @@ +--- +outline: deep +--- + +# Configuration File + +RedisShake uses the [TOML](https://toml.io/cn/) language for writing, and all configuration parameters are explained in all.toml. + +The configuration file is composed as follows: + +```toml +function = "..." + +[xxx_reader] +... + +[xxx_writer] +... + +[advanced] +... +``` + +Under normal usage, you only need to write the `xxx_reader` and `xxx_writer` parts. The `function` and `advanced` parts are for advanced usage, and users can configure them according to their needs. + +## function Configuration + +Refer to [What is function](../function/introduction.md). + +## reader Configuration + +RedisShake provides different Readers to interface with different sources, see the Reader section for configuration details: + +* [Sync Reader](../reader/sync_reader.md) +* [Scan Reader](../reader/scan_reader.md) +* [RDB Reader](../reader/rdb_reader.md) + +## writer Configuration + +RedisShake provides different Writers to interface with different targets, see the Writer section for configuration details: + +* [Redis Writer](../writer/redis_writer.md) + +## advanced Configuration + +```toml +[advanced] +dir = "data" +ncpu = 3 # runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores + +pprof_port = 0 # pprof port, 0 means disable +status_port = 0 # status port, 0 means disable + +# log +log_file = "shake.log" +log_level = "info" # debug, info or warn +log_interval = 5 # in seconds + +# redis-shake gets key and value from rdb file, and uses RESTORE command to +# create the key in target redis. Redis RESTORE will return a "Target key name +# is busy" error when key already exists. You can use this configuration item +# to change the default behavior of restore: +# panic: redis-shake will stop when meet "Target key name is busy" error. +# rewrite: redis-shake will replace the key with new value. +# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. +rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip + +# redis-shake uses pipeline to improve sending performance. +# This item limits the maximum number of commands in a pipeline. +pipeline_count_limit = 1024 + +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default. This amount is normally 1gb. +target_redis_client_max_querybuf_len = 1024_000_000 + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. +target_redis_proto_max_bulk_len = 512_000_000 + +# If the source is Elasticache or MemoryDB, you can set this item. +aws_psync = "" +``` \ No newline at end of file diff --git a/docs/src/en/guide/getting-started.md b/docs/src/en/guide/getting-started.md new file mode 100644 index 00000000..26e95291 --- /dev/null +++ b/docs/src/en/guide/getting-started.md @@ -0,0 +1,45 @@ +# Quick Start + +## Installation + +### Download Binary Package + +Directly download the binary package from [Release](https://github.com/tair-opensource/RedisShake/releases). + +### Compile from Source Code + +To compile from the source code, make sure you have set up the Golang environment on your local machine: + +```shell +git clone https://github.com/alibaba/RedisShake +cd RedisShake +sh build.sh +``` + +## Usage + +Assume you have two Redis instances: + +* Instance A: 127.0.0.1:6379 +* Instance B: 127.0.0.1:6380 + +Create a new configuration file `shake.toml`: + +```toml +[sync_reader] +address = "127.0.0.1:6379" + +[redis_writer] +address = "127.0.0.1:6380" +``` + +To start RedisShake, run the following command: + +```shell +./redis-shake shake.toml +``` + +## Precautions + +1. Do not run two RedisShake processes in the same directory, as the temporary files generated during runtime may be overwritten, leading to abnormal behavior. +2. Do not downgrade the Redis version, such as from 6.0 to 5.0, because each major version of RedisShake introduces some new commands and encoding methods. If the version is lowered, it may lead to incompatibility. \ No newline at end of file diff --git a/docs/src/en/guide/image.png b/docs/src/en/guide/image.png new file mode 100644 index 00000000..725251fa Binary files /dev/null and b/docs/src/en/guide/image.png differ diff --git a/docs/src/en/guide/introduction.md b/docs/src/en/guide/introduction.md new file mode 100644 index 00000000..40e13675 --- /dev/null +++ b/docs/src/en/guide/introduction.md @@ -0,0 +1,39 @@ +--- +outline: deep +--- + +# What is RedisShake + +RedisShake is a tool for processing and migrating Redis data, offering the following features: + +1. **Redis Compatibility**: RedisShake is compatible with Redis versions from 2.8 to 7.2 and supports various deployment methods, including standalone, master-slave, sentinel, and cluster. +2. **Cloud Service Compatibility**: RedisShake seamlessly works with popular Redis-like databases provided by mainstream cloud service providers, including but not limited to: + - [Alibaba Cloud - ApsaraDB for Redis](https://www.alibabacloud.com/product/apsaradb-for-redis) + - [Alibaba Cloud - Tair](https://www.alibabacloud.com/product/tair) + - [AWS - ElastiCache](https://aws.amazon.com/elasticache/) + - [AWS - MemoryDB](https://aws.amazon.com/memorydb/) +3. **Module Compatibility**: RedisShake is compatible with [TairString](https://github.com/tair-opensource/TairString), [TairZSet](https://github.com/tair-opensource/TairZset), and [TairHash](https://github.com/tair-opensource/TairHash) modules. +4. **Various Export Modes**: RedisShake supports PSync, RDB, and Scan export modes. +5. **Data Processing**: RedisShake implements data filtering and transformation through custom scripts. + +## Contributions + +We welcome contributions from the community. For significant changes, please open an issue first to discuss what you would like to change. We are particularly interested in: + +1. Adding support for more modules +2. Enhancing the support for Readers and Writers +3. Sharing your Lua scripts and best practices + +## History + +RedisShake is a project actively maintained by the Alibaba Cloud [Tair Team](https://github.com/tair-opensource). Its evolution can be traced back to its initial version, which was branched out from [redis-port](https://github.com/CodisLabs/redis-port). + +Versions (configurations are not interchangeable between different versions): + +- The [RedisShake 2.x](https://github.com/tair-opensource/RedisShake/tree/v2) version brought a series of improvements and updates, enhancing its overall stability and performance. +- The [RedisShake 3.x](https://github.com/tair-opensource/RedisShake/tree/v3) version was a significant milestone, with the entire codebase being completely rewritten and optimized for better efficiency and availability. +- The [RedisShake 4.x](https://github.com/tair-opensource/RedisShake/tree/v4) version further enhanced features related to the [Reader](../reader/scan_reader.md), configuration, observability, and [function](../function/introduction.md). + +## License + +RedisShake is open-source under the [MIT License](https://github.com/tair-opensource/RedisShake/blob/v2/license.txt). \ No newline at end of file diff --git a/docs/src/en/guide/mode.md b/docs/src/en/guide/mode.md new file mode 100644 index 00000000..a404e422 --- /dev/null +++ b/docs/src/en/guide/mode.md @@ -0,0 +1,53 @@ +--- +outline: deep +--- + +# Migration Mode Selection + +## Overview + +Currently, RedisShake has three migration modes: `PSync`, `RDB`, and `SCAN`, corresponding to [`sync_reader`](../reader/sync_reader.md), [`rdb_reader`](../reader/rdb_reader.md), and [`scan_reader`](../reader/scan_reader.md) respectively. + +* For scenarios of recovering data from backups, you can use `rdb_reader`. +* For data migration scenarios, `sync_reader` should be the preferred choice. Some cloud vendors do not provide support for the PSync protocol, in which case `scan_reader` can be chosen. +* For long-term data synchronization scenarios, RedisShake currently cannot handle them because the PSync protocol is not reliable. When the replication connection is disconnected, RedisShake will not be able to reconnect to the source database. If the demand for availability is not high, you can use `scan_reader`. If the write volume is not large and there are no large keys, `scan_reader` can also be considered. + +Different modes have their pros and cons, and you need to check each Reader section for more information. + +## Redis Cluster Architecture + +When the source Redis is deployed in a cluster architecture, you can use `sync_reader` or `scan_reader`. Both have switches in their configuration items to enable cluster mode, which will automatically obtain all nodes in the cluster through the `cluster nodes` command and establish connections. + +## Redis Sentinel Architecture + +When the source Redis is deployed in a sentinel architecture and RedisShake uses `sync_reader` to connect to the master, it will be treated as a slave by the master and may be elected as the new master by the sentinel. + +To avoid this, you should choose a replica as the source. + +## Cloud Redis Service + +Mainstream cloud vendors all provide Redis services, but there are several reasons that make using RedisShake on these services more complex: +1. Engine restrictions. Some self-developed Redis-like databases do not support the PSync protocol. +2. Architecture restrictions. Many cloud vendors support proxy mode, i.e., adding a Proxy component between the user and the Redis service. Because of the existence of the Proxy component, the PSync protocol cannot be supported. +3. Security restrictions. In native Redis, the PSync protocol will basically trigger fork(2), leading to memory bloat and increased user request latency. In worse cases, it may even lead to out of memory. Although there are solutions to alleviate these issues, not all cloud vendors have invested in this area. +4. Business strategies. Many users use RedisShake to migrate off the cloud or switch clouds, so some cloud vendors do not want users to use RedisShake, thus blocking the PSync protocol. + +The following will introduce some RedisShake usage schemes in special scenarios based on practical experience. + +### Alibaba Cloud Redis & Tair + +Alibaba Cloud Redis and Tair both support the PSync protocol, and `sync_reader` is recommended. Users need to create an account with replication permissions. RedisShake can use this account for data synchronization. The specific creation steps can be found in [Create and manage database accounts](https://help.aliyun.com/zh/redis/user-guide/create-and-manage-database-accounts). + +Exceptions: +1. Version 2.8 Redis instances do not support the creation of accounts with replication permissions. You need to [upgrade to a major version](https://help.aliyun.com/zh/redis/user-guide/upgrade-the-major-version-1). +2. Cluster architecture Redis and Tair instances do not support the PSync protocol under [proxy mode](https://help.aliyun.com/zh/redis/product-overview/cluster-master-replica-instances#section-h69-izd-531). +3. Read-write separation architecture does not support the PSync protocol. + +In scenarios where the PSync protocol is not supported, `scan_reader` can be used. It should be noted that `scan_reader` will put significant pressure on the source database. + +### AWS ElastiCache and MemoryDB + +`sync_reader` is preferred. AWS ElastiCache and MemoryDB do not enable the PSync protocol by default, but you can request to enable the PSync protocol by submitting a ticket. AWS will provide a renamed PSync command in the ticket, such as `xhma21yfkssync` and `nmfu2bl5osync`. This command has the same effect as the `psync` command, just with a different name. +Users only need to modify the `aws_psync` configuration item in the RedisShake configuration file. For a single instance, write one pair of `ip:port@cmd`. For cluster instances, write all `ip:port@cmd`, separated by commas. + +When it is inconvenient to submit a ticket, you can use `scan_reader`. It should be noted that `scan_reader` will put significant pressure on the source database. diff --git a/docs/src/en/index.md b/docs/src/en/index.md new file mode 100644 index 00000000..cff1513b --- /dev/null +++ b/docs/src/en/index.md @@ -0,0 +1,24 @@ +--- +# https://vitepress.dev/reference/default-theme-home-page +layout: home + +hero: + name: "RedisShake" + # text: "用于 Redis-like 数据库的数据迁移与处理服务" + tagline: 用于 Redis-like 数据库的数据迁移与处理服务 + actions: + - theme: brand + text: Get Started + link: /en/guide/getting-started + - theme: alt + text: What is RedisShake + link: /en/guide/introduction +features: + - title: Data Migration + details: Supports sync, scan, and restore modes for data migration + - title: Data Processing + details: Supports data filtering and modification using lua scripts + - title: Compatibility + details: Compatible with various Redis deployment forms and mainstream cloud vendor's Redis-like databases +--- + diff --git a/docs/src/en/others/modules.md b/docs/src/en/others/modules.md new file mode 100644 index 00000000..6309515c --- /dev/null +++ b/docs/src/en/others/modules.md @@ -0,0 +1,43 @@ +--- +outline: deep +--- + +# Redis Modules + +Redis Modules 是 Redis 4.0 版本引入的一个新特性，它允许开发者扩展 Redis 的功能。通过创建模块，开发者可以定义新的命令，数据类型，甚至改变 Redis 的行为。因此，Redis Modules 可以极大地增强 Redis 的灵活性和可扩展性。 + +由于 Redis Modules 可以定义新的数据类型和命令，RedisShake 需要对这些新的数据类型和命令进行专门的处理，才能正确地迁移或同步这些数据。否则，如果 RedisShake 不理解这些新的数据类型和命令，它可能会无法正确地处理这些数据，或者在处理过程中出错。因此，对于使用了 Redis Modules 的 Redis 实例，一般需要 RedisShake 为其使用的 Module 提供相应的适配器，以便正确地处理这些自定义的数据类型和命令。 + +## 已支持的 Redis Modules 列表 + +- [TairHash](https://github.com/tair-opensource/TairHash)：支持 field 级别设置过期和版本的 Hash 数据结构。 +- [TairString](https://github.com/tair-opensource/TairString)：支持版本的 String 结构，可以实现分布式锁/乐观锁。 +- [TairZset](https://github.com/tair-opensource/TairZset)：支持最多 256 维的 double 排序，可以实现多维排行榜。 + +## 如何支持新的 Redis Modules + +### 核心流程 + +相关代码在`internal\rdb`目录下，如需要支持其它 Redis Modules 类型，可分解为以下三个步骤： +- 从rdb文件中正确读入 + - RedisShake 中已经 对 redis module 自定的几种类型进行了封装，从 rdb 文件进行读取时，可直接借助于已经封装好的函数进行读取（`internal\rdb\structure\module2_struct.go`） +- 构建一个合适的中间数据结构类型，用于存储相应数据（key + value） +- 大小key 的处理 + - 小key + - 在实际工作中，执行`LoadFromBuffer`函数从rdb读入数据时，其对应的 value 值会流动到两个地方，一个是直接存储在缓存区中一份，用于小 key 发送时直接读取（与`restore`命令有关），一个流动到上述的中间数据结构中，被下述的 `rewrite`函数使用 + - 大key + - 借助于` rewrite` 函数，从上述的中间数据结构中读取，并拆分为对应的命令进行发送 + +![module-supported.jpg](/public/module-supported.jpg) + +### 补充命令测试 +为了确保正常，需要在` tests\helpers\commands` 里面添加对应 module 的命令，来测试相关命令可以在 rdb、sync、scan 三个模式下工作正常。测试框架具体见[pybbt](https://pypi.org/project/pybbt/)，具体思想——借助于redis-py 包，对其进行封装，模拟客户端发送命令，然后比对实际的返回值与已有的返回值。 + +### 补充命令列表 +RedisShake 在针对大 key 进行传输时，会查命令表格`RedisShake\internal\commands\table.go`，检查命令的合规性，因此在添加新 module 时，需要将对应的命令加入表格，具体可参照`RedisShake\scripts`部分代码 + +### 补充 ci +在 ci 测试中，需要添加对自定义 module 的编译，具体可见` ci.yml` 内容 + + + diff --git a/docs/src/en/reader/aof_reader.md b/docs/src/en/reader/aof_reader.md new file mode 100644 index 00000000..4d44b62f --- /dev/null +++ b/docs/src/en/reader/aof_reader.md @@ -0,0 +1,19 @@ +# aof_reader + +## Introduction + +Can use ` aof_ Reader 'to read data from the AOF file and then write it to the target end. +It is commonly used to recover data from backup files and also supports data flash back. + +## configuration + +```toml +[aof_reader] +aoffilepath="/tmp/appendonly.aof.manifest" or single-aof: "/tmp/appendonly.aof" +aoftimestamp="0" +``` + +*An absolute path should be passed in. + +##The main process is as follows: +![aof_reader.jpg](/public/aof_reader.jpg) \ No newline at end of file diff --git a/docs/src/en/reader/rdb_reader.md b/docs/src/en/reader/rdb_reader.md new file mode 100644 index 00000000..e2db7402 --- /dev/null +++ b/docs/src/en/reader/rdb_reader.md @@ -0,0 +1,14 @@ +# rdb_reader + +## 介绍 + +可以使用 `rdb_reader` 来从 RDB 文件中读取数据，然后写入目标端。常见于从备份文件中恢复数据。 + +## 配置 + +```toml +[rdb_reader] +filepath = "/tmp/dump.rdb" +``` + +* 应传入绝对路径。 diff --git a/docs/src/en/reader/scan_reader.md b/docs/src/en/reader/scan_reader.md new file mode 100644 index 00000000..ea61718e --- /dev/null +++ b/docs/src/en/reader/scan_reader.md @@ -0,0 +1,41 @@ +# Scan Reader + +## 介绍 + +::: tip +本方案为次选方案，当可以使用 [`sync_reader`](sync_reader.md) 时，请优选 [`sync_reader`](sync_reader.md)。 +::: + +`scan_reader` 通过 `SCAN` 命令遍历源端数据库中的所有 Key，并使用 `DUMP` 与 `RESTORE` 命令来读取与写入 Key 的内容。 + +注意： +1. Redis 的 `SCAN` 命令只保证 `SCAN` 的开始与结束之前均存在的 Key 一定会被返回，但是新写入的 Key 有可能会被遗漏，期间删除的 Key 也可能已经被写入目的端。可以通过 `ksn` 配置解决 +2. `SCAN` 命令与 `DUMP` 命令会占用源端数据库较多的 CPU 资源。 + + + +## 配置 + +```toml +[scan_reader] +cluster = false # set to true if source is a redis cluster +address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false +ksn = false # set to true to enabled Redis keyspace notifications (KSN) subscription +``` + +* `cluster`：源端是否为集群 +* `address`：源端地址, 当源端为集群时，`address` 为集群中的任意一个节点即可 +* 鉴权： + * 当源端使用 ACL 账号时，配置 `username` 和 `password` + * 当源端使用传统账号时，仅配置 `password` + * 当源端无鉴权时，不配置 `username` 和 `password` +* `tls`：源端是否开启 TLS/SSL，不需要配置证书因为 RedisShake 没有校验服务器证书 +* `ksn`：开启 `ksn` 参数后 RedisShake 会在 `SCAN` 之前使用 [Redis keyspace notifications](https://redis.io/docs/manual/keyspace-notifications/) +能力来订阅 Key 的变化。当 Key 发生变化时，RedisShake 会使用 `DUMP` 与 `RESTORE` 命令来从源端读取 Key 的内容，并写入目标端。 + +::: warning +Redis keyspace notifications 不会感知到 `FLUSHALL` 与 `FLUSHDB` 命令，因此在使用 `ksn` 参数时，需要确保源端数据库不会执行这两个命令。 +::: diff --git a/docs/src/en/reader/sync_reader.md b/docs/src/en/reader/sync_reader.md new file mode 100644 index 00000000..7aaf82e2 --- /dev/null +++ b/docs/src/en/reader/sync_reader.md @@ -0,0 +1,37 @@ +# Sync Reader + +## Introduction + +When the source database is compatible with the PSync protocol, `sync_reader` is recommended. Databases compatible with the PSync protocol include: + +* Redis +* Tair +* ElastiCache (partially compatible) +* MemoryDB (partially compatible) + +Advantages: Best data consistency, minimal impact on the source database, and allows for seamless switching. + +Principle: RedisShake simulates a Slave connecting to the Master node, and the Master will send data to RedisShake, which includes both full and incremental parts. The full data is an RDB file, and the incremental data is an AOF data stream. RedisShake will accept both full and incremental data and temporarily store them on the hard disk. During the full synchronization phase, RedisShake first parses the RDB file into individual Redis commands, then sends these commands to the destination. During the incremental synchronization phase, RedisShake continues to synchronize the AOF data stream to the destination. + +## Configuration + +```toml +[sync_reader] +cluster = false # set to true if source is a redis cluster +address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false +sync_rdb = true # set to false if you don't want to sync rdb +sync_aof = true # set to false if you don't want to sync aof +``` + +* `cluster`: Whether the source is a cluster +* `address`: Source address, when the source is a cluster, `address` can be set to any node in the cluster +* Authentication: + * When the source uses ACL accounts, configure `username` and `password` + * When the source uses traditional accounts, only configure `password` + * When the source does not require authentication, do not configure `username` and `password` +* `tls`: Whether the source has enabled TLS/SSL, no need to configure a certificate because RedisShake does not verify the server certificate +* `sync_rdb`: Whether to synchronize RDB, when set to false, RedisShake will skip the full synchronization phase +* `sync_aof`: Whether to synchronize AOF, when set to false, RedisShake will skip the incremental synchronization phase, at which point RedisShake will exit after the full synchronization phase is complete. \ No newline at end of file diff --git a/docs/src/en/writer/redis_writer.md b/docs/src/en/writer/redis_writer.md new file mode 100644 index 00000000..80e870e4 --- /dev/null +++ b/docs/src/en/writer/redis_writer.md @@ -0,0 +1,28 @@ +# Redis Writer + +## 介绍 + +`redis_writer` 用于将数据写入 Redis-like 数据库。 + +## 配置 + +```toml +[redis_writer] +cluster = false +address = "127.0.0.1:6379" # when cluster is true, address is one of the cluster node +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false +``` + +* `cluster`：是否为集群。 +* `address`：连接地址。当目的端为集群时，`address` 填写集群中的任意一个节点即可 +* 鉴权： + * 当使用 ACL 账号体系时，配置 `username` 和 `password` + * 当使用传统账号体系时，仅配置 `password` + * 当无鉴权时，不配置 `username` 和 `password` +* `tls`：是否开启 TLS/SSL，不需要配置证书因为 RedisShake 没有校验服务器证书 + +注意事项： +1. 当目的端为集群时，应保证源端发过来的命令满足 [Key 的哈希值属于同一个 slot](https://redis.io/docs/reference/cluster-spec/#implemented-subset)。 +2. 应尽量保证目的端版本大于等于源端版本，否则可能会出现不支持的命令。如确实需要降低版本，可以设置 `target_redis_proto_max_bulk_len` 为 0，来避免使用 `restore` 命令恢复数据。 diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 00000000..07072da1 --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,24 @@ +--- +# https://vitepress.dev/reference/default-theme-home-page +layout: home + +hero: + name: "RedisShake" + # text: "用于 Redis-like 数据库的数据迁移与处理服务" + tagline: 用于 Redis-like 数据库的数据迁移与处理服务 + actions: + - theme: brand + text: 快速上手 + link: /zh/guide/getting-started + - theme: alt + text: 什么是 RedisShake + link: /zh/guide/introduction +features: + - title: 数据迁移 + details: 支持 sync、scan 和 restore 三种数据迁移模式 + - title: 数据加工 + details: 支持使用 lua 脚本对数据进行过滤与修改 + - title: 兼容 + details: 兼容多种 Redis 部署形态，兼容主流云厂商的 Redis-like 数据库 +--- + diff --git a/docs/src/public/aof_reader.jpg b/docs/src/public/aof_reader.jpg new file mode 100644 index 00000000..1a3db65e Binary files /dev/null and b/docs/src/public/aof_reader.jpg differ diff --git a/docs/src/public/favicon.ico b/docs/src/public/favicon.ico new file mode 100644 index 00000000..8b342de0 Binary files /dev/null and b/docs/src/public/favicon.ico differ diff --git a/docs/src/public/module-supported.jpg b/docs/src/public/module-supported.jpg new file mode 100644 index 00000000..80862a24 Binary files /dev/null and b/docs/src/public/module-supported.jpg differ diff --git a/docs/src/zh/function/best_practices.md b/docs/src/zh/function/best_practices.md new file mode 100644 index 00000000..f663956a --- /dev/null +++ b/docs/src/zh/function/best_practices.md @@ -0,0 +1,99 @@ +--- +outline: deep +--- + +# 最佳实践 + +## 过滤 + +### 过滤 Key + +```lua +local prefix = "user:" +local prefix_len = #prefix + +if string.sub(KEYS[1], 1, prefix_len) ~= prefix then + return +end + +shake.call(DB, ARGV) +``` + +效果是只将 key 以 `user:` 开头的源数据写入到目标端。没有考虑 `mset` 等多 key 命令的情况。 + +### 过滤 DB + +```lua +shake.log(DB) +if DB == 0 +then + return +end +shake.call(DB, ARGV) +``` + +效果是丢弃源端 `db` 0 的数据，将其他 `db` 的数据写入到目标端。 + + +### 过滤某类数据结构 + +可以通过 `GROUP` 变量来判断数据结构类型，支持的数据结构类型有：`STRING`、`LIST`、`SET`、`ZSET`、`HASH`、`SCRIPTING` 等。 + +#### 过滤 Hash 类型数据 +```lua +if GROUP == "HASH" then + return +end +shake.call(DB, ARGV) +``` + +效果是丢弃源端的 `hash` 类型数据，将其他数据写入到目标端。 + +#### 过滤 [LUA 脚本](https://redis.io/docs/interact/programmability/eval-intro/) + +```lua +if GROUP == "SCRIPTING" then + return +end +shake.call(DB, ARGV) +``` + +效果是丢弃源端的 `lua` 脚本，将其他数据写入到目标端。常见于主从同步至集群时，存在集群不支持的 LUA 脚本。 + +## 修改 + +### 修改 Key 的前缀 + +```lua +local prefix_old = "prefix_old_" +local prefix_new = "prefix_new_" + +shake.log("old=" .. table.concat(ARGV, " ")) + +for i, index in ipairs(KEY_INDEXES) do + local key = ARGV[index] + if string.sub(key, 1, #prefix_old) == prefix_old then + ARGV[index] = prefix_new .. string.sub(key, #prefix_old + 1) + end +end + +shake.log("new=" .. table.concat(ARGV, " ")) +shake.call(DB, ARGV) +``` +效果是将源端的 key `prefix_old_key` 写入到目标端的 key `prefix_new_key`。 + +### 交换 DB + +```lua +local db1 = 1 +local db2 = 2 + +if DB == db1 then + DB = db2 +elseif DB == db2 then + DB = db1 +end +shake.call(DB, ARGV) +``` + +效果是将源端的 `db 1` 写入到目标端的 `db 2`，将源端的 `db 2` 写入到目标端的 `db 1`, 其他 `db` 不变。 \ No newline at end of file diff --git a/docs/src/zh/function/introduction.md b/docs/src/zh/function/introduction.md new file mode 100644 index 00000000..579d0eec --- /dev/null +++ b/docs/src/zh/function/introduction.md @@ -0,0 +1,56 @@ +--- +outline: deep +--- + +# 什么是 function + +RedisShake 通过提供 function 功能，实现了的 [ETL(提取-转换-加载)](https://en.wikipedia.org/wiki/Extract,_transform,_load) 中的 `transform` 能力。通过利用 function 可以实现类似功能： +* 更改数据所属的 `db`，比如将源端的 `db 0` 写入到目的端的 `db 1`。 +* 对数据进行筛选，例如，只将 key 以 `user:` 开头的源数据写入到目标端。 +* 改变 Key 的前缀，例如，将源端的 key `prefix_old_key` 写入到目标端的 key `prefix_new_key`。 +* ... + +要使用 function 功能，只需编写一份 lua 脚本。RedisShake 在从源端获取数据后，会将数据转换为 Redis 命令。然后，它会处理这些命令，从中解析出 `KEYS`、`ARGV`、`SLOTS`、`GROUP` 等信息，并将这些信息传递给 lua 脚本。lua 脚本会处理这些数据，并返回处理后的命令。最后，RedisShake 会将处理后的数据写入到目标端。 + +以下是一个具体的例子： +```toml +function = """ +shake.log(DB) +if DB == 0 +then + return +end +shake.call(DB, ARGV) +""" + +[sync_reader] +address = "127.0.0.1:6379" + +[redis_writer] +address = "127.0.0.1:6380" +``` +`DB` 是 RedisShake 提供的信息，表示当前数据所属的 db。`shake.log` 用于打印日志，`shake.call` 用于调用 Redis 命令。上述脚本的目的是丢弃源端 `db` 0 的数据，将其他 `db` 的数据写入到目标端。 + +除了 `DB`，还有其他信息如 `KEYS`、`ARGV`、`SLOTS`、`GROUP` 等，可供调用的函数有 `shake.log` 和 `shake.call`，具体请参考 [function API](#function-api)。 + +关于更多的示例，可以参考 [最佳实践](./best_practices.md)。 + +## function API + +### 变量 + +因为有些命令中含有多个 key，比如 `mset` 等命令。所以，`KEYS`、`KEY_INDEXES`、`SLOTS` 这三个变量都是数组类型。如果确认命令只有一个 key，可以直接使用 `KEYS[1]`、`KEY_INDEXES[1]`、`SLOTS[1]`。 + +| 变量 | 类型 | 示例 | 描述 | +|-|-|-|-----| +| DB | number | 1 | 命令所属的 `db` | +| GROUP | string | "LIST" | 命令所属的 `group`，符合 [Command key specifications](https://redis.io/docs/reference/key-specs/)，可以在 [commands](https://github.com/tair-opensource/RedisShake/tree/v4/scripts/commands) 中查询每个命令的 `group` 字段 | +| CMD | string | "XGROUP-DELCONSUMER" | 命令的名称 | +| KEYS | table | \{"key1", "key2"\} | 命令的所有 Key | +| KEY_INDEXES | table | \{2, 4\} | 命令的所有 Key 在 `ARGV` 中的索引 | +| SLOTS | table | \{9189, 4998\} | 当前命令的所有 Key 所属的 [slot](https://redis.io/docs/reference/cluster-spec/#key-distribution-model) | +| ARGV | table | \{"mset", "key1", "value1", "key2", "value2"\} | 命令的所有参数 | + +### 函数 +* `shake.call(DB, ARGV)`：返回一个 Redis 命令，RedisShake 会将该命令写入目标端。 +* `shake.log(msg)`：打印日志。 diff --git a/docs/src/zh/guide/config.md b/docs/src/zh/guide/config.md new file mode 100644 index 00000000..206a5575 --- /dev/null +++ b/docs/src/zh/guide/config.md @@ -0,0 +1,83 @@ +--- +outline: deep +--- + +# 配置文件 + +RedisShake 使用 [TOML](https://toml.io/cn/) 语言书写，所有的配置参数在 all.toml 中均有说明。 + +配置文件的组成如下： + +```toml +function = "..." + +[xxx_reader] +... + +[xxx_writer] +... + +[advanced] +... +``` + +一般用法下，只需要书写 `xxx_reader`、`xxx_writer` 两个部分即可，`function` 和 `advanced` 部分为进阶用法，用户可以根据自己的需求进行配置。 + +## function 配置 + +参考 [什么是 function](../function/introduction.md)。 + +## reader 配置 + +RedisShake 提供了不同的 Reader 用来对接不同的源端，配置详见 Reader 章节： + +* [Sync Reader](../reader/sync_reader.md) +* [Scan Reader](../reader/scan_reader.md) +* [RDB Reader](../reader/rdb_reader.md) +* [AOF Reader](../reader/aof_reader.md) + +## writer 配置 + +RedisShake 提供了不同的 Writer 用来对接不同的目标端，配置详见 Writer 章节： + +* [Redis Writer](../writer/redis_writer.md) + +## advanced 配置 + +```toml +[advanced] +dir = "data" +ncpu = 3 # runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores + +pprof_port = 0 # pprof port, 0 means disable +status_port = 0 # status port, 0 means disable + +# log +log_file = "shake.log" +log_level = "info" # debug, info or warn +log_interval = 5 # in seconds + +# redis-shake gets key and value from rdb file, and uses RESTORE command to +# create the key in target redis. Redis RESTORE will return a "Target key name +# is busy" error when key already exists. You can use this configuration item +# to change the default behavior of restore: +# panic: redis-shake will stop when meet "Target key name is busy" error. +# rewrite: redis-shake will replace the key with new value. +# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. +rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip + +# redis-shake uses pipeline to improve sending performance. +# This item limits the maximum number of commands in a pipeline. +pipeline_count_limit = 1024 + +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default. This amount is normally 1gb. +target_redis_client_max_querybuf_len = 1024_000_000 + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. +target_redis_proto_max_bulk_len = 512_000_000 + +# If the source is Elasticache or MemoryDB, you can set this item. +aws_psync = "" +``` \ No newline at end of file diff --git a/docs/src/zh/guide/getting-started.md b/docs/src/zh/guide/getting-started.md new file mode 100644 index 00000000..d640771f --- /dev/null +++ b/docs/src/zh/guide/getting-started.md @@ -0,0 +1,45 @@ +# 快速上手 + +## 安装 + +### 下载二进制包 + +直接从 [Release](https://github.com/tair-opensource/RedisShake/releases) 下载二进制包。 + +### 从源代码编译 + +要从源代码编译，确保您在本地机器上设置了 Golang 环境： + +```shell +git clone https://github.com/alibaba/RedisShake +cd RedisShake +sh build.sh +``` + +## 使用 + +假设你有两个 Redis 实例： + +* 实例 A：127.0.0.1:6379 +* 实例 B：127.0.0.1:6380 + +创建一个新的配置文件 `shake.toml`： + +```toml +[sync_reader] +address = "127.0.0.1:6379" + +[redis_writer] +address = "127.0.0.1:6380" +``` + +要启动 RedisShake，运行以下命令： + +```shell +./redis-shake shake.toml +``` + +## 注意事项 + +1. 不要在同一个目录运行两个 RedisShake 进程，因为运行时产生的临时文件可能会被覆盖，导致异常行为。 +2. 不要降低 Redis 版本，比如从 6.0 降到 5.0，因为 RedisShake 每个大版本都会引入一些新的命令和新的编码方式，如果降低版本，可能会导致不兼容。 diff --git a/docs/src/zh/guide/introduction.md b/docs/src/zh/guide/introduction.md new file mode 100644 index 00000000..feaaee96 --- /dev/null +++ b/docs/src/zh/guide/introduction.md @@ -0,0 +1,43 @@ +--- +outline: deep +--- + +# 什么是 RedisShake + +RedisShake 是一个用于处理和迁移 Redis 数据的工具，它提供以下特性： + +1. **Redis 兼容性**：RedisShake 兼容从 2.8 到 7.2 的 Redis 版本，并支持各种部署方式，包括单机，主从，哨兵和集群。 +2. **云服务兼容性**：RedisShake 与主流云服务提供商提供的流行 Redis-like 数据库无缝工作，包括但不限于： + - [阿里云-云数据库 Redis 版](https://www.aliyun.com/product/redis) + - [阿里云-云原生内存数据库Tair](https://www.aliyun.com/product/apsaradb/kvstore/tair) + - [AWS - ElastiCache](https://aws.amazon.com/elasticache/) + - [AWS - MemoryDB](https://aws.amazon.com/memorydb/) +3. **Module 兼容**：RedisShake + 与 [TairString](https://github.com/tair-opensource/TairString)，[TairZSet](https://github.com/tair-opensource/TairZset) + 和 [TairHash](https://github.com/tair-opensource/TairHash) 模块兼容。 +4. **多种导出模式**：RedisShake 支持 PSync，RDB 和 Scan 导出模式。 +5. **数据处理**：RedisShake 通过自定义脚本实现数据过滤和转换。 + +## 贡献 + +我们欢迎社区的贡献。对于重大变更，请先开一个 issue 来讨论你想要改变的内容。我们特别感兴趣的是： + +1. 添加对更多模块的支持 +2. 提高对 Readers 和 Writers 的支持 +3. 分享你的 Lua 脚本和最佳实践 + +## 历史 + +RedisShake 是阿里云 [Tair 团队](https://github.com/tair-opensource) +积极维护的一个项目。它的演变可以追溯到其初始版本，该版本是从 [redis-port](https://github.com/CodisLabs/redis-port) 分支出来的。 + +版本（不同版本间配置不通用）： + +- [RedisShake 2.x](https://github.com/tair-opensource/RedisShake/tree/v2) 版本带来了一系列的改进和更新，提高了其整体稳定性和性能。 +- [RedisShake 3.x](https://github.com/tair-opensource/RedisShake/tree/v3) 版本是一个重要的里程碑，整个代码库被完全重写和优化，具有更好的效率和可用性。 +- [RedisShake 4.x](https://github.com/tair-opensource/RedisShake/tree/v4) 版本 + ，进一步增强了 [Reader](../reader/scan_reader.md)、配置、可观察性和 [function](../function/introduction.md) 相关的特性。 + +## 许可证 + +RedisShake 在 [MIT 许可证](https://github.com/tair-opensource/RedisShake/blob/v2/license.txt) 下开源。 \ No newline at end of file diff --git a/docs/src/zh/guide/mode.md b/docs/src/zh/guide/mode.md new file mode 100644 index 00000000..1bb444c3 --- /dev/null +++ b/docs/src/zh/guide/mode.md @@ -0,0 +1,60 @@ +--- +outline: deep +--- + +# 迁移模式选择 + +## 概述 + +目前 RedisShake 有三种迁移模式：`PSync`、`RDB` 和 +`SCAN`，分别对应 [`sync_reader`](../reader/sync_reader.md)、[`rdb_reader`](../reader/rdb_reader.md) +和 [`scan_reader`](../reader/scan_reader.md)。 + +* 对于从备份中恢复数据的场景，可以使用 `rdb_reader`。 +* 对于数据迁移场景，优先选择 `sync_reader`。一些云厂商没有提供 PSync 协议支持，可以选择`scan_reader`。 +* 对于长期的数据同步场景，RedisShake 目前没有能力承接，因为 PSync 协议并不可靠，当复制连接断开时，RedisShake 将无法重新连接至源端数据库。如果对于可用性要求不高，可以使用 `scan_reader`。如果写入量不大，且不存在大 key，也可以考虑 `scan_reader`。 + +不同模式各有优缺点，需要查看各 Reader 章节了解更多信息。 + +## Redis Cluster 架构 + +当源端 Redis 以 cluster 架构部署时，可以使用 `sync_reader` 或者 `scan_reader`。两者配置项中均有开关支持开启 cluster 模式，会通过 `cluster nodes` 命令自动获取集群中的所有节点，并建立连接。 + +## Redis Sentinel 架构 + +当源端 Redis 以 sentinel 架构部署且 RedisShake 使用 `sync_reader` 连接主库时，会被主库当做 slave，从而有可能被 sentinel 选举为新的 master。 + +为了避免这种情况，应选择备库作为源端。 + +## 云 Redis 服务 + +主流云厂商都提供了 Redis 服务，不过有几个原因导致在这些服务上使用 RedisShake 较为复杂： +1. 引擎限制。存在一些自研的 Redis-like 数据库没有兼容 PSync 协议。 +2. 架构限制。较多云厂商支持代理模式，即在用户与 Redis 服务之间增加 Proxy 组件。因为 Proxy 组件的存在，所以 PSync 协议无法支持。 +3. 安全限制。在原生 Redis 中 PSync 协议基本会触发 fork(2)，会导致内存膨胀与用户请求延迟增加，较坏情况下甚至会发生 out of memory。尽管这些都有方案缓解，但并不是所有云厂商都有这方面的投入。 +4. 商业策略。较多用户使用 RedisShake 是为了下云或者换云，所以部分云厂商并不希望用户使用 RedisShake，从而屏蔽了 PSync 协议。 + +下文会结合实践经验，介绍一些特殊场景下的 RedisShake 使用方案。 + +### 阿里云「云数据库 Redis」与「云原生内存数据库Tair」 + +「云数据库 Redis」与「云原生内存数据库Tair」都支持 PSync 协议，推荐使用 `sync_reader`。用户需要创建一个具有复制权限的账号（可以执行 PSync 命令），RedisShake 使用该账号进行数据同步，具体创建步骤见 [创建与管理账号](https://help.aliyun.com/zh/redis/user-guide/create-and-manage-database-accounts)。 + +例外情况： +1. 2.8 版本的 Redis 实例不支持创建复制权限的账号，需要 [升级大版本](https://help.aliyun.com/zh/redis/user-guide/upgrade-the-major-version-1)。 +2. 集群架构的 Reids 与 Tair 实例在 [代理模式](https://help.aliyun.com/zh/redis/product-overview/cluster-master-replica-instances#section-h69-izd-531) 下不支持 PSync 协议。 +3. 读写分离架构不支持 PSync 协议。 + +在不支持 PSync 协议的场景下，可以使用 `scan_reader`。需要注意的是，`scan_reader` 会对源库造成较大的压力。 + +### AWS ElastiCache and MemoryDB + +优选 `sync_reader`, AWS ElastiCache and MemoryDB 默认情况下没有开启 PSync 协议，但是可以通过提交工单的方式请求开启 PSync 协议。AWS 会在工单中给出一份重命名的 PSync 命令，比如 `xhma21yfkssync` 和 `nmfu2bl5osync`。此命令效果等同于 `psync` 命令，只是名字不一样。 +用户修改 RedisShake 配置文件中的 `aws_psync` 配置项即可。对于单实例只写一对 `ip:port@cmd` 即可，对于集群实例，需要写上所有的 `ip:port@cmd`，以逗号分隔。 + +不方便提交工单时，可以使用 `scan_reader`。需要注意的是，`scan_reader` 会对源库造成较大的压力。 + + + + + diff --git a/docs/src/zh/others/modules.md b/docs/src/zh/others/modules.md new file mode 100644 index 00000000..6309515c --- /dev/null +++ b/docs/src/zh/others/modules.md @@ -0,0 +1,43 @@ +--- +outline: deep +--- + +# Redis Modules + +Redis Modules 是 Redis 4.0 版本引入的一个新特性，它允许开发者扩展 Redis 的功能。通过创建模块，开发者可以定义新的命令，数据类型，甚至改变 Redis 的行为。因此，Redis Modules 可以极大地增强 Redis 的灵活性和可扩展性。 + +由于 Redis Modules 可以定义新的数据类型和命令，RedisShake 需要对这些新的数据类型和命令进行专门的处理，才能正确地迁移或同步这些数据。否则，如果 RedisShake 不理解这些新的数据类型和命令，它可能会无法正确地处理这些数据，或者在处理过程中出错。因此，对于使用了 Redis Modules 的 Redis 实例，一般需要 RedisShake 为其使用的 Module 提供相应的适配器，以便正确地处理这些自定义的数据类型和命令。 + +## 已支持的 Redis Modules 列表 + +- [TairHash](https://github.com/tair-opensource/TairHash)：支持 field 级别设置过期和版本的 Hash 数据结构。 +- [TairString](https://github.com/tair-opensource/TairString)：支持版本的 String 结构，可以实现分布式锁/乐观锁。 +- [TairZset](https://github.com/tair-opensource/TairZset)：支持最多 256 维的 double 排序，可以实现多维排行榜。 + +## 如何支持新的 Redis Modules + +### 核心流程 + +相关代码在`internal\rdb`目录下，如需要支持其它 Redis Modules 类型，可分解为以下三个步骤： +- 从rdb文件中正确读入 + - RedisShake 中已经 对 redis module 自定的几种类型进行了封装，从 rdb 文件进行读取时，可直接借助于已经封装好的函数进行读取（`internal\rdb\structure\module2_struct.go`） +- 构建一个合适的中间数据结构类型，用于存储相应数据（key + value） +- 大小key 的处理 + - 小key + - 在实际工作中，执行`LoadFromBuffer`函数从rdb读入数据时，其对应的 value 值会流动到两个地方，一个是直接存储在缓存区中一份，用于小 key 发送时直接读取（与`restore`命令有关），一个流动到上述的中间数据结构中，被下述的 `rewrite`函数使用 + - 大key + - 借助于` rewrite` 函数，从上述的中间数据结构中读取，并拆分为对应的命令进行发送 + +![module-supported.jpg](/public/module-supported.jpg) + +### 补充命令测试 +为了确保正常，需要在` tests\helpers\commands` 里面添加对应 module 的命令，来测试相关命令可以在 rdb、sync、scan 三个模式下工作正常。测试框架具体见[pybbt](https://pypi.org/project/pybbt/)，具体思想——借助于redis-py 包，对其进行封装，模拟客户端发送命令，然后比对实际的返回值与已有的返回值。 + +### 补充命令列表 +RedisShake 在针对大 key 进行传输时，会查命令表格`RedisShake\internal\commands\table.go`，检查命令的合规性，因此在添加新 module 时，需要将对应的命令加入表格，具体可参照`RedisShake\scripts`部分代码 + +### 补充 ci +在 ci 测试中，需要添加对自定义 module 的编译，具体可见` ci.yml` 内容 + + + diff --git a/docs/src/zh/reader/aof_reader.md b/docs/src/zh/reader/aof_reader.md new file mode 100644 index 00000000..13182de4 --- /dev/null +++ b/docs/src/zh/reader/aof_reader.md @@ -0,0 +1,18 @@ +# aof_reader + +## 介绍 + +可以使用 `aof_reader` 来从 AOF 文件中读取数据，然后写入目标端。常见于从备份文件中恢复数据，还支持数据闪回。 + +## 配置 + +```toml +[aof_reader] +aoffilepath="/tmp/appendonly.aof.manifest" 或者单aof文件 "/tmp/appendonly.aof" +aoftimestamp="0" +``` + +* 应传入绝对路径。 + +## 主要流程如下： +![aof_reader.jpg](/public/aof_reader.jpg) \ No newline at end of file diff --git a/docs/src/zh/reader/rdb_reader.md b/docs/src/zh/reader/rdb_reader.md new file mode 100644 index 00000000..e2db7402 --- /dev/null +++ b/docs/src/zh/reader/rdb_reader.md @@ -0,0 +1,14 @@ +# rdb_reader + +## 介绍 + +可以使用 `rdb_reader` 来从 RDB 文件中读取数据，然后写入目标端。常见于从备份文件中恢复数据。 + +## 配置 + +```toml +[rdb_reader] +filepath = "/tmp/dump.rdb" +``` + +* 应传入绝对路径。 diff --git a/docs/src/zh/reader/scan_reader.md b/docs/src/zh/reader/scan_reader.md new file mode 100644 index 00000000..ea61718e --- /dev/null +++ b/docs/src/zh/reader/scan_reader.md @@ -0,0 +1,41 @@ +# Scan Reader + +## 介绍 + +::: tip +本方案为次选方案，当可以使用 [`sync_reader`](sync_reader.md) 时，请优选 [`sync_reader`](sync_reader.md)。 +::: + +`scan_reader` 通过 `SCAN` 命令遍历源端数据库中的所有 Key，并使用 `DUMP` 与 `RESTORE` 命令来读取与写入 Key 的内容。 + +注意： +1. Redis 的 `SCAN` 命令只保证 `SCAN` 的开始与结束之前均存在的 Key 一定会被返回，但是新写入的 Key 有可能会被遗漏，期间删除的 Key 也可能已经被写入目的端。可以通过 `ksn` 配置解决 +2. `SCAN` 命令与 `DUMP` 命令会占用源端数据库较多的 CPU 资源。 + + + +## 配置 + +```toml +[scan_reader] +cluster = false # set to true if source is a redis cluster +address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false +ksn = false # set to true to enabled Redis keyspace notifications (KSN) subscription +``` + +* `cluster`：源端是否为集群 +* `address`：源端地址, 当源端为集群时，`address` 为集群中的任意一个节点即可 +* 鉴权： + * 当源端使用 ACL 账号时，配置 `username` 和 `password` + * 当源端使用传统账号时，仅配置 `password` + * 当源端无鉴权时，不配置 `username` 和 `password` +* `tls`：源端是否开启 TLS/SSL，不需要配置证书因为 RedisShake 没有校验服务器证书 +* `ksn`：开启 `ksn` 参数后 RedisShake 会在 `SCAN` 之前使用 [Redis keyspace notifications](https://redis.io/docs/manual/keyspace-notifications/) +能力来订阅 Key 的变化。当 Key 发生变化时，RedisShake 会使用 `DUMP` 与 `RESTORE` 命令来从源端读取 Key 的内容，并写入目标端。 + +::: warning +Redis keyspace notifications 不会感知到 `FLUSHALL` 与 `FLUSHDB` 命令，因此在使用 `ksn` 参数时，需要确保源端数据库不会执行这两个命令。 +::: diff --git a/docs/src/zh/reader/sync_reader.md b/docs/src/zh/reader/sync_reader.md new file mode 100644 index 00000000..5b624a3a --- /dev/null +++ b/docs/src/zh/reader/sync_reader.md @@ -0,0 +1,37 @@ +# Sync Reader + +## 介绍 + +当源端数据库兼容 PSync 协议时，推荐使用 `sync_reader`。兼容 PSync 协议的数据库有： + +* Redis +* Tair +* ElastiCache 部分兼容 +* MemoryDB 部分兼容 + +优势：数据一致性最佳，对源库影响小，可以实现不停机的切换 + +原理：RedisShake 模拟 Slave 连接到 Master 节点，Master 会向 RedisShake 发送数据，数据包含全量与增量两部分。全量是一个 RDB 文件，增量是 AOF 数据流，RedisShake 会接受全量与增量将其暂存到硬盘上。全量同步阶段：RedisShake 首先会将 RDB 文件解析为一条条的 Redis 命令，然后将这些命令发送至目的端。增量同步阶段：RedisShake 会持续将 AOF 数据流同步至目的端。 + +## 配置 + +```toml +[sync_reader] +cluster = false # set to true if source is a redis cluster +address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false +sync_rdb = true # set to false if you don't want to sync rdb +sync_aof = true # set to false if you don't want to sync aof +``` + +* `cluster`：源端是否为集群 +* `address`：源端地址, 当源端为集群时，`address` 为集群中的任意一个节点即可 +* 鉴权： + * 当源端使用 ACL 账号时，配置 `username` 和 `password` + * 当源端使用传统账号时，仅配置 `password` + * 当源端无鉴权时，不配置 `username` 和 `password` +* `tls`：源端是否开启 TLS/SSL，不需要配置证书因为 RedisShake 没有校验服务器证书 +* `sync_rdb`：是否同步 RDB，设置为 false 时，RedisShake 会跳过全量同步阶段 +* `sync_aof`：是否同步 AOF，设置为 false 时，RedisShake 会跳过增量同步阶段，此时 RedisShake 会在全量同步阶段结束后退出 \ No newline at end of file diff --git a/docs/src/zh/writer/redis_writer.md b/docs/src/zh/writer/redis_writer.md new file mode 100644 index 00000000..80e870e4 --- /dev/null +++ b/docs/src/zh/writer/redis_writer.md @@ -0,0 +1,28 @@ +# Redis Writer + +## 介绍 + +`redis_writer` 用于将数据写入 Redis-like 数据库。 + +## 配置 + +```toml +[redis_writer] +cluster = false +address = "127.0.0.1:6379" # when cluster is true, address is one of the cluster node +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false +``` + +* `cluster`：是否为集群。 +* `address`：连接地址。当目的端为集群时，`address` 填写集群中的任意一个节点即可 +* 鉴权： + * 当使用 ACL 账号体系时，配置 `username` 和 `password` + * 当使用传统账号体系时，仅配置 `password` + * 当无鉴权时，不配置 `username` 和 `password` +* `tls`：是否开启 TLS/SSL，不需要配置证书因为 RedisShake 没有校验服务器证书 + +注意事项： +1. 当目的端为集群时，应保证源端发过来的命令满足 [Key 的哈希值属于同一个 slot](https://redis.io/docs/reference/cluster-spec/#implemented-subset)。 +2. 应尽量保证目的端版本大于等于源端版本，否则可能会出现不支持的命令。如确实需要降低版本，可以设置 `target_redis_proto_max_bulk_len` 为 0，来避免使用 `restore` 命令恢复数据。 diff --git a/filters/aliyun.lua b/filters/aliyun.lua deleted file mode 100644 index 8ab684b6..00000000 --- a/filters/aliyun.lua +++ /dev/null @@ -1,8 +0,0 @@ --- Aliyun Redis 4.0: skip OPINFO command -function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) - if cmd_name == "OPINFO" then - return 1, db_id -- disallow - else - return 0, db_id -- allow - end -end \ No newline at end of file diff --git a/filters/aws.lua b/filters/aws.lua deleted file mode 100644 index 23101c39..00000000 --- a/filters/aws.lua +++ /dev/null @@ -1,8 +0,0 @@ --- ElastiCache: skip REPLCONF command -function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) - if cmd_name == "REPLCONF" then - return 1, db_id -- disallow - else - return 0, db_id -- allow - end -end \ No newline at end of file diff --git a/filters/key_prefix.lua b/filters/key_prefix.lua deleted file mode 100644 index e17d18c2..00000000 --- a/filters/key_prefix.lua +++ /dev/null @@ -1,12 +0,0 @@ --- skip keys prefixed with ABC -function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) - if #keys ~= 1 then - return 0, db_id -- allow - end - - if string.sub(keys[1], 0, 3) == "ABC" then - return 1, db_id -- disallow - end - - return 0, db_id -- allow -end \ No newline at end of file diff --git a/filters/print.lua b/filters/print.lua deleted file mode 100644 index ff9e7fc4..00000000 --- a/filters/print.lua +++ /dev/null @@ -1,24 +0,0 @@ ---- function name must be `filter` ---- ---- arguments: ---- @id number: the sequence of the cmd ---- @is_base boolean: whether the command is decoded from dump.rdb file ---- @group string: the group of cmd ---- @cmd_name string: cmd name ---- @keys table: keys of the command ---- @slots table: slots of the command ---- @db_id: database id ---- @timestamp_ms number: timestamp in milliseconds, 0 if not available - ---- return: ---- @code number: ---- * 0: allow ---- * 1: disallow ---- * 2: error occurred ---- @db_id number: redirection database id - -function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) - print(string.format("lua filter. id=[%d], is_base=[%s], db_id=[%d], group=[%s], cmd_name=[%s], keys=[%s], slots=[%s], timestamp_ms=[%d]", - id, tostring(is_base), db_id, group, cmd_name, table.concat(keys, ", "), table.concat(slots, ", "), timestamp_ms)) - return 0, db_id -end \ No newline at end of file diff --git a/filters/skip_scripts.lua b/filters/skip_scripts.lua deleted file mode 100644 index 7e0dafad..00000000 --- a/filters/skip_scripts.lua +++ /dev/null @@ -1,8 +0,0 @@ --- skip all scripts included LUA scripts and Redis Functions. -function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) - if group == "SCRIPTING" then - return 1, db_id -- disallow - else - return 0, db_id -- allow - end -end \ No newline at end of file diff --git a/filters/swap_db.lua b/filters/swap_db.lua deleted file mode 100644 index 2cc6b1e9..00000000 --- a/filters/swap_db.lua +++ /dev/null @@ -1,12 +0,0 @@ ---- dbid: 0 -> 1 ---- dbid: 1 -> 0 ---- dbid: others -> drop -function filter(id, is_base, group, cmd_name, keys, slots, db_id, timestamp_ms) - if db_id == 0 then - return 0, 1 - elseif db_id == 1 then - return 0, 0 - else - return 1, db_id - end -end \ No newline at end of file diff --git a/go.mod b/go.mod index 4dc27cf2..03f9bdc3 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,33 @@ -module github.com/alibaba/RedisShake +module RedisShake -go 1.17 +go 1.20 require ( - github.com/pelletier/go-toml/v2 v2.0.0-beta.3 + github.com/dustin/go-humanize v1.0.1 + github.com/go-stack/stack v1.8.1 + github.com/mcuadros/go-defaults v1.2.0 github.com/rs/zerolog v1.28.0 + github.com/spf13/viper v1.15.0 + github.com/theckman/go-flock v0.8.1 github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/spf13/afero v1.9.3 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.4.2 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.12.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 55959bc7..2d2336c2 100644 --- a/go.sum +++ b/go.sum @@ -1,33 +1,503 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/pelletier/go-toml/v2 v2.0.0-beta.3 h1:PNCTU4naEJ8mKal97P3A2qDU74QRQGlv4FXiL1XDqi4= -github.com/pelletier/go-toml/v2 v2.0.0-beta.3/go.mod h1:aNseLYu/uKskg0zpr/kbr2z8yGuWtotWf/0BpGIAL2Y= +github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk56oZtc= +github.com/mcuadros/go-defaults v1.2.0/go.mod h1:WEZtHEVIGYVDqkKSWBdWKUVdRyKlMfulPaGDWIVeCWY= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942 h1:t0lM6y/M5IiUZyvbBTcngso8SZEZICH7is9B6g/obVU= -github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/theckman/go-flock v0.8.1 h1:kTixuOsFBOtGYSTLRLWK6GOs1hk/8OD11sR1pDd0dl4= +github.com/theckman/go-flock v0.8.1/go.mod h1:kjuth3y9VJ2aNlkNEO99G/8lp9fMIKaGyBmh84IBheM= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= -golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/aof/aof.go b/internal/aof/aof.go new file mode 100644 index 00000000..78625cde --- /dev/null +++ b/internal/aof/aof.go @@ -0,0 +1,155 @@ +package aof + +import ( + "bufio" + "io" + "os" + "strconv" + "strings" + + "RedisShake/internal/entry" + "RedisShake/internal/log" +) + +const ( + AOFNotExist = 1 + AOFOpenErr = 3 + AOFOK = 0 + AOFEmpty = 2 + AOFFailed = 4 + AOFTruncated = 5 + SizeMax = 128 +) + +type Loader struct { + filPath string + ch chan *entry.Entry +} + +func NewLoader(filPath string, ch chan *entry.Entry) *Loader { + ld := new(Loader) + ld.ch = ch + ld.filPath = filPath + return ld +} + +func ReadCompleteLine(reader *bufio.Reader) ([]byte, error) { + line, isPrefix, err := reader.ReadLine() + if err != nil { + return nil, err + } + + for isPrefix { + var additional []byte + additional, isPrefix, err = reader.ReadLine() + if err != nil { + return nil, err + } + line = append(line, additional...) + } + + return line, err +} + +func (ld *Loader) LoadSingleAppendOnlyFile(AOFTimeStamp int64) int { + ret := AOFOK + AOFFilepath := ld.filPath + fp, err := os.Open(AOFFilepath) + if err != nil { + if os.IsNotExist(err) { + if _, err := os.Stat(AOFFilepath); err == nil || !os.IsNotExist(err) { + log.Infof("Fatal error: can't open the append log File %v for reading: %v", AOFFilepath, err.Error()) + return AOFOpenErr + } else { + log.Infof("The append log File %v doesn't exist: %v", AOFFilepath, err.Error()) + return AOFNotExist + } + + } + defer fp.Close() + + stat, _ := fp.Stat() + if stat.Size() == 0 { + return AOFEmpty + } + } + reader := bufio.NewReader(fp) + for { + + line, err := ReadCompleteLine(reader) + { + if err != nil { + if err == io.EOF { + break + } else { + log.Infof("Unrecoverable error reading the append only File %v: %v", AOFFilepath, err) + ret = AOFFailed + return ret + } + } else { + _, errs := fp.Seek(0, io.SeekCurrent) + if errs != nil { + log.Infof("Unrecoverable error reading the append only File %v: %v", AOFFilepath, errs) + ret = AOFFailed + return ret + } + } + + if line[0] == '#' { + if AOFTimeStamp != 0 && strings.HasPrefix(string(line), "#TS:") { + var ts int64 + ts, err = strconv.ParseInt(strings.TrimPrefix(string(line), "#TS:"), 10, 64) + if err != nil { + log.Panicf("Invalid timestamp annotation") + } + + if ts > AOFTimeStamp { + ret = AOFTruncated + log.Infof("Reached recovery timestamp: %s, subsequent data will no longer be read.", line) + return ret + } + } + continue + } + if line[0] != '*' { + log.Panicf("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", AOFFilepath) + } + argc, _ := strconv.ParseInt(string(line[1:]), 10, 64) + if argc < 1 { + log.Panicf("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", AOFFilepath) + } + if argc > int64(SizeMax) { + log.Panicf("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", AOFFilepath) + } + e := entry.NewEntry() + var argv []string + + for j := 0; j < int(argc); j++ { + line, err := ReadCompleteLine(reader) + if err != nil || line[0] != '$' { + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", AOFFilepath) + ret = AOFFailed + return ret + } + v64, _ := strconv.ParseInt(string(line[1:]), 10, 64) + argString := make([]byte, v64+2) + argString, err = ReadCompleteLine(reader) + if err != nil { + log.Infof("Unrecoverable error reading the append only File %v: %v", AOFFilepath, err) + ret = AOFFailed + return ret + } + argString = argString[:v64] + argv = append(argv, string(argString)) + } + + for _, value := range argv { + e.Argv = append(e.Argv, value) + } + ld.ch <- e + + } + + } + return ret +} diff --git a/internal/client/func.go b/internal/client/func.go index dcd0c1e0..9195683a 100644 --- a/internal/client/func.go +++ b/internal/client/func.go @@ -1,9 +1,10 @@ package client import ( + "RedisShake/internal/client/proto" + "RedisShake/internal/log" "bytes" - "github.com/alibaba/RedisShake/internal/client/proto" - "github.com/alibaba/RedisShake/internal/log" + "strings" ) func EncodeArgv(argv []string, buf *bytes.Buffer) { @@ -15,6 +16,12 @@ func EncodeArgv(argv []string, buf *bytes.Buffer) { } err := writer.WriteArgs(argvInterface) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } } + +// IsCluster is for determining whether the server is in cluster mode. +func (r *Redis) IsCluster() bool { + reply := r.DoWithStringReply("INFO", "Cluster") + return strings.Contains(reply, "cluster_enabled:1") +} diff --git a/internal/client/redis.go b/internal/client/redis.go index 4d4d567c..5c1cb463 100644 --- a/internal/client/redis.go +++ b/internal/client/redis.go @@ -1,10 +1,10 @@ package client import ( + "RedisShake/internal/client/proto" + "RedisShake/internal/log" "bufio" "crypto/tls" - "github.com/alibaba/RedisShake/internal/client/proto" - "github.com/alibaba/RedisShake/internal/log" "net" "strconv" "time" @@ -17,19 +17,19 @@ type Redis struct { protoWriter *proto.Writer } -func NewRedisClient(address string, username string, password string, isTls bool) *Redis { +func NewRedisClient(address string, username string, password string, Tls bool) *Redis { r := new(Redis) var conn net.Conn var dialer net.Dialer var err error dialer.Timeout = 3 * time.Second - if isTls { + if Tls { conn, err = tls.DialWithDialer(&dialer, "tcp", address, &tls.Config{InsecureSkipVerify: true}) } else { conn, err = dialer.Dial("tcp", address) } if err != nil { - log.PanicError(err) + log.Panicf("dial failed. address=[%s], tls=[%v], err=[%v]", address, Tls, err) } r.reader = bufio.NewReader(conn) @@ -48,14 +48,10 @@ func NewRedisClient(address string, username string, password string, isTls bool if reply != "OK" { log.Panicf("auth failed with reply: %s", reply) } - log.Infof("auth successful. address=[%s]", address) - } else { - log.Infof("no password. address=[%s]", address) } // ping to test connection reply := r.DoWithStringReply("ping") - if reply != "PONG" { panic("ping failed with reply: " + reply) } @@ -68,12 +64,22 @@ func (r *Redis) DoWithStringReply(args ...string) string { replyInterface, err := r.Receive() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } reply := replyInterface.(string) return reply } +func (r *Redis) Do(args ...string) interface{} { + r.Send(args...) + + reply, err := r.Receive() + if err != nil { + log.Panicf(err.Error()) + } + return reply +} + func (r *Redis) Send(args ...string) { argsInterface := make([]interface{}, len(args)) for inx, item := range args { @@ -81,7 +87,7 @@ func (r *Redis) Send(args ...string) { } err := r.protoWriter.WriteArgs(argsInterface) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.flush() } @@ -89,7 +95,7 @@ func (r *Redis) Send(args ...string) { func (r *Redis) SendBytes(buf []byte) { _, err := r.writer.Write(buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.flush() } @@ -97,7 +103,7 @@ func (r *Redis) SendBytes(buf []byte) { func (r *Redis) flush() { err := r.writer.Flush() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } } @@ -105,6 +111,14 @@ func (r *Redis) Receive() (interface{}, error) { return r.protoReader.ReadReply() } +func (r *Redis) ReceiveString() string { + reply, err := r.Receive() + if err != nil { + log.Panicf(err.Error()) + } + return reply.(string) +} + func (r *Redis) BufioReader() *bufio.Reader { return r.reader } @@ -120,7 +134,7 @@ func (r *Redis) Scan(cursor uint64) (newCursor uint64, keys []string) { r.Send("scan", strconv.FormatUint(cursor, 10), "count", "2048") reply, err := r.Receive() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } array := reply.([]interface{}) @@ -131,7 +145,7 @@ func (r *Redis) Scan(cursor uint64) (newCursor uint64, keys []string) { // cursor newCursor, err = strconv.ParseUint(array[0].(string), 10, 64) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } // keys keys = make([]string, 0) diff --git a/internal/client/reply.go b/internal/client/reply.go index ef385e71..b9a00cb2 100644 --- a/internal/client/reply.go +++ b/internal/client/reply.go @@ -1,10 +1,10 @@ package client -import "github.com/alibaba/RedisShake/internal/log" +import "RedisShake/internal/log" func ArrayString(replyInterface interface{}, err error) []string { if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } replyArray := replyInterface.([]interface{}) replyArrayString := make([]string, len(replyArray)) diff --git a/internal/commands/keys.go b/internal/commands/keys.go index 0899e055..7d13d21d 100644 --- a/internal/commands/keys.go +++ b/internal/commands/keys.go @@ -1,16 +1,16 @@ package commands import ( + "RedisShake/internal/log" + "RedisShake/internal/utils" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/utils" "math" "strconv" "strings" ) // CalcKeys https://redis.io/docs/reference/key-specs/ -func CalcKeys(argv []string) (cmaName string, group string, keys []string) { +func CalcKeys(argv []string) (cmaName string, group string, keys []string, keysIndexes []int) { argc := len(argv) group = "unknown" cmaName = strings.ToUpper(argv[0]) @@ -64,6 +64,7 @@ func CalcKeys(argv []string) (cmaName string, group string, keys []string) { keyStep := spec.findKeysRangeKeyStep for inx := begin; inx <= lastKeyInx && limitCount > 0; inx += keyStep { keys = append(keys, argv[inx]) + keysIndexes = append(keysIndexes, inx+1) limitCount -= 1 } case "keynum": @@ -73,12 +74,13 @@ func CalcKeys(argv []string) (cmaName string, group string, keys []string) { } keyCount, err := strconv.Atoi(argv[keynumIdx]) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } firstKey := spec.findKeysKeynumFirstKey step := spec.findKeysKeynumKeyStep for inx := begin + firstKey; keyCount > 0; inx += step { keys = append(keys, argv[inx]) + keysIndexes = append(keysIndexes, inx+1) keyCount -= 1 } default: diff --git a/internal/commands/keys_test.go b/internal/commands/keys_test.go index 84ba5050..e6c7bd74 100644 --- a/internal/commands/keys_test.go +++ b/internal/commands/keys_test.go @@ -18,25 +18,25 @@ func testEq(a, b []string) bool { func TestCalcKeys(t *testing.T) { // SET - cmd, group, keys := CalcKeys([]string{"SET", "key", "value"}) + cmd, group, keys, _ := CalcKeys([]string{"SET", "key", "value"}) if cmd != "SET" || group != "STRING" || !testEq(keys, []string{"key"}) { t.Errorf("CalcKeys(SET key value) failed. cmd=%s, group=%s, keys=%v", cmd, group, keys) } // MSET - cmd, group, keys = CalcKeys([]string{"MSET", "key1", "value1", "key2", "value2"}) + cmd, group, keys, _ = CalcKeys([]string{"MSET", "key1", "value1", "key2", "value2"}) if cmd != "MSET" || group != "STRING" || !testEq(keys, []string{"key1", "key2"}) { t.Errorf("CalcKeys(MSET key1 value1 key2 value2) failed. cmd=%s, group=%s, keys=%v", cmd, group, keys) } // XADD - cmd, group, keys = CalcKeys([]string{"XADD", "key", "*", "field1", "value1", "field2", "value2"}) + cmd, group, keys, _ = CalcKeys([]string{"XADD", "key", "*", "field1", "value1", "field2", "value2"}) if cmd != "XADD" || group != "STREAM" || !testEq(keys, []string{"key"}) { t.Errorf("CalcKeys(XADD key * field1 value1 field2 value2) failed. cmd=%s, group=%s, keys=%v", cmd, group, keys) } // ZUNIONSTORE - cmd, group, keys = CalcKeys([]string{"ZUNIONSTORE", "key", "2", "key1", "key2"}) + cmd, group, keys, _ = CalcKeys([]string{"ZUNIONSTORE", "key", "2", "key1", "key2"}) if cmd != "ZUNIONSTORE" || group != "SORTED_SET" || !testEq(keys, []string{"key", "key1", "key2"}) { t.Errorf("CalcKeys(ZUNIONSTORE key 2 key1 key2) failed. cmd=%s, group=%s, keys=%v", cmd, group, keys) } diff --git a/internal/commands/table.go b/internal/commands/table.go index 686e9ade..41ce47e8 100644 --- a/internal/commands/table.go +++ b/internal/commands/table.go @@ -1,26 +1,628 @@ package commands var containers = map[string]bool{ - "XINFO": true, + "ACL": true, + "CLIENT": true, + "CLUSTER": true, "COMMAND": true, - "FUNCTION": true, "CONFIG": true, - "MODULE": true, - "MEMORY": true, + "FUNCTION": true, "LATENCY": true, - "SCRIPT": true, - "ACL": true, - "CLUSTER": true, - "CLIENT": true, - "XGROUP": true, - "PUBSUB": true, + "MEMORY": true, + "MODULE": true, "OBJECT": true, + "PUBSUB": true, + "SCRIPT": true, "SENTINEL": true, "SLOWLOG": true, + "XGROUP": true, + "XINFO": true, } var redisCommands = map[string]redisCommand{ - "LLEN": { - "LIST", + "ACL-CAT": { + "SERVER", + []keySpec{}, + }, + "ACL-DELUSER": { + "SERVER", + []keySpec{}, + }, + "ACL-DRYRUN": { + "SERVER", + []keySpec{}, + }, + "ACL-GENPASS": { + "SERVER", + []keySpec{}, + }, + "ACL-GETUSER": { + "SERVER", + []keySpec{}, + }, + "ACL-HELP": { + "SERVER", + []keySpec{}, + }, + "ACL-LIST": { + "SERVER", + []keySpec{}, + }, + "ACL-LOAD": { + "SERVER", + []keySpec{}, + }, + "ACL-LOG": { + "SERVER", + []keySpec{}, + }, + "ACL-SAVE": { + "SERVER", + []keySpec{}, + }, + "ACL-SETUSER": { + "SERVER", + []keySpec{}, + }, + "ACL-USERS": { + "SERVER", + []keySpec{}, + }, + "ACL-WHOAMI": { + "SERVER", + []keySpec{}, + }, + "ACL": { + "SERVER", + []keySpec{}, + }, + "BGREWRITEAOF": { + "SERVER", + []keySpec{}, + }, + "BGSAVE": { + "SERVER", + []keySpec{}, + }, + "COMMAND-COUNT": { + "SERVER", + []keySpec{}, + }, + "COMMAND-DOCS": { + "SERVER", + []keySpec{}, + }, + "COMMAND-GETKEYS": { + "SERVER", + []keySpec{}, + }, + "COMMAND-GETKEYSANDFLAGS": { + "SERVER", + []keySpec{}, + }, + "COMMAND-HELP": { + "SERVER", + []keySpec{}, + }, + "COMMAND-INFO": { + "SERVER", + []keySpec{}, + }, + "COMMAND-LIST": { + "SERVER", + []keySpec{}, + }, + "COMMAND": { + "SERVER", + []keySpec{}, + }, + "CONFIG-GET": { + "SERVER", + []keySpec{}, + }, + "CONFIG-HELP": { + "SERVER", + []keySpec{}, + }, + "CONFIG-RESETSTAT": { + "SERVER", + []keySpec{}, + }, + "CONFIG-REWRITE": { + "SERVER", + []keySpec{}, + }, + "CONFIG-SET": { + "SERVER", + []keySpec{}, + }, + "CONFIG": { + "SERVER", + []keySpec{}, + }, + "DBSIZE": { + "SERVER", + []keySpec{}, + }, + "DEBUG": { + "SERVER", + []keySpec{}, + }, + "FAILOVER": { + "SERVER", + []keySpec{}, + }, + "FLUSHALL": { + "SERVER", + []keySpec{}, + }, + "FLUSHDB": { + "SERVER", + []keySpec{}, + }, + "INFO": { + "SERVER", + []keySpec{}, + }, + "LASTSAVE": { + "SERVER", + []keySpec{}, + }, + "LATENCY-DOCTOR": { + "SERVER", + []keySpec{}, + }, + "LATENCY-GRAPH": { + "SERVER", + []keySpec{}, + }, + "LATENCY-HELP": { + "SERVER", + []keySpec{}, + }, + "LATENCY-HISTOGRAM": { + "SERVER", + []keySpec{}, + }, + "LATENCY-HISTORY": { + "SERVER", + []keySpec{}, + }, + "LATENCY-LATEST": { + "SERVER", + []keySpec{}, + }, + "LATENCY-RESET": { + "SERVER", + []keySpec{}, + }, + "LATENCY": { + "SERVER", + []keySpec{}, + }, + "LOLWUT": { + "SERVER", + []keySpec{}, + }, + "MEMORY-DOCTOR": { + "SERVER", + []keySpec{}, + }, + "MEMORY-HELP": { + "SERVER", + []keySpec{}, + }, + "MEMORY-MALLOC-STATS": { + "SERVER", + []keySpec{}, + }, + "MEMORY-PURGE": { + "SERVER", + []keySpec{}, + }, + "MEMORY-STATS": { + "SERVER", + []keySpec{}, + }, + "MEMORY-USAGE": { + "SERVER", + []keySpec{ + { + "index", + 2, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "MEMORY": { + "SERVER", + []keySpec{}, + }, + "MODULE-HELP": { + "SERVER", + []keySpec{}, + }, + "MODULE-LIST": { + "SERVER", + []keySpec{}, + }, + "MODULE-LOAD": { + "SERVER", + []keySpec{}, + }, + "MODULE-LOADEX": { + "SERVER", + []keySpec{}, + }, + "MODULE-UNLOAD": { + "SERVER", + []keySpec{}, + }, + "MODULE": { + "SERVER", + []keySpec{}, + }, + "MONITOR": { + "SERVER", + []keySpec{}, + }, + "PSYNC": { + "SERVER", + []keySpec{}, + }, + "REPLCONF": { + "SERVER", + []keySpec{}, + }, + "REPLICAOF": { + "SERVER", + []keySpec{}, + }, + "RESTORE-ASKING": { + "SERVER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "ROLE": { + "SERVER", + []keySpec{}, + }, + "SAVE": { + "SERVER", + []keySpec{}, + }, + "SHUTDOWN": { + "SERVER", + []keySpec{}, + }, + "SLAVEOF": { + "SERVER", + []keySpec{}, + }, + "SLOWLOG-GET": { + "SERVER", + []keySpec{}, + }, + "SLOWLOG-HELP": { + "SERVER", + []keySpec{}, + }, + "SLOWLOG-LEN": { + "SERVER", + []keySpec{}, + }, + "SLOWLOG-RESET": { + "SERVER", + []keySpec{}, + }, + "SLOWLOG": { + "SERVER", + []keySpec{}, + }, + "SWAPDB": { + "SERVER", + []keySpec{}, + }, + "SYNC": { + "SERVER", + []keySpec{}, + }, + "TIME": { + "SERVER", + []keySpec{}, + }, + "APPEND": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "DECR": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "DECRBY": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "GET": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "GETDEL": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "GETEX": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "GETRANGE": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "GETSET": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "INCR": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "INCRBY": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "INCRBYFLOAT": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "LCS": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 1, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "MGET": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + -1, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "MSET": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + -1, + 2, + 0, + 0, + 0, + 0, + }, + }, + }, + "MSETNX": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + -1, + 2, + 0, + 0, + 0, + 0, + }, + }, + }, + "PSETEX": { + "STRING", []keySpec{ { "index", @@ -37,8 +639,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BRPOPLPUSH": { - "LIST", + "SET": { + "STRING", []keySpec{ { "index", @@ -53,9 +655,86 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + }, + }, + "SETEX": { + "STRING", + []keySpec{ { "index", - 2, + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "SETNX": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "SETRANGE": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "STRLEN": { + "STRING", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "SUBSTR": { + "STRING", + []keySpec{ + { + "index", + 1, "", 0, "range", @@ -68,147 +747,228 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "LMPOP": { - "LIST", - []keySpec{ - { - "index", - 1, - "", - 0, - "keynum", - 0, - 0, - 0, - 0, - 1, - 1, - }, - }, + "ASKING": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-ADDSLOTS": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-ADDSLOTSRANGE": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-BUMPEPOCH": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-COUNT-FAILURE-REPORTS": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-COUNTKEYSINSLOT": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-DELSLOTS": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-DELSLOTSRANGE": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-FAILOVER": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-FLUSHSLOTS": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-FORGET": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-GETKEYSINSLOT": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-HELP": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-INFO": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-KEYSLOT": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-LINKS": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-MEET": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-MYID": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-NODES": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-REPLICAS": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-REPLICATE": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-RESET": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-SAVECONFIG": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-SET-CONFIG-EPOCH": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-SETSLOT": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-SHARDS": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-SLAVES": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER-SLOTS": { + "CLUSTER", + []keySpec{}, + }, + "CLUSTER": { + "CLUSTER", + []keySpec{}, + }, + "READONLY": { + "CLUSTER", + []keySpec{}, + }, + "READWRITE": { + "CLUSTER", + []keySpec{}, + }, + "AUTH": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-CACHING": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-GETNAME": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-GETREDIR": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-HELP": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-ID": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-INFO": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-KILL": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-LIST": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-NO-EVICT": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-PAUSE": { + "CONNECTION", + []keySpec{}, + }, + "CLIENT-REPLY": { + "CONNECTION", + []keySpec{}, }, - "LSET": { - "LIST", - []keySpec{ - { - "index", - 1, - "", - 0, - "range", - 0, - 1, - 0, - 0, - 0, - 0, - }, - }, + "CLIENT-SETNAME": { + "CONNECTION", + []keySpec{}, }, - "BLMPOP": { - "LIST", - []keySpec{ - { - "index", - 2, - "", - 0, - "keynum", - 0, - 0, - 0, - 0, - 1, - 1, - }, - }, + "CLIENT-TRACKING": { + "CONNECTION", + []keySpec{}, }, - "LINDEX": { - "LIST", - []keySpec{ - { - "index", - 1, - "", - 0, - "range", - 0, - 1, - 0, - 0, - 0, - 0, - }, - }, + "CLIENT-TRACKINGINFO": { + "CONNECTION", + []keySpec{}, }, - "LPOS": { - "LIST", - []keySpec{ - { - "index", - 1, - "", - 0, - "range", - 0, - 1, - 0, - 0, - 0, - 0, - }, - }, + "CLIENT-UNBLOCK": { + "CONNECTION", + []keySpec{}, }, - "RPOPLPUSH": { - "LIST", - []keySpec{ - { - "index", - 1, - "", - 0, - "range", - 0, - 1, - 0, - 0, - 0, - 0, - }, - { - "index", - 2, - "", - 0, - "range", - 0, - 1, - 0, - 0, - 0, - 0, - }, - }, + "CLIENT-UNPAUSE": { + "CONNECTION", + []keySpec{}, }, - "LTRIM": { - "LIST", - []keySpec{ - { - "index", - 1, - "", - 0, - "range", - 0, - 1, - 0, - 0, - 0, - 0, - }, - }, + "CLIENT": { + "CONNECTION", + []keySpec{}, }, - "LPUSH": { - "LIST", + "ECHO": { + "CONNECTION", + []keySpec{}, + }, + "HELLO": { + "CONNECTION", + []keySpec{}, + }, + "PING": { + "CONNECTION", + []keySpec{}, + }, + "QUIT": { + "CONNECTION", + []keySpec{}, + }, + "RESET": { + "CONNECTION", + []keySpec{}, + }, + "SELECT": { + "CONNECTION", + []keySpec{}, + }, + "BITCOUNT": { + "BITMAP", []keySpec{ { "index", @@ -225,8 +985,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BRPOP": { - "LIST", + "BITFIELD": { + "BITMAP", []keySpec{ { "index", @@ -234,7 +994,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -2, + 0, 1, 0, 0, @@ -243,8 +1003,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "LINSERT": { - "LIST", + "BITFIELD_RO": { + "BITMAP", []keySpec{ { "index", @@ -261,12 +1021,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "LRANGE": { - "LIST", + "BITOP": { + "BITMAP", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -277,18 +1037,13 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "LREM": { - "LIST", - []keySpec{ { "index", - 1, + 3, "", 0, "range", - 0, + -1, 1, 0, 0, @@ -297,8 +1052,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "RPUSH": { - "LIST", + "BITPOS": { + "BITMAP", []keySpec{ { "index", @@ -315,8 +1070,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "RPOP": { - "LIST", + "GETBIT": { + "BITMAP", []keySpec{ { "index", @@ -333,8 +1088,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "LPOP": { - "LIST", + "SETBIT": { + "BITMAP", []keySpec{ { "index", @@ -351,7 +1106,7 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "LMOVE": { + "BLMOVE": { "LIST", []keySpec{ { @@ -382,56 +1137,25 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "RPUSHX": { - "LIST", - []keySpec{ - { - "index", - 1, - "", - 0, - "range", - 0, - 1, - 0, - 0, - 0, - 0, - }, - }, - }, - "BLMOVE": { - "LIST", - []keySpec{ - { - "index", - 1, - "", - 0, - "range", - 0, - 1, - 0, - 0, - 0, - 0, - }, + "BLMPOP": { + "LIST", + []keySpec{ { "index", 2, "", 0, - "range", - 0, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "LPUSHX": { + "BLPOP": { "LIST", []keySpec{ { @@ -440,7 +1164,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - 0, + -2, 1, 0, 0, @@ -449,7 +1173,7 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BLPOP": { + "BRPOP": { "LIST", []keySpec{ { @@ -465,251 +1189,24 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "MODULE-UNLOAD": { - "SERVER", - []keySpec{}, - }, - "PSYNC": { - "SERVER", - []keySpec{}, - }, - "ACL-WHOAMI": { - "SERVER", - []keySpec{}, - }, - "ACL-GETUSER": { - "SERVER", - []keySpec{}, - }, - "MEMORY-STATS": { - "SERVER", - []keySpec{}, - }, - "LATENCY-HELP": { - "SERVER", - []keySpec{}, - }, - "MONITOR": { - "SERVER", - []keySpec{}, - }, - "BGREWRITEAOF": { - "SERVER", - []keySpec{}, - }, - "LATENCY-GRAPH": { - "SERVER", - []keySpec{}, - }, - "REPLCONF": { - "SERVER", - []keySpec{}, - }, - "LATENCY-HISTORY": { - "SERVER", - []keySpec{}, - }, - "FLUSHDB": { - "SERVER", - []keySpec{}, - }, - "SHUTDOWN": { - "SERVER", - []keySpec{}, - }, - "ROLE": { - "SERVER", - []keySpec{}, - }, - "ACL-SAVE": { - "SERVER", - []keySpec{}, - }, - "LATENCY-LATEST": { - "SERVER", - []keySpec{}, - }, - "LATENCY-HISTOGRAM": { - "SERVER", - []keySpec{}, - }, - "DEBUG": { - "SERVER", - []keySpec{}, - }, - "COMMAND-GETKEYSANDFLAGS": { - "SERVER", - []keySpec{}, - }, - "CONFIG": { - "SERVER", - []keySpec{}, - }, - "ACL-LOG": { - "SERVER", - []keySpec{}, - }, - "CONFIG-HELP": { - "SERVER", - []keySpec{}, - }, - "ACL": { - "SERVER", - []keySpec{}, - }, - "MEMORY": { - "SERVER", - []keySpec{}, - }, - "ACL-CAT": { - "SERVER", - []keySpec{}, - }, - "SAVE": { - "SERVER", - []keySpec{}, - }, - "LOLWUT": { - "SERVER", - []keySpec{}, - }, - "LATENCY-RESET": { - "SERVER", - []keySpec{}, - }, - "MEMORY-PURGE": { - "SERVER", - []keySpec{}, - }, - "COMMAND-DOCS": { - "SERVER", - []keySpec{}, - }, - "ACL-DRYRUN": { - "SERVER", - []keySpec{}, - }, - "SWAPDB": { - "SERVER", - []keySpec{}, - }, - "SYNC": { - "SERVER", - []keySpec{}, - }, - "ACL-USERS": { - "SERVER", - []keySpec{}, - }, - "ACL-SETUSER": { - "SERVER", - []keySpec{}, - }, - "MODULE-HELP": { - "SERVER", - []keySpec{}, - }, - "ACL-LOAD": { - "SERVER", - []keySpec{}, - }, - "COMMAND-COUNT": { - "SERVER", - []keySpec{}, - }, - "COMMAND-HELP": { - "SERVER", - []keySpec{}, - }, - "ACL-HELP": { - "SERVER", - []keySpec{}, - }, - "MODULE-LOAD": { - "SERVER", - []keySpec{}, - }, - "SLOWLOG": { - "SERVER", - []keySpec{}, - }, - "TIME": { - "SERVER", - []keySpec{}, - }, - "CONFIG-REWRITE": { - "SERVER", - []keySpec{}, - }, - "COMMAND": { - "SERVER", - []keySpec{}, - }, - "SLOWLOG-RESET": { - "SERVER", - []keySpec{}, - }, - "SLAVEOF": { - "SERVER", - []keySpec{}, - }, - "ACL-DELUSER": { - "SERVER", - []keySpec{}, - }, - "FLUSHALL": { - "SERVER", - []keySpec{}, - }, - "CONFIG-RESETSTAT": { - "SERVER", - []keySpec{}, - }, - "LATENCY-DOCTOR": { - "SERVER", - []keySpec{}, - }, - "MEMORY-DOCTOR": { - "SERVER", - []keySpec{}, - }, - "INFO": { - "SERVER", - []keySpec{}, - }, - "MODULE": { - "SERVER", - []keySpec{}, - }, - "BGSAVE": { - "SERVER", - []keySpec{}, - }, - "MODULE-LOADEX": { - "SERVER", - []keySpec{}, - }, - "MEMORY-HELP": { - "SERVER", - []keySpec{}, - }, - "ACL-GENPASS": { - "SERVER", - []keySpec{}, - }, - "DBSIZE": { - "SERVER", - []keySpec{}, - }, - "SLOWLOG-GET": { - "SERVER", - []keySpec{}, + }, }, - "MEMORY-USAGE": { - "SERVER", + "BRPOPLPUSH": { + "LIST", []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, { "index", 2, @@ -725,52 +1222,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "COMMAND-GETKEYS": { - "SERVER", - []keySpec{}, - }, - "LATENCY": { - "SERVER", - []keySpec{}, - }, - "COMMAND-INFO": { - "SERVER", - []keySpec{}, - }, - "ACL-LIST": { - "SERVER", - []keySpec{}, - }, - "LASTSAVE": { - "SERVER", - []keySpec{}, - }, - "MODULE-LIST": { - "SERVER", - []keySpec{}, - }, - "SLOWLOG-HELP": { - "SERVER", - []keySpec{}, - }, - "COMMAND-LIST": { - "SERVER", - []keySpec{}, - }, - "CONFIG-GET": { - "SERVER", - []keySpec{}, - }, - "MEMORY-MALLOC-STATS": { - "SERVER", - []keySpec{}, - }, - "SLOWLOG-LEN": { - "SERVER", - []keySpec{}, - }, - "RESTORE-ASKING": { - "SERVER", + "LINDEX": { + "LIST", []keySpec{ { "index", @@ -787,148 +1240,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "CONFIG-SET": { - "SERVER", - []keySpec{}, - }, - "REPLICAOF": { - "SERVER", - []keySpec{}, - }, - "FAILOVER": { - "SERVER", - []keySpec{}, - }, - "READONLY": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-MYID": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-ADDSLOTS": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-KEYSLOT": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-FORGET": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-MEET": { - "CLUSTER", - []keySpec{}, - }, - "READWRITE": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-SLOTS": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-REPLICATE": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-LINKS": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-DELSLOTS": { - "CLUSTER", - []keySpec{}, - }, - "ASKING": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-COUNTKEYSINSLOT": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-SHARDS": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-BUMPEPOCH": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-COUNT-FAILURE-REPORTS": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-SLAVES": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-ADDSLOTSRANGE": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-INFO": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-GETKEYSINSLOT": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-SETSLOT": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-DELSLOTSRANGE": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-HELP": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-FAILOVER": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-SAVECONFIG": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-FLUSHSLOTS": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-SET-CONFIG-EPOCH": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-REPLICAS": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-RESET": { - "CLUSTER", - []keySpec{}, - }, - "CLUSTER-NODES": { - "CLUSTER", - []keySpec{}, - }, - "WAIT": { - "GENERIC", - []keySpec{}, - }, - "DUMP": { - "GENERIC", + "LINSERT": { + "LIST", []keySpec{ { "index", @@ -945,8 +1258,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PTTL": { - "GENERIC", + "LLEN": { + "LIST", []keySpec{ { "index", @@ -963,8 +1276,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "TOUCH": { - "GENERIC", + "LMOVE": { + "LIST", []keySpec{ { "index", @@ -972,21 +1285,16 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -1, + 0, 1, 0, 0, 0, 0, }, - }, - }, - "RESTORE": { - "GENERIC", - []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -999,26 +1307,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "UNLINK": { - "GENERIC", + "LMPOP": { + "LIST", []keySpec{ { "index", 1, "", 0, - "range", - -1, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "TTL": { - "GENERIC", + "LPOP": { + "LIST", []keySpec{ { "index", @@ -1035,8 +1343,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "MOVE": { - "GENERIC", + "LPOS": { + "LIST", []keySpec{ { "index", @@ -1053,12 +1361,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "OBJECT-FREQ": { - "GENERIC", + "LPUSH": { + "LIST", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -1071,8 +1379,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "COPY": { - "GENERIC", + "LPUSHX": { + "LIST", []keySpec{ { "index", @@ -1087,9 +1395,14 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + }, + }, + "LRANGE": { + "LIST", + []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -1102,8 +1415,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PERSIST": { - "GENERIC", + "LREM": { + "LIST", []keySpec{ { "index", @@ -1120,12 +1433,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "OBJECT-REFCOUNT": { - "GENERIC", + "LSET": { + "LIST", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -1138,12 +1451,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "OBJECT-IDLETIME": { - "GENERIC", + "LTRIM": { + "LIST", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -1156,8 +1469,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "EXPIRETIME": { - "GENERIC", + "RPOP": { + "LIST", []keySpec{ { "index", @@ -1174,16 +1487,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "OBJECT": { - "GENERIC", - []keySpec{}, - }, - "KEYS": { - "GENERIC", - []keySpec{}, - }, - "DEL": { - "GENERIC", + "RPOPLPUSH": { + "LIST", []keySpec{ { "index", @@ -1191,21 +1496,16 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -1, + 0, 1, 0, 0, 0, 0, }, - }, - }, - "PEXPIREAT": { - "GENERIC", - []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -1218,12 +1518,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "OBJECT-HELP": { - "GENERIC", - []keySpec{}, - }, - "PEXPIRETIME": { - "GENERIC", + "RPUSH": { + "LIST", []keySpec{ { "index", @@ -1240,12 +1536,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "RANDOMKEY": { - "GENERIC", - []keySpec{}, - }, - "RENAME": { - "GENERIC", + "RPUSHX": { + "LIST", []keySpec{ { "index", @@ -1260,23 +1552,28 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + }, + }, + "BZMPOP": { + "SORTED_SET", + []keySpec{ { "index", 2, "", 0, - "range", - 0, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "RENAMENX": { - "GENERIC", + "BZPOPMAX": { + "SORTED_SET", []keySpec{ { "index", @@ -1284,20 +1581,25 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - 0, + -2, 1, 0, 0, 0, 0, }, + }, + }, + "BZPOPMIN": { + "SORTED_SET", + []keySpec{ { "index", - 2, + 1, "", 0, "range", - 0, + -2, 1, 0, 0, @@ -1306,8 +1608,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PEXPIRE": { - "GENERIC", + "ZADD": { + "SORTED_SET", []keySpec{ { "index", @@ -1324,12 +1626,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SCAN": { - "GENERIC", - []keySpec{}, - }, - "TYPE": { - "GENERIC", + "ZCARD": { + "SORTED_SET", []keySpec{ { "index", @@ -1346,12 +1644,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "OBJECT-ENCODING": { - "GENERIC", + "ZCOUNT": { + "SORTED_SET", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -1364,26 +1662,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "EXISTS": { - "GENERIC", + "ZDIFF": { + "SORTED_SET", []keySpec{ { "index", 1, "", 0, - "range", - -1, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "EXPIRE": { - "GENERIC", + "ZDIFFSTORE": { + "SORTED_SET", []keySpec{ { "index", @@ -1398,10 +1696,23 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + { + "index", + 2, + "", + 0, + "keynum", + 0, + 0, + 0, + 0, + 1, + 1, + }, }, }, - "EXPIREAT": { - "GENERIC", + "ZINCRBY": { + "SORTED_SET", []keySpec{ { "index", @@ -1418,44 +1729,44 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "MSETNX": { - "STRING", + "ZINTER": { + "SORTED_SET", []keySpec{ { "index", 1, "", 0, - "range", - -1, - 2, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "GETEX": { - "STRING", + "ZINTERCARD": { + "SORTED_SET", []keySpec{ { "index", 1, "", 0, - "range", - 0, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "GET": { - "STRING", + "ZINTERSTORE": { + "SORTED_SET", []keySpec{ { "index", @@ -1470,28 +1781,23 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "INCRBYFLOAT": { - "STRING", - []keySpec{ { "index", - 1, + 2, "", 0, - "range", - 0, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "MSET": { - "STRING", + "ZLEXCOUNT": { + "SORTED_SET", []keySpec{ { "index", @@ -1499,8 +1805,8 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -1, - 2, + 0, + 1, 0, 0, 0, @@ -1508,26 +1814,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "MGET": { - "STRING", + "ZMPOP": { + "SORTED_SET", []keySpec{ { "index", 1, "", 0, - "range", - -1, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "SET": { - "STRING", + "ZMSCORE": { + "SORTED_SET", []keySpec{ { "index", @@ -1544,8 +1850,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SUBSTR": { - "STRING", + "ZPOPMAX": { + "SORTED_SET", []keySpec{ { "index", @@ -1562,8 +1868,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "DECRBY": { - "STRING", + "ZPOPMIN": { + "SORTED_SET", []keySpec{ { "index", @@ -1580,8 +1886,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "INCRBY": { - "STRING", + "ZRANDMEMBER": { + "SORTED_SET", []keySpec{ { "index", @@ -1598,8 +1904,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SETEX": { - "STRING", + "ZRANGE": { + "SORTED_SET", []keySpec{ { "index", @@ -1616,8 +1922,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GETRANGE": { - "STRING", + "ZRANGEBYLEX": { + "SORTED_SET", []keySpec{ { "index", @@ -1634,8 +1940,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "DECR": { - "STRING", + "ZRANGEBYSCORE": { + "SORTED_SET", []keySpec{ { "index", @@ -1652,8 +1958,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "STRLEN": { - "STRING", + "ZRANGESTORE": { + "SORTED_SET", []keySpec{ { "index", @@ -1668,14 +1974,9 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "INCR": { - "STRING", - []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -1688,8 +1989,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PSETEX": { - "STRING", + "ZRANK": { + "SORTED_SET", []keySpec{ { "index", @@ -1706,8 +2007,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GETSET": { - "STRING", + "ZREM": { + "SORTED_SET", []keySpec{ { "index", @@ -1724,8 +2025,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SETRANGE": { - "STRING", + "ZREMRANGEBYLEX": { + "SORTED_SET", []keySpec{ { "index", @@ -1742,8 +2043,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "LCS": { - "STRING", + "ZREMRANGEBYRANK": { + "SORTED_SET", []keySpec{ { "index", @@ -1751,7 +2052,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - 1, + 0, 1, 0, 0, @@ -1760,8 +2061,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "APPEND": { - "STRING", + "ZREMRANGEBYSCORE": { + "SORTED_SET", []keySpec{ { "index", @@ -1778,8 +2079,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SETNX": { - "STRING", + "ZREVRANGE": { + "SORTED_SET", []keySpec{ { "index", @@ -1796,8 +2097,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GETDEL": { - "STRING", + "ZREVRANGEBYLEX": { + "SORTED_SET", []keySpec{ { "index", @@ -1814,8 +2115,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SDIFFSTORE": { - "SET", + "ZREVRANGEBYSCORE": { + "SORTED_SET", []keySpec{ { "index", @@ -1830,13 +2131,18 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + }, + }, + "ZREVRANK": { + "SORTED_SET", + []keySpec{ { "index", - 2, + 1, "", 0, "range", - -1, + 0, 1, 0, 0, @@ -1845,8 +2151,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SSCAN": { - "SET", + "ZSCAN": { + "SORTED_SET", []keySpec{ { "index", @@ -1863,8 +2169,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SINTERSTORE": { - "SET", + "ZSCORE": { + "SORTED_SET", []keySpec{ { "index", @@ -1879,23 +2185,28 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + }, + }, + "ZUNION": { + "SORTED_SET", + []keySpec{ { "index", - 2, + 1, "", 0, - "range", - -1, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "SCARD": { - "SET", + "ZUNIONSTORE": { + "SORTED_SET", []keySpec{ { "index", @@ -1910,28 +2221,23 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "SINTER": { - "SET", - []keySpec{ { "index", - 1, + 2, "", 0, - "range", - -1, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "SUNIONSTORE": { - "SET", + "COPY": { + "GENERIC", []keySpec{ { "index", @@ -1952,7 +2258,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -1, + 0, 1, 0, 0, @@ -1961,8 +2267,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SUNION": { - "SET", + "DEL": { + "GENERIC", []keySpec{ { "index", @@ -1979,8 +2285,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SPOP": { - "SET", + "DUMP": { + "GENERIC", []keySpec{ { "index", @@ -1997,26 +2303,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SINTERCARD": { - "SET", + "EXISTS": { + "GENERIC", []keySpec{ { "index", 1, "", 0, - "keynum", + "range", + -1, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "SMISMEMBER": { - "SET", + "EXPIRE": { + "GENERIC", []keySpec{ { "index", @@ -2033,8 +2339,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SMEMBERS": { - "SET", + "EXPIREAT": { + "GENERIC", []keySpec{ { "index", @@ -2051,8 +2357,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SADD": { - "SET", + "EXPIRETIME": { + "GENERIC", []keySpec{ { "index", @@ -2069,8 +2375,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SDIFF": { - "SET", + "KEYS": { + "GENERIC", + []keySpec{}, + }, + "MOVE": { + "GENERIC", []keySpec{ { "index", @@ -2078,7 +2388,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -1, + 0, 1, 0, 0, @@ -2087,12 +2397,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SREM": { - "SET", + "OBJECT-ENCODING": { + "GENERIC", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -2105,12 +2415,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SISMEMBER": { - "SET", + "OBJECT-FREQ": { + "GENERIC", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -2123,12 +2433,16 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SMOVE": { - "SET", + "OBJECT-HELP": { + "GENERIC", + []keySpec{}, + }, + "OBJECT-IDLETIME": { + "GENERIC", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -2139,6 +2453,11 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + }, + }, + "OBJECT-REFCOUNT": { + "GENERIC", + []keySpec{ { "index", 2, @@ -2154,8 +2473,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SRANDMEMBER": { - "SET", + "OBJECT": { + "GENERIC", + []keySpec{}, + }, + "PERSIST": { + "GENERIC", []keySpec{ { "index", @@ -2172,8 +2495,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GEORADIUS": { - "GEO", + "PEXPIRE": { + "GENERIC", []keySpec{ { "index", @@ -2188,24 +2511,16 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + }, + }, + "PEXPIREAT": { + "GENERIC", + []keySpec{ { - "keyword", - 0, - "STORE", - 6, - "range", - 0, + "index", 1, + "", 0, - 0, - 0, - 0, - }, - { - "keyword", - 0, - "STOREDIST", - 6, "range", 0, 1, @@ -2216,8 +2531,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GEOHASH": { - "GEO", + "PEXPIRETIME": { + "GENERIC", []keySpec{ { "index", @@ -2234,8 +2549,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GEODIST": { - "GEO", + "PTTL": { + "GENERIC", []keySpec{ { "index", @@ -2252,8 +2567,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GEORADIUS_RO": { - "GEO", + "RANDOMKEY": { + "GENERIC", + []keySpec{}, + }, + "RENAME": { + "GENERIC", []keySpec{ { "index", @@ -2268,14 +2587,9 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "GEOADD": { - "GEO", - []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -2288,8 +2602,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GEOSEARCHSTORE": { - "GEO", + "RENAMENX": { + "GENERIC", []keySpec{ { "index", @@ -2319,8 +2633,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GEORADIUSBYMEMBER": { - "GEO", + "RESTORE": { + "GENERIC", []keySpec{ { "index", @@ -2335,24 +2649,38 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + }, + }, + "SCAN": { + "GENERIC", + []keySpec{}, + }, + "TOUCH": { + "GENERIC", + []keySpec{ { - "keyword", + "index", + 1, + "", 0, - "STORE", - 5, "range", - 0, + -1, 1, 0, 0, 0, 0, }, + }, + }, + "TTL": { + "GENERIC", + []keySpec{ { - "keyword", + "index", + 1, + "", 0, - "STOREDIST", - 5, "range", 0, 1, @@ -2363,8 +2691,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GEOSEARCH": { - "GEO", + "TYPE": { + "GENERIC", []keySpec{ { "index", @@ -2381,8 +2709,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GEOPOS": { - "GEO", + "UNLINK": { + "GENERIC", []keySpec{ { "index", @@ -2390,7 +2718,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - 0, + -1, 1, 0, 0, @@ -2399,8 +2727,28 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GEORADIUSBYMEMBER_RO": { - "GEO", + "WAIT": { + "GENERIC", + []keySpec{}, + }, + "DISCARD": { + "TRANSACTIONS", + []keySpec{}, + }, + "EXEC": { + "TRANSACTIONS", + []keySpec{}, + }, + "MULTI": { + "TRANSACTIONS", + []keySpec{}, + }, + "UNWATCH": { + "TRANSACTIONS", + []keySpec{}, + }, + "WATCH": { + "TRANSACTIONS", []keySpec{ { "index", @@ -2408,7 +2756,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - 0, + -1, 1, 0, 0, @@ -2417,107 +2765,99 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BZPOPMIN": { - "SORTED_SET", + "EVAL": { + "SCRIPTING", []keySpec{ { "index", - 1, + 2, "", 0, - "range", - -2, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "ZPOPMAX": { - "SORTED_SET", + "EVAL_RO": { + "SCRIPTING", []keySpec{ { "index", - 1, + 2, "", 0, - "range", - 0, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "ZREMRANGEBYSCORE": { - "SORTED_SET", + "EVALSHA": { + "SCRIPTING", []keySpec{ { "index", - 1, + 2, "", 0, - "range", - 0, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "ZRANGESTORE": { - "SORTED_SET", + "EVALSHA_RO": { + "SCRIPTING", []keySpec{ - { - "index", - 1, - "", - 0, - "range", - 0, - 1, - 0, - 0, - 0, - 0, - }, { "index", 2, "", 0, - "range", - 0, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "ZINTERSTORE": { - "SORTED_SET", + "FCALL": { + "SCRIPTING", []keySpec{ { "index", - 1, + 2, "", 0, - "range", - 0, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, + }, + }, + "FCALL_RO": { + "SCRIPTING", + []keySpec{ { "index", 2, @@ -2533,8 +2873,76 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZREVRANGEBYSCORE": { - "SORTED_SET", + "FUNCTION-DELETE": { + "SCRIPTING", + []keySpec{}, + }, + "FUNCTION-DUMP": { + "SCRIPTING", + []keySpec{}, + }, + "FUNCTION-FLUSH": { + "SCRIPTING", + []keySpec{}, + }, + "FUNCTION-HELP": { + "SCRIPTING", + []keySpec{}, + }, + "FUNCTION-KILL": { + "SCRIPTING", + []keySpec{}, + }, + "FUNCTION-LIST": { + "SCRIPTING", + []keySpec{}, + }, + "FUNCTION-LOAD": { + "SCRIPTING", + []keySpec{}, + }, + "FUNCTION-RESTORE": { + "SCRIPTING", + []keySpec{}, + }, + "FUNCTION-STATS": { + "SCRIPTING", + []keySpec{}, + }, + "FUNCTION": { + "SCRIPTING", + []keySpec{}, + }, + "SCRIPT-DEBUG": { + "SCRIPTING", + []keySpec{}, + }, + "SCRIPT-EXISTS": { + "SCRIPTING", + []keySpec{}, + }, + "SCRIPT-FLUSH": { + "SCRIPTING", + []keySpec{}, + }, + "SCRIPT-HELP": { + "SCRIPTING", + []keySpec{}, + }, + "SCRIPT-KILL": { + "SCRIPTING", + []keySpec{}, + }, + "SCRIPT-LOAD": { + "SCRIPTING", + []keySpec{}, + }, + "SCRIPT": { + "SCRIPTING", + []keySpec{}, + }, + "EXHSET": { + "TAIRHASH", []keySpec{ { "index", @@ -2551,8 +2959,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BZPOPMAX": { - "SORTED_SET", + "EXSET": { + "TAIRSTRING", []keySpec{ { "index", @@ -2560,7 +2968,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -2, + 0, 1, 0, 0, @@ -2569,8 +2977,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZLEXCOUNT": { - "SORTED_SET", + "EXZADD": { + "TAIRZSET", []keySpec{ { "index", @@ -2587,8 +2995,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZREVRANK": { - "SORTED_SET", + "GEOADD": { + "GEO", []keySpec{ { "index", @@ -2605,8 +3013,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZPOPMIN": { - "SORTED_SET", + "GEODIST": { + "GEO", []keySpec{ { "index", @@ -2623,8 +3031,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZINCRBY": { - "SORTED_SET", + "GEOHASH": { + "GEO", []keySpec{ { "index", @@ -2641,8 +3049,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZDIFFSTORE": { - "SORTED_SET", + "GEOPOS": { + "GEO", []keySpec{ { "index", @@ -2657,23 +3065,10 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - { - "index", - 2, - "", - 0, - "keynum", - 0, - 0, - 0, - 0, - 1, - 1, - }, }, }, - "ZUNIONSTORE": { - "SORTED_SET", + "GEORADIUS": { + "GEO", []keySpec{ { "index", @@ -2689,28 +3084,10 @@ var redisCommands = map[string]redisCommand{ 0, }, { - "index", - 2, - "", - 0, - "keynum", - 0, - 0, - 0, - 0, - 1, - 1, - }, - }, - }, - "ZRANGE": { - "SORTED_SET", - []keySpec{ - { - "index", - 1, - "", + "keyword", 0, + "STORE", + 6, "range", 0, 1, @@ -2719,16 +3096,11 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "ZRANGEBYSCORE": { - "SORTED_SET", - []keySpec{ { - "index", - 1, - "", + "keyword", 0, + "STOREDIST", + 6, "range", 0, 1, @@ -2739,26 +3111,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZDIFF": { - "SORTED_SET", + "GEORADIUS_RO": { + "GEO", []keySpec{ { "index", 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "ZSCORE": { - "SORTED_SET", + "GEORADIUSBYMEMBER": { + "GEO", []keySpec{ { "index", @@ -2773,34 +3145,24 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "ZREMRANGEBYLEX": { - "SORTED_SET", - []keySpec{ { - "index", - 1, - "", + "keyword", 0, + "STORE", + 5, "range", 0, 1, 0, 0, - 0, - 0, - }, - }, - }, - "ZREVRANGE": { - "SORTED_SET", - []keySpec{ + 0, + 0, + }, { - "index", - 1, - "", + "keyword", 0, + "STOREDIST", + 5, "range", 0, 1, @@ -2811,8 +3173,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZREVRANGEBYLEX": { - "SORTED_SET", + "GEORADIUSBYMEMBER_RO": { + "GEO", []keySpec{ { "index", @@ -2829,26 +3191,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZMPOP": { - "SORTED_SET", + "GEOSEARCH": { + "GEO", []keySpec{ { "index", 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "ZRANK": { - "SORTED_SET", + "GEOSEARCHSTORE": { + "GEO", []keySpec{ { "index", @@ -2863,14 +3225,9 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "ZMSCORE": { - "SORTED_SET", - []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -2883,26 +3240,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BZMPOP": { - "SORTED_SET", + "HDEL": { + "HASH", []keySpec{ { "index", - 2, + 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "ZCOUNT": { - "SORTED_SET", + "HEXISTS": { + "HASH", []keySpec{ { "index", @@ -2919,8 +3276,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZSCAN": { - "SORTED_SET", + "HGET": { + "HASH", []keySpec{ { "index", @@ -2937,26 +3294,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZUNION": { - "SORTED_SET", + "HGETALL": { + "HASH", []keySpec{ { "index", 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "ZRANDMEMBER": { - "SORTED_SET", + "HINCRBY": { + "HASH", []keySpec{ { "index", @@ -2973,26 +3330,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZINTER": { - "SORTED_SET", + "HINCRBYFLOAT": { + "HASH", []keySpec{ { "index", 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "ZADD": { - "SORTED_SET", + "HKEYS": { + "HASH", []keySpec{ { "index", @@ -3009,8 +3366,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZREMRANGEBYRANK": { - "SORTED_SET", + "HLEN": { + "HASH", []keySpec{ { "index", @@ -3027,26 +3384,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZINTERCARD": { - "SORTED_SET", + "HMGET": { + "HASH", []keySpec{ { "index", 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "ZRANGEBYLEX": { - "SORTED_SET", + "HMSET": { + "HASH", []keySpec{ { "index", @@ -3063,8 +3420,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZREM": { - "SORTED_SET", + "HRANDFIELD": { + "HASH", []keySpec{ { "index", @@ -3081,8 +3438,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "ZCARD": { - "SORTED_SET", + "HSCAN": { + "HASH", []keySpec{ { "index", @@ -3099,276 +3456,134 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SENTINEL-MASTER": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-SIMULATE-FAILURE": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-FAILOVER": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-REPLICAS": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-MASTERS": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-MYID": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-PENDING-SCRIPTS": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-DEBUG": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-INFO-CACHE": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-REMOVE": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-IS-MASTER-DOWN-BY-ADDR": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-FLUSHCONFIG": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-GET-MASTER-ADDR-BY-NAME": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-CONFIG": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-SENTINELS": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-CKQUORUM": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-MONITOR": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-SLAVES": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-RESET": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-HELP": { - "SENTINEL", - []keySpec{}, - }, - "SENTINEL-SET": { - "SENTINEL", - []keySpec{}, - }, - "FUNCTION-FLUSH": { - "SCRIPTING", - []keySpec{}, - }, - "FUNCTION-LIST": { - "SCRIPTING", - []keySpec{}, - }, - "FUNCTION": { - "SCRIPTING", - []keySpec{}, - }, - "SCRIPT": { - "SCRIPTING", - []keySpec{}, - }, - "EVALSHA_RO": { - "SCRIPTING", + "HSET": { + "HASH", []keySpec{ { "index", - 2, + 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "FUNCTION-DELETE": { - "SCRIPTING", - []keySpec{}, - }, - "FUNCTION-STATS": { - "SCRIPTING", - []keySpec{}, - }, - "FUNCTION-RESTORE": { - "SCRIPTING", - []keySpec{}, - }, - "FUNCTION-LOAD": { - "SCRIPTING", - []keySpec{}, - }, - "FUNCTION-HELP": { - "SCRIPTING", - []keySpec{}, - }, - "SCRIPT-KILL": { - "SCRIPTING", - []keySpec{}, - }, - "SCRIPT-FLUSH": { - "SCRIPTING", - []keySpec{}, - }, - "EVAL_RO": { - "SCRIPTING", + "HSETNX": { + "HASH", []keySpec{ { "index", - 2, + 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "FUNCTION-KILL": { - "SCRIPTING", - []keySpec{}, - }, - "EVALSHA": { - "SCRIPTING", + "HSTRLEN": { + "HASH", []keySpec{ { "index", - 2, + 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "FCALL": { - "SCRIPTING", + "HVALS": { + "HASH", []keySpec{ { "index", - 2, + 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "SCRIPT-LOAD": { - "SCRIPTING", - []keySpec{}, - }, - "SCRIPT-HELP": { - "SCRIPTING", - []keySpec{}, - }, - "FUNCTION-DUMP": { - "SCRIPTING", - []keySpec{}, - }, - "SCRIPT-DEBUG": { - "SCRIPTING", - []keySpec{}, - }, - "SCRIPT-EXISTS": { - "SCRIPTING", - []keySpec{}, - }, - "EVAL": { - "SCRIPTING", + "PFADD": { + "HYPERLOGLOG", []keySpec{ { "index", - 2, + 1, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, + }, + }, + }, + "PFCOUNT": { + "HYPERLOGLOG", + []keySpec{ + { + "index", 1, + "", + 0, + "range", + -1, 1, + 0, + 0, + 0, + 0, }, }, }, - "FCALL_RO": { - "SCRIPTING", + "PFDEBUG": { + "HYPERLOGLOG", []keySpec{ { "index", 2, "", 0, - "keynum", + "range", + 0, + 1, 0, 0, 0, 0, - 1, - 1, }, }, }, - "XGROUP-HELP": { - "STREAM", - []keySpec{}, - }, - "XCLAIM": { - "STREAM", + "PFMERGE": { + "HYPERLOGLOG", []keySpec{ { "index", @@ -3383,14 +3598,71 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + { + "index", + 2, + "", + 0, + "range", + -1, + 1, + 0, + 0, + 0, + 0, + }, }, }, - "XINFO-GROUPS": { - "STREAM", + "PFSELFTEST": { + "HYPERLOGLOG", + []keySpec{}, + }, + "PSUBSCRIBE": { + "PUBSUB", + []keySpec{}, + }, + "PUBLISH": { + "PUBSUB", + []keySpec{}, + }, + "PUBSUB-CHANNELS": { + "PUBSUB", + []keySpec{}, + }, + "PUBSUB-HELP": { + "PUBSUB", + []keySpec{}, + }, + "PUBSUB-NUMPAT": { + "PUBSUB", + []keySpec{}, + }, + "PUBSUB-NUMSUB": { + "PUBSUB", + []keySpec{}, + }, + "PUBSUB-SHARDCHANNELS": { + "PUBSUB", + []keySpec{}, + }, + "PUBSUB-SHARDNUMSUB": { + "PUBSUB", + []keySpec{}, + }, + "PUBSUB": { + "PUBSUB", + []keySpec{}, + }, + "PUNSUBSCRIBE": { + "PUBSUB", + []keySpec{}, + }, + "SPUBLISH": { + "PUBSUB", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -3403,34 +3675,38 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XREADGROUP": { - "STREAM", + "SSUBSCRIBE": { + "PUBSUB", []keySpec{ { - "keyword", + "index", + 1, + "", 0, - "STREAMS", - 4, "range", -1, 1, - 2, + 0, 0, 0, 0, }, }, }, - "XINFO-CONSUMERS": { - "STREAM", + "SUBSCRIBE": { + "PUBSUB", + []keySpec{}, + }, + "SUNSUBSCRIBE": { + "PUBSUB", []keySpec{ { "index", - 2, + 1, "", 0, "range", - 0, + -1, 1, 0, 0, @@ -3439,12 +3715,16 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XGROUP-DELCONSUMER": { - "STREAM", + "UNSUBSCRIBE": { + "PUBSUB", + []keySpec{}, + }, + "SADD": { + "SET", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -3457,8 +3737,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XACK": { - "STREAM", + "SCARD": { + "SET", []keySpec{ { "index", @@ -3475,26 +3755,26 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XREAD": { - "STREAM", + "SDIFF": { + "SET", []keySpec{ { - "keyword", - 0, - "STREAMS", + "index", 1, + "", + 0, "range", -1, 1, - 2, + 0, 0, 0, 0, }, }, }, - "XLEN": { - "STREAM", + "SDIFFSTORE": { + "SET", []keySpec{ { "index", @@ -3509,10 +3789,23 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + { + "index", + 2, + "", + 0, + "range", + -1, + 1, + 0, + 0, + 0, + 0, + }, }, }, - "XTRIM": { - "STREAM", + "SINTER": { + "SET", []keySpec{ { "index", @@ -3520,7 +3813,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - 0, + -1, 1, 0, 0, @@ -3529,38 +3822,30 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XREVRANGE": { - "STREAM", + "SINTERCARD": { + "SET", []keySpec{ { "index", 1, "", 0, - "range", - 0, - 1, + "keynum", 0, 0, 0, 0, + 1, + 1, }, }, }, - "XGROUP": { - "STREAM", - []keySpec{}, - }, - "XINFO": { - "STREAM", - []keySpec{}, - }, - "XGROUP-CREATE": { - "STREAM", + "SINTERSTORE": { + "SET", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -3571,18 +3856,13 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "XDEL": { - "STREAM", - []keySpec{ { "index", - 1, + 2, "", 0, "range", - 0, + -1, 1, 0, 0, @@ -3591,8 +3871,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XAUTOCLAIM": { - "STREAM", + "SISMEMBER": { + "SET", []keySpec{ { "index", @@ -3609,16 +3889,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XINFO-HELP": { - "STREAM", - []keySpec{}, - }, - "XGROUP-DESTROY": { - "STREAM", + "SMEMBERS": { + "SET", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -3631,8 +3907,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XADD": { - "STREAM", + "SMISMEMBER": { + "SET", []keySpec{ { "index", @@ -3649,8 +3925,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XSETID": { - "STREAM", + "SMOVE": { + "SET", []keySpec{ { "index", @@ -3665,14 +3941,9 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, - }, - }, - "XPENDING": { - "STREAM", - []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -3685,12 +3956,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XGROUP-SETID": { - "STREAM", + "SPOP": { + "SET", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -3703,12 +3974,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XINFO-STREAM": { - "STREAM", + "SRANDMEMBER": { + "SET", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -3721,12 +3992,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XGROUP-CREATECONSUMER": { - "STREAM", + "SREM": { + "SET", []keySpec{ { "index", - 2, + 1, "", 0, "range", @@ -3739,8 +4010,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "XRANGE": { - "STREAM", + "SSCAN": { + "SET", []keySpec{ { "index", @@ -3757,20 +4028,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "DISCARD": { - "TRANSACTIONS", - []keySpec{}, - }, - "EXEC": { - "TRANSACTIONS", - []keySpec{}, - }, - "MULTI": { - "TRANSACTIONS", - []keySpec{}, - }, - "WATCH": { - "TRANSACTIONS", + "SUNION": { + "SET", []keySpec{ { "index", @@ -3787,12 +4046,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "UNWATCH": { - "TRANSACTIONS", - []keySpec{}, - }, - "HEXISTS": { - "HASH", + "SUNIONSTORE": { + "SET", []keySpec{ { "index", @@ -3807,10 +4062,111 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + { + "index", + 2, + "", + 0, + "range", + -1, + 1, + 0, + 0, + 0, + 0, + }, }, }, - "HVALS": { - "HASH", + "SENTINEL-CKQUORUM": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-CONFIG": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-DEBUG": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-FAILOVER": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-FLUSHCONFIG": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-GET-MASTER-ADDR-BY-NAME": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-HELP": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-INFO-CACHE": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-IS-MASTER-DOWN-BY-ADDR": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-MASTER": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-MASTERS": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-MONITOR": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-MYID": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-PENDING-SCRIPTS": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-REMOVE": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-REPLICAS": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-RESET": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-SENTINELS": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-SET": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-SIMULATE-FAILURE": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL-SLAVES": { + "SENTINEL", + []keySpec{}, + }, + "SENTINEL": { + "SENTINEL", + []keySpec{}, + }, + "XACK": { + "STREAM", []keySpec{ { "index", @@ -3827,8 +4183,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HMGET": { - "HASH", + "XADD": { + "STREAM", []keySpec{ { "index", @@ -3845,8 +4201,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HMSET": { - "HASH", + "XAUTOCLAIM": { + "STREAM", []keySpec{ { "index", @@ -3863,8 +4219,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HINCRBYFLOAT": { - "HASH", + "XCLAIM": { + "STREAM", []keySpec{ { "index", @@ -3881,8 +4237,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HDEL": { - "HASH", + "XDEL": { + "STREAM", []keySpec{ { "index", @@ -3899,12 +4255,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HGETALL": { - "HASH", + "XGROUP-CREATE": { + "STREAM", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -3917,12 +4273,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HSTRLEN": { - "HASH", + "XGROUP-CREATECONSUMER": { + "STREAM", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -3935,12 +4291,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HKEYS": { - "HASH", + "XGROUP-DELCONSUMER": { + "STREAM", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -3953,12 +4309,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HRANDFIELD": { - "HASH", + "XGROUP-DESTROY": { + "STREAM", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -3971,12 +4327,16 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HLEN": { - "HASH", + "XGROUP-HELP": { + "STREAM", + []keySpec{}, + }, + "XGROUP-SETID": { + "STREAM", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -3989,12 +4349,16 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HGET": { - "HASH", + "XGROUP": { + "STREAM", + []keySpec{}, + }, + "XINFO-CONSUMERS": { + "STREAM", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -4007,12 +4371,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HSETNX": { - "HASH", + "XINFO-GROUPS": { + "STREAM", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -4025,12 +4389,16 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HSET": { - "HASH", + "XINFO-HELP": { + "STREAM", + []keySpec{}, + }, + "XINFO-STREAM": { + "STREAM", []keySpec{ { "index", - 1, + 2, "", 0, "range", @@ -4043,8 +4411,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HINCRBY": { - "HASH", + "XINFO": { + "STREAM", + []keySpec{}, + }, + "XLEN": { + "STREAM", []keySpec{ { "index", @@ -4061,8 +4433,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "HSCAN": { - "HASH", + "XPENDING": { + "STREAM", []keySpec{ { "index", @@ -4079,8 +4451,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BITCOUNT": { - "BITMAP", + "XRANGE": { + "STREAM", []keySpec{ { "index", @@ -4097,47 +4469,52 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "SETBIT": { - "BITMAP", + "XREAD": { + "STREAM", []keySpec{ { - "index", - 1, - "", + "keyword", 0, + "STREAMS", + 1, "range", - 0, + -1, 1, - 0, + 2, 0, 0, 0, }, }, }, - "BITOP": { - "BITMAP", + "XREADGROUP": { + "STREAM", []keySpec{ { - "index", - 2, - "", + "keyword", 0, + "STREAMS", + 4, "range", - 0, + -1, 1, - 0, + 2, 0, 0, 0, }, + }, + }, + "XREVRANGE": { + "STREAM", + []keySpec{ { "index", - 3, + 1, "", 0, "range", - -1, + 0, 1, 0, 0, @@ -4146,8 +4523,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "GETBIT": { - "BITMAP", + "XSETID": { + "STREAM", []keySpec{ { "index", @@ -4164,8 +4541,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BITFIELD_RO": { - "BITMAP", + "XTRIM": { + "STREAM", []keySpec{ { "index", @@ -4182,8 +4559,9 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BITFIELD": { - "BITMAP", + + "BF.ADD": { + "BLOOM FILTER", []keySpec{ { "index", @@ -4200,8 +4578,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "BITPOS": { - "BITMAP", + "BF.CARD": { + "BLOOM FILTER", []keySpec{ { "index", @@ -4218,20 +4596,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PUBSUB-NUMSUB": { - "PUBSUB", - []keySpec{}, - }, - "PUBSUB": { - "PUBSUB", - []keySpec{}, - }, - "PUNSUBSCRIBE": { - "PUBSUB", - []keySpec{}, - }, - "SPUBLISH": { - "PUBSUB", + "BF.EXISTS": { + "BLOOM FILTER", []keySpec{ { "index", @@ -4248,40 +4614,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PUBSUB-SHARDNUMSUB": { - "PUBSUB", - []keySpec{}, - }, - "PUBSUB-SHARDCHANNELS": { - "PUBSUB", - []keySpec{}, - }, - "SUBSCRIBE": { - "PUBSUB", - []keySpec{}, - }, - "PUBSUB-HELP": { - "PUBSUB", - []keySpec{}, - }, - "PUBLISH": { - "PUBSUB", - []keySpec{}, - }, - "PUBSUB-NUMPAT": { - "PUBSUB", - []keySpec{}, - }, - "UNSUBSCRIBE": { - "PUBSUB", - []keySpec{}, - }, - "PSUBSCRIBE": { - "PUBSUB", - []keySpec{}, - }, - "SSUBSCRIBE": { - "PUBSUB", + "BF.INFO": { + "BLOOM FILTER", []keySpec{ { "index", @@ -4289,7 +4623,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -1, + 0, 1, 0, 0, @@ -4298,12 +4632,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PUBSUB-CHANNELS": { - "PUBSUB", - []keySpec{}, - }, - "SUNSUBSCRIBE": { - "PUBSUB", + "BF.INSERT": { + "BLOOM FILTER", []keySpec{ { "index", @@ -4311,7 +4641,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -1, + 0, 1, 0, 0, @@ -4320,104 +4650,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "CLIENT-UNBLOCK": { - "CONNECTION", - []keySpec{}, - }, - "ECHO": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-ID": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-SETNAME": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-LIST": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-INFO": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-GETNAME": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-HELP": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-TRACKINGINFO": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-NO-EVICT": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-PAUSE": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-REPLY": { - "CONNECTION", - []keySpec{}, - }, - "HELLO": { - "CONNECTION", - []keySpec{}, - }, - "QUIT": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-KILL": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-CACHING": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-GETREDIR": { - "CONNECTION", - []keySpec{}, - }, - "AUTH": { - "CONNECTION", - []keySpec{}, - }, - "PING": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-UNPAUSE": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT": { - "CONNECTION", - []keySpec{}, - }, - "RESET": { - "CONNECTION", - []keySpec{}, - }, - "CLIENT-TRACKING": { - "CONNECTION", - []keySpec{}, - }, - "SELECT": { - "CONNECTION", - []keySpec{}, - }, - "PFCOUNT": { - "HYPERLOGLOG", + "BF.LOADCHUNK": { + "BLOOM FILTER", []keySpec{ { "index", @@ -4425,7 +4659,7 @@ var redisCommands = map[string]redisCommand{ "", 0, "range", - -1, + 0, 1, 0, 0, @@ -4434,8 +4668,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PFMERGE": { - "HYPERLOGLOG", + "BF.MADD": { + "BLOOM FILTER", []keySpec{ { "index", @@ -4450,13 +4684,18 @@ var redisCommands = map[string]redisCommand{ 0, 0, }, + }, + }, + "BF.MEXISTS": { + "BLOOM FILTER", + []keySpec{ { "index", - 2, + 1, "", 0, "range", - -1, + 0, 1, 0, 0, @@ -4465,8 +4704,8 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PFADD": { - "HYPERLOGLOG", + "BF.RESERVE": { + "BLOOM FILTER", []keySpec{ { "index", @@ -4483,16 +4722,12 @@ var redisCommands = map[string]redisCommand{ }, }, }, - "PFSELFTEST": { - "HYPERLOGLOG", - []keySpec{}, - }, - "PFDEBUG": { - "HYPERLOGLOG", + "BF.SCANDUMP": { + "BLOOM FILTER", []keySpec{ { "index", - 2, + 1, "", 0, "range", diff --git a/internal/config/config.go b/internal/config/config.go index eb303abc..f55643db 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,147 +1,102 @@ package config import ( - "bytes" + "RedisShake/internal/log" "fmt" - "github.com/pelletier/go-toml/v2" - "io/ioutil" + "github.com/mcuadros/go-defaults" + "github.com/rs/zerolog" + "github.com/spf13/viper" "os" - "runtime" + "strings" ) -type tomlSource struct { - // sync mode - Version float32 `toml:"version"` - Address string `toml:"address"` - Username string `toml:"username"` - Password string `toml:"password"` - IsTLS bool `toml:"tls"` - ElastiCachePSync string `toml:"elasticache_psync"` - - // restore mode - RDBFilePath string `toml:"rdb_file_path"` -} - -type tomlTarget struct { - Type string `toml:"type"` - Version float32 `toml:"version"` - Username string `toml:"username"` - Address string `toml:"address"` - Password string `toml:"password"` - IsTLS bool `toml:"tls"` -} +type AdvancedOptions struct { + Dir string `mapstructure:"dir" default:"data"` -type tomlAdvanced struct { - Dir string `toml:"dir"` + Ncpu int `mapstructure:"ncpu" default:"0"` - Ncpu int `toml:"ncpu"` - - PprofPort int `toml:"pprof_port"` - MetricsPort int `toml:"metrics_port"` + PprofPort int `mapstructure:"pprof_port" default:"0"` + StatusPort int `mapstructure:"status_port" default:"6479"` // log - LogFile string `toml:"log_file"` - LogLevel string `toml:"log_level"` - LogInterval int `toml:"log_interval"` - - // rdb restore - RDBRestoreCommandBehavior string `toml:"rdb_restore_command_behavior"` + LogFile string `mapstructure:"log_file" default:"shake.log"` + LogLevel string `mapstructure:"log_level" default:"info"` + LogInterval int `mapstructure:"log_interval" default:"5"` + + // redis-shake gets key and value from rdb file, and uses RESTORE command to + // create the key in target redis. Redis RESTORE will return a "Target key name + // is busy" error when key already exists. You can use this configuration item + // to change the default behavior of restore: + // panic: redis-shake will stop when meet "Target key name is busy" error. + // rewrite: redis-shake will replace the key with new value. + // ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. + RDBRestoreCommandBehavior string `mapstructure:"rdb_restore_command_behavior" default:"panic"` + + PipelineCountLimit uint64 `mapstructure:"pipeline_count_limit" default:"1024"` + TargetRedisClientMaxQuerybufLen int64 `mapstructure:"target_redis_client_max_querybuf_len" default:"1024000000"` + TargetRedisProtoMaxBulkLen uint64 `mapstructure:"target_redis_proto_max_bulk_len" default:"512000000"` + + AwsPSync string `mapstructure:"aws_psync" default:""` // 10.0.0.1:6379@nmfu2sl5osync,10.0.0.1:6379@xhma21xfkssync +} - // for writer - PipelineCountLimit uint64 `toml:"pipeline_count_limit"` - TargetRedisClientMaxQuerybufLen uint64 `toml:"target_redis_client_max_querybuf_len"` - TargetRedisProtoMaxBulkLen uint64 `toml:"target_redis_proto_max_bulk_len"` +type ModuleOptions struct { + TargetMBbloomVersion int `mapstructure:"target_mbbloom_version" default:"0"` // v1.0.0 <=> 10000 } -type tomlShakeConfig struct { - Type string - Source tomlSource - Target tomlTarget - Advanced tomlAdvanced +func (opt *AdvancedOptions) GetPSyncCommand(address string) string { + items := strings.Split(opt.AwsPSync, ",") + for _, item := range items { + if strings.HasPrefix(item, address) { + return strings.Split(item, "@")[1] + } + } + log.Panicf("can not find aws psync command. address=[%s],aws_psync=[%s]", address, opt.AwsPSync) + return "" } -var Config tomlShakeConfig - -func init() { - Config.Type = "sync" - - // source - Config.Source.Version = 5.0 - Config.Source.Address = "" - Config.Source.Username = "" - Config.Source.Password = "" - Config.Source.IsTLS = false - Config.Source.ElastiCachePSync = "" - // restore - Config.Source.RDBFilePath = "" - - // target - Config.Target.Type = "standalone" - Config.Target.Version = 5.0 - Config.Target.Address = "" - Config.Target.Username = "" - Config.Target.Password = "" - Config.Target.IsTLS = false - - // advanced - Config.Advanced.Dir = "data" - Config.Advanced.Ncpu = 4 - Config.Advanced.PprofPort = 0 - Config.Advanced.MetricsPort = 0 - Config.Advanced.LogFile = "redis-shake.log" - Config.Advanced.LogLevel = "info" - Config.Advanced.LogInterval = 5 - Config.Advanced.RDBRestoreCommandBehavior = "rewrite" - Config.Advanced.PipelineCountLimit = 1024 - Config.Advanced.TargetRedisClientMaxQuerybufLen = 1024 * 1000 * 1000 - Config.Advanced.TargetRedisProtoMaxBulkLen = 512 * 1000 * 1000 +type ShakeOptions struct { + Function string `mapstructure:"function" default:""` + Advanced AdvancedOptions + Module ModuleOptions } -func LoadFromFile(filename string) { +var Opt ShakeOptions - buf, err := ioutil.ReadFile(filename) - if err != nil { - panic(err.Error()) - } +func LoadConfig() *viper.Viper { + defaults.SetDefaults(&Opt) - decoder := toml.NewDecoder(bytes.NewReader(buf)) - decoder.SetStrict(true) - err = decoder.Decode(&Config) - if err != nil { - missingError, ok := err.(*toml.StrictMissingError) - if ok { - panic(fmt.Sprintf("decode config error:\n%s", missingError.String())) - } - panic(err.Error()) - } - - // dir - err = os.MkdirAll(Config.Advanced.Dir, os.ModePerm) - if err != nil { - panic(err.Error()) - } - err = os.Chdir(Config.Advanced.Dir) - if err != nil { - panic(err.Error()) + v := viper.New() + if len(os.Args) > 2 { + fmt.Println("Usage: redis-shake [config file]") + fmt.Println("Example: ") + fmt.Println(" redis-shake sync.toml # load config from sync.toml") + fmt.Println(" redis-shake # load config from environment variables") + os.Exit(1) } - - // cpu core - var ncpu int - if Config.Advanced.Ncpu == 0 { - ncpu = runtime.NumCPU() - } else { - ncpu = Config.Advanced.Ncpu + consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: "2006-01-02 15:04:05"} + logger := zerolog.New(consoleWriter).With().Timestamp().Logger() + // load config from file + if len(os.Args) == 2 { + logger.Info().Msgf("load config from file: %s", os.Args[1]) + configFile := os.Args[1] + v.SetConfigFile(configFile) + err := v.ReadInConfig() + if err != nil { + panic(err) + } } - runtime.GOMAXPROCS(ncpu) - if Config.Source.Version < 2.8 { - panic("source redis version must be greater than 2.8") - } - if Config.Target.Version < 2.8 { - panic("target redis version must be greater than 2.8") + // load config from environment variables + if len(os.Args) == 1 { + logger.Warn().Msg("load config from environment variables") + v.SetConfigType("env") + v.AutomaticEnv() } - if Config.Type != "sync" && Config.Type != "restore" && Config.Type != "scan" { - panic("type must be sync/restore/scan") + // unmarshal config + err := v.Unmarshal(&Opt) + if err != nil { + panic(err) } + return v } diff --git a/internal/entry/entry.go b/internal/entry/entry.go index 28dfc46a..e81a51d8 100644 --- a/internal/entry/entry.go +++ b/internal/entry/entry.go @@ -1,22 +1,25 @@ package entry -import "fmt" +import ( + "RedisShake/internal/client/proto" + "RedisShake/internal/commands" + "RedisShake/internal/log" + "bytes" + "strings" +) type Entry struct { - Id uint64 - IsBase bool // whether the command is decoded from dump.rdb file - DbId int - Argv []string - TimestampMs uint64 - - CmdName string - Group string - Keys []string - Slots []int - - // for statistics - Offset int64 - EncodedSize uint64 // the size of the entry after encode + DbId int // required + Argv []string // required + + CmdName string + Group string + Keys []string + KeyIndexes []int + Slots []int + + // for stat + SerializedSize int64 } func NewEntry() *Entry { @@ -24,6 +27,31 @@ func NewEntry() *Entry { return e } -func (e *Entry) ToString() string { - return fmt.Sprintf("%v", e.Argv) +func (e *Entry) String() string { + str := strings.Join(e.Argv, " ") + if len(str) > 100 { + str = str[:100] + "..." + } + return str +} + +func (e *Entry) Serialize() []byte { + buf := new(bytes.Buffer) + writer := proto.NewWriter(buf) + argvInterface := make([]interface{}, len(e.Argv)) + + for inx, item := range e.Argv { + argvInterface[inx] = item + } + err := writer.WriteArgs(argvInterface) + if err != nil { + log.Panicf(err.Error()) + } + e.SerializedSize = int64(buf.Len()) + return buf.Bytes() +} + +func (e *Entry) Parse() { + e.CmdName, e.Group, e.Keys, e.KeyIndexes = commands.CalcKeys(e.Argv) + e.Slots = commands.CalcSlots(e.Keys) } diff --git a/internal/filter/filter.go b/internal/filter/filter.go deleted file mode 100644 index 0d93a309..00000000 --- a/internal/filter/filter.go +++ /dev/null @@ -1,55 +0,0 @@ -package filter - -import ( - "github.com/alibaba/RedisShake/internal/entry" - lua "github.com/yuin/gopher-lua" -) - -const ( - Allow = 0 - Disallow = 1 - Error = 2 -) - -var luaInstance *lua.LState - -func LoadFromFile(luaFile string) { - luaInstance = lua.NewState() - err := luaInstance.DoFile(luaFile) - if err != nil { - panic(err) - } -} - -func Filter(e *entry.Entry) int { - if luaInstance == nil { - return Allow - } - keys := luaInstance.NewTable() - for _, key := range e.Keys { - keys.Append(lua.LString(key)) - } - - slots := luaInstance.NewTable() - for _, slot := range e.Slots { - slots.Append(lua.LNumber(slot)) - } - - f := luaInstance.GetGlobal("filter") - luaInstance.Push(f) - luaInstance.Push(lua.LNumber(e.Id)) // id - luaInstance.Push(lua.LBool(e.IsBase)) // is_base - luaInstance.Push(lua.LString(e.Group)) // group - luaInstance.Push(lua.LString(e.CmdName)) // cmd name - luaInstance.Push(keys) // keys - luaInstance.Push(slots) // slots - luaInstance.Push(lua.LNumber(e.DbId)) // dbid - luaInstance.Push(lua.LNumber(e.TimestampMs)) // timestamp_ms - - luaInstance.Call(8, 2) - - code := int(luaInstance.Get(1).(lua.LNumber)) - e.DbId = int(luaInstance.Get(2).(lua.LNumber)) - luaInstance.Pop(2) - return code -} diff --git a/internal/function/function.go b/internal/function/function.go new file mode 100644 index 00000000..4a13393d --- /dev/null +++ b/internal/function/function.go @@ -0,0 +1,90 @@ +package function + +import ( + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + lua "github.com/yuin/gopher-lua" + "strings" +) + +var luaString string + +func Init() { + luaString = config.Opt.Function + luaString = strings.TrimSpace(luaString) + if len(luaString) == 0 { + log.Infof("no function script") + return + } +} + +// DB +// GROUP +// CMD +// KEYS +// KEY_INDEXES +// SLOTS +// ARGV + +// shake.call(DB, ARGV) +// shake.log() + +func RunFunction(e *entry.Entry) []*entry.Entry { + entries := make([]*entry.Entry, 0) + if len(luaString) == 0 { + entries = append(entries, e) + return entries + } + + L := lua.NewState() + L.SetGlobal("DB", lua.LNumber(e.DbId)) + L.SetGlobal("GROUP", lua.LString(e.Group)) + L.SetGlobal("CMD", lua.LString(e.CmdName)) + keys := L.NewTable() + for _, key := range e.Keys { + keys.Append(lua.LString(key)) + } + L.SetGlobal("KEYS", keys) + slots := L.NewTable() + for _, slot := range e.Slots { + slots.Append(lua.LNumber(slot)) + } + keyIndexes := L.NewTable() + for _, keyIndex := range e.KeyIndexes { + keyIndexes.Append(lua.LNumber(keyIndex)) + } + L.SetGlobal("KEY_INDEXES", keyIndexes) + L.SetGlobal("SLOTS", slots) + argv := L.NewTable() + for _, arg := range e.Argv { + argv.Append(lua.LString(arg)) + } + L.SetGlobal("ARGV", argv) + shake := L.NewTypeMetatable("shake") + L.SetGlobal("shake", shake) + + L.SetField(shake, "call", L.NewFunction(func(ls *lua.LState) int { + db := ls.ToInt(1) + argv := ls.ToTable(2) + var argvStrings []string + for i := 1; i <= argv.Len(); i++ { + argvStrings = append(argvStrings, argv.RawGetInt(i).String()) + } + entries = append(entries, &entry.Entry{ + DbId: db, + Argv: argvStrings, + }) + return 0 + })) + L.SetField(shake, "log", L.NewFunction(func(ls *lua.LState) int { + log.Infof("lua log: %v", ls.ToString(1)) + return 0 + })) + err := L.DoString(luaString) + if err != nil { + log.Panicf("load function script failed: %v", err) + } + + return entries +} diff --git a/internal/log/func.go b/internal/log/func.go index f9604648..2d6aa150 100644 --- a/internal/log/func.go +++ b/internal/log/func.go @@ -1,16 +1,10 @@ package log import ( - "runtime/debug" - "strings" + "github.com/go-stack/stack" + "os" ) -func Assert(condition bool, msg string) { - if !condition { - Panicf("Assert failed: %s", msg) - } -} - func Debugf(format string, args ...interface{}) { logger.Debug().Msgf(format, args...) } @@ -24,20 +18,10 @@ func Warnf(format string, args ...interface{}) { } func Panicf(format string, args ...interface{}) { - stack := string(debug.Stack()) - stack = strings.ReplaceAll(stack, "\n\t", "]<-") - stack = strings.ReplaceAll(stack, "\n", " [") - logger.Info().Msg(stack) - - logger.Panic().Msgf(format, args...) -} - -func PanicError(err error) { - Panicf(err.Error()) -} - -func PanicIfError(err error) { - if err != nil { - PanicError(err) + frames := stack.Trace().TrimRuntime() + for _, frame := range frames { + logger.Warn().Msgf("%+v -> %n()", frame, frame) } + logger.Error().Msgf(format, args...) + os.Exit(1) } diff --git a/internal/log/init.go b/internal/log/init.go index 793f0c58..0f7d5305 100644 --- a/internal/log/init.go +++ b/internal/log/init.go @@ -2,17 +2,16 @@ package log import ( "fmt" - "github.com/alibaba/RedisShake/internal/config" "github.com/rs/zerolog" "os" + "path/filepath" ) var logger zerolog.Logger -func Init() { - +func Init(level string, file string, dir string) { // log level - switch config.Config.Advanced.LogLevel { + switch level { case "debug": zerolog.SetGlobalLevel(zerolog.DebugLevel) case "info": @@ -20,15 +19,31 @@ func Init() { case "warn": zerolog.SetGlobalLevel(zerolog.WarnLevel) default: - panic(fmt.Sprintf("unknown log level: %s", config.Config.Advanced.LogLevel)) + panic(fmt.Sprintf("unknown log level: %s", level)) + } + + // dir + dir, err := filepath.Abs(dir) + if err != nil { + panic(fmt.Sprintf("failed to determine current directory: %v", err)) + } + err = os.RemoveAll(dir) + if err != nil { + panic(fmt.Sprintf("remove dir failed. dir=[%s], error=[%v]", dir, err)) + } + err = os.MkdirAll(dir, 0777) + if err != nil { + panic(fmt.Sprintf("mkdir failed. dir=[%s], error=[%v]", dir, err)) } + path := filepath.Join(dir, file) // log file consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: "2006-01-02 15:04:05"} - fileWriter, err := os.OpenFile(config.Config.Advanced.LogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + fileWriter, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) if err != nil { - panic(fmt.Sprintf("open log file failed: %s", err)) + panic(fmt.Sprintf("open log file failed. file=[%s], err=[%s]", path, err)) } multi := zerolog.MultiLevelWriter(consoleWriter, fileWriter) logger = zerolog.New(multi).With().Timestamp().Logger() + Infof("log_level: [%v], log_file: [%v]", level, path) } diff --git a/internal/rdb/rdb.go b/internal/rdb/rdb.go index 45bd426b..81fa05a2 100644 --- a/internal/rdb/rdb.go +++ b/internal/rdb/rdb.go @@ -1,16 +1,15 @@ package rdb import ( + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" + "RedisShake/internal/rdb/types" + "RedisShake/internal/utils" "bufio" "bytes" "encoding/binary" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" - "github.com/alibaba/RedisShake/internal/rdb/types" - "github.com/alibaba/RedisShake/internal/statistics" - "github.com/alibaba/RedisShake/internal/utils" "io" "os" "strconv" @@ -32,7 +31,7 @@ const ( ) type Loader struct { - replStreamDbId int // https://github.com/alibaba/RedisShake/pull/430#issuecomment-1099014464 + replStreamDbId int // https://RedisShake/pull/430#issuecomment-1099014464 nowDBId int expireMs int64 @@ -44,15 +43,22 @@ type Loader struct { ch chan *entry.Entry dumpBuffer bytes.Buffer + + name string + updateFunc func(int64) } -func NewLoader(filPath string, ch chan *entry.Entry) *Loader { +func NewLoader(name string, updateFunc func(int64), filPath string, ch chan *entry.Entry) *Loader { ld := new(Loader) ld.ch = ch ld.filPath = filPath + ld.name = name + ld.updateFunc = updateFunc return ld } +// ParseRDB parse rdb file +// return repl stream db id func (ld *Loader) ParseRDB() int { var err error ld.fp, err = os.OpenFile(ld.filPath, os.O_RDONLY, 0666) @@ -66,43 +72,41 @@ func (ld *Loader) ParseRDB() int { } }() rd := bufio.NewReader(ld.fp) - //magic + version + // magic + version buf := make([]byte, 9) _, err = io.ReadFull(rd, buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } if !bytes.Equal(buf[:5], []byte("REDIS")) { log.Panicf("verify magic string, invalid file format. bytes=[%v]", buf[:5]) } version, err := strconv.Atoi(string(buf[5:])) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } - log.Infof("RDB version: %d", version) + log.Debugf("[%s] RDB version: %d", ld.name, version) // read entries ld.parseRDBEntry(rd) - // force update rdb_sent_size for issue: https://github.com/alibaba/RedisShake/issues/485 - fi, err := os.Stat(ld.filPath) - if err != nil { - log.Panicf("NewRDBReader: os.Stat error: %s", err.Error()) - } - statistics.Metrics.RdbSendSize = uint64(fi.Size()) return ld.replStreamDbId } func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { // for stat - UpdateRDBSentSize := func() { + updateProcessSize := func() { + if ld.updateFunc == nil { + return + } offset, err := ld.fp.Seek(0, io.SeekCurrent) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } - statistics.UpdateRDBSentSize(uint64(offset)) + ld.updateFunc(offset) } - defer UpdateRDBSentSize() + defer updateProcessSize() + // read one entry tick := time.Tick(time.Second * 1) for true { @@ -119,22 +123,21 @@ func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { var err error ld.replStreamDbId, err = strconv.Atoi(value) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } - log.Infof("RDB repl-stream-db: %d", ld.replStreamDbId) + log.Debugf("[%s] RDB repl-stream-db: [%s]", ld.name, value) } else if key == "lua" { e := entry.NewEntry() e.Argv = []string{"script", "load", value} - e.IsBase = true ld.ch <- e - log.Infof("LUA script: [%s]", value) + log.Debugf("[%s] LUA script: [%s]", ld.name, value) } else { - log.Infof("RDB AUX fields. key=[%s], value=[%s]", key, value) + log.Debugf("[%s] RDB AUX: key=[%s], value=[%s]", ld.name, key, value) } case kFlagResizeDB: dbSize := structure.ReadLength(rd) expireSize := structure.ReadLength(rd) - log.Infof("RDB resize db. db_size=[%d], expire_size=[%d]", dbSize, expireSize) + log.Debugf("[%s] RDB resize db: db_size=[%d], expire_size=[%d]", ld.name, dbSize, expireSize) case kFlagExpireMs: ld.expireMs = int64(structure.ReadUint64(rd)) - time.Now().UnixMilli() if ld.expireMs < 0 { @@ -154,40 +157,37 @@ func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { var value bytes.Buffer anotherReader := io.TeeReader(rd, &value) o := types.ParseObject(anotherReader, typeByte, key) - if uint64(value.Len()) > config.Config.Advanced.TargetRedisProtoMaxBulkLen { + if uint64(value.Len()) > config.Opt.Advanced.TargetRedisProtoMaxBulkLen { cmds := o.Rewrite() for _, cmd := range cmds { e := entry.NewEntry() - e.IsBase = true e.DbId = ld.nowDBId e.Argv = cmd ld.ch <- e } if ld.expireMs != 0 { e := entry.NewEntry() - e.IsBase = true e.DbId = ld.nowDBId e.Argv = []string{"PEXPIRE", key, strconv.FormatInt(ld.expireMs, 10)} ld.ch <- e } } else { e := entry.NewEntry() - e.IsBase = true e.DbId = ld.nowDBId v := ld.createValueDump(typeByte, value.Bytes()) e.Argv = []string{"restore", key, strconv.FormatInt(ld.expireMs, 10), v} - if config.Config.Advanced.RDBRestoreCommandBehavior == "rewrite" { - if config.Config.Target.Version < 3.0 { - log.Panicf("RDB restore command behavior is rewrite, but target redis version is %f, not support REPLACE modifier", config.Config.Target.Version) - } + if config.Opt.Advanced.RDBRestoreCommandBehavior == "rewrite" { + //if config.Opt.Target.Version < 3.0 { + // log.Panicf("RDB restore command behavior is rewrite, but target redis version is %f, not support REPLACE modifier", config.Config.Target.Version) + //} e.Argv = append(e.Argv, "replace") } - if ld.idle != 0 && config.Config.Target.Version >= 5.0 { - e.Argv = append(e.Argv, "idletime", strconv.FormatInt(ld.idle, 10)) - } - if ld.freq != 0 && config.Config.Target.Version >= 5.0 { - e.Argv = append(e.Argv, "freq", strconv.FormatInt(ld.freq, 10)) - } + //if ld.idle != 0 && config.Config.Target.Version >= 5.0 { + // e.Argv = append(e.Argv, "idletime", strconv.FormatInt(ld.idle, 10)) + //} + //if ld.freq != 0 && config.Config.Target.Version >= 5.0 { + // e.Argv = append(e.Argv, "freq", strconv.FormatInt(ld.freq, 10)) + //} ld.ch <- e } ld.expireMs = 0 @@ -196,7 +196,7 @@ func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { } select { case <-tick: - UpdateRDBSentSize() + updateProcessSize() default: } } diff --git a/internal/rdb/structure/byte.go b/internal/rdb/structure/byte.go index d52df8ec..d8b0bed5 100644 --- a/internal/rdb/structure/byte.go +++ b/internal/rdb/structure/byte.go @@ -1,7 +1,7 @@ package structure import ( - "github.com/alibaba/RedisShake/internal/log" + "RedisShake/internal/log" "io" ) @@ -14,7 +14,7 @@ func ReadBytes(rd io.Reader, n int) []byte { buf := make([]byte, n) _, err := io.ReadFull(rd, buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } return buf } diff --git a/internal/rdb/structure/float.go b/internal/rdb/structure/float.go index 8351d96a..a254f863 100644 --- a/internal/rdb/structure/float.go +++ b/internal/rdb/structure/float.go @@ -1,8 +1,8 @@ package structure import ( + "RedisShake/internal/log" "encoding/binary" - "github.com/alibaba/RedisShake/internal/log" "io" "math" "strconv" @@ -27,7 +27,7 @@ func ReadFloat(rd io.Reader) float64 { v, err := strconv.ParseFloat(string(buf), 64) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } return v } @@ -37,7 +37,7 @@ func ReadDouble(rd io.Reader) float64 { var buf = make([]byte, 8) _, err := io.ReadFull(rd, buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } num := binary.LittleEndian.Uint64(buf) return math.Float64frombits(num) diff --git a/internal/rdb/structure/length.go b/internal/rdb/structure/length.go index e62543fe..dc598c08 100644 --- a/internal/rdb/structure/length.go +++ b/internal/rdb/structure/length.go @@ -1,9 +1,9 @@ package structure import ( + "RedisShake/internal/log" "encoding/binary" "fmt" - "github.com/alibaba/RedisShake/internal/log" "io" ) @@ -22,7 +22,7 @@ func ReadLength(rd io.Reader) uint64 { log.Panicf("illegal length special=true, encoding: %d", length) } if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } return length } diff --git a/internal/rdb/structure/listpack.go b/internal/rdb/structure/listpack.go index 2d30374a..ab4600a5 100644 --- a/internal/rdb/structure/listpack.go +++ b/internal/rdb/structure/listpack.go @@ -1,8 +1,8 @@ package structure import ( + "RedisShake/internal/log" "bufio" - "github.com/alibaba/RedisShake/internal/log" "io" "math" "strconv" diff --git a/internal/rdb/structure/module2_struct.go b/internal/rdb/structure/module2_struct.go new file mode 100644 index 00000000..ca3da026 --- /dev/null +++ b/internal/rdb/structure/module2_struct.go @@ -0,0 +1,71 @@ +package structure + +import ( + "fmt" + "io" + "log" + "strconv" +) + +const ( + rdbModuleOpcodeEOF = 0 // End of module value. + rdbModuleOpcodeSINT = 1 // Signed integer. + rdbModuleOpcodeUINT = 2 // Unsigned integer. + rdbModuleOpcodeFLOAT = 3 // Float. + rdbModuleOpcodeDOUBLE = 4 // Double. + rdbModuleOpcodeSTRING = 5 // String. +) + +func ReadModuleUnsigned(rd io.Reader) string { + opcode := ReadByte(rd) + if opcode != rdbModuleOpcodeUINT { + log.Panicf("Unknown module unsigned encode type") + } + value := ReadLength(rd) + return strconv.FormatUint(value, 10) +} + +func ReadModuleSigned(rd io.Reader) string { + opcode := ReadByte(rd) + if opcode != rdbModuleOpcodeSINT { + log.Panicf("Unknown module signed encode type") + } + value := ReadLength(rd) + return strconv.FormatUint(value, 10) +} + +func ReadModuleFloat(rd io.Reader) string { + opcode := ReadByte(rd) + if opcode != rdbModuleOpcodeDOUBLE { + + log.Panicf("Unknown module double encode type") + } + value := ReadDouble(rd) + return fmt.Sprintf("%f", value) +} + +func ReadModuleDouble(rd io.Reader) string { + opcode := ReadByte(rd) + if opcode != rdbModuleOpcodeDOUBLE { + log.Panicf("Unknown module double encode type") + } + value := ReadDouble(rd) + return fmt.Sprintf("%f", value) +} + +func ReadModuleString(rd io.Reader) string { + opcode := ReadByte(rd) + if opcode != rdbModuleOpcodeSTRING { + log.Panicf("Unknown module string encode type") + } + return ReadString(rd) +} + +func ReadModuleEof(rd io.Reader) error { + eof := ReadLength(rd) + if eof != rdbModuleOpcodeEOF { + log.Panicf("The RDB file is not teminated by the proper module value EOF marker") + } + return nil + +} diff --git a/internal/rdb/structure/string.go b/internal/rdb/structure/string.go index f53559e3..101148a1 100644 --- a/internal/rdb/structure/string.go +++ b/internal/rdb/structure/string.go @@ -1,7 +1,7 @@ package structure import ( - "github.com/alibaba/RedisShake/internal/log" + "RedisShake/internal/log" "io" "strconv" ) @@ -16,7 +16,7 @@ const ( func ReadString(rd io.Reader) string { length, special, err := readEncodedLength(rd) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } if special { switch length { diff --git a/internal/rdb/structure/ziplist.go b/internal/rdb/structure/ziplist.go index 141ea026..4dcd37f3 100644 --- a/internal/rdb/structure/ziplist.go +++ b/internal/rdb/structure/ziplist.go @@ -1,9 +1,9 @@ package structure import ( + "RedisShake/internal/log" "bufio" "encoding/binary" - "github.com/alibaba/RedisShake/internal/log" "io" "strconv" "strings" diff --git a/internal/rdb/types/hash.go b/internal/rdb/types/hash.go index e27b67e8..2d373ae6 100644 --- a/internal/rdb/types/hash.go +++ b/internal/rdb/types/hash.go @@ -1,8 +1,8 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/rdb/types/interface.go b/internal/rdb/types/interface.go index 7d9fb3d5..03ba67e3 100644 --- a/internal/rdb/types/interface.go +++ b/internal/rdb/types/interface.go @@ -1,7 +1,7 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" + "RedisShake/internal/log" "io" ) @@ -91,8 +91,7 @@ func ParseObject(rd io.Reader, typeByte byte, key string) RedisObject { o.LoadFromBuffer(rd, key, typeByte) return o case rdbTypeModule, rdbTypeModule2: // module - o := new(ModuleObject) - o.LoadFromBuffer(rd, key, typeByte) + o := PareseModuleType(rd, key, typeByte) return o } log.Panicf("unknown type byte: %d", typeByte) diff --git a/internal/rdb/types/list.go b/internal/rdb/types/list.go index 57e23c08..4f6b8dff 100644 --- a/internal/rdb/types/list.go +++ b/internal/rdb/types/list.go @@ -1,8 +1,8 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/rdb/types/mbbloom.go b/internal/rdb/types/mbbloom.go new file mode 100644 index 00000000..5704c2a0 --- /dev/null +++ b/internal/rdb/types/mbbloom.go @@ -0,0 +1,223 @@ +package types + +import ( + "RedisShake/internal/config" + "RedisShake/internal/rdb/structure" + "io" + "strconv" + "unsafe" +) + +// BloomObject for MBbloom-- at https://github.com/RedisBloom/RedisBloom +type BloomObject struct { + encver int + key string + sb chain +} + +type chain struct { + filters []link + size uint64 + nfilters uint64 + options uint64 + growth uint64 +} + +type link struct { + inner bloom + size uint64 +} + +type bloom struct { + hashes uint64 + n2 uint64 + entries uint64 + err float64 + bpe float64 + bf string + bits uint64 +} + +type dumpedChainHeader struct { + size uint64 + nfilters uint32 + options uint32 + growth uint32 +} + +type dumpedChainLink struct { + bytes uint64 + bits uint64 + size uint64 + err float64 + bpe float64 + hashes uint32 + entries uint32 + enthigh uint32 + n2 uint8 +} + +const ( + BF_MIN_OPTIONS_ENC = 2 + BF_MIN_GROWTH_ENC = 4 +) + +const ( + DUMPED_CHAIN_HEADER_SIZE = 20 + DUMPED_CHAIN_LINK_SIZE = 53 + DUMPED_CHAIN_HEADER_SIZE_V3 = 16 + DUMPED_CHAIN_LINK_SIZE_V3 = 49 +) + +const MAX_SCANDUMP_SIZE = 10485760 // 10MB + +func (o *BloomObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { + o.key = key + var sb chain + sb.size = readUnsigned(rd) + sb.nfilters = readUnsigned(rd) + if o.encver >= BF_MIN_OPTIONS_ENC { + sb.options = readUnsigned(rd) + } + if o.encver >= BF_MIN_GROWTH_ENC { + sb.growth = readUnsigned(rd) + } else { + sb.growth = 2 + } + for i := uint64(0); i < sb.nfilters; i++ { + var lb link + bm := &lb.inner + bm.entries = readUnsigned(rd) + bm.err = readDouble(rd) + bm.hashes = readUnsigned(rd) + bm.bpe = readDouble(rd) + if o.encver == 0 { + bm.bits = uint64(float64(bm.entries) * bm.bpe) + } else { + bm.bits = readUnsigned(rd) + bm.n2 = readUnsigned(rd) + } + bm.bf = structure.ReadModuleString(rd) + lb.size = readUnsigned(rd) + sb.filters = append(sb.filters, lb) + } + o.sb = sb + structure.ReadModuleEof(rd) + return +} + +func readUnsigned(rd io.Reader) uint64 { + v := structure.ReadModuleUnsigned(rd) + u, err := strconv.ParseUint(v, 10, 64) + if err != nil { + panic(err) + } + return u +} + +func readDouble(rd io.Reader) float64 { + v := structure.ReadModuleDouble(rd) + f, err := strconv.ParseFloat(v, 64) + if err != nil { + panic(err) + } + return f +} + +func (o *BloomObject) Rewrite() []RedisCmd { + var cs []RedisCmd + var h string + if ver := config.Opt.Module.TargetMBbloomVersion; ver > 20200 { + h = getEncodedHeader(&o.sb, true, true) + } else if ver == 20200 { + h = getEncodedHeader(&o.sb, true, false) + } else if ver >= 10000 { + h = getEncodedHeader(&o.sb, false, false) + } else if o.encver < BF_MIN_GROWTH_ENC { + h = getEncodedHeader(&o.sb, false, false) + } else { + h = getEncodedHeader(&o.sb, true, true) + } + cmd := RedisCmd{"BF.LOADCHUNK", o.key, "1", h} + cs = append(cs, cmd) + curIter := uint64(1) + for { + c := getEncodedChunk(&o.sb, &curIter, MAX_SCANDUMP_SIZE) + if c == "" { + break + } + cmd := RedisCmd{"BF.LOADCHUNK", o.key, strconv.FormatUint(curIter, 10), c} + cs = append(cs, cmd) + } + return cs +} + +func getEncodedHeader(sb *chain, withGrowth, bigEntries bool) string { + var hs uint64 = DUMPED_CHAIN_HEADER_SIZE_V3 + if withGrowth { + hs = DUMPED_CHAIN_HEADER_SIZE + } + var ls uint64 = DUMPED_CHAIN_LINK_SIZE_V3 + if bigEntries { + ls = DUMPED_CHAIN_LINK_SIZE + } + h := make([]byte, hs+ls*sb.nfilters) + ph := (*dumpedChainHeader)(unsafe.Pointer(&h[0])) + ph.size = sb.size + ph.nfilters = uint32(sb.nfilters) + ph.options = uint32(sb.options) + if withGrowth { + ph.growth = uint32(sb.growth) + } + for i := uint64(0); i < sb.nfilters; i++ { + pl := (*dumpedChainLink)(unsafe.Add(unsafe.Pointer(&h[0]), hs+ls*i)) + sl := sb.filters[i] + pl.bytes = uint64(len(sl.inner.bf)) + pl.bits = sl.inner.bits + pl.size = sl.size + pl.err = sl.inner.err + pl.hashes = uint32(sl.inner.hashes) + pl.bpe = sl.inner.bpe + if bigEntries { + *(*uint64)(unsafe.Pointer(&pl.entries)) = sl.inner.entries + pl.n2 = uint8(sl.inner.n2) + } else { + pl.entries = uint32(sl.inner.entries) + *(*uint8)(unsafe.Pointer(&pl.enthigh)) = uint8(sl.inner.n2) + } + } + return *(*string)(unsafe.Pointer(&h)) +} + +func getEncodedChunk(sb *chain, curIter *uint64, maxChunkSize uint64) string { + pl, off := getLinkPos(sb, *curIter) + if pl == nil { + *curIter = 0 + return "" + } + l := maxChunkSize + lr := uint64(len(pl.inner.bf)) - off + if lr < l { + l = lr + } + *curIter += l + return pl.inner.bf[off : off+l] +} + +func getLinkPos(sb *chain, curIter uint64) (pl *link, offset uint64) { + curIter-- + var seekPos uint64 + for i := uint64(0); i < sb.nfilters; i++ { + if seekPos+uint64(len(sb.filters[i].inner.bf)) > curIter { + pl = &sb.filters[i] + break + } else { + seekPos += uint64(len(sb.filters[i].inner.bf)) + } + } + if pl == nil { + return + } + offset = curIter - seekPos + return +} diff --git a/internal/rdb/types/module2.go b/internal/rdb/types/module2.go index 26ab46e0..fad8caa8 100644 --- a/internal/rdb/types/module2.go +++ b/internal/rdb/types/module2.go @@ -1,40 +1,44 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" "io" + + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" ) -type ModuleObject struct { +type ModuleObject interface { + RedisObject } -func (o *ModuleObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { +func PareseModuleType(rd io.Reader, key string, typeByte byte) ModuleObject { if typeByte == rdbTypeModule { log.Panicf("module type with version 1 is not supported, key=[%s]", key) } moduleId := structure.ReadLength(rd) moduleName := moduleTypeNameByID(moduleId) - opcode := structure.ReadByte(rd) - for opcode != rdbModuleOpcodeEOF { - switch opcode { - case rdbModuleOpcodeSINT: - case rdbModuleOpcodeUINT: - structure.ReadLength(rd) - case rdbModuleOpcodeFLOAT: - structure.ReadFloat(rd) - case rdbModuleOpcodeDOUBLE: - structure.ReadDouble(rd) - case rdbModuleOpcodeSTRING: - structure.ReadString(rd) - default: - log.Panicf("unknown module opcode=[%d], module name=[%s]", opcode, moduleName) - } - opcode = structure.ReadByte(rd) + switch moduleName { + case "exstrtype": + o := new(TairStringObject) + o.LoadFromBuffer(rd, key, typeByte) + return o + case "tairhash-": + o := new(TairHashObject) + o.LoadFromBuffer(rd, key, typeByte) + return o + case "tairzset_": + o := new(TairZsetObject) + o.LoadFromBuffer(rd, key, typeByte) + return o + case "MBbloom--": + o := new(BloomObject) + o.encver = int(moduleId & 1023) + o.LoadFromBuffer(rd, key, typeByte) + return o + default: + log.Panicf("unsupported module type: %s", moduleName) + return nil + } -} -func (o *ModuleObject) Rewrite() []RedisCmd { - log.Panicf("module Rewrite not implemented") - return nil } diff --git a/internal/rdb/types/set.go b/internal/rdb/types/set.go index ac02f378..2f83b843 100644 --- a/internal/rdb/types/set.go +++ b/internal/rdb/types/set.go @@ -1,8 +1,8 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/rdb/types/stream.go b/internal/rdb/types/stream.go index 9b0a6b03..402affbc 100644 --- a/internal/rdb/types/stream.go +++ b/internal/rdb/types/stream.go @@ -1,10 +1,10 @@ package types import ( + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "encoding/binary" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" "io" "strconv" ) diff --git a/internal/rdb/types/string.go b/internal/rdb/types/string.go index 2adf51c6..6321fd36 100644 --- a/internal/rdb/types/string.go +++ b/internal/rdb/types/string.go @@ -1,7 +1,7 @@ package types import ( - "github.com/alibaba/RedisShake/internal/rdb/structure" + "RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/rdb/types/tairhash.go b/internal/rdb/types/tairhash.go new file mode 100644 index 00000000..7476bd82 --- /dev/null +++ b/internal/rdb/types/tairhash.go @@ -0,0 +1,57 @@ +package types + +import ( + "io" + "strconv" + + "RedisShake/internal/rdb/structure" +) + +type TairHashValue struct { + skey string + version string + expire string + fieldValue string +} + +type TairHashObject struct { + dictSize string + key string + value []TairHashValue +} + +func (o *TairHashObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { + o.dictSize = structure.ReadModuleUnsigned(rd) + o.key = structure.ReadModuleString(rd) + + size, _ := strconv.Atoi(o.dictSize) + for i := 0; i < size; i++ { + hashValue := TairHashValue{ + structure.ReadModuleString(rd), + structure.ReadModuleUnsigned(rd), + structure.ReadModuleUnsigned(rd), + structure.ReadModuleString(rd), + } + o.value = append(o.value, hashValue) + } + structure.ReadModuleEof(rd) +} + +func (o *TairHashObject) Rewrite() []RedisCmd { + var cmds []RedisCmd + size, _ := strconv.Atoi(o.dictSize) + for i := 0; i < size; i++ { + cmd := []string{} + expire, _ := strconv.Atoi(o.value[i].expire) + if expire == 0 { + cmd = append(cmd, "EXHSET", o.key, o.value[i].skey, o.value[i].fieldValue) + } else { + cmd = append(cmd, "EXHSET", o.key, o.value[i].skey, o.value[i].fieldValue, + "ABS", o.value[i].version, + "PXAT", o.value[i].expire) + } + + cmds = append(cmds, cmd) + } + return cmds +} diff --git a/internal/rdb/types/tairstring.go b/internal/rdb/types/tairstring.go new file mode 100644 index 00000000..e5db7dbd --- /dev/null +++ b/internal/rdb/types/tairstring.go @@ -0,0 +1,32 @@ +package types + +import ( + "io" + + "RedisShake/internal/rdb/structure" +) + +type TairStringValue struct { + version string + flags string + tairValue string +} + +type TairStringObject struct { + value TairStringValue + key string +} + +func (o *TairStringObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { + o.key = key + o.value.version = structure.ReadModuleUnsigned(rd) + o.value.flags = structure.ReadModuleUnsigned(rd) + o.value.tairValue = structure.ReadModuleString(rd) + structure.ReadModuleEof(rd) +} + +func (o *TairStringObject) Rewrite() []RedisCmd { + cmd := RedisCmd{} + cmd = append(cmd, "EXSET", o.key, o.value.tairValue, "ABS", o.value.version, "FLAGS", o.value.flags) + return []RedisCmd{cmd} +} diff --git a/internal/rdb/types/tairzset.go b/internal/rdb/types/tairzset.go new file mode 100644 index 00000000..52f75b54 --- /dev/null +++ b/internal/rdb/types/tairzset.go @@ -0,0 +1,46 @@ +package types + +import ( + "io" + "strconv" + "strings" + + "RedisShake/internal/rdb/structure" +) + +type TairZsetObject struct { + key string + length string + scoreNum string + value map[string][]string +} + +func (o *TairZsetObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { + o.key = key + o.length = structure.ReadModuleUnsigned(rd) + o.scoreNum = structure.ReadModuleUnsigned(rd) + + len, _ := strconv.Atoi(o.length) + scoreNum, _ := strconv.Atoi(o.scoreNum) + valueMap := make(map[string][]string) + for i := 0; i < len; i++ { + key := structure.ReadModuleString(rd) + values := []string{} + for j := 0; j < scoreNum; j++ { + values = append(values, structure.ReadModuleDouble(rd)) + } + valueMap[key] = values + } + o.value = valueMap + structure.ReadModuleEof(rd) +} + +func (o *TairZsetObject) Rewrite() []RedisCmd { + var cmds []RedisCmd + for k, v := range o.value { + score := strings.Join(v, "#") + cmd := RedisCmd{"EXZADD", o.key, score, k} + cmds = append(cmds, cmd) + } + return cmds +} diff --git a/internal/rdb/types/zset.go b/internal/rdb/types/zset.go index 34f99b37..16f1d8a1 100644 --- a/internal/rdb/types/zset.go +++ b/internal/rdb/types/zset.go @@ -1,9 +1,9 @@ package types import ( + "RedisShake/internal/log" + "RedisShake/internal/rdb/structure" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" "io" ) diff --git a/internal/reader/aof_reader.go b/internal/reader/aof_reader.go new file mode 100644 index 00000000..e5299989 --- /dev/null +++ b/internal/reader/aof_reader.go @@ -0,0 +1,104 @@ +package reader + +import ( + "RedisShake/internal/aof" + "path/filepath" + + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/utils" + + "github.com/dustin/go-humanize" +) + +type AOFReaderOptions struct { + Filepath string `mapstructure:"filepath" default:""` + AOFTimestamp int64 `mapstructure:"timestamp" default:"0"` +} + +type aofReader struct { + path string + ch chan *entry.Entry + + stat struct { + AOFName string `json:"aof_name"` + AOFStatus string `json:"aof_status"` + AOFFilepath string `json:"aof_file_path"` + AOFFileSizeBytes int64 `json:"aof_file_size_bytes"` + AOFFileSizeHuman string `json:"aof_file_size_human"` + AOFFileSentBytes int64 `json:"aof_file_sent_bytes"` + AOFFileSentHuman string `json:"aof_file_sent_human"` + AOFPercent string `json:"aof_percent"` + AOFTimestamp int64 `json:"aof_time_stamp"` + } +} + +func (r *aofReader) Status() interface{} { + return r.stat +} + +func (r *aofReader) StatusString() string { + return r.stat.AOFStatus +} + +func (r *aofReader) StatusConsistent() bool { + return r.stat.AOFFileSentBytes == r.stat.AOFFileSizeBytes +} + +func NewAOFReader(opts *AOFReaderOptions) Reader { + log.Infof("NewAOFReader: path=[%s]", opts.Filepath) + absolutePath, err := filepath.Abs(opts.Filepath) + if err != nil { + log.Panicf("NewAOFReader: filepath.Abs error: %s", err.Error()) + } + log.Infof("NewAOFReader: absolute path=[%s]", absolutePath) + r := &aofReader{ + path: absolutePath, + ch: make(chan *entry.Entry), + } + r.stat.AOFName = "aof_reader" + r.stat.AOFStatus = "init" + r.stat.AOFFilepath = absolutePath + r.stat.AOFFileSizeBytes = int64(utils.GetFileSize(absolutePath)) + r.stat.AOFFileSizeHuman = humanize.Bytes(uint64(r.stat.AOFFileSizeBytes)) + r.stat.AOFTimestamp = opts.AOFTimestamp + return r +} + +func (r *aofReader) StartRead() chan *entry.Entry { + //init entry + r.ch = make(chan *entry.Entry, 1024) + + // start read aof + go func() { + aofFileInfo := NewAOFFileInfo(r.path, r.ch) + // try load manifest file + aofFileInfo.AOFLoadManifestFromDisk() + manifestInfo := aofFileInfo.AOFManifest + if manifestInfo == nil { // load single aof file + log.Infof("start send single AOF path=[%s]", r.path) + aofLoader := aof.NewLoader(r.path, r.ch) + ret := aofLoader.LoadSingleAppendOnlyFile(r.stat.AOFTimestamp) + if ret == AOFOk || ret == AOFTruncated { + log.Infof("The AOF File was successfully loaded") + } else { + log.Infof("There was an error opening the AOF File.") + } + log.Infof("Send single AOF finished. path=[%s]", r.path) + close(r.ch) + } else { + aofLoader := NewAOFFileInfo(r.path, r.ch) + ret := aofLoader.LoadAppendOnlyFile(manifestInfo, r.stat.AOFTimestamp) + if ret == AOFOk || ret == AOFTruncated { + log.Infof("The AOF File was successfully loaded") + } else { + log.Infof("There was an error opening the AOF File.") + } + log.Infof("Send multi-part AOF finished. path=[%s]", r.path) + close(r.ch) + } + + }() + + return r.ch +} diff --git a/internal/reader/interface.go b/internal/reader/interface.go index 9d583e43..7ddc0d0a 100644 --- a/internal/reader/interface.go +++ b/internal/reader/interface.go @@ -1,7 +1,11 @@ package reader -import "github.com/alibaba/RedisShake/internal/entry" +import ( + "RedisShake/internal/entry" + "RedisShake/internal/status" +) type Reader interface { + status.Statusable StartRead() chan *entry.Entry } diff --git a/internal/reader/parsing_aof.go b/internal/reader/parsing_aof.go new file mode 100644 index 00000000..329f8b43 --- /dev/null +++ b/internal/reader/parsing_aof.go @@ -0,0 +1,743 @@ +package reader + +import ( + "RedisShake/internal/aof" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "bufio" + "bytes" + "container/list" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + "unicode" +) + +const ( + AOFManifestFileTypeBase = "b" /* Base File */ + AOFManifestTypeHist = "h" /* History File */ + AOFManifestTypeIncr = "i" /* INCR File */ + AOFNotExist = 1 + AOFOpenErr = 3 + AOFOk = 0 + AOFEmpty = 2 + AOFFailed = 4 + AOFTruncated = 5 + AOFManifestKeyFileName = "File" + AOFManifestKeyFileSeq = "seq" + AOFManifestKeyFileType = "type" +) + +func Ustime() int64 { + tv := time.Now() + ust := int64(tv.UnixNano()) / 1000 + return ust + +} + +func MakePath(Paths string, FileName string) string { + return path.Join(Paths, FileName) +} + +func StringNeedsRepr(s string) int { + sLen := len(s) + point := 0 + for sLen > 0 { + if s[point] == '\\' || s[point] == '"' || s[point] == '\n' || s[point] == '\r' || + s[point] == '\t' || s[point] == '\a' || s[point] == '\b' || !unicode.IsPrint(rune(s[point])) || unicode.IsSpace(rune(s[point])) { + return 1 + } + sLen-- + point++ + } + + return 0 +} + +type INFO struct { + AOFDirName string + AOFManifest *AOFManifest + AOFFileName string + AOFCurrentSize int64 + AOFRewriteBaseSize int64 + updateLoadingFile string + ch chan *entry.Entry +} + +func (aofInfo *INFO) GetAOFDirName() string { + return aofInfo.AOFDirName +} + +func NewAOFFileInfo(aofFilePath string, ch chan *entry.Entry) *INFO { + return &INFO{ + AOFDirName: filepath.Dir(aofFilePath), + AOFManifest: nil, + AOFFileName: filepath.Base(aofFilePath), + AOFCurrentSize: 0, + AOFRewriteBaseSize: 0, + ch: ch, + } +} + +func (a *AOFInfo) GetAOFInfoName() string { + return a.FileName +} + +func DirExists(dName string) int { + _, err := os.Stat(dName) + if err != nil { + return 0 + } + + return 1 +} + +func FileExist(FileName string) int { + _, err := os.Stat(FileName) + if err != nil { + return 0 + } + + return 1 +} + +func IsHexDigit(c byte) bool { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F') +} + +func HexDigitToInt(c byte) int { + switch c { + case '0': + return 0 + case '1': + return 1 + case '2': + return 2 + case '3': + return 3 + case '4': + return 4 + case '5': + return 5 + case '6': + return 6 + case '7': + return 7 + case '8': + return 8 + case '9': + return 9 + case 'a', 'A': + return 10 + case 'b', 'B': + return 11 + case 'c', 'C': + return 12 + case 'd', 'D': + return 13 + case 'e', 'E': + return 14 + case 'f', 'F': + return 15 + default: + return 0 + } +} + +func SplitArgs(line string) ([]string, int) { + var p string = line + var Current string + var vector []string + argc := 0 + i := 0 + lens := len(p) + for { //SKIP BLANKS + for i < lens && unicode.IsSpace(rune(p[i])) { + i++ + } + if i < lens { + inq := false // Set to true if we are in "quotes" + insq := false // Set to true if we are in 'single quotes' + done := false + + for !done { + if inq { + + if p[i] == '\\' && (p[i+1]) == 'x' && IsHexDigit(p[i+2]) && IsHexDigit(p[i+3]) { + _, err1 := strconv.ParseInt(string(p[i+2]), 16, 64) + _, err2 := strconv.ParseInt(string(p[i+3]), 16, 64) + if err1 == nil && err2 == nil { + hexadecimal := (HexDigitToInt((p[i+2])) * 16) + HexDigitToInt(p[i+3]) + Current = Current + fmt.Sprint(hexadecimal) + i += 3 + } + + } else if p[i] == '\\' && i+1 < lens { + var c byte + i++ + switch p[i] { + case 'n': + c = '\n' + case 'r': + c = 'r' + case 'a': + c = '\a' + default: + c = p[i] + } + Current += string(c) + } else if p[i] == '"' { + if i+1 < lens && !unicode.IsSpace(rune(p[i+1])) { + return nil, 0 + } + done = true + } else if i >= lens { + return nil, 0 + } else { + Current += string(p[i]) + } + } else if insq { + if p[i] == '\\' && p[i+1] == '\'' { + i++ + Current += "'" + } else if p[i] == '\'' { + if i+1 < lens && !unicode.IsSpace(rune(p[i+1])) { + return nil, 0 + } + done = true + } else if i >= lens { + return nil, 0 + } else { + Current += string(p[i]) + } + + } else { + switch p[i] { + case ' ', '\n', '\r', '\t', '\000': + done = true + case '"': + inq = true + case '\'': + insq = true + default: + Current += string(p[i]) + } + } + if i < lens { + i++ + } + if i == lens { + done = true + } + } + + vector = append(vector, Current) + argc++ + Current = "" + + } else { + return vector, argc + } + + } +} + +func StringCatPrintf(s string, fmtStr string, args ...interface{}) string { + result := fmt.Sprintf(fmtStr, args...) + if s == "" { + return result + } else { + return s + result + } +} + +func StringCatRepr(s string, p string, length int) string { + s = s + ("\"") + for i := 0; i < length; i++ { + switch p[i] { + case '\\', '"': + s = StringCatPrintf(s, "\\%c", p[i]) + case '\n': + s = s + "\\n" + case '\r': + s = s + "\\r" + case '\t': + s = s + "\\t" + case '\a': + s = s + "\\a" + case '\b': + s = s + "\\b" + default: + if strconv.IsPrint(rune(p[i])) { + s = s + string(p[i]) + } else { + s = s + "\\x%02x" + } + } + } + return s + "\"" +} + +func (aofInfo *INFO) UpdateLoadingFileName(FileName string) { + aofInfo.updateLoadingFile = FileName +} + +// AOFInfo AOF manifest definition +type AOFInfo struct { + FileName string + FileSeq int64 + AOFFileType string +} + +func AOFInfoCreate() *AOFInfo { + return new(AOFInfo) +} + +func AOFInfoDup(orig *AOFInfo) *AOFInfo { + if orig == nil { + log.Panicf("Assertion failed: orig != nil") + } + ai := AOFInfoCreate() + ai.FileName = orig.FileName + ai.FileSeq = orig.FileSeq + ai.AOFFileType = orig.AOFFileType + return ai +} + +func AOFInfoFormat(buf string, ai *AOFInfo) string { + var aofManifestTostring string + if StringNeedsRepr(ai.FileName) == 1 { + aofManifestTostring = StringCatRepr("", ai.FileName, len(ai.FileName)) + } + var ret string + if aofManifestTostring != "" { + ret = StringCatPrintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, aofManifestTostring, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) + } else { + ret = StringCatPrintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, ai.FileName, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) + } + return ret +} + +func PathIsBaseName(Path string) bool { + return strings.IndexByte(Path, '/') == -1 && strings.IndexByte(Path, '\\') == -1 +} + +func AOFLoadManifestFromFile(amFilepath string) *AOFManifest { + var maxSeq int64 + am := AOFManifestCreate() + fp, err := os.Open(amFilepath) + if err != nil { + log.Panicf("Fatal error:can't open the AOF manifest %v for reading: %v", amFilepath, err) + } + defer fp.Close() + var argv []string + var ai *AOFInfo + var line string + lineNum := 0 + fpReader := bufio.NewReader(fp) + for { + buf, err := fpReader.ReadString('\n') + if err != nil { + if err == io.EOF { + if lineNum == 0 { + log.Infof("Found an empty AOF manifest") + am = nil + return am + } else { + break + } + + } else { + log.Infof("Reading the manifest file, at line %d", lineNum) + log.Infof("Read AOF manifest failed") + am = nil + return am + + } + } + + lineNum++ + if buf[0] == '#' { + continue + } + if !strings.Contains(buf, "\n") { + log.Infof("Reading the manifest file, at line %d", lineNum) + log.Infof("The AOF manifest File contains too long line") + return nil + } + line = strings.Trim(buf, " \t\r\n") + if len(line) == 0 { + log.Infof("Reading the manifest file, at line %d", lineNum) + log.Infof("Invalid AOF manifest File format") + return nil + } + argc := 0 + argv, argc = SplitArgs(line) + + if argc < 6 || argc%2 != 0 { + log.Infof("Reading the manifest file, at line %d", lineNum) + log.Infof("Invalid AOF manifest File format") + am = nil + return am + } + ai = AOFInfoCreate() + for i := 0; i < argc; i += 2 { + if strings.EqualFold(argv[i], AOFManifestKeyFileName) { + ai.FileName = argv[i+1] + if !PathIsBaseName(ai.FileName) { + log.Infof("Reading the manifest file, at line %d", lineNum) + log.Panicf("File can't be a path, just a Filename") + } + } else if strings.EqualFold(argv[i], AOFManifestKeyFileSeq) { + ai.FileSeq, _ = strconv.ParseInt(argv[i+1], 10, 64) + } else if strings.EqualFold(argv[i], AOFManifestKeyFileType) { + ai.AOFFileType = string(argv[i+1][0]) + } + } + if ai.FileName == "" || ai.FileSeq == 0 || ai.AOFFileType == "" { + log.Infof("Reading the manifest file, at line %d", lineNum) + log.Panicf("Invalid AOF manifest File format") + } + if ai.AOFFileType == AOFManifestFileTypeBase { + if am.BaseAOFInfo != nil { + log.Infof("Reading the manifest file, at line %d", lineNum) + log.Panicf("Found duplicate Base File information") + } + am.BaseAOFInfo = ai + am.CurrBaseFileSeq = ai.FileSeq + } else if ai.AOFFileType == AOFManifestTypeHist { + am.HistoryList.PushBack(ai) + } else if ai.AOFFileType == AOFManifestTypeIncr { + if ai.FileSeq <= maxSeq { + log.Infof("Reading the manifest file, at line %d", lineNum) + log.Panicf("Found a non-monotonic sequence number") + } + am.incrAOFList.PushBack(ai) + am.CurrIncrFileSeq = ai.FileSeq + maxSeq = ai.FileSeq + } else { + log.Infof("Reading the manifest file, at line %d", lineNum) + log.Panicf("Unknown AOF File type") + } + ai = nil + } + return am +} + +type AOFManifest struct { + BaseAOFInfo *AOFInfo + incrAOFList *list.List + HistoryList *list.List + CurrBaseFileSeq int64 + CurrIncrFileSeq int64 +} + +func AOFManifestCreate() *AOFManifest { + am := &AOFManifest{ + incrAOFList: list.New(), + HistoryList: list.New(), + } + return am +} + +func GetAOFManifestAsString(am *AOFManifest) string { + if am == nil { + panic("am is nil") + } + var buf string + if am.BaseAOFInfo != nil { + buf = AOFInfoFormat(buf, am.BaseAOFInfo) + } + for ln := am.HistoryList.Front(); ln != nil; ln = ln.Next() { + buf = AOFInfoFormat(buf, ln.Value.(*AOFInfo)) + } + for ln := am.incrAOFList.Front(); ln != nil; ln = ln.Next() { + buf = AOFInfoFormat(buf, ln.Value.(*AOFInfo)) + } + return buf + +} + +func (aofInfo *INFO) AOFLoadManifestFromDisk() { + if DirExists(aofInfo.AOFDirName) == 0 { + log.Infof("The AOF Directory %v doesn't exist", aofInfo.AOFDirName) + return + } + aofInfo.AOFManifest = AOFManifestCreate() + amFilepath := path.Join(aofInfo.AOFDirName, aofInfo.AOFFileName) + if FileExist(amFilepath) == 0 { + log.Infof("The AOF Directory %v doesn't exist", aofInfo.AOFDirName) + return + } + + am := AOFLoadManifestFromFile(amFilepath) + aofInfo.AOFManifest = am +} + +func (aofInfo *INFO) GetAOFManifestFileName() string { + return aofInfo.AOFFileName +} + +func (aofInfo *INFO) AOFFileExist(FileName string) int { + Filepath := path.Join(aofInfo.AOFDirName, FileName) + ret := FileExist(Filepath) + return ret +} + +func (aofInfo *INFO) GetAppendOnlyFileSize(FileName string, status *int) int64 { + var size int64 + + AOFFilePath := path.Join(aofInfo.AOFDirName, FileName) + + stat, err := os.Stat(AOFFilePath) + if err != nil { + if status != nil { + if os.IsNotExist(err) { + *status = AOFNotExist + } else { + *status = AOFOpenErr + } + } + log.Panicf("Unable to obtain the AOF File %v length. stat: %v", FileName, err.Error()) + size = 0 + } else { + if status != nil { + *status = AOFOk + } + size = stat.Size() + } + return size +} + +func (aofInfo *INFO) GetBaseAndIncrAppendOnlyFilesSize(am *AOFManifest, status *int) int64 { + var size int64 + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType != AOFManifestFileTypeBase { + log.Panicf("File type must be Base.") + } + size += aofInfo.GetAppendOnlyFileSize(am.BaseAOFInfo.FileName, status) + if *status != AOFOk { + return 0 + } + } + + for ln := am.HistoryList.Front(); ln != nil; ln = ln.Next() { + ai := ln.Value.(*AOFInfo) + if ai.AOFFileType != AOFManifestTypeIncr { + log.Panicf("File type must be Incr") + } + size += aofInfo.GetAppendOnlyFileSize(ai.FileName, status) + if *status != AOFOk { + return 0 + } + } + return size +} + +func GetBaseAndIncrAppendOnlyFilesNum(am *AOFManifest) int { + num := 0 + if am.BaseAOFInfo != nil { + num++ + } + if am.incrAOFList != nil { + num += am.incrAOFList.Len() + } + return num +} + +func GetHistoryAndIncrAppendOnlyFilesNum(am *AOFManifest) int { + num := 0 + if am.HistoryList != nil { + num += am.HistoryList.Len() + } + if am.incrAOFList != nil { + num += am.incrAOFList.Len() + } + return num +} + +func (aofInfo *INFO) LoadAppendOnlyFile(am *AOFManifest, AOFTimeStamp int64) int { + if am == nil { + log.Panicf("AOFManifest is null") + } + status := AOFOk + ret := AOFOk + var start int64 + var totalSize int64 = 0 + var BaseSize int64 = 0 + var AOFName string + var totalNum, AOFNum int + + if am.BaseAOFInfo == nil && am.incrAOFList == nil { + return AOFNotExist + } + + totalNum = GetBaseAndIncrAppendOnlyFilesNum(am) + if totalNum <= 0 { + log.Panicf("Assertion failed: IncrAppendOnlyFilestotalNum > 0") + } + + totalSize = aofInfo.GetBaseAndIncrAppendOnlyFilesSize(am, &status) + if status != AOFOk { + if status == AOFNotExist { + status = AOFFailed + } + return status + } else if totalSize == 0 { + return AOFEmpty + } + + log.Infof("The AOF File starts loading.") + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType == AOFManifestFileTypeBase { + AOFName = am.BaseAOFInfo.FileName + aofInfo.UpdateLoadingFileName(AOFName) + BaseSize = aofInfo.GetAppendOnlyFileSize(AOFName, nil) + start = Ustime() + ret = aofInfo.ParsingSingleAppendOnlyFile(AOFName, 0) //Currently, RDB files cannot be restored at a point in time. + if ret == AOFOk || (ret == AOFTruncated) { + log.Infof("DB loaded from Base File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) + } + if ret == AOFEmpty { + ret = AOFOk + } + if ret == AOFOpenErr || ret == AOFFailed { + if ret == AOFOk || ret == AOFTruncated { + log.Infof("The AOF File was successfully loaded") + } else { + if ret == AOFOpenErr { + log.Panicf("There was an error opening the AOF File.") + } else { + log.Panicf("Failed to open AOF File.") + } + } + return ret + } + } + totalNum-- + } else { + totalNum = GetHistoryAndIncrAppendOnlyFilesNum(am) + log.Infof("The BaseAOF file does not exist. Start loading the HistoryAOF and IncrAOF files.") + if am.HistoryList.Len() > 0 { + for ln := am.HistoryList.Front(); ln != nil; ln = ln.Next() { + ai := ln.Value.(*AOFInfo) + if ai.AOFFileType != AOFManifestTypeHist { + log.Panicf("The manifestType must be Hist") + } + AOFName = ai.FileName + aofInfo.UpdateLoadingFileName(AOFName) + AOFNum++ + start = Ustime() + ret = aofInfo.ParsingSingleAppendOnlyFile(AOFName, AOFTimeStamp) + if ret == AOFOk || (ret == AOFTruncated) { + log.Infof("DB loaded from History File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) + return ret + } + if ret == AOFEmpty { + ret = AOFOk + } + if ret == AOFOpenErr || ret == AOFFailed { + if ret == AOFOpenErr { + log.Panicf("There was an error opening the AOF File.") + } else { + log.Infof("Failed to open AOF File.") + } + return ret + } + totalNum-- + } + } + + } + + if am.incrAOFList.Len() > 0 { + for ln := am.incrAOFList.Front(); ln != nil; ln = ln.Next() { + ai := ln.Value.(*AOFInfo) + if ai.AOFFileType != AOFManifestTypeIncr { + log.Panicf("The manifestType must be Incr") + } + AOFName = ai.FileName + aofInfo.UpdateLoadingFileName(AOFName) + AOFNum++ + start = Ustime() + ret = aofInfo.ParsingSingleAppendOnlyFile(AOFName, AOFTimeStamp) + if ret == AOFOk || (ret == AOFTruncated) { + log.Infof("DB loaded from incr File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) + return ret + } + if ret == AOFEmpty { + ret = AOFOk + } + if ret == AOFOpenErr || ret == AOFFailed { + if ret == AOFOpenErr { + log.Panicf("There was an error opening the AOF File.") + } else { + log.Infof("Failed to open AOF File.") + } + return ret + } + totalNum-- + } + } + if totalNum == 0 { + log.Infof("All AOF files have been sent.") + } else { + log.Panicf("There are still %d AOF files that were not successfully sent.", totalNum) + } + aofInfo.AOFCurrentSize = totalSize + aofInfo.AOFRewriteBaseSize = BaseSize + + log.Infof("The AOF File loading end.") + return ret + +} + +func (aofInfo *INFO) ParsingSingleAppendOnlyFile(FileName string, AOFTimeStamp int64) int { + ret := AOFOk + AOFFilepath := path.Join(aofInfo.AOFDirName, FileName) + println(AOFFilepath) + fp, err := os.Open(AOFFilepath) + if err != nil { + if os.IsNotExist(err) { + if _, err := os.Stat(AOFFilepath); err == nil || !os.IsNotExist(err) { + log.Infof("Fatal error: can't open the append log File %v for reading: %v", FileName, err.Error()) + return AOFOpenErr + } else { + log.Infof("The append log File %v doesn't exist: %v", FileName, err.Error()) + return AOFNotExist + } + + } + + stat, _ := fp.Stat() + if stat.Size() == 0 { + return AOFEmpty + } + } + defer fp.Close() + sig := make([]byte, 5) + if n, err := fp.Read(sig); err != nil || n != 5 || !bytes.Equal(sig, []byte("REDIS")) { + if _, err := fp.Seek(0, 0); err != nil { + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed + return ret + } + } else { //Skipped RDB checksum and has not been processed yet. + log.Infof("Reading RDB Base File on AOF loading...") + rdbOpt := RdbReaderOptions{Filepath: AOFFilepath} + ldRDB := NewRDBReader(&rdbOpt) + ldRDB.StartRead() + return AOFOk + } + // load single aof file + aofSingleReader := aof.NewLoader(MakePath(aofInfo.AOFDirName, FileName), aofInfo.ch) + ret = aofSingleReader.LoadSingleAppendOnlyFile(AOFTimeStamp) + return ret + +} diff --git a/internal/reader/psync.go b/internal/reader/psync.go deleted file mode 100644 index 9745945c..00000000 --- a/internal/reader/psync.go +++ /dev/null @@ -1,247 +0,0 @@ -package reader - -import ( - "bufio" - "github.com/alibaba/RedisShake/internal/client" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb" - "github.com/alibaba/RedisShake/internal/reader/rotate" - "github.com/alibaba/RedisShake/internal/statistics" - "io" - "io/ioutil" - "os" - "strconv" - "strings" - "time" -) - -type psyncReader struct { - client *client.Redis - address string - ch chan *entry.Entry - DbId int - - rd *bufio.Reader - receivedOffset int64 - elastiCachePSync string -} - -func NewPSyncReader(address string, username string, password string, isTls bool, ElastiCachePSync string) Reader { - r := new(psyncReader) - r.address = address - r.elastiCachePSync = ElastiCachePSync - r.client = client.NewRedisClient(address, username, password, isTls) - r.rd = r.client.BufioReader() - log.Infof("psyncReader connected to redis successful. address=[%s]", address) - return r -} - -func (r *psyncReader) StartRead() chan *entry.Entry { - r.ch = make(chan *entry.Entry, 1024) - - go func() { - r.clearDir() - go r.sendReplconfAck() - r.saveRDB() - startOffset := r.receivedOffset - go r.saveAOF(r.rd) - r.sendRDB() - time.Sleep(1 * time.Second) // wait for saveAOF create aof file - r.sendAOF(startOffset) - }() - - return r.ch -} - -func (r *psyncReader) clearDir() { - files, err := ioutil.ReadDir("./") - if err != nil { - log.PanicError(err) - } - - for _, f := range files { - if strings.HasSuffix(f.Name(), ".rdb") || strings.HasSuffix(f.Name(), ".aof") { - err = os.Remove(f.Name()) - if err != nil { - log.PanicError(err) - } - log.Warnf("remove file. filename=[%s]", f.Name()) - } - } -} - -func (r *psyncReader) saveRDB() { - log.Infof("start save RDB. address=[%s]", r.address) - argv := []string{"replconf", "listening-port", "10007"} // 10007 is magic number - log.Infof("send %v", argv) - reply := r.client.DoWithStringReply(argv...) - if reply != "OK" { - log.Warnf("send replconf command to redis server failed. address=[%s], reply=[%s], error=[]", r.address, reply) - } - - // send psync - argv = []string{"PSYNC", "?", "-1"} - if r.elastiCachePSync != "" { - argv = []string{r.elastiCachePSync, "?", "-1"} - } - r.client.Send(argv...) - log.Infof("send %v", argv) - // format: \n\n\n$\r\n - for true { - // \n\n\n$ - b, err := r.rd.ReadByte() - if err != nil { - log.PanicError(err) - } - if b == '\n' { - continue - } - if b == '-' { - reply, err := r.rd.ReadString('\n') - if err != nil { - log.PanicError(err) - } - reply = strings.TrimSpace(reply) - log.Panicf("psync error. address=[%s], reply=[%s]", r.address, reply) - } - if b != '+' { - log.Panicf("invalid psync reply. address=[%s], b=[%s]", r.address, string(b)) - } - break - } - reply, err := r.rd.ReadString('\n') - if err != nil { - log.PanicError(err) - } - reply = strings.TrimSpace(reply) - log.Infof("receive [%s]", reply) - masterOffset, err := strconv.Atoi(strings.Split(reply, " ")[2]) - if err != nil { - log.PanicError(err) - } - r.receivedOffset = int64(masterOffset) - - log.Infof("source db is doing bgsave. address=[%s]", r.address) - statistics.Metrics.IsDoingBgsave = true - - timeStart := time.Now() - // format: \n\n\n$\r\n - for true { - // \n\n\n$ - b, err := r.rd.ReadByte() - if err != nil { - log.PanicError(err) - } - if b == '\n' { - continue - } - if b != '$' { - log.Panicf("invalid rdb format. address=[%s], b=[%s]", r.address, string(b)) - } - break - } - statistics.Metrics.IsDoingBgsave = false - log.Infof("source db bgsave finished. timeUsed=[%.2f]s, address=[%s]", time.Since(timeStart).Seconds(), r.address) - lengthStr, err := r.rd.ReadString('\n') - if err != nil { - log.PanicError(err) - } - lengthStr = strings.TrimSpace(lengthStr) - length, err := strconv.ParseInt(lengthStr, 10, 64) - if err != nil { - log.PanicError(err) - } - log.Infof("received rdb length. length=[%d]", length) - statistics.SetRDBFileSize(uint64(length)) - - // create rdb file - rdbFilePath := "dump.rdb" - log.Infof("create dump.rdb file. filename_path=[%s]", rdbFilePath) - rdbFileHandle, err := os.OpenFile(rdbFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) - if err != nil { - log.PanicError(err) - } - - // read rdb - remainder := length - const bufSize int64 = 32 * 1024 * 1024 // 32MB - buf := make([]byte, bufSize) - for remainder != 0 { - readOnce := bufSize - if remainder < readOnce { - readOnce = remainder - } - n, err := r.rd.Read(buf[:readOnce]) - if err != nil { - log.PanicError(err) - } - remainder -= int64(n) - statistics.UpdateRDBReceivedSize(uint64(length - remainder)) - _, err = rdbFileHandle.Write(buf[:n]) - if err != nil { - log.PanicError(err) - } - } - err = rdbFileHandle.Close() - if err != nil { - log.PanicError(err) - } - log.Infof("save RDB finished. address=[%s], total_bytes=[%d]", r.address, length) -} - -func (r *psyncReader) saveAOF(rd io.Reader) { - log.Infof("start save AOF. address=[%s]", r.address) - // create aof file - aofWriter := rotate.NewAOFWriter(r.receivedOffset) - defer aofWriter.Close() - buf := make([]byte, 16*1024) // 16KB is enough for writing file - for { - n, err := rd.Read(buf) - if err != nil { - log.PanicError(err) - } - r.receivedOffset += int64(n) - statistics.UpdateAOFReceivedOffset(uint64(r.receivedOffset)) - aofWriter.Write(buf[:n]) - } -} - -func (r *psyncReader) sendRDB() { - // start parse rdb - log.Infof("start send RDB. address=[%s]", r.address) - rdbLoader := rdb.NewLoader("dump.rdb", r.ch) - r.DbId = rdbLoader.ParseRDB() - log.Infof("send RDB finished. address=[%s], repl-stream-db=[%d]", r.address, r.DbId) -} - -func (r *psyncReader) sendAOF(offset int64) { - aofReader := rotate.NewAOFReader(offset) - defer aofReader.Close() - r.client.SetBufioReader(bufio.NewReader(aofReader)) - for { - argv := client.ArrayString(r.client.Receive()) - // select - if strings.EqualFold(argv[0], "select") { - DbId, err := strconv.Atoi(argv[1]) - if err != nil { - log.PanicError(err) - } - r.DbId = DbId - continue - } - - e := entry.NewEntry() - e.Argv = argv - e.DbId = r.DbId - e.Offset = aofReader.Offset() - r.ch <- e - } -} - -func (r *psyncReader) sendReplconfAck() { - for range time.Tick(time.Millisecond * 100) { - // send ack receivedOffset - r.client.Send("replconf", "ack", strconv.FormatInt(r.receivedOffset, 10)) - } -} diff --git a/internal/reader/rdb_reader.go b/internal/reader/rdb_reader.go index ee6444ed..23b5a30f 100644 --- a/internal/reader/rdb_reader.go +++ b/internal/reader/rdb_reader.go @@ -1,48 +1,72 @@ package reader import ( - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb" - "github.com/alibaba/RedisShake/internal/statistics" - "os" - "path/filepath" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/rdb" + "RedisShake/internal/utils" + "fmt" + "github.com/dustin/go-humanize" ) -type rdbReader struct { - path string - ch chan *entry.Entry +type RdbReaderOptions struct { + Filepath string `mapstructure:"filepath" default:""` } -func NewRDBReader(path string) Reader { - log.Infof("NewRDBReader: path=[%s]", path) - absolutePath, err := filepath.Abs(path) - if err != nil { - log.Panicf("NewRDBReader: filepath.Abs error: %s", err.Error()) +type rdbReader struct { + ch chan *entry.Entry + + stat struct { + Name string `json:"name"` + Status string `json:"status"` + Filepath string `json:"filepath"` + FileSizeBytes int64 `json:"file_size_bytes"` + FileSizeHuman string `json:"file_size_human"` + FileSentBytes int64 `json:"file_sent_bytes"` + FileSentHuman string `json:"file_sent_human"` + Percent string `json:"percent"` } - log.Infof("NewRDBReader: absolute path=[%s]", absolutePath) +} + +func NewRDBReader(opts *RdbReaderOptions) Reader { + absolutePath := utils.GetAbsPath(opts.Filepath) r := new(rdbReader) - r.path = absolutePath + r.stat.Name = "rdb_reader" + r.stat.Status = "init" + r.stat.Filepath = absolutePath + r.stat.FileSizeBytes = int64(utils.GetFileSize(absolutePath)) + r.stat.FileSizeHuman = humanize.Bytes(uint64(r.stat.FileSizeBytes)) return r } func (r *rdbReader) StartRead() chan *entry.Entry { + log.Infof("[%s] start read", r.stat.Name) r.ch = make(chan *entry.Entry, 1024) + updateFunc := func(offset int64) { + r.stat.FileSentBytes = offset + r.stat.FileSentHuman = humanize.Bytes(uint64(offset)) + r.stat.Percent = fmt.Sprintf("%.2f%%", float64(offset)/float64(r.stat.FileSizeBytes)*100) + r.stat.Status = fmt.Sprintf("[%s] rdb file synced: %s", r.stat.Name, r.stat.Percent) + } + rdbLoader := rdb.NewLoader(r.stat.Name, updateFunc, r.stat.Filepath, r.ch) go func() { - // start parse rdb - log.Infof("start send RDB. path=[%s]", r.path) - fi, err := os.Stat(r.path) - if err != nil { - log.Panicf("NewRDBReader: os.Stat error: %s", err.Error()) - } - statistics.Metrics.RdbFileSize = uint64(fi.Size()) - statistics.Metrics.RdbReceivedSize = uint64(fi.Size()) - rdbLoader := rdb.NewLoader(r.path, r.ch) _ = rdbLoader.ParseRDB() - log.Infof("send RDB finished. path=[%s]", r.path) + log.Infof("[%s] rdb file parse done", r.stat.Name) close(r.ch) }() return r.ch } + +func (r *rdbReader) Status() interface{} { + return r.stat +} + +func (r *rdbReader) StatusString() string { + return r.stat.Status +} + +func (r *rdbReader) StatusConsistent() bool { + return r.stat.FileSentBytes == r.stat.FileSizeBytes +} diff --git a/internal/reader/scan_cluster_reader.go b/internal/reader/scan_cluster_reader.go new file mode 100644 index 00000000..3a3f1151 --- /dev/null +++ b/internal/reader/scan_cluster_reader.go @@ -0,0 +1,67 @@ +package reader + +import ( + "RedisShake/internal/entry" + "RedisShake/internal/utils" + "fmt" + "sync" +) + +type scanClusterReader struct { + readers []Reader + statusId int +} + +func NewScanClusterReader(opts *ScanReaderOptions) Reader { + addresses, _ := utils.GetRedisClusterNodes(opts.Address, opts.Username, opts.Password, opts.Tls) + + rd := &scanClusterReader{} + for _, address := range addresses { + theOpts := *opts + theOpts.Address = address + rd.readers = append(rd.readers, NewScanStandaloneReader(&theOpts)) + } + return rd +} + +func (rd *scanClusterReader) StartRead() chan *entry.Entry { + ch := make(chan *entry.Entry, 1024) + var wg sync.WaitGroup + for _, r := range rd.readers { + wg.Add(1) + go func(r Reader) { + for e := range r.StartRead() { + ch <- e + } + wg.Done() + }(r) + } + go func() { + wg.Wait() + close(ch) + }() + return ch +} + +func (rd *scanClusterReader) Status() interface{} { + stat := make([]interface{}, 0) + for _, r := range rd.readers { + stat = append(stat, r.Status()) + } + return stat +} + +func (rd *scanClusterReader) StatusString() string { + rd.statusId += 1 + rd.statusId %= len(rd.readers) + return fmt.Sprintf("src-%d, %s", rd.statusId, rd.readers[rd.statusId].StatusString()) +} + +func (rd *scanClusterReader) StatusConsistent() bool { + for _, r := range rd.readers { + if !r.StatusConsistent() { + return false + } + } + return true +} diff --git a/internal/reader/scan_reader.go b/internal/reader/scan_reader.go deleted file mode 100644 index b43d0297..00000000 --- a/internal/reader/scan_reader.go +++ /dev/null @@ -1,145 +0,0 @@ -package reader - -import ( - "strconv" - "strings" - - "github.com/alibaba/RedisShake/internal/client" - "github.com/alibaba/RedisShake/internal/client/proto" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/statistics" -) - -const ( - // cluster_enabled: Indicate Redis cluster is enabled. reference from https://redis.io/commands/info/ - clusterMode = "cluster_enabled:1" -) - -type dbKey struct { - db int - key string - isSelect bool -} - -type scanReader struct { - address string - - // client for scan keys - clientScan *client.Redis - innerChannel chan *dbKey - isCluster bool - - // client for dump keys - clientDump *client.Redis - clientDumpDbid int - ch chan *entry.Entry -} - -func NewScanReader(address string, username string, password string, isTls bool) Reader { - r := new(scanReader) - r.address = address - r.clientScan = client.NewRedisClient(address, username, password, isTls) - r.clientDump = client.NewRedisClient(address, username, password, isTls) - log.Infof("scanReader connected to redis successful. address=[%s]", address) - - r.isCluster = r.IsCluster() - return r -} - -// IsCluster is for determining whether the server is in cluster mode. -func (r *scanReader) IsCluster() bool { - reply := r.clientScan.DoWithStringReply("INFO", "Cluster") - return strings.Contains(reply, clusterMode) -} - -func (r *scanReader) StartRead() chan *entry.Entry { - r.ch = make(chan *entry.Entry, 1024) - r.innerChannel = make(chan *dbKey, 1024) - go r.scan() - go r.fetch() - return r.ch -} - -func (r *scanReader) scan() { - scanDbIdUpper := 15 - if r.isCluster { - log.Infof("scanReader node are in cluster mode, only scan db 0") - scanDbIdUpper = 0 - } - for dbId := 0; dbId <= scanDbIdUpper; dbId++ { - if !r.isCluster { - reply := r.clientScan.DoWithStringReply("SELECT", strconv.Itoa(dbId)) - if reply != "OK" { - log.Panicf("scanReader select db failed. db=[%d]", dbId) - } - - r.clientDump.Send("SELECT", strconv.Itoa(dbId)) - r.innerChannel <- &dbKey{dbId, "", true} - } - - var cursor uint64 = 0 - for { - var keys []string - cursor, keys = r.clientScan.Scan(cursor) - for _, key := range keys { - r.clientDump.Send("DUMP", key) - r.clientDump.Send("PTTL", key) - r.innerChannel <- &dbKey{dbId, key, false} - } - - // stat - statistics.Metrics.ScanDbId = dbId - statistics.Metrics.ScanCursor = cursor - - if cursor == 0 { - break - } - } - } - close(r.innerChannel) -} - -func (r *scanReader) fetch() { - var id uint64 = 0 - for item := range r.innerChannel { - if item.isSelect { - // select - receive, err := client.String(r.clientDump.Receive()) - if err != nil { - log.Panicf("scanReader select db failed. db=[%d], err=[%v]", item.db, err) - } - if receive != "OK" { - log.Panicf("scanReader select db failed. db=[%d]", item.db) - } - } else { - // dump - receive, err := client.String(r.clientDump.Receive()) - if err != proto.Nil && err != nil { // error! - log.PanicIfError(err) - } - - // pttl - pttl, pttlErr := client.Int64(r.clientDump.Receive()) - log.PanicIfError(pttlErr) - if pttl < 0 { - pttl = 0 - } - - if err == proto.Nil { // key not exist - continue - } - - id += 1 - argv := []string{"RESTORE", item.key, strconv.FormatInt(pttl, 10), receive} - r.ch <- &entry.Entry{ - Id: id, - IsBase: false, - DbId: item.db, - Argv: argv, - } - } - } - log.Infof("scanReader fetch finished. address=[%s]", r.address) - close(r.ch) -} diff --git a/internal/reader/scan_standalone_reader.go b/internal/reader/scan_standalone_reader.go new file mode 100644 index 00000000..8d7cfb7c --- /dev/null +++ b/internal/reader/scan_standalone_reader.go @@ -0,0 +1,215 @@ +package reader + +import ( + "RedisShake/internal/client" + "RedisShake/internal/client/proto" + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/rdb/types" + "RedisShake/internal/utils" + "fmt" + "math/bits" + "regexp" + "strconv" + "strings" +) + +type ScanReaderOptions struct { + Cluster bool `mapstructure:"cluster" default:"false"` + Address string `mapstructure:"address" default:""` + Username string `mapstructure:"username" default:""` + Password string `mapstructure:"password" default:""` + Tls bool `mapstructure:"tls" default:"false"` + KSN bool `mapstructure:"ksn" default:"false"` +} + +type dbKey struct { + db int + key string +} + +type scanStandaloneReader struct { + dbs []int + opts *ScanReaderOptions + ch chan *entry.Entry + keyQueue *utils.UniqueQueue + + stat struct { + Name string `json:"name"` + ScanFinished bool `json:"scan_finished"` + ScanDbId int `json:"scan_dbId"` + ScanCursor uint64 `json:"scan_cursor"` + ScanPercentByDbId string `json:"scan_percent"` + NeedUpdateCount int64 `json:"need_update_count"` + } +} + +func NewScanStandaloneReader(opts *ScanReaderOptions) Reader { + r := new(scanStandaloneReader) + // dbs + c := client.NewRedisClient(opts.Address, opts.Username, opts.Password, opts.Tls) + if c.IsCluster() { // not use opts.Cluster, because user may use standalone mode to scan a cluster node + r.dbs = []int{0} + } else { + r.dbs = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15} + } + r.opts = opts + r.ch = make(chan *entry.Entry, 1024) + r.stat.Name = "reader_" + strings.Replace(opts.Address, ":", "_", -1) + r.keyQueue = utils.NewUniqueQueue(100000) // cache 100000 keys + return r +} + +func (r *scanStandaloneReader) StartRead() chan *entry.Entry { + r.subscript() + go r.scan() + go r.fetch() + return r.ch +} + +func (r *scanStandaloneReader) subscript() { + if !r.opts.KSN { + return + } + c := client.NewRedisClient(r.opts.Address, r.opts.Username, r.opts.Password, r.opts.Tls) + c.Send("psubscribe", "__keyevent@*__:*") + + go func() { + _, err := c.Receive() + if err != nil { + log.Panicf(err.Error()) + } + regex := regexp.MustCompile(`\d+`) + for { + resp, err := c.Receive() + if err != nil { + log.Panicf(err.Error()) + } + key := resp.([]interface{})[3].(string) + dbId := regex.FindString(resp.([]interface{})[2].(string)) + dbIdInt, err := strconv.Atoi(dbId) + if err != nil { + log.Panicf(err.Error()) + } + r.keyQueue.Put(dbKey{db: dbIdInt, key: key}) + } + }() +} + +func (r *scanStandaloneReader) scan() { + c := client.NewRedisClient(r.opts.Address, r.opts.Username, r.opts.Password, r.opts.Tls) + for dbId := range r.dbs { + if dbId != 0 { + reply := c.DoWithStringReply("SELECT", strconv.Itoa(dbId)) + if reply != "OK" { + log.Panicf("scanStandaloneReader select db failed. db=[%d]", dbId) + } + } + + var cursor uint64 = 0 + for { + var keys []string + cursor, keys = c.Scan(cursor) + for _, key := range keys { + r.keyQueue.Put(dbKey{dbId, key}) // pass value not pointer + } + + // stat + r.stat.ScanCursor = cursor + r.stat.ScanDbId = dbId + r.stat.ScanPercentByDbId = fmt.Sprintf("%.2f%%", float64(bits.Reverse64(cursor))/float64(^uint(0))*100) + + if cursor == 0 { + break + } + } + } + r.stat.ScanFinished = true + if !r.opts.KSN { + r.keyQueue.Close() + } +} + +func (r *scanStandaloneReader) fetch() { + nowDbId := 0 + c := client.NewRedisClient(r.opts.Address, r.opts.Username, r.opts.Password, r.opts.Tls) + for item := range r.keyQueue.Ch { + r.stat.NeedUpdateCount = int64(r.keyQueue.Len()) + dbId := item.(dbKey).db + key := item.(dbKey).key + if nowDbId != dbId { + reply := c.DoWithStringReply("SELECT", strconv.Itoa(dbId)) + if reply != "OK" { + log.Panicf("scanStandaloneReader select db failed. db=[%d]", dbId) + } + nowDbId = dbId + } + // dump + c.Send("DUMP", key) + c.Send("PTTL", key) + iDump, err1 := c.Receive() + iPttl, err2 := c.Receive() + if err1 == proto.Nil { + continue // key not exist + } else if err1 != nil { + log.Panicf(err1.Error()) + } else if err2 != nil { + log.Panicf(err2.Error()) + } + dump := iDump.(string) + pttl := int(iPttl.(int64)) + if pttl == -2 { + continue // key not exist + } + if pttl == -1 { + pttl = 0 // -1 means no expire + } + if uint64(len(dump)) > config.Opt.Advanced.TargetRedisProtoMaxBulkLen { + log.Warnf("key=[%s] dump len=[%d] too large, split it. This is not a good practice in Redis.", key, len(dump)) + typeByte := dump[0] + anotherReader := strings.NewReader(dump[1 : len(dump)-10]) + o := types.ParseObject(anotherReader, typeByte, key) + cmds := o.Rewrite() + for _, cmd := range cmds { + e := entry.NewEntry() + e.DbId = dbId + e.Argv = cmd + r.ch <- e + } + if pttl != 0 { + e := entry.NewEntry() + e.DbId = dbId + e.Argv = []string{"PEXPIRE", key, strconv.Itoa(pttl)} + r.ch <- e + } + } else { + argv := []string{"RESTORE", key, strconv.Itoa(pttl), dump} + if config.Opt.Advanced.RDBRestoreCommandBehavior == "rewrite" { + argv = append(argv, "replace") + } + r.ch <- &entry.Entry{ + DbId: dbId, + Argv: argv, + } + } + } + + log.Infof("[%s] scanStandaloneReader fetch finished.", r.stat.Name) + close(r.ch) +} + +func (r *scanStandaloneReader) Status() interface{} { + return r.stat +} + +func (r *scanStandaloneReader) StatusString() string { + if r.stat.ScanFinished { + return fmt.Sprintf("need_update_count=[%d]", r.stat.NeedUpdateCount) + } + return fmt.Sprintf("scan_dbid=[%d], scan_percent=[%s], need_update_count=[%d]", r.stat.ScanDbId, r.stat.ScanPercentByDbId, r.stat.NeedUpdateCount) +} + +func (r *scanStandaloneReader) StatusConsistent() bool { + return r.stat.ScanFinished && r.stat.NeedUpdateCount == 0 +} diff --git a/internal/reader/sync_cluster_reader.go b/internal/reader/sync_cluster_reader.go new file mode 100644 index 00000000..0a98be5f --- /dev/null +++ b/internal/reader/sync_cluster_reader.go @@ -0,0 +1,71 @@ +package reader + +import ( + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/utils" + "fmt" + "sync" +) + +type syncClusterReader struct { + readers []Reader + statusId int +} + +func NewSyncClusterReader(opts *SyncReaderOptions) Reader { + addresses, _ := utils.GetRedisClusterNodes(opts.Address, opts.Username, opts.Password, opts.Tls) + log.Debugf("get redis cluster nodes:") + for _, address := range addresses { + log.Debugf("%s", address) + } + rd := &syncClusterReader{} + for _, address := range addresses { + theOpts := *opts + theOpts.Address = address + rd.readers = append(rd.readers, NewSyncStandaloneReader(&theOpts)) + } + return rd +} + +func (rd *syncClusterReader) StartRead() chan *entry.Entry { + ch := make(chan *entry.Entry, 1024) + var wg sync.WaitGroup + for _, r := range rd.readers { + wg.Add(1) + go func(r Reader) { + defer wg.Done() + for e := range r.StartRead() { + ch <- e + } + }(r) + } + go func() { + wg.Wait() + close(ch) + }() + return ch +} + +func (rd *syncClusterReader) Status() interface{} { + stat := make([]interface{}, 0) + for _, r := range rd.readers { + stat = append(stat, r.Status()) + } + return stat +} + +func (rd *syncClusterReader) StatusString() string { + rd.statusId += 1 + rd.statusId %= len(rd.readers) + return fmt.Sprintf("src-%d, %s", rd.statusId, rd.readers[rd.statusId].StatusString()) +} + +func (rd *syncClusterReader) StatusConsistent() bool { + for _, r := range rd.readers { + if !r.StatusConsistent() { + return false + } + } + return true +} diff --git a/internal/reader/sync_standalone_reader.go b/internal/reader/sync_standalone_reader.go new file mode 100644 index 00000000..fc969702 --- /dev/null +++ b/internal/reader/sync_standalone_reader.go @@ -0,0 +1,319 @@ +package reader + +import ( + "RedisShake/internal/client" + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/rdb" + "RedisShake/internal/utils" + "RedisShake/internal/utils/file_rotate" + "bufio" + "fmt" + "github.com/dustin/go-humanize" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +type SyncReaderOptions struct { + Cluster bool `mapstructure:"cluster" default:"false"` + Address string `mapstructure:"address" default:""` + Username string `mapstructure:"username" default:""` + Password string `mapstructure:"password" default:""` + Tls bool `mapstructure:"tls" default:"false"` + SyncRdb bool `mapstructure:"sync_rdb" default:"true"` + SyncAof bool `mapstructure:"sync_aof" default:"true"` +} + +type State string + +const ( + kHandShake State = "hand shaking" + kWaitBgsave State = "waiting bgsave" + kReceiveRdb State = "receiving rdb" + kSyncRdb State = "syncing rdb" + kSyncAof State = "syncing aof" +) + +type syncStandaloneReader struct { + opts *SyncReaderOptions + client *client.Redis + + ch chan *entry.Entry + DbId int + + rd *bufio.Reader + + stat struct { + Name string `json:"name"` + Address string `json:"address"` + Dir string `json:"dir"` + + // status + Status State `json:"status"` + + // rdb info + RdbFilePath string `json:"rdb_file_path"` + RdbFileSizeBytes int64 `json:"rdb_file_size_bytes"` // bytes of the rdb file + RdbFileSizeHuman string `json:"rdb_file_size_human"` + RdbReceivedBytes int64 `json:"rdb_received_bytes"` // bytes of RDB received from master + RdbReceivedHuman string `json:"rdb_received_human"` + RdbSentBytes int64 `json:"rdb_sent_bytes"` // bytes of RDB sent to chan + RdbSentHuman string `json:"rdb_sent_human"` + + // aof info + AofReceivedOffset int64 `json:"aof_received_offset"` // offset of AOF received from master + AofSentOffset int64 `json:"aof_sent_offset"` // offset of AOF sent to chan + AofReceivedBytes int64 `json:"aof_received_bytes"` // bytes of AOF received from master + AofReceivedHuman string `json:"aof_received_human"` + } +} + +func NewSyncStandaloneReader(opts *SyncReaderOptions) Reader { + r := new(syncStandaloneReader) + r.opts = opts + r.client = client.NewRedisClient(opts.Address, opts.Username, opts.Password, opts.Tls) + r.rd = r.client.BufioReader() + r.stat.Name = "reader_" + strings.Replace(opts.Address, ":", "_", -1) + r.stat.Address = opts.Address + r.stat.Status = kHandShake + r.stat.Dir = utils.GetAbsPath(r.stat.Name) + utils.CreateEmptyDir(r.stat.Dir) + return r +} + +func (r *syncStandaloneReader) StartRead() chan *entry.Entry { + r.ch = make(chan *entry.Entry, 1024) + go func() { + r.sendReplconfListenPort() + r.sendPSync() + go r.sendReplconfAck() // start sent replconf ack + r.receiveRDB() + startOffset := r.stat.AofReceivedOffset + go r.receiveAOF(r.rd) + if r.opts.SyncRdb { + r.sendRDB() + } + if r.opts.SyncAof { + r.stat.Status = kSyncAof + r.sendAOF(startOffset) + } + close(r.ch) + }() + + return r.ch +} + +func (r *syncStandaloneReader) sendReplconfListenPort() { + // use status_port as redis-shake port + argv := []string{"replconf", "listening-port", strconv.Itoa(config.Opt.Advanced.StatusPort)} + r.client.Send(argv...) + _, err := r.client.Receive() + if err != nil { + log.Warnf("[%s] send replconf command to redis server failed. error=[%v]", r.stat.Name, err) + } +} + +func (r *syncStandaloneReader) sendPSync() { + // send PSync + argv := []string{"PSYNC", "?", "-1"} + if config.Opt.Advanced.AwsPSync != "" { + argv = []string{config.Opt.Advanced.GetPSyncCommand(r.stat.Address), "?", "-1"} + } + r.client.Send(argv...) + + // format: \n\n\n+\r\n + for { + bytes, err := r.rd.Peek(1) + if err != nil { + log.Panicf(err.Error()) + } + if bytes[0] != '\n' { + break + } + } + reply := r.client.ReceiveString() + masterOffset, err := strconv.Atoi(strings.Split(reply, " ")[2]) + if err != nil { + log.Panicf(err.Error()) + } + r.stat.AofReceivedOffset = int64(masterOffset) +} + +func (r *syncStandaloneReader) receiveRDB() { + log.Debugf("[%s] source db is doing bgsave.", r.stat.Name) + r.stat.Status = kWaitBgsave + timeStart := time.Now() + // format: \n\n\n$\r\n + for { + b, err := r.rd.ReadByte() + if err != nil { + log.Panicf(err.Error()) + } + if b == '\n' { // heartbeat + continue + } + if b != '$' { + log.Panicf("[%s] invalid rdb format. b=[%s]", r.stat.Name, string(b)) + } + break + } + log.Debugf("[%s] source db bgsave finished. timeUsed=[%.2f]s", r.stat.Name, time.Since(timeStart).Seconds()) + lengthStr, err := r.rd.ReadString('\n') + if err != nil { + log.Panicf(err.Error()) + } + lengthStr = strings.TrimSpace(lengthStr) + length, err := strconv.ParseInt(lengthStr, 10, 64) + if err != nil { + log.Panicf(err.Error()) + } + log.Debugf("[%s] rdb file size: [%v]", r.stat.Name, humanize.IBytes(uint64(length))) + r.stat.RdbFileSizeBytes = length + r.stat.RdbFileSizeHuman = humanize.IBytes(uint64(length)) + + // create rdb file + r.stat.RdbFilePath, err = filepath.Abs(r.stat.Name + "/dump.rdb") + if err != nil { + log.Panicf(err.Error()) + } + timeStart = time.Now() + log.Debugf("[%s] start receiving RDB. path=[%s]", r.stat.Name, r.stat.RdbFilePath) + rdbFileHandle, err := os.OpenFile(r.stat.RdbFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + log.Panicf(err.Error()) + } + + // receive rdb + r.stat.Status = kReceiveRdb + remainder := length + const bufSize int64 = 32 * 1024 * 1024 // 32MB + buf := make([]byte, bufSize) + for remainder != 0 { + readOnce := bufSize + if remainder < readOnce { + readOnce = remainder + } + n, err := r.rd.Read(buf[:readOnce]) + if err != nil { + log.Panicf(err.Error()) + } + remainder -= int64(n) + _, err = rdbFileHandle.Write(buf[:n]) + if err != nil { + log.Panicf(err.Error()) + } + + r.stat.RdbReceivedBytes += int64(n) + r.stat.RdbReceivedHuman = humanize.IBytes(uint64(r.stat.RdbReceivedBytes)) + } + err = rdbFileHandle.Close() + if err != nil { + log.Panicf(err.Error()) + } + log.Debugf("[%s] save RDB finished. timeUsed=[%.2f]s", r.stat.Name, time.Since(timeStart).Seconds()) +} + +func (r *syncStandaloneReader) receiveAOF(rd io.Reader) { + log.Debugf("[%s] start receiving aof data, and save to file", r.stat.Name) + aofWriter := rotate.NewAOFWriter(r.stat.Name, r.stat.Dir, r.stat.AofReceivedOffset) + defer aofWriter.Close() + buf := make([]byte, 16*1024) // 16KB is enough for writing file + for { + n, err := rd.Read(buf) + if err != nil { + log.Panicf(err.Error()) + } + r.stat.AofReceivedBytes += int64(n) + r.stat.AofReceivedHuman = humanize.IBytes(uint64(r.stat.AofReceivedBytes)) + aofWriter.Write(buf[:n]) + r.stat.AofReceivedOffset += int64(n) + } +} + +func (r *syncStandaloneReader) sendRDB() { + // start parse rdb + log.Debugf("[%s] start sending RDB to target", r.stat.Name) + r.stat.Status = kSyncRdb + updateFunc := func(offset int64) { + r.stat.RdbSentBytes = offset + r.stat.RdbSentHuman = humanize.IBytes(uint64(offset)) + } + rdbLoader := rdb.NewLoader(r.stat.Name, updateFunc, r.stat.RdbFilePath, r.ch) + r.DbId = rdbLoader.ParseRDB() + log.Debugf("[%s] send RDB finished", r.stat.Name) +} + +func (r *syncStandaloneReader) sendAOF(offset int64) { + time.Sleep(1 * time.Second) // wait for receiveAOF create aof file + aofReader := rotate.NewAOFReader(r.stat.Name, r.stat.Dir, offset) + defer aofReader.Close() + r.client.SetBufioReader(bufio.NewReader(aofReader)) + for { + argv := client.ArrayString(r.client.Receive()) + r.stat.AofSentOffset = aofReader.Offset() + // select + if strings.EqualFold(argv[0], "select") { + DbId, err := strconv.Atoi(argv[1]) + if err != nil { + log.Panicf(err.Error()) + } + r.DbId = DbId + continue + } + // ping + if strings.EqualFold(argv[0], "ping") { + continue + } + // replconf @AWS + if strings.EqualFold(argv[0], "replconf") { + continue + } + // opinfo @Aliyun + if strings.EqualFold(argv[0], "opinfo") { + continue + } + // sentinel + if strings.EqualFold(argv[0], "publish") && strings.EqualFold(argv[1], "__sentinel__:hello") { + continue + } + + e := entry.NewEntry() + e.Argv = argv + e.DbId = r.DbId + r.ch <- e + } +} + +// sendReplconfAck send replconf ack to master to keep heartbeat between redis-shake and source redis. +func (r *syncStandaloneReader) sendReplconfAck() { + for range time.Tick(time.Millisecond * 100) { + if r.stat.AofReceivedOffset != 0 { + r.client.Send("replconf", "ack", strconv.FormatInt(r.stat.AofReceivedOffset, 10)) + } + } +} + +func (r *syncStandaloneReader) Status() interface{} { + return r.stat +} + +func (r *syncStandaloneReader) StatusString() string { + if r.stat.Status == kSyncRdb { + return fmt.Sprintf("%s, size=[%s/%s]", r.stat.Status, r.stat.RdbSentHuman, r.stat.RdbFileSizeHuman) + } + if r.stat.Status == kSyncAof { + return fmt.Sprintf("%s, diff=[%v]", r.stat.Status, -r.stat.AofSentOffset+r.stat.AofReceivedOffset) + } + return string(r.stat.Status) +} + +func (r *syncStandaloneReader) StatusConsistent() bool { + return r.stat.AofReceivedOffset != 0 && + r.stat.AofReceivedOffset == r.stat.AofSentOffset && + len(r.ch) == 0 +} diff --git a/internal/statistics/statistics.go b/internal/statistics/statistics.go deleted file mode 100644 index 9af801d5..00000000 --- a/internal/statistics/statistics.go +++ /dev/null @@ -1,157 +0,0 @@ -package statistics - -import ( - "encoding/json" - "fmt" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/log" - "math/bits" - "net/http" - "strings" - "time" -) - -type metrics struct { - // info - Address string `json:"address"` - - // entries - EntryId uint64 `json:"entry_id"` - AllowEntriesCount uint64 `json:"allow_entries_count"` - DisallowEntriesCount uint64 `json:"disallow_entries_count"` - - // rdb - IsDoingBgsave bool `json:"is_doing_bgsave"` - RdbFileSize uint64 `json:"rdb_file_size"` - RdbReceivedSize uint64 `json:"rdb_received_size"` - RdbSendSize uint64 `json:"rdb_send_size"` - - // aof - AofReceivedOffset uint64 `json:"aof_received_offset"` - AofAppliedOffset uint64 `json:"aof_applied_offset"` - - // for performance debug - InQueueEntriesCount uint64 `json:"in_queue_entries_count"` - UnansweredBytesCount uint64 `json:"unanswered_bytes_count"` - - // scan cursor - ScanDbId int `json:"scan_db_id"` - ScanCursor uint64 `json:"scan_cursor"` - - // for log - Msg string `json:"msg"` -} - -var Metrics = &metrics{} - -func Handler(w http.ResponseWriter, _ *http.Request) { - w.Header().Add("Content-Type", "application/json") - err := json.NewEncoder(w).Encode(Metrics) - if err != nil { - log.PanicError(err) - } -} - -func Init() { - go func() { - seconds := config.Config.Advanced.LogInterval - if seconds <= 0 { - log.Infof("statistics disabled. seconds=[%d]", seconds) - } - - lastAllowEntriesCount := Metrics.AllowEntriesCount - lastDisallowEntriesCount := Metrics.DisallowEntriesCount - - for range time.Tick(time.Duration(seconds) * time.Second) { - // scan - if config.Config.Type == "scan" { - Metrics.Msg = fmt.Sprintf("syncing. dbId=[%d], percent=[%.2f]%%, allowOps=[%.2f], disallowOps=[%.2f], entryId=[%d], InQueueEntriesCount=[%d], unansweredBytesCount=[%d]bytes", - Metrics.ScanDbId, - float64(bits.Reverse64(Metrics.ScanCursor))/float64(^uint(0))*100, - float32(Metrics.AllowEntriesCount-lastAllowEntriesCount)/float32(seconds), - float32(Metrics.DisallowEntriesCount-lastDisallowEntriesCount)/float32(seconds), - Metrics.EntryId, - Metrics.InQueueEntriesCount, - Metrics.UnansweredBytesCount) - log.Infof(strings.Replace(Metrics.Msg, "%", "%%", -1)) - lastAllowEntriesCount = Metrics.AllowEntriesCount - lastDisallowEntriesCount = Metrics.DisallowEntriesCount - continue - } - // sync or restore - if Metrics.RdbFileSize == 0 { - Metrics.Msg = "source db is doing bgsave" - } else if Metrics.RdbSendSize > Metrics.RdbReceivedSize { - Metrics.Msg = fmt.Sprintf("receiving rdb. percent=[%.2f]%%, rdbFileSize=[%.3f]G, rdbReceivedSize=[%.3f]G", - float64(Metrics.RdbReceivedSize)/float64(Metrics.RdbFileSize)*100, - float64(Metrics.RdbFileSize)/1024/1024/1024, - float64(Metrics.RdbReceivedSize)/1024/1024/1024) - } else if Metrics.RdbFileSize > Metrics.RdbSendSize { - Metrics.Msg = fmt.Sprintf("syncing rdb. percent=[%.2f]%%, allowOps=[%.2f], disallowOps=[%.2f], entryId=[%d], InQueueEntriesCount=[%d], unansweredBytesCount=[%d]bytes, rdbFileSize=[%.3f]G, rdbSendSize=[%.3f]G", - float64(Metrics.RdbSendSize)*100/float64(Metrics.RdbFileSize), - float32(Metrics.AllowEntriesCount-lastAllowEntriesCount)/float32(seconds), - float32(Metrics.DisallowEntriesCount-lastDisallowEntriesCount)/float32(seconds), - Metrics.EntryId, - Metrics.InQueueEntriesCount, - Metrics.UnansweredBytesCount, - float64(Metrics.RdbFileSize)/1024/1024/1024, - float64(Metrics.RdbSendSize)/1024/1024/1024) - } else { - Metrics.Msg = fmt.Sprintf("syncing aof. allowOps=[%.2f], disallowOps=[%.2f], entryId=[%d], InQueueEntriesCount=[%d], unansweredBytesCount=[%d]bytes, diff=[%d], aofReceivedOffset=[%d], aofAppliedOffset=[%d]", - float32(Metrics.AllowEntriesCount-lastAllowEntriesCount)/float32(seconds), - float32(Metrics.DisallowEntriesCount-lastDisallowEntriesCount)/float32(seconds), - Metrics.EntryId, - Metrics.InQueueEntriesCount, - Metrics.UnansweredBytesCount, - Metrics.AofReceivedOffset-Metrics.AofAppliedOffset, - Metrics.AofReceivedOffset, - Metrics.AofAppliedOffset) - } - log.Infof(strings.Replace(Metrics.Msg, "%", "%%", -1)) - lastAllowEntriesCount = Metrics.AllowEntriesCount - lastDisallowEntriesCount = Metrics.DisallowEntriesCount - } - }() -} - -// entry id - -func UpdateEntryId(id uint64) { - Metrics.EntryId = id -} -func AddAllowEntriesCount() { - Metrics.AllowEntriesCount++ -} -func AddDisallowEntriesCount() { - Metrics.DisallowEntriesCount++ -} - -// rdb - -func SetRDBFileSize(size uint64) { - Metrics.RdbFileSize = size -} -func UpdateRDBReceivedSize(size uint64) { - Metrics.RdbReceivedSize = size -} -func UpdateRDBSentSize(offset uint64) { - Metrics.RdbSendSize = offset -} - -// aof - -func UpdateAOFReceivedOffset(offset uint64) { - Metrics.AofReceivedOffset = offset -} -func UpdateAOFAppliedOffset(offset uint64) { - Metrics.AofAppliedOffset = offset -} - -// for debug - -func UpdateInQueueEntriesCount(count uint64) { - Metrics.InQueueEntriesCount = count -} -func UpdateUnansweredBytesCount(count uint64) { - Metrics.UnansweredBytesCount = count -} diff --git a/internal/status/entry_count.go b/internal/status/entry_count.go new file mode 100644 index 00000000..02a933cc --- /dev/null +++ b/internal/status/entry_count.go @@ -0,0 +1,35 @@ +package status + +import ( + "fmt" + "time" +) + +type EntryCount struct { + ReadCount uint64 `json:"read_count"` + ReadOps float64 `json:"read_ops"` + WriteCount uint64 `json:"write_count"` + WriteOps float64 `json:"write_ops"` + + // update ops + lastReadCount uint64 + lastWriteCount uint64 + lastUpdateTimestampSec float64 +} + +// call this function every second +func (e *EntryCount) updateOPS() { + nowTimestampSec := float64(time.Now().UnixNano()) / 1e9 + if e.lastUpdateTimestampSec != 0 { + timeIntervalSec := nowTimestampSec - e.lastUpdateTimestampSec + e.ReadOps = float64(e.ReadCount-e.lastReadCount) / timeIntervalSec + e.WriteOps = float64(e.WriteCount-e.lastWriteCount) / timeIntervalSec + e.lastReadCount = e.ReadCount + e.lastWriteCount = e.WriteCount + } + e.lastUpdateTimestampSec = nowTimestampSec +} + +func (e *EntryCount) String() string { + return fmt.Sprintf("read_count=[%d], read_ops=[%.2f], write_count=[%d], write_ops=[%.2f]", e.ReadCount, e.ReadOps, e.WriteCount, e.WriteOps) +} diff --git a/internal/status/handler.go b/internal/status/handler.go new file mode 100644 index 00000000..c0069cf8 --- /dev/null +++ b/internal/status/handler.go @@ -0,0 +1,53 @@ +package status + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "encoding/json" + "fmt" + "net/http" + "time" +) + +func Handler(w http.ResponseWriter, _ *http.Request) { + w.Header().Add("Content-Type", "application/json") + + bytesChannel := make(chan []byte, 1) + + ch <- func() { + stat.Consistent = theReader.StatusConsistent() && theWriter.StatusConsistent() + jsonBytes, err := json.Marshal(stat) + if err != nil { + log.Warnf("marshal status info failed, err=[%v]", err) + bytesChannel <- []byte(fmt.Sprintf(`{"error": "%v"}`, err)) + return + } + bytesChannel <- jsonBytes + } + + select { + case bytes := <-bytesChannel: + _, err := w.Write(bytes) + if err != nil { + log.Warnf("write status info failed, err=[%v]", err) + } + case <-time.After(time.Second * 3): + log.Warnf("write status info timeout") + w.WriteHeader(http.StatusRequestTimeout) + } +} + +func setStatusPort() { + if config.Opt.Advanced.StatusPort != 0 { + go func() { + addr := fmt.Sprintf(":%d", config.Opt.Advanced.StatusPort) + if err := http.ListenAndServe(addr, http.HandlerFunc(Handler)); err != nil { + log.Panicf(err.Error()) + } + }() + log.Infof("status information: http://localhost:%v", config.Opt.Advanced.StatusPort) + log.Infof("status information: watch -n 0.3 'curl -s http://localhost:%v | python -m json.tool'", config.Opt.Advanced.StatusPort) + } else { + log.Infof("not set status port") + } +} diff --git a/internal/status/status.go b/internal/status/status.go new file mode 100644 index 00000000..a83a4412 --- /dev/null +++ b/internal/status/status.go @@ -0,0 +1,118 @@ +package status + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "time" +) + +type Statusable interface { + Status() interface{} + StatusString() string + StatusConsistent() bool +} + +type Stat struct { + Time string `json:"start_time"` + Consistent bool `json:"consistent"` + // function + TotalEntriesCount EntryCount `json:"total_entries_count"` + PerCmdEntriesCount map[string]EntryCount `json:"per_cmd_entries_count"` + // reader + Reader interface{} `json:"reader"` + // writer + Writer interface{} `json:"writer"` +} + +var ch = make(chan func(), 1000) +var stat = new(Stat) +var theReader Statusable +var theWriter Statusable + +func AddReadCount(cmd string) { + ch <- func() { + if stat.PerCmdEntriesCount == nil { + stat.PerCmdEntriesCount = make(map[string]EntryCount) + } + cmdEntryCount, ok := stat.PerCmdEntriesCount[cmd] + if !ok { + cmdEntryCount = EntryCount{} + stat.PerCmdEntriesCount[cmd] = cmdEntryCount + } + stat.TotalEntriesCount.ReadCount += 1 + cmdEntryCount.ReadCount += 1 + stat.PerCmdEntriesCount[cmd] = cmdEntryCount + } +} + +func AddWriteCount(cmd string) { + ch <- func() { + if stat.PerCmdEntriesCount == nil { + stat.PerCmdEntriesCount = make(map[string]EntryCount) + } + cmdEntryCount, ok := stat.PerCmdEntriesCount[cmd] + if !ok { + cmdEntryCount = EntryCount{} + stat.PerCmdEntriesCount[cmd] = cmdEntryCount + } + stat.TotalEntriesCount.WriteCount += 1 + cmdEntryCount.WriteCount += 1 + stat.PerCmdEntriesCount[cmd] = cmdEntryCount + } +} + +func Init(r Statusable, w Statusable) { + theReader = r + theWriter = w + setStatusPort() + stat.Time = time.Now().Format("2006-01-02 15:04:05") + + // for update reader/writer stat + go func() { + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + lastConsistent := false + for { + select { + case <-ticker.C: + ch <- func() { + // update reader/writer stat + stat.Reader = theReader.Status() + stat.Writer = theWriter.Status() + stat.Consistent = lastConsistent && theReader.StatusConsistent() && theWriter.StatusConsistent() + lastConsistent = stat.Consistent + // update OPS + stat.TotalEntriesCount.updateOPS() + for _, cmdEntryCount := range stat.PerCmdEntriesCount { + cmdEntryCount.updateOPS() + } + } + } + } + }() + + // for log to screen + go func() { + if config.Opt.Advanced.LogInterval <= 0 { + log.Infof("log interval is 0, will not log to screen") + return + } + ticker := time.NewTicker(time.Duration(config.Opt.Advanced.LogInterval) * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + ch <- func() { + log.Infof("%s, %s", stat.TotalEntriesCount.String(), theReader.StatusString()) + } + } + } + }() + + // run all func in ch + go func() { + for f := range ch { + f() + } + }() +} diff --git a/internal/utils/UniqueQueue.go b/internal/utils/UniqueQueue.go new file mode 100644 index 00000000..140f8202 --- /dev/null +++ b/internal/utils/UniqueQueue.go @@ -0,0 +1,50 @@ +package utils + +import ( + "sync" +) + +type UniqueQueue struct { + innerChannel chan interface{} + set map[interface{}]bool + lock sync.Mutex + closed bool + Ch chan interface{} +} + +func NewUniqueQueue(size int) *UniqueQueue { + mc := new(UniqueQueue) + mc.innerChannel = make(chan interface{}, size) + mc.Ch = make(chan interface{}) + mc.set = make(map[interface{}]bool) + go func() { + for item := range mc.innerChannel { + mc.lock.Lock() + delete(mc.set, item) + mc.lock.Unlock() + mc.Ch <- item + } + close(mc.Ch) + }() + return mc +} + +func (mc *UniqueQueue) Put(item interface{}) { + mc.lock.Lock() + if _, ok := mc.set[item]; ok { + mc.lock.Unlock() + return + } else { + mc.set[item] = true + mc.lock.Unlock() + mc.innerChannel <- item + } +} + +func (mc *UniqueQueue) Len() int { + return len(mc.innerChannel) +} + +func (mc *UniqueQueue) Close() { + close(mc.innerChannel) +} diff --git a/internal/utils/cluster_nodes.go b/internal/utils/cluster_nodes.go new file mode 100644 index 00000000..d98ad643 --- /dev/null +++ b/internal/utils/cluster_nodes.go @@ -0,0 +1,78 @@ +package utils + +import ( + "RedisShake/internal/client" + "RedisShake/internal/log" + "fmt" + "strconv" + "strings" +) + +func GetRedisClusterNodes(address string, username string, password string, Tls bool) (addresses []string, slots [][]int) { + c := client.NewRedisClient(address, username, password, Tls) + reply := c.DoWithStringReply("cluster", "nodes") + reply = strings.TrimSpace(reply) + slotsCount := 0 + for _, line := range strings.Split(reply, "\n") { + line = strings.TrimSpace(line) + words := strings.Split(line, " ") + if !strings.Contains(words[2], "master") { + continue + } + if len(words) < 8 { + log.Panicf("invalid cluster nodes line: %s", line) + } + log.Infof("redisClusterWriter load cluster nodes. line=%v", line) + + // address + address := strings.Split(words[1], "@")[0] + // handle ipv6 address + tok := strings.Split(address, ":") + if len(tok) > 2 { + // ipv6 address + port := tok[len(tok)-1] + + ipv6Addr := strings.Join(tok[:len(tok)-1], ":") + address = fmt.Sprintf("[%s]:%s", ipv6Addr, port) + } + if len(words) < 9 { + log.Warnf("the current master node does not hold any slots. address=[%v]", address) + continue + } + addresses = append(addresses, address) + + // parse slots + slot := make([]int, 0) + for i := 8; i < len(words); i++ { + words[i] = strings.TrimSpace(words[i]) + var start, end int + var err error + if strings.Contains(words[i], "-") { + seg := strings.Split(words[i], "-") + start, err = strconv.Atoi(seg[0]) + if err != nil { + log.Panicf(err.Error()) + } + end, err = strconv.Atoi(seg[1]) + if err != nil { + log.Panicf(err.Error()) + } + } else { + start, err = strconv.Atoi(words[i]) + if err != nil { + log.Panicf(err.Error()) + } + end = start + } + for j := start; j <= end; j++ { + slot = append(slot, j) + slotsCount++ + } + slots = append(slots, slot) + } + } + if slotsCount != 16384 { + log.Panicf("invalid cluster nodes slots. slots_count=%v, address=%v", slotsCount, address) + } + return addresses, slots +} diff --git a/internal/utils/file.go b/internal/utils/file.go index 47fb7c07..28d3fec5 100644 --- a/internal/utils/file.go +++ b/internal/utils/file.go @@ -1,18 +1,49 @@ package utils import ( - "github.com/alibaba/RedisShake/internal/log" + "RedisShake/internal/log" "os" + "path/filepath" ) -func DoesFileExist(fileName string) bool { - _, err := os.Stat(fileName) +func CreateEmptyDir(dir string) { + if IsExist(dir) { + err := os.RemoveAll(dir) + if err != nil { + log.Panicf("remove dir failed. dir=[%s], error=[%v]", dir, err) + } + } + err := os.MkdirAll(dir, 0777) + if err != nil { + log.Panicf("mkdir failed. dir=[%s], error=[%v]", dir, err) + } + log.Debugf("CreateEmptyDir: dir=[%s]", dir) +} + +func IsExist(path string) bool { + _, err := os.Stat(path) if err != nil { if os.IsNotExist(err) { return false } else { - log.PanicError(err) + log.Panicf(err.Error()) } } return true } + +func GetFileSize(path string) uint64 { + fi, err := os.Stat(path) + if err != nil { + log.Panicf(err.Error()) + } + return uint64(fi.Size()) +} + +func GetAbsPath(path string) string { + absolutePath, err := filepath.Abs(path) + if err != nil { + log.Panicf(err.Error()) + } + return absolutePath +} diff --git a/internal/reader/rotate/aof_reader.go b/internal/utils/file_rotate/aof_reader.go similarity index 55% rename from internal/reader/rotate/aof_reader.go rename to internal/utils/file_rotate/aof_reader.go index f5ba676c..1d82d20d 100644 --- a/internal/reader/rotate/aof_reader.go +++ b/internal/utils/file_rotate/aof_reader.go @@ -1,44 +1,48 @@ package rotate import ( + "RedisShake/internal/log" + "RedisShake/internal/utils" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/utils" "io" "os" "time" ) type AOFReader struct { + name string + dir string file *os.File offset int64 pos int64 - filename string + filepath string } -func NewAOFReader(offset int64) *AOFReader { +func NewAOFReader(name string, dir string, offset int64) *AOFReader { r := new(AOFReader) + r.name = name + r.dir = dir r.openFile(offset) return r } func (r *AOFReader) openFile(offset int64) { - r.filename = fmt.Sprintf("%d.aof", offset) + r.filepath = fmt.Sprintf("%s/%d.aof", r.dir, r.offset) var err error - r.file, err = os.OpenFile(r.filename, os.O_RDONLY, 0644) + r.file, err = os.OpenFile(r.filepath, os.O_RDONLY, 0644) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.offset = offset r.pos = 0 - log.Infof("AOFReader open file. aof_filename=[%s]", r.filename) + log.Debugf("[%s] open file for read. filename=[%s]", r.name, r.filepath) } func (r *AOFReader) readNextFile(offset int64) { - filename := fmt.Sprintf("%d.aof", offset) - if utils.DoesFileExist(filename) { + filepath := fmt.Sprintf("%s/%d.aof", r.dir, r.offset) + if utils.IsExist(filepath) { r.Close() - err := os.Remove(r.filename) + err := os.Remove(r.filepath) if err != nil { return } @@ -49,18 +53,18 @@ func (r *AOFReader) readNextFile(offset int64) { func (r *AOFReader) Read(buf []byte) (n int, err error) { n, err = r.file.Read(buf) for err == io.EOF { - if r.filename != fmt.Sprintf("%d.aof", r.offset) { + if r.filepath != fmt.Sprintf("%s/%d.aof", r.dir, r.offset) { r.readNextFile(r.offset) } time.Sleep(time.Millisecond * 10) _, err = r.file.Seek(0, 1) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } n, err = r.file.Read(buf) } if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.offset += int64(n) r.pos += int64(n) @@ -77,8 +81,8 @@ func (r *AOFReader) Close() { } err := r.file.Close() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } r.file = nil - log.Infof("AOFReader close file. aof_filename=[%s]", r.filename) + log.Infof("[%s] close file. filename=[%s]", r.name, r.filepath) } diff --git a/internal/reader/rotate/aof_writer.go b/internal/utils/file_rotate/aof_writer.go similarity index 55% rename from internal/reader/rotate/aof_writer.go rename to internal/utils/file_rotate/aof_writer.go index 0e58474f..18c12b86 100644 --- a/internal/reader/rotate/aof_writer.go +++ b/internal/utils/file_rotate/aof_writer.go @@ -1,42 +1,47 @@ package rotate import ( + "RedisShake/internal/log" "fmt" - "github.com/alibaba/RedisShake/internal/log" "os" ) const MaxFileSize = 1024 * 1024 * 1024 // 1G type AOFWriter struct { + name string + dir string + file *os.File offset int64 - filename string + filepath string filesize int64 } -func NewAOFWriter(offset int64) *AOFWriter { - w := &AOFWriter{} +func NewAOFWriter(name string, dir string, offset int64) *AOFWriter { + w := new(AOFWriter) + w.name = name + w.dir = dir w.openFile(offset) return w } func (w *AOFWriter) openFile(offset int64) { - w.filename = fmt.Sprintf("%d.aof", offset) + w.filepath = fmt.Sprintf("%s/%d.aof", w.dir, w.offset) var err error - w.file, err = os.OpenFile(w.filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + w.file, err = os.OpenFile(w.filepath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } w.offset = offset w.filesize = 0 - log.Infof("AOFWriter open file. filename=[%s]", w.filename) + log.Debugf("[%s] open file for write. filename=[%s]", w.name, w.filepath) } func (w *AOFWriter) Write(buf []byte) { _, err := w.file.Write(buf) if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } w.offset += int64(len(buf)) w.filesize += int64(len(buf)) @@ -46,7 +51,7 @@ func (w *AOFWriter) Write(buf []byte) { } err = w.file.Sync() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } } @@ -56,11 +61,11 @@ func (w *AOFWriter) Close() { } err := w.file.Sync() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } err = w.file.Close() if err != nil { - log.PanicError(err) + log.Panicf(err.Error()) } - log.Infof("AOFWriter close file. filename=[%s], filesize=[%d]", w.filename, w.filesize) + log.Infof("[%s] close file. filename=[%s], filesize=[%d]", w.name, w.filepath, w.filesize) } diff --git a/internal/utils/filelock.go b/internal/utils/filelock.go new file mode 100644 index 00000000..69b1acb4 --- /dev/null +++ b/internal/utils/filelock.go @@ -0,0 +1,46 @@ +package utils + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "github.com/theckman/go-flock" + "os" + "path/filepath" +) + +var filelock *flock.Flock + +func ChdirAndAcquireFileLock() { + // dir + dir, err := filepath.Abs(config.Opt.Advanced.Dir) + if err != nil { + log.Panicf("failed to determine current directory: %v", err) + } + // create dir + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + log.Panicf("failed to create dir. dir: %v, err: %v", dir, err) + } + filelock = flock.New(filepath.Join(dir, "pid.lockfile")) + locked, err := filelock.TryLock() + if err != nil { + log.Panicf("failed to lock pid file: %v", err) + } + if !locked { + log.Warnf("failed to lock pid file") + } + err = os.Chdir(dir) // change dir + if err != nil { + log.Panicf("failed to change dir. dir: %v, err: %v", dir, err) + } + log.Infof("changed work dir to [%s]", dir) +} + +func ReleaseFileLock() { + if filelock != nil { + err := filelock.Unlock() + if err != nil { + log.Warnf("failed to unlock pid file: %v", err) + } + } +} diff --git a/internal/utils/ncpu.go b/internal/utils/ncpu.go new file mode 100644 index 00000000..ba3c502e --- /dev/null +++ b/internal/utils/ncpu.go @@ -0,0 +1,17 @@ +package utils + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "runtime" +) + +func SetNcpu() { + if config.Opt.Advanced.Ncpu != 0 { + log.Infof("set ncpu to %d", config.Opt.Advanced.Ncpu) + runtime.GOMAXPROCS(config.Opt.Advanced.Ncpu) + log.Infof("set GOMAXPROCS to %v", config.Opt.Advanced.Ncpu) + } else { + log.Infof("GOMAXPROCS defaults to the value of runtime.NumCPU [%v]", runtime.NumCPU()) + } +} diff --git a/internal/utils/pprof.go b/internal/utils/pprof.go new file mode 100644 index 00000000..a7884307 --- /dev/null +++ b/internal/utils/pprof.go @@ -0,0 +1,23 @@ +package utils + +import ( + "RedisShake/internal/config" + "RedisShake/internal/log" + "fmt" + "net/http" +) + +func SetPprofPort() { + // pprof_port + if config.Opt.Advanced.PprofPort != 0 { + go func() { + err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Opt.Advanced.PprofPort), nil) + if err != nil { + log.Panicf(err.Error()) + } + }() + log.Infof("pprof information: http://localhost:%d/debug/pprof/", config.Opt.Advanced.PprofPort) + } else { + log.Infof("not set pprof port") + } +} diff --git a/internal/writer/interface.go b/internal/writer/interface.go index 679d2001..523f14fb 100644 --- a/internal/writer/interface.go +++ b/internal/writer/interface.go @@ -1,8 +1,12 @@ package writer -import "github.com/alibaba/RedisShake/internal/entry" +import ( + "RedisShake/internal/entry" + "RedisShake/internal/status" +) type Writer interface { + status.Statusable Write(entry *entry.Entry) Close() } diff --git a/internal/writer/redis.go b/internal/writer/redis.go deleted file mode 100644 index c3502451..00000000 --- a/internal/writer/redis.go +++ /dev/null @@ -1,96 +0,0 @@ -package writer - -import ( - "bytes" - "github.com/alibaba/RedisShake/internal/client" - "github.com/alibaba/RedisShake/internal/client/proto" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/statistics" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" -) - -type redisWriter struct { - client *client.Redis - DbId int - - cmdBuffer *bytes.Buffer - chWaitReply chan *entry.Entry - chWg sync.WaitGroup - - UpdateUnansweredBytesCount uint64 // have sent in bytes -} - -func NewRedisWriter(address string, username string, password string, isTls bool) Writer { - rw := new(redisWriter) - rw.client = client.NewRedisClient(address, username, password, isTls) - log.Infof("redisWriter connected to redis successful. address=[%s]", address) - rw.cmdBuffer = new(bytes.Buffer) - rw.chWaitReply = make(chan *entry.Entry, config.Config.Advanced.PipelineCountLimit) - rw.chWg.Add(1) - go rw.flushInterval() - return rw -} - -func (w *redisWriter) Write(e *entry.Entry) { - // switch db if we need - if w.DbId != e.DbId { - w.switchDbTo(e.DbId) - } - - // send - w.cmdBuffer.Reset() - client.EncodeArgv(e.Argv, w.cmdBuffer) - e.EncodedSize = uint64(w.cmdBuffer.Len()) - for e.EncodedSize+atomic.LoadUint64(&w.UpdateUnansweredBytesCount) > config.Config.Advanced.TargetRedisClientMaxQuerybufLen { - time.Sleep(1 * time.Nanosecond) - } - w.chWaitReply <- e - atomic.AddUint64(&w.UpdateUnansweredBytesCount, e.EncodedSize) - w.client.SendBytes(w.cmdBuffer.Bytes()) -} - -func (w *redisWriter) switchDbTo(newDbId int) { - w.client.Send("select", strconv.Itoa(newDbId)) - w.DbId = newDbId - w.chWaitReply <- &entry.Entry{ - Argv: []string{"select", strconv.Itoa(newDbId)}, - CmdName: "select", - } -} - -func (w *redisWriter) flushInterval() { - for e := range w.chWaitReply { - reply, err := w.client.Receive() - if err == proto.Nil { - log.Warnf("redisWriter receive nil reply. argv=%v", e.Argv) - } else if err != nil { - if err.Error() == "BUSYKEY Target key name already exists." { - if config.Config.Advanced.RDBRestoreCommandBehavior == "skip" { - log.Warnf("redisWriter received BUSYKEY reply. argv=%v", e.Argv) - } else if config.Config.Advanced.RDBRestoreCommandBehavior == "panic" { - log.Panicf("redisWriter received BUSYKEY reply. argv=%v", e.Argv) - } - } else { - log.Panicf("redisWriter received error. error=[%v], argv=%v, slots=%v, reply=[%v]", err, e.Argv, e.Slots, reply) - } - } - if strings.EqualFold(e.CmdName, "select") { // skip select command - continue - } - atomic.AddUint64(&w.UpdateUnansweredBytesCount, ^(e.EncodedSize - 1)) - statistics.UpdateAOFAppliedOffset(uint64(e.Offset)) - statistics.UpdateUnansweredBytesCount(atomic.LoadUint64(&w.UpdateUnansweredBytesCount)) - } - w.chWg.Done() -} - -func (w *redisWriter) Close() { - close(w.chWaitReply) - w.chWg.Wait() -} diff --git a/internal/writer/redis_cluster.go b/internal/writer/redis_cluster.go deleted file mode 100644 index de77a36a..00000000 --- a/internal/writer/redis_cluster.go +++ /dev/null @@ -1,121 +0,0 @@ -package writer - -import ( - "fmt" - "github.com/alibaba/RedisShake/internal/client" - "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/log" - "strconv" - "strings" -) - -const KeySlots = 16384 - -type RedisClusterWriter struct { - addresses []string - writers []Writer - router [KeySlots]Writer -} - -func NewRedisClusterWriter(address string, username string, password string, isTls bool) Writer { - rw := new(RedisClusterWriter) - - rw.loadClusterNodes(address, username, password, isTls) - - log.Infof("redisClusterWriter connected to redis cluster successful. addresses=%v", rw.addresses) - return rw -} - -func (r *RedisClusterWriter) loadClusterNodes(address string, username string, password string, isTls bool) { - client_ := client.NewRedisClient(address, username, password, isTls) - reply := client_.DoWithStringReply("cluster", "nodes") - reply = strings.TrimSpace(reply) - for _, line := range strings.Split(reply, "\n") { - line = strings.TrimSpace(line) - words := strings.Split(line, " ") - if !strings.Contains(words[2], "master") { - continue - } - if len(words) < 9 { - log.Panicf("invalid cluster nodes line: %s", line) - } - log.Infof("redisClusterWriter load cluster nodes. line=%v", line) - // address - address := strings.Split(words[1], "@")[0] - - // handle ipv6 address - tok := strings.Split(address, ":") - if len(tok) > 2 { - // ipv6 address - port := tok[len(tok)-1] - - ipv6Addr := strings.Join(tok[:len(tok)-1], ":") - address = fmt.Sprintf("[%s]:%s", ipv6Addr, port) - } - - r.addresses = append(r.addresses, address) - // writers - redisWriter := NewRedisWriter(address, username, password, isTls) - r.writers = append(r.writers, redisWriter) - // parse slots - for i := 8; i < len(words); i++ { - words[i] = strings.TrimSpace(words[i]) - var start, end int - var err error - if strings.Contains(words[i], "-") { - seg := strings.Split(words[i], "-") - start, err = strconv.Atoi(seg[0]) - if err != nil { - log.PanicError(err) - } - end, err = strconv.Atoi(seg[1]) - if err != nil { - log.PanicError(err) - } - } else { - start, err = strconv.Atoi(words[i]) - if err != nil { - log.PanicError(err) - } - end = start - } - for j := start; j <= end; j++ { - if r.router[j] != nil { - log.Panicf("redisClusterWriter: slot %d already occupied", j) - } - r.router[j] = redisWriter - } - } - } - for i := 0; i < KeySlots; i++ { - if r.router[i] == nil { - log.Panicf("redisClusterWriter: slot %d not occupied", i) - } - } -} - -func (r *RedisClusterWriter) Write(entry *entry.Entry) { - if len(entry.Slots) == 0 { - for _, writer := range r.writers { - writer.Write(entry) - } - return - } - - lastSlot := -1 - for _, slot := range entry.Slots { - if lastSlot == -1 { - lastSlot = slot - } - if slot != lastSlot { - log.Panicf("CROSSSLOT Keys in request don't hash to the same slot. argv=%v", entry.Argv) - } - } - r.router[lastSlot].Write(entry) -} - -func (r *RedisClusterWriter) Close() { - for _, writer := range r.writers { - writer.Close() - } -} diff --git a/internal/writer/redis_cluster_writer.go b/internal/writer/redis_cluster_writer.go new file mode 100644 index 00000000..76853907 --- /dev/null +++ b/internal/writer/redis_cluster_writer.go @@ -0,0 +1,103 @@ +package writer + +import ( + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/utils" +) + +const KeySlots = 16384 + +type RedisClusterWriter struct { + addresses []string + writers []Writer + router [KeySlots]Writer + + stat []interface{} +} + +func NewRedisClusterWriter(opts *RedisWriterOptions) Writer { + rw := new(RedisClusterWriter) + rw.loadClusterNodes(opts) + log.Infof("redisClusterWriter connected to redis cluster successful. addresses=%v", rw.addresses) + return rw +} + +func (r *RedisClusterWriter) Close() { + for _, writer := range r.writers { + writer.Close() + } +} + +func (r *RedisClusterWriter) loadClusterNodes(opts *RedisWriterOptions) { + addresses, slots := utils.GetRedisClusterNodes(opts.Address, opts.Username, opts.Password, opts.Tls) + r.addresses = addresses + for i, address := range addresses { + theOpts := *opts + theOpts.Address = address + redisWriter := NewRedisStandaloneWriter(&theOpts) + r.writers = append(r.writers, redisWriter) + for _, s := range slots[i] { + if r.router[s] != nil { + log.Panicf("redisClusterWriter: slot %d already occupied", s) + } + r.router[s] = redisWriter + } + } + + for i := 0; i < KeySlots; i++ { + if r.router[i] == nil { + log.Panicf("redisClusterWriter: slot %d not occupied", i) + } + } +} + +func (r *RedisClusterWriter) Write(entry *entry.Entry) { + if len(entry.Slots) == 0 { + for _, writer := range r.writers { + writer.Write(entry) + } + return + } + + lastSlot := -1 + for _, slot := range entry.Slots { + if lastSlot == -1 { + lastSlot = slot + } + if slot != lastSlot { + log.Panicf("CROSSSLOT Keys in request don't hash to the same slot. argv=%v", entry.Argv) + } + } + r.router[lastSlot].Write(entry) +} + +func (r *RedisClusterWriter) Consistent() bool { + for _, writer := range r.writers { + if !writer.StatusConsistent() { + return false + } + } + return true +} + +func (r *RedisClusterWriter) Status() interface{} { + r.stat = make([]interface{}, 0) + for _, writer := range r.writers { + r.stat = append(r.stat, writer.Status()) + } + return r.stat +} + +func (r *RedisClusterWriter) StatusString() string { + return "[redis_cluster_writer] writing to redis cluster" +} + +func (r *RedisClusterWriter) StatusConsistent() bool { + for _, writer := range r.writers { + if !writer.StatusConsistent() { + return false + } + } + return true +} diff --git a/internal/writer/redis_standalone_writer.go b/internal/writer/redis_standalone_writer.go new file mode 100644 index 00000000..f4f57e3c --- /dev/null +++ b/internal/writer/redis_standalone_writer.go @@ -0,0 +1,120 @@ +package writer + +import ( + "RedisShake/internal/client" + "RedisShake/internal/client/proto" + "RedisShake/internal/config" + "RedisShake/internal/entry" + "RedisShake/internal/log" + "fmt" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +type RedisWriterOptions struct { + Cluster bool `mapstructure:"cluster" default:"false"` + Address string `mapstructure:"address" default:""` + Username string `mapstructure:"username" default:""` + Password string `mapstructure:"password" default:""` + Tls bool `mapstructure:"tls" default:"false"` +} + +type redisStandaloneWriter struct { + address string + client *client.Redis + DbId int + + chWaitReply chan *entry.Entry + chWg sync.WaitGroup + + stat struct { + Name string `json:"name"` + UnansweredBytes int64 `json:"unanswered_bytes"` + UnansweredEntries int64 `json:"unanswered_entries"` + } +} + +func NewRedisStandaloneWriter(opts *RedisWriterOptions) Writer { + rw := new(redisStandaloneWriter) + rw.address = opts.Address + rw.stat.Name = "writer_" + strings.Replace(opts.Address, ":", "_", -1) + rw.client = client.NewRedisClient(opts.Address, opts.Username, opts.Password, opts.Tls) + rw.chWaitReply = make(chan *entry.Entry, config.Opt.Advanced.PipelineCountLimit) + rw.chWg.Add(1) + go rw.processReply() + return rw +} + +func (w *redisStandaloneWriter) Close() { + close(w.chWaitReply) + w.chWg.Wait() +} + +func (w *redisStandaloneWriter) Write(e *entry.Entry) { + // switch db if we need + if w.DbId != e.DbId { + w.switchDbTo(e.DbId) + } + + // send + bytes := e.Serialize() + for e.SerializedSize+atomic.LoadInt64(&w.stat.UnansweredBytes) > config.Opt.Advanced.TargetRedisClientMaxQuerybufLen { + time.Sleep(1 * time.Nanosecond) + } + log.Debugf("[%s] send cmd. cmd=[%s]", w.stat.Name, e.String()) + w.chWaitReply <- e + atomic.AddInt64(&w.stat.UnansweredBytes, e.SerializedSize) + atomic.AddInt64(&w.stat.UnansweredEntries, 1) + w.client.SendBytes(bytes) +} + +func (w *redisStandaloneWriter) switchDbTo(newDbId int) { + log.Debugf("[%s] switch db to [%d]", w.stat.Name, newDbId) + w.client.Send("select", strconv.Itoa(newDbId)) + w.DbId = newDbId + w.chWaitReply <- &entry.Entry{ + Argv: []string{"select", strconv.Itoa(newDbId)}, + CmdName: "select", + } +} + +func (w *redisStandaloneWriter) processReply() { + for e := range w.chWaitReply { + reply, err := w.client.Receive() + log.Debugf("[%s] receive reply. reply=[%v], cmd=[%s]", w.stat.Name, reply, e.String()) + if err == proto.Nil { + log.Warnf("[%s] receive nil reply. cmd=[%s]", w.stat.Name, e.String()) + } else if err != nil { + if err.Error() == "BUSYKEY Target key name already exists." { + if config.Opt.Advanced.RDBRestoreCommandBehavior == "skip" { + log.Debugf("[%s] redisStandaloneWriter received BUSYKEY reply. cmd=[%s]", w.stat.Name, e.String()) + } else if config.Opt.Advanced.RDBRestoreCommandBehavior == "panic" { + log.Panicf("[%s] redisStandaloneWriter received BUSYKEY reply. cmd=[%s]", w.stat.Name, e.String()) + } + } else { + log.Panicf("[%s] receive reply failed. cmd=[%s], error=[%v]", w.stat.Name, e.String(), err) + } + } + if strings.EqualFold(e.CmdName, "select") { // skip select command + continue + } + atomic.AddInt64(&w.stat.UnansweredBytes, -e.SerializedSize) + atomic.AddInt64(&w.stat.UnansweredEntries, -1) + } + w.chWg.Done() +} + +func (w *redisStandaloneWriter) Status() interface{} { + return w.stat +} + +func (w *redisStandaloneWriter) StatusString() string { + return fmt.Sprintf("[%s]: unanswered_entries=%d", w.stat.Name, atomic.LoadInt64(&w.stat.UnansweredEntries)) +} + +func (w *redisStandaloneWriter) StatusConsistent() bool { + return atomic.LoadInt64(&w.stat.UnansweredBytes) == 0 && atomic.LoadInt64(&w.stat.UnansweredEntries) == 0 +} diff --git a/restore.toml b/restore.toml deleted file mode 100644 index 6ed1d1ae..00000000 --- a/restore.toml +++ /dev/null @@ -1,54 +0,0 @@ -type = "restore" - -[source] -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -# Path to the dump.rdb file. Absolute path or relative path. Note -# that relative paths are relative to the dir directory. -rdb_file_path = "dump.rdb" - -[target] -type = "standalone" # standalone or cluster -# When the target is a cluster, write the address of one of the nodes. -# redis-shake will obtain other nodes through the `cluster nodes` command. -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -address = "127.0.0.1:6379" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false - -[advanced] -dir = "data" - -# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores -ncpu = 3 - -# pprof port, 0 means disable -pprof_port = 0 - -# metric port, 0 means disable -metrics_port = 0 - -# log -log_file = "redis-shake.log" -log_level = "info" # debug, info or warn -log_interval = 5 # in seconds - -# redis-shake gets key and value from rdb file, and uses RESTORE command to -# create the key in target redis. Redis RESTORE will return a "Target key name -# is busy" error when key already exists. You can use this configuration item -# to change the default behavior of restore: -# panic: redis-shake will stop when meet "Target key name is busy" error. -# rewrite: redis-shake will replace the key with new value. -# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. -rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip - -# pipeline -pipeline_count_limit = 1024 - -# Client query buffers accumulate new commands. They are limited to a fixed -# amount by default. This amount is normally 1gb. -target_redis_client_max_querybuf_len = 1024_000_000 - -# In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited to 512 mb. -target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file diff --git a/scan.toml b/scan.toml deleted file mode 100644 index 25f61add..00000000 --- a/scan.toml +++ /dev/null @@ -1,55 +0,0 @@ -type = "scan" - -[source] -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -address = "127.0.0.1:6379" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false - -[target] -type = "standalone" # "standalone" or "cluster" -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -# When the target is a cluster, write the address of one of the nodes. -# redis-shake will obtain other nodes through the `cluster nodes` command. -address = "127.0.0.1:6380" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false - -[advanced] -dir = "data" - -# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores -ncpu = 0 - -# pprof port, 0 means disable -pprof_port = 0 - -# metric port, 0 means disable -metrics_port = 0 - -# log -log_file = "redis-shake.log" -log_level = "info" # debug, info or warn -log_interval = 5 # in seconds - -# redis-shake gets key and value from rdb file, and uses RESTORE command to -# create the key in target redis. Redis RESTORE will return a "Target key name -# is busy" error when key already exists. You can use this configuration item -# to change the default behavior of restore: -# panic: redis-shake will stop when meet "Target key name is busy" error. -# rewrite: redis-shake will replace the key with new value. -# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. -rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip - -# pipeline -pipeline_count_limit = 1024 - -# Client query buffers accumulate new commands. They are limited to a fixed -# amount by default. This amount is normally 1gb. -target_redis_client_max_querybuf_len = 1024_000_000 - -# In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited to 512 mb. -target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file diff --git a/scripts/cluster_helper/cluster_helper.py b/scripts/cluster_helper/cluster_helper.py deleted file mode 100644 index 8a9a68ec..00000000 --- a/scripts/cluster_helper/cluster_helper.py +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 -import datetime -import os -import shutil -import signal -import sys -import time -from pathlib import Path - -import redis -import requests -import toml - -from launcher import Launcher - -USAGE = """ -cluster_helper is a helper script to start many redis-shake for syncing from cluster. - -Usage: - $ python3 cluster_helper.py ./bin/redis-shake sync.toml - or - $ python3 cluster_helper.py ./bin/redis-shake sync.toml ./bin/filters/key_prefix.lua -""" - -REDIS_SHAKE_PATH = "" -LUA_FILTER_PATH = "" -SLEEP_SECONDS = 5 -stopped = False -toml_template = {} - - -class Shake: - def __init__(self): - self.metrics_port = 0 - self.launcher = None - - -nodes = {} - - -def parse_args(): - if len(sys.argv) != 3 and len(sys.argv) != 4: - print(USAGE) - exit(1) - global REDIS_SHAKE_PATH, LUA_FILTER_PATH, toml_template - - # 1. check redis-shake path - REDIS_SHAKE_PATH = sys.argv[1] - if not Path(REDIS_SHAKE_PATH).is_file(): - print(f"redis-shake path [{REDIS_SHAKE_PATH}] is not a file") - print(USAGE) - exit(1) - print(f"redis-shake path: {REDIS_SHAKE_PATH}") - REDIS_SHAKE_PATH = os.path.abspath(REDIS_SHAKE_PATH) - print(f"redis-shake abs path: {REDIS_SHAKE_PATH}") - - # 2. check and load toml file - toml_template = toml.load(sys.argv[2]) - print(toml_template) - if "username" not in toml_template["source"]: - toml_template["source"]["username"] = "" - if "password" not in toml_template["source"]: - toml_template["source"]["password"] = "" - if "tls" not in toml_template["source"]: - toml_template["source"]["tls"] = False - if "advanced" not in toml_template: - toml_template["advanced"] = {} - - # 3. check filter - if len(sys.argv) == 4: - LUA_FILTER_PATH = sys.argv[3] - if not Path(LUA_FILTER_PATH).is_file(): - print(f"filter path [{LUA_FILTER_PATH}] is not a file") - print(USAGE) - exit(1) - print(f"filter path: {LUA_FILTER_PATH}") - LUA_FILTER_PATH = os.path.abspath(LUA_FILTER_PATH) - print(f"filter abs path: {LUA_FILTER_PATH}") - - -def stop(): - for shake in nodes.values(): - shake.launcher.stop() - exit(0) - - -def loop(): - while True: - if stopped: - stop() - print( - f"================ {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ================" - ) - - metrics = [] - for address, shake in nodes.items(): - try: - ret = requests.get(f"http://localhost:{shake.metrics_port}").json() - metrics.append(ret) - except requests.exceptions.RequestException as e: - print(f"get metrics from [{address}] failed: {e}") - - for metric in sorted(metrics, key=lambda x: x["address"]): - print(f"{metric['address']} {metric['msg']} ") - - if len(metrics) == 0: - print("no redis-shake is running") - break - - time.sleep(SLEEP_SECONDS) - - -def main(): - parse_args() - - # parse args - address = toml_template["source"]["address"] - host, port = address.split(":") - username = toml_template["source"]["username"] - password = toml_template["source"]["password"] - tls = toml_template["source"]["tls"] - print( - f"host: {host}, port: {port}, username: {username}, password: {password}, tls: {tls}" - ) - cluster = redis.RedisCluster( - host=host, port=port, username=username, password=password, ssl=tls - ) - print("cluster nodes:", cluster.cluster_nodes()) - - # parse cluster nodes - for address, node in cluster.cluster_nodes().items(): - if "master" in node["flags"]: - nodes[address] = Shake() - print(f"addresses:") - for k in nodes.keys(): - print(k) - - # create workdir and start redis-shake - if os.path.exists("data"): - shutil.rmtree("data") - os.mkdir("data") - os.chdir("data") - start_port = ( - 11007 - if toml_template.get("advanced").get("metrics_port", 0) == 0 - else toml_template["advanced"]["metrics_port"] - ) - for address in nodes.keys(): - workdir = address.replace(".", "_").replace(":", "_") - - os.mkdir(workdir) - tmp_toml = toml_template - tmp_toml["source"]["address"] = address - start_port += 1 - tmp_toml["advanced"]["metrics_port"] = start_port - - with open(f"{workdir}/sync.toml", "w") as f: - toml.dump(tmp_toml, f) - - # start redis-shake - args = [REDIS_SHAKE_PATH, f"sync.toml"] - if LUA_FILTER_PATH != "": - args.append(LUA_FILTER_PATH) - launcher = Launcher(args=args, work_dir=workdir) - nodes[address].launcher = launcher - nodes[address].metrics_port = start_port - - signal.signal(signal.SIGINT, signal_handler) - print("start syncing...") - print("sleep 3 seconds to wait redis-shake start") - time.sleep(3) - loop() - for node in nodes.values(): - node.launcher.stop() - - -def signal_handler(sig, frame): - global stopped - print("\nYou pressed Ctrl+C!") - stopped = True - - -if __name__ == "__main__": - main() diff --git a/scripts/cluster_helper/launcher.py b/scripts/cluster_helper/launcher.py deleted file mode 100644 index f1da8944..00000000 --- a/scripts/cluster_helper/launcher.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import signal -import subprocess -from pathlib import Path - - -class Launcher: - def __init__(self, args, work_dir): - self.started = True - self.args = args - self.work_dir = work_dir - if not os.path.exists(work_dir): - Path(self.work_dir).mkdir(parents=True, exist_ok=True) - self.stdout_file = open(work_dir + "/stdout", 'a') - self.stderr_file = open(work_dir + "/stderr", 'a') - self.process = subprocess.Popen(self.args, stdout=self.stdout_file, - stderr=self.stderr_file, cwd=self.work_dir, - encoding="utf-8") - - def __del__(self): - assert not self.started, "Every Launcher should be closed manually! work_dir:" + self.work_dir - - def get_pid(self): - return self.process.pid - - def stop(self): - if self.started: - self.started = False - print(f"Waiting for process {self.process.pid} to exit...") - self.stdout_file.close() - self.stderr_file.close() - self.process.send_signal(signal.SIGINT) - self.process.wait() - print(f"process {self.process.pid} exited.") diff --git a/scripts/cluster_helper/requirements.txt b/scripts/cluster_helper/requirements.txt deleted file mode 100644 index 5cd405fc..00000000 --- a/scripts/cluster_helper/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -redis==4.3.4 -requests==2.27.1 -toml==0.10.2 diff --git a/scripts/commands/exhset.json b/scripts/commands/exhset.json new file mode 100644 index 00000000..708c8968 --- /dev/null +++ b/scripts/commands/exhset.json @@ -0,0 +1,28 @@ +{ + "EXHSET": { + "summary": "TairHash, Insert a field into the TairHash specified by the key", + "complexity": "O(1)", + "group": "TairHash", + "function": "exhsetCommand", + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ] + } +} diff --git a/scripts/commands/exset.json b/scripts/commands/exset.json new file mode 100644 index 00000000..5bbad89b --- /dev/null +++ b/scripts/commands/exset.json @@ -0,0 +1,28 @@ +{ + "EXSET": { + "summary": "TairString, Set the value of a key", + "complexity": "O(1)", + "group": "TairString", + "function": "exsetCommand", + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ] + } +} diff --git a/scripts/commands/exzadd.json b/scripts/commands/exzadd.json new file mode 100644 index 00000000..fc311830 --- /dev/null +++ b/scripts/commands/exzadd.json @@ -0,0 +1,28 @@ +{ + "EXZADD": { + "summary": "TairZset, Adds all the specified members with the specified (multi)scores to the tairzset stored at key", + "complexity": "O(N)", + "group": "TairZset", + "function": "exzaddCommand", + "key_specs": [ + { + "flags": [ + "RW", + "UPDATE" + ], + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + } + ] + } +} diff --git a/scripts/gen_table_go_from_table_json.py b/scripts/gen_table_go_from_table_json.py index 831a1d6a..ca3cb893 100644 --- a/scripts/gen_table_go_from_table_json.py +++ b/scripts/gen_table_go_from_table_json.py @@ -51,6 +51,188 @@ fp.write('},\n') fp.write('},\n') fp.write('},\n') +fp.write(""" + "BF.ADD": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.CARD": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.EXISTS": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.INFO": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.INSERT": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.LOADCHUNK": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.MADD": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.MEXISTS": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.RESERVE": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.SCANDUMP": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, +""") fp.write('}\n') fp.close() os.system("go fmt table.go") diff --git a/scripts/gen_table_json_from_commands.py b/scripts/gen_table_json_from_commands.py index 06785318..22e9386f 100644 --- a/scripts/gen_table_json_from_commands.py +++ b/scripts/gen_table_json_from_commands.py @@ -5,33 +5,33 @@ files = os.listdir(commands_dir) table = {} container = set() +files = sorted(files) for file in files: - j = json.load(open(f"{commands_dir}/{file}")) - cmd_name = list(j.keys())[0] - j = j[cmd_name] + content = json.load(open(f"{commands_dir}/{file}")) + for cmd_name, j in content.items(): + print(cmd_name) + if cmd_name in ("SORT", "SORT_RO", "MIGRATE"): + continue - print(cmd_name) - if cmd_name in ("SORT", "SORT_RO", "MIGRATE"): - continue + group = j["group"] + key_specs = [] + if "key_specs" in j: + for key_spec in j["key_specs"]: + begin_search = key_spec["begin_search"] + find_keys = key_spec["find_keys"] + key_specs.append({ + "begin_search": begin_search, + "find_keys": find_keys + }) + if "container" in j: + cmd_name = j["container"] + "-" + cmd_name + container.add(j["container"]) - group = j["group"] - key_specs = [] - if "key_specs" in j: - for key_spec in j["key_specs"]: - begin_search = key_spec["begin_search"] - find_keys = key_spec["find_keys"] - key_specs.append({ - "begin_search": begin_search, - "find_keys": find_keys - }) - if "container" in j: - cmd_name = j["container"] + "-" + cmd_name - container.add(j["container"]) - - if group not in table: - table[group] = {} - table[group][cmd_name] = key_specs + if group not in table: + table[group] = {} + table[group][cmd_name] = key_specs +container = sorted(container) with open("table.json", "w") as f: json.dump({ "table": table, diff --git a/scripts/table.json b/scripts/table.json index 11987b5c..af15fc1a 100644 --- a/scripts/table.json +++ b/scripts/table.json @@ -1,11 +1,62 @@ { "table": { - "list": { - "LLEN": [ + "server": { + "ACL-CAT": [], + "ACL-DELUSER": [], + "ACL-DRYRUN": [], + "ACL-GENPASS": [], + "ACL-GETUSER": [], + "ACL-HELP": [], + "ACL-LIST": [], + "ACL-LOAD": [], + "ACL-LOG": [], + "ACL-SAVE": [], + "ACL-SETUSER": [], + "ACL-USERS": [], + "ACL-WHOAMI": [], + "ACL": [], + "BGREWRITEAOF": [], + "BGSAVE": [], + "COMMAND-COUNT": [], + "COMMAND-DOCS": [], + "COMMAND-GETKEYS": [], + "COMMAND-GETKEYSANDFLAGS": [], + "COMMAND-HELP": [], + "COMMAND-INFO": [], + "COMMAND-LIST": [], + "COMMAND": [], + "CONFIG-GET": [], + "CONFIG-HELP": [], + "CONFIG-RESETSTAT": [], + "CONFIG-REWRITE": [], + "CONFIG-SET": [], + "CONFIG": [], + "DBSIZE": [], + "DEBUG": [], + "FAILOVER": [], + "FLUSHALL": [], + "FLUSHDB": [], + "INFO": [], + "LASTSAVE": [], + "LATENCY-DOCTOR": [], + "LATENCY-GRAPH": [], + "LATENCY-HELP": [], + "LATENCY-HISTOGRAM": [], + "LATENCY-HISTORY": [], + "LATENCY-LATEST": [], + "LATENCY-RESET": [], + "LATENCY": [], + "LOLWUT": [], + "MEMORY-DOCTOR": [], + "MEMORY-HELP": [], + "MEMORY-MALLOC-STATS": [], + "MEMORY-PURGE": [], + "MEMORY-STATS": [], + "MEMORY-USAGE": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -17,7 +68,18 @@ } } ], - "BRPOPLPUSH": [ + "MEMORY": [], + "MODULE-HELP": [], + "MODULE-LIST": [], + "MODULE-LOAD": [], + "MODULE-LOADEX": [], + "MODULE-UNLOAD": [], + "MODULE": [], + "MONITOR": [], + "PSYNC": [], + "REPLCONF": [], + "REPLICAOF": [], + "RESTORE-ASKING": [ { "begin_search": { "index": { @@ -31,11 +93,27 @@ "limit": 0 } } - }, + } + ], + "ROLE": [], + "SAVE": [], + "SHUTDOWN": [], + "SLAVEOF": [], + "SLOWLOG-GET": [], + "SLOWLOG-HELP": [], + "SLOWLOG-LEN": [], + "SLOWLOG-RESET": [], + "SLOWLOG": [], + "SWAPDB": [], + "SYNC": [], + "TIME": [] + }, + "string": { + "APPEND": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -47,23 +125,7 @@ } } ], - "LMPOP": [ - { - "begin_search": { - "index": { - "pos": 1 - } - }, - "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 - } - } - } - ], - "LSET": [ + "DECR": [ { "begin_search": { "index": { @@ -79,23 +141,7 @@ } } ], - "BLMPOP": [ - { - "begin_search": { - "index": { - "pos": 2 - } - }, - "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 - } - } - } - ], - "LINDEX": [ + "DECRBY": [ { "begin_search": { "index": { @@ -111,7 +157,7 @@ } } ], - "LPOS": [ + "GET": [ { "begin_search": { "index": { @@ -127,7 +173,7 @@ } } ], - "RPOPLPUSH": [ + "GETDEL": [ { "begin_search": { "index": { @@ -141,11 +187,13 @@ "limit": 0 } } - }, + } + ], + "GETEX": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -157,7 +205,7 @@ } } ], - "LTRIM": [ + "GETRANGE": [ { "begin_search": { "index": { @@ -173,7 +221,7 @@ } } ], - "LPUSH": [ + "GETSET": [ { "begin_search": { "index": { @@ -189,7 +237,7 @@ } } ], - "BRPOP": [ + "INCR": [ { "begin_search": { "index": { @@ -198,14 +246,14 @@ }, "find_keys": { "range": { - "lastkey": -2, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "LINSERT": [ + "INCRBY": [ { "begin_search": { "index": { @@ -221,7 +269,7 @@ } } ], - "LRANGE": [ + "INCRBYFLOAT": [ { "begin_search": { "index": { @@ -237,7 +285,7 @@ } } ], - "LREM": [ + "LCS": [ { "begin_search": { "index": { @@ -246,14 +294,14 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": 1, "step": 1, "limit": 0 } } } ], - "RPUSH": [ + "MGET": [ { "begin_search": { "index": { @@ -262,14 +310,14 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "RPOP": [ + "MSET": [ { "begin_search": { "index": { @@ -278,14 +326,14 @@ }, "find_keys": { "range": { - "lastkey": 0, - "step": 1, + "lastkey": -1, + "step": 2, "limit": 0 } } } ], - "LPOP": [ + "MSETNX": [ { "begin_search": { "index": { @@ -294,14 +342,14 @@ }, "find_keys": { "range": { - "lastkey": 0, - "step": 1, + "lastkey": -1, + "step": 2, "limit": 0 } } } ], - "LMOVE": [ + "PSETEX": [ { "begin_search": { "index": { @@ -315,11 +363,13 @@ "limit": 0 } } - }, + } + ], + "SET": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -331,7 +381,7 @@ } } ], - "RPUSHX": [ + "SETEX": [ { "begin_search": { "index": { @@ -347,7 +397,7 @@ } } ], - "BLMOVE": [ + "SETNX": [ { "begin_search": { "index": { @@ -361,11 +411,13 @@ "limit": 0 } } - }, + } + ], + "SETRANGE": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -377,7 +429,7 @@ } } ], - "LPUSHX": [ + "STRLEN": [ { "begin_search": { "index": { @@ -393,7 +445,7 @@ } } ], - "BLPOP": [ + "SUBSTR": [ { "begin_search": { "index": { @@ -402,7 +454,7 @@ }, "find_keys": { "range": { - "lastkey": -2, + "lastkey": 0, "step": 1, "limit": 0 } @@ -410,72 +462,71 @@ } ] }, - "server": { - "MODULE-UNLOAD": [], - "PSYNC": [], - "ACL-WHOAMI": [], - "ACL-GETUSER": [], - "MEMORY-STATS": [], - "LATENCY-HELP": [], - "MONITOR": [], - "BGREWRITEAOF": [], - "LATENCY-GRAPH": [], - "REPLCONF": [], - "LATENCY-HISTORY": [], - "FLUSHDB": [], - "SHUTDOWN": [], - "ROLE": [], - "ACL-SAVE": [], - "LATENCY-LATEST": [], - "LATENCY-HISTOGRAM": [], - "DEBUG": [], - "COMMAND-GETKEYSANDFLAGS": [], - "CONFIG": [], - "ACL-LOG": [], - "CONFIG-HELP": [], - "ACL": [], - "MEMORY": [], - "ACL-CAT": [], - "SAVE": [], - "LOLWUT": [], - "LATENCY-RESET": [], - "MEMORY-PURGE": [], - "COMMAND-DOCS": [], - "ACL-DRYRUN": [], - "SWAPDB": [], - "SYNC": [], - "ACL-USERS": [], - "ACL-SETUSER": [], - "MODULE-HELP": [], - "ACL-LOAD": [], - "COMMAND-COUNT": [], - "COMMAND-HELP": [], - "ACL-HELP": [], - "MODULE-LOAD": [], - "SLOWLOG": [], - "TIME": [], - "CONFIG-REWRITE": [], - "COMMAND": [], - "SLOWLOG-RESET": [], - "SLAVEOF": [], - "ACL-DELUSER": [], - "FLUSHALL": [], - "CONFIG-RESETSTAT": [], - "LATENCY-DOCTOR": [], - "MEMORY-DOCTOR": [], - "INFO": [], - "MODULE": [], - "BGSAVE": [], - "MODULE-LOADEX": [], - "MEMORY-HELP": [], - "ACL-GENPASS": [], - "DBSIZE": [], - "SLOWLOG-GET": [], - "MEMORY-USAGE": [ + "cluster": { + "ASKING": [], + "CLUSTER-ADDSLOTS": [], + "CLUSTER-ADDSLOTSRANGE": [], + "CLUSTER-BUMPEPOCH": [], + "CLUSTER-COUNT-FAILURE-REPORTS": [], + "CLUSTER-COUNTKEYSINSLOT": [], + "CLUSTER-DELSLOTS": [], + "CLUSTER-DELSLOTSRANGE": [], + "CLUSTER-FAILOVER": [], + "CLUSTER-FLUSHSLOTS": [], + "CLUSTER-FORGET": [], + "CLUSTER-GETKEYSINSLOT": [], + "CLUSTER-HELP": [], + "CLUSTER-INFO": [], + "CLUSTER-KEYSLOT": [], + "CLUSTER-LINKS": [], + "CLUSTER-MEET": [], + "CLUSTER-MYID": [], + "CLUSTER-NODES": [], + "CLUSTER-REPLICAS": [], + "CLUSTER-REPLICATE": [], + "CLUSTER-RESET": [], + "CLUSTER-SAVECONFIG": [], + "CLUSTER-SET-CONFIG-EPOCH": [], + "CLUSTER-SETSLOT": [], + "CLUSTER-SHARDS": [], + "CLUSTER-SLAVES": [], + "CLUSTER-SLOTS": [], + "CLUSTER": [], + "READONLY": [], + "READWRITE": [] + }, + "connection": { + "AUTH": [], + "CLIENT-CACHING": [], + "CLIENT-GETNAME": [], + "CLIENT-GETREDIR": [], + "CLIENT-HELP": [], + "CLIENT-ID": [], + "CLIENT-INFO": [], + "CLIENT-KILL": [], + "CLIENT-LIST": [], + "CLIENT-NO-EVICT": [], + "CLIENT-PAUSE": [], + "CLIENT-REPLY": [], + "CLIENT-SETNAME": [], + "CLIENT-TRACKING": [], + "CLIENT-TRACKINGINFO": [], + "CLIENT-UNBLOCK": [], + "CLIENT-UNPAUSE": [], + "CLIENT": [], + "ECHO": [], + "HELLO": [], + "PING": [], + "QUIT": [], + "RESET": [], + "SELECT": [] + }, + "bitmap": { + "BITCOUNT": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -487,18 +538,7 @@ } } ], - "COMMAND-GETKEYS": [], - "LATENCY": [], - "COMMAND-INFO": [], - "ACL-LIST": [], - "LASTSAVE": [], - "MODULE-LIST": [], - "SLOWLOG-HELP": [], - "COMMAND-LIST": [], - "CONFIG-GET": [], - "MEMORY-MALLOC-STATS": [], - "SLOWLOG-LEN": [], - "RESTORE-ASKING": [ + "BITFIELD": [ { "begin_search": { "index": { @@ -514,46 +554,7 @@ } } ], - "CONFIG-SET": [], - "REPLICAOF": [], - "FAILOVER": [] - }, - "cluster": { - "READONLY": [], - "CLUSTER-MYID": [], - "CLUSTER-ADDSLOTS": [], - "CLUSTER-KEYSLOT": [], - "CLUSTER-FORGET": [], - "CLUSTER-MEET": [], - "READWRITE": [], - "CLUSTER-SLOTS": [], - "CLUSTER-REPLICATE": [], - "CLUSTER-LINKS": [], - "CLUSTER-DELSLOTS": [], - "ASKING": [], - "CLUSTER-COUNTKEYSINSLOT": [], - "CLUSTER-SHARDS": [], - "CLUSTER-BUMPEPOCH": [], - "CLUSTER-COUNT-FAILURE-REPORTS": [], - "CLUSTER": [], - "CLUSTER-SLAVES": [], - "CLUSTER-ADDSLOTSRANGE": [], - "CLUSTER-INFO": [], - "CLUSTER-GETKEYSINSLOT": [], - "CLUSTER-SETSLOT": [], - "CLUSTER-DELSLOTSRANGE": [], - "CLUSTER-HELP": [], - "CLUSTER-FAILOVER": [], - "CLUSTER-SAVECONFIG": [], - "CLUSTER-FLUSHSLOTS": [], - "CLUSTER-SET-CONFIG-EPOCH": [], - "CLUSTER-REPLICAS": [], - "CLUSTER-RESET": [], - "CLUSTER-NODES": [] - }, - "generic": { - "WAIT": [], - "DUMP": [ + "BITFIELD_RO": [ { "begin_search": { "index": { @@ -569,11 +570,11 @@ } } ], - "PTTL": [ + "BITOP": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -583,13 +584,11 @@ "limit": 0 } } - } - ], - "TOUCH": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 3 } }, "find_keys": { @@ -601,7 +600,7 @@ } } ], - "RESTORE": [ + "BITPOS": [ { "begin_search": { "index": { @@ -617,7 +616,7 @@ } } ], - "UNLINK": [ + "GETBIT": [ { "begin_search": { "index": { @@ -626,14 +625,14 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "TTL": [ + "SETBIT": [ { "begin_search": { "index": { @@ -648,8 +647,10 @@ } } } - ], - "MOVE": [ + ] + }, + "list": { + "BLMOVE": [ { "begin_search": { "index": { @@ -663,9 +664,7 @@ "limit": 0 } } - } - ], - "OBJECT-FREQ": [ + }, { "begin_search": { "index": { @@ -681,7 +680,23 @@ } } ], - "COPY": [ + "BLMPOP": [ + { + "begin_search": { + "index": { + "pos": 2 + } + }, + "find_keys": { + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 + } + } + } + ], + "BLPOP": [ { "begin_search": { "index": { @@ -690,28 +705,30 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -2, "step": 1, "limit": 0 } } - }, + } + ], + "BRPOP": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -2, "step": 1, "limit": 0 } } } ], - "PERSIST": [ + "BRPOPLPUSH": [ { "begin_search": { "index": { @@ -725,9 +742,7 @@ "limit": 0 } } - } - ], - "OBJECT-REFCOUNT": [ + }, { "begin_search": { "index": { @@ -743,11 +758,11 @@ } } ], - "OBJECT-IDLETIME": [ + "LINDEX": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -759,7 +774,7 @@ } } ], - "EXPIRETIME": [ + "LINSERT": [ { "begin_search": { "index": { @@ -775,9 +790,7 @@ } } ], - "OBJECT": [], - "KEYS": [], - "DEL": [ + "LLEN": [ { "begin_search": { "index": { @@ -786,14 +799,14 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "PEXPIREAT": [ + "LMOVE": [ { "begin_search": { "index": { @@ -807,14 +820,11 @@ "limit": 0 } } - } - ], - "OBJECT-HELP": [], - "PEXPIRETIME": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -826,8 +836,7 @@ } } ], - "RANDOMKEY": [], - "RENAME": [ + "LMPOP": [ { "begin_search": { "index": { @@ -835,17 +844,19 @@ } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } - }, + } + ], + "LPOP": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -857,7 +868,7 @@ } } ], - "RENAMENX": [ + "LPOS": [ { "begin_search": { "index": { @@ -871,11 +882,13 @@ "limit": 0 } } - }, + } + ], + "LPUSH": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -887,7 +900,7 @@ } } ], - "PEXPIRE": [ + "LPUSHX": [ { "begin_search": { "index": { @@ -903,8 +916,7 @@ } } ], - "SCAN": [], - "TYPE": [ + "LRANGE": [ { "begin_search": { "index": { @@ -920,11 +932,11 @@ } } ], - "OBJECT-ENCODING": [ + "LREM": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -936,7 +948,7 @@ } } ], - "EXISTS": [ + "LSET": [ { "begin_search": { "index": { @@ -945,14 +957,14 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "EXPIRE": [ + "LTRIM": [ { "begin_search": { "index": { @@ -968,7 +980,7 @@ } } ], - "EXPIREAT": [ + "RPOP": [ { "begin_search": { "index": { @@ -983,10 +995,8 @@ } } } - ] - }, - "string": { - "MSETNX": [ + ], + "RPOPLPUSH": [ { "begin_search": { "index": { @@ -995,18 +1005,16 @@ }, "find_keys": { "range": { - "lastkey": -1, - "step": 2, + "lastkey": 0, + "step": 1, "limit": 0 } } - } - ], - "GETEX": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -1018,7 +1026,7 @@ } } ], - "GET": [ + "RPUSH": [ { "begin_search": { "index": { @@ -1034,7 +1042,7 @@ } } ], - "INCRBYFLOAT": [ + "RPUSHX": [ { "begin_search": { "index": { @@ -1049,8 +1057,26 @@ } } } + ] + }, + "sorted_set": { + "BZMPOP": [ + { + "begin_search": { + "index": { + "pos": 2 + } + }, + "find_keys": { + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 + } + } + } ], - "MSET": [ + "BZPOPMAX": [ { "begin_search": { "index": { @@ -1059,14 +1085,14 @@ }, "find_keys": { "range": { - "lastkey": -1, - "step": 2, + "lastkey": -2, + "step": 1, "limit": 0 } } } ], - "MGET": [ + "BZPOPMIN": [ { "begin_search": { "index": { @@ -1075,14 +1101,14 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": -2, "step": 1, "limit": 0 } } } ], - "SET": [ + "ZADD": [ { "begin_search": { "index": { @@ -1098,7 +1124,7 @@ } } ], - "SUBSTR": [ + "ZCARD": [ { "begin_search": { "index": { @@ -1114,7 +1140,7 @@ } } ], - "DECRBY": [ + "ZCOUNT": [ { "begin_search": { "index": { @@ -1130,7 +1156,23 @@ } } ], - "INCRBY": [ + "ZDIFF": [ + { + "begin_search": { + "index": { + "pos": 1 + } + }, + "find_keys": { + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 + } + } + } + ], + "ZDIFFSTORE": [ { "begin_search": { "index": { @@ -1144,25 +1186,23 @@ "limit": 0 } } - } - ], - "SETEX": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } } ], - "GETRANGE": [ + "ZINCRBY": [ { "begin_search": { "index": { @@ -1178,7 +1218,7 @@ } } ], - "DECR": [ + "ZINTER": [ { "begin_search": { "index": { @@ -1186,15 +1226,15 @@ } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } } ], - "STRLEN": [ + "ZINTERCARD": [ { "begin_search": { "index": { @@ -1202,15 +1242,15 @@ } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } } ], - "INCR": [ + "ZINTERSTORE": [ { "begin_search": { "index": { @@ -1224,25 +1264,23 @@ "limit": 0 } } - } - ], - "PSETEX": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } } ], - "GETSET": [ + "ZLEXCOUNT": [ { "begin_search": { "index": { @@ -1258,7 +1296,7 @@ } } ], - "SETRANGE": [ + "ZMPOP": [ { "begin_search": { "index": { @@ -1266,15 +1304,15 @@ } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } } ], - "LCS": [ + "ZMSCORE": [ { "begin_search": { "index": { @@ -1283,14 +1321,14 @@ }, "find_keys": { "range": { - "lastkey": 1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "APPEND": [ + "ZPOPMAX": [ { "begin_search": { "index": { @@ -1306,7 +1344,7 @@ } } ], - "SETNX": [ + "ZPOPMIN": [ { "begin_search": { "index": { @@ -1322,7 +1360,7 @@ } } ], - "GETDEL": [ + "ZRANDMEMBER": [ { "begin_search": { "index": { @@ -1337,10 +1375,8 @@ } } } - ] - }, - "set": { - "SDIFFSTORE": [ + ], + "ZRANGE": [ { "begin_search": { "index": { @@ -1354,23 +1390,25 @@ "limit": 0 } } - }, + } + ], + "ZRANGEBYLEX": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "SSCAN": [ + "ZRANGEBYSCORE": [ { "begin_search": { "index": { @@ -1386,7 +1424,7 @@ } } ], - "SINTERSTORE": [ + "ZRANGESTORE": [ { "begin_search": { "index": { @@ -1409,14 +1447,14 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "SCARD": [ + "ZRANK": [ { "begin_search": { "index": { @@ -1432,7 +1470,7 @@ } } ], - "SINTER": [ + "ZREM": [ { "begin_search": { "index": { @@ -1441,14 +1479,14 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "SUNIONSTORE": [ + "ZREMRANGEBYLEX": [ { "begin_search": { "index": { @@ -1462,23 +1500,9 @@ "limit": 0 } } - }, - { - "begin_search": { - "index": { - "pos": 2 - } - }, - "find_keys": { - "range": { - "lastkey": -1, - "step": 1, - "limit": 0 - } - } } ], - "SUNION": [ + "ZREMRANGEBYRANK": [ { "begin_search": { "index": { @@ -1487,14 +1511,14 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "SPOP": [ + "ZREMRANGEBYSCORE": [ { "begin_search": { "index": { @@ -1510,7 +1534,7 @@ } } ], - "SINTERCARD": [ + "ZREVRANGE": [ { "begin_search": { "index": { @@ -1518,15 +1542,15 @@ } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "SMISMEMBER": [ + "ZREVRANGEBYLEX": [ { "begin_search": { "index": { @@ -1542,7 +1566,7 @@ } } ], - "SMEMBERS": [ + "ZREVRANGEBYSCORE": [ { "begin_search": { "index": { @@ -1558,7 +1582,7 @@ } } ], - "SADD": [ + "ZREVRANK": [ { "begin_search": { "index": { @@ -1574,7 +1598,7 @@ } } ], - "SDIFF": [ + "ZSCAN": [ { "begin_search": { "index": { @@ -1583,14 +1607,14 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "SREM": [ + "ZSCORE": [ { "begin_search": { "index": { @@ -1606,7 +1630,7 @@ } } ], - "SISMEMBER": [ + "ZUNION": [ { "begin_search": { "index": { @@ -1614,15 +1638,15 @@ } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } } ], - "SMOVE": [ + "ZUNIONSTORE": [ { "begin_search": { "index": { @@ -1644,33 +1668,17 @@ } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 - } - } - } - ], - "SRANDMEMBER": [ - { - "begin_search": { - "index": { - "pos": 1 - } - }, - "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } } ] }, - "geo": { - "GEORADIUS": [ + "generic": { + "COPY": [ { "begin_search": { "index": { @@ -1687,9 +1695,8 @@ }, { "begin_search": { - "keyword": { - "keyword": "STORE", - "startfrom": 6 + "index": { + "pos": 2 } }, "find_keys": { @@ -1699,24 +1706,25 @@ "limit": 0 } } - }, + } + ], + "DEL": [ { "begin_search": { - "keyword": { - "keyword": "STOREDIST", - "startfrom": 6 + "index": { + "pos": 1 } }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "GEOHASH": [ + "DUMP": [ { "begin_search": { "index": { @@ -1732,7 +1740,7 @@ } } ], - "GEODIST": [ + "EXISTS": [ { "begin_search": { "index": { @@ -1741,14 +1749,14 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "GEORADIUS_RO": [ + "EXPIRE": [ { "begin_search": { "index": { @@ -1764,7 +1772,7 @@ } } ], - "GEOADD": [ + "EXPIREAT": [ { "begin_search": { "index": { @@ -1780,7 +1788,7 @@ } } ], - "GEOSEARCHSTORE": [ + "EXPIRETIME": [ { "begin_search": { "index": { @@ -1794,11 +1802,14 @@ "limit": 0 } } - }, + } + ], + "KEYS": [], + "MOVE": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -1810,11 +1821,11 @@ } } ], - "GEORADIUSBYMEMBER": [ + "OBJECT-ENCODING": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -1824,12 +1835,13 @@ "limit": 0 } } - }, + } + ], + "OBJECT-FREQ": [ { "begin_search": { - "keyword": { - "keyword": "STORE", - "startfrom": 5 + "index": { + "pos": 2 } }, "find_keys": { @@ -1839,12 +1851,14 @@ "limit": 0 } } - }, + } + ], + "OBJECT-HELP": [], + "OBJECT-IDLETIME": [ { "begin_search": { - "keyword": { - "keyword": "STOREDIST", - "startfrom": 5 + "index": { + "pos": 2 } }, "find_keys": { @@ -1856,11 +1870,11 @@ } } ], - "GEOSEARCH": [ + "OBJECT-REFCOUNT": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -1872,7 +1886,8 @@ } } ], - "GEOPOS": [ + "OBJECT": [], + "PERSIST": [ { "begin_search": { "index": { @@ -1888,7 +1903,7 @@ } } ], - "GEORADIUSBYMEMBER_RO": [ + "PEXPIRE": [ { "begin_search": { "index": { @@ -1903,10 +1918,8 @@ } } } - ] - }, - "sorted_set": { - "BZPOPMIN": [ + ], + "PEXPIREAT": [ { "begin_search": { "index": { @@ -1915,14 +1928,14 @@ }, "find_keys": { "range": { - "lastkey": -2, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "ZPOPMAX": [ + "PEXPIRETIME": [ { "begin_search": { "index": { @@ -1938,7 +1951,7 @@ } } ], - "ZREMRANGEBYSCORE": [ + "PTTL": [ { "begin_search": { "index": { @@ -1954,7 +1967,8 @@ } } ], - "ZRANGESTORE": [ + "RANDOMKEY": [], + "RENAME": [ { "begin_search": { "index": { @@ -1984,7 +1998,7 @@ } } ], - "ZINTERSTORE": [ + "RENAMENX": [ { "begin_search": { "index": { @@ -2006,15 +2020,15 @@ } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "ZREVRANGEBYSCORE": [ + "RESTORE": [ { "begin_search": { "index": { @@ -2030,7 +2044,8 @@ } } ], - "BZPOPMAX": [ + "SCAN": [], + "TOUCH": [ { "begin_search": { "index": { @@ -2039,14 +2054,14 @@ }, "find_keys": { "range": { - "lastkey": -2, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "ZLEXCOUNT": [ + "TTL": [ { "begin_search": { "index": { @@ -2062,7 +2077,7 @@ } } ], - "ZREVRANK": [ + "TYPE": [ { "begin_search": { "index": { @@ -2078,7 +2093,7 @@ } } ], - "ZPOPMIN": [ + "UNLINK": [ { "begin_search": { "index": { @@ -2087,14 +2102,21 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "ZINCRBY": [ + "WAIT": [] + }, + "transactions": { + "DISCARD": [], + "EXEC": [], + "MULTI": [], + "UNWATCH": [], + "WATCH": [ { "begin_search": { "index": { @@ -2103,28 +2125,48 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } + ] + }, + "scripting": { + "EVAL": [ + { + "begin_search": { + "index": { + "pos": 2 + } + }, + "find_keys": { + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 + } + } + } ], - "ZDIFFSTORE": [ + "EVAL_RO": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } - }, + } + ], + "EVALSHA": [ { "begin_search": { "index": { @@ -2140,21 +2182,23 @@ } } ], - "ZUNIONSTORE": [ + "EVALSHA_RO": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } - }, + } + ], + "FCALL": [ { "begin_search": { "index": { @@ -2170,7 +2214,42 @@ } } ], - "ZRANGE": [ + "FCALL_RO": [ + { + "begin_search": { + "index": { + "pos": 2 + } + }, + "find_keys": { + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 + } + } + } + ], + "FUNCTION-DELETE": [], + "FUNCTION-DUMP": [], + "FUNCTION-FLUSH": [], + "FUNCTION-HELP": [], + "FUNCTION-KILL": [], + "FUNCTION-LIST": [], + "FUNCTION-LOAD": [], + "FUNCTION-RESTORE": [], + "FUNCTION-STATS": [], + "FUNCTION": [], + "SCRIPT-DEBUG": [], + "SCRIPT-EXISTS": [], + "SCRIPT-FLUSH": [], + "SCRIPT-HELP": [], + "SCRIPT-KILL": [], + "SCRIPT-LOAD": [], + "SCRIPT": [] + }, + "TairHash": { + "EXHSET": [ { "begin_search": { "index": { @@ -2185,8 +2264,10 @@ } } } - ], - "ZRANGEBYSCORE": [ + ] + }, + "TairString": { + "EXSET": [ { "begin_search": { "index": { @@ -2201,8 +2282,10 @@ } } } - ], - "ZDIFF": [ + ] + }, + "TairZset": { + "EXZADD": [ { "begin_search": { "index": { @@ -2210,15 +2293,17 @@ } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } - ], - "ZSCORE": [ + ] + }, + "geo": { + "GEOADD": [ { "begin_search": { "index": { @@ -2234,7 +2319,7 @@ } } ], - "ZREMRANGEBYLEX": [ + "GEODIST": [ { "begin_search": { "index": { @@ -2250,7 +2335,7 @@ } } ], - "ZREVRANGE": [ + "GEOHASH": [ { "begin_search": { "index": { @@ -2266,7 +2351,7 @@ } } ], - "ZREVRANGEBYLEX": [ + "GEOPOS": [ { "begin_search": { "index": { @@ -2282,7 +2367,7 @@ } } ], - "ZMPOP": [ + "GEORADIUS": [ { "begin_search": { "index": { @@ -2290,15 +2375,45 @@ } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + }, + { + "begin_search": { + "keyword": { + "keyword": "STORE", + "startfrom": 6 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + }, + { + "begin_search": { + "keyword": { + "keyword": "STOREDIST", + "startfrom": 6 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "ZRANK": [ + "GEORADIUS_RO": [ { "begin_search": { "index": { @@ -2314,7 +2429,7 @@ } } ], - "ZMSCORE": [ + "GEORADIUSBYMEMBER": [ { "begin_search": { "index": { @@ -2328,25 +2443,39 @@ "limit": 0 } } - } - ], - "BZMPOP": [ + }, { "begin_search": { - "index": { - "pos": 2 + "keyword": { + "keyword": "STORE", + "startfrom": 5 } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 + } + } + }, + { + "begin_search": { + "keyword": { + "keyword": "STOREDIST", + "startfrom": 5 + } + }, + "find_keys": { + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "ZCOUNT": [ + "GEORADIUSBYMEMBER_RO": [ { "begin_search": { "index": { @@ -2362,7 +2491,7 @@ } } ], - "ZSCAN": [ + "GEOSEARCH": [ { "begin_search": { "index": { @@ -2378,7 +2507,7 @@ } } ], - "ZUNION": [ + "GEOSEARCHSTORE": [ { "begin_search": { "index": { @@ -2386,19 +2515,17 @@ } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } - } - ], - "ZRANDMEMBER": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -2409,8 +2536,10 @@ } } } - ], - "ZINTER": [ + ] + }, + "hash": { + "HDEL": [ { "begin_search": { "index": { @@ -2418,15 +2547,15 @@ } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "ZADD": [ + "HEXISTS": [ { "begin_search": { "index": { @@ -2442,7 +2571,7 @@ } } ], - "ZREMRANGEBYRANK": [ + "HGET": [ { "begin_search": { "index": { @@ -2458,7 +2587,7 @@ } } ], - "ZINTERCARD": [ + "HGETALL": [ { "begin_search": { "index": { @@ -2466,15 +2595,15 @@ } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "ZRANGEBYLEX": [ + "HINCRBY": [ { "begin_search": { "index": { @@ -2490,7 +2619,7 @@ } } ], - "ZREM": [ + "HINCRBYFLOAT": [ { "begin_search": { "index": { @@ -2506,7 +2635,7 @@ } } ], - "ZCARD": [ + "HKEYS": [ { "begin_search": { "index": { @@ -2521,150 +2650,104 @@ } } } - ] - }, - "sentinel": { - "SENTINEL-MASTER": [], - "SENTINEL-SIMULATE-FAILURE": [], - "SENTINEL-FAILOVER": [], - "SENTINEL-REPLICAS": [], - "SENTINEL-MASTERS": [], - "SENTINEL-MYID": [], - "SENTINEL-PENDING-SCRIPTS": [], - "SENTINEL-DEBUG": [], - "SENTINEL-INFO-CACHE": [], - "SENTINEL-REMOVE": [], - "SENTINEL-IS-MASTER-DOWN-BY-ADDR": [], - "SENTINEL-FLUSHCONFIG": [], - "SENTINEL-GET-MASTER-ADDR-BY-NAME": [], - "SENTINEL-CONFIG": [], - "SENTINEL-SENTINELS": [], - "SENTINEL-CKQUORUM": [], - "SENTINEL-MONITOR": [], - "SENTINEL-SLAVES": [], - "SENTINEL-RESET": [], - "SENTINEL": [], - "SENTINEL-HELP": [], - "SENTINEL-SET": [] - }, - "scripting": { - "FUNCTION-FLUSH": [], - "FUNCTION-LIST": [], - "FUNCTION": [], - "SCRIPT": [], - "EVALSHA_RO": [ + ], + "HLEN": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "FUNCTION-DELETE": [], - "FUNCTION-STATS": [], - "FUNCTION-RESTORE": [], - "FUNCTION-LOAD": [], - "FUNCTION-HELP": [], - "SCRIPT-KILL": [], - "SCRIPT-FLUSH": [], - "EVAL_RO": [ + "HMGET": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "FUNCTION-KILL": [], - "EVALSHA": [ + "HMSET": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "FCALL": [ + "HRANDFIELD": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "SCRIPT-LOAD": [], - "SCRIPT-HELP": [], - "FUNCTION-DUMP": [], - "SCRIPT-DEBUG": [], - "SCRIPT-EXISTS": [], - "EVAL": [ + "HSCAN": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } ], - "FCALL_RO": [ + "HSET": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { - "keynum": { - "keynumidx": 0, - "firstkey": 1, - "step": 1 + "range": { + "lastkey": 0, + "step": 1, + "limit": 0 } } } - ] - }, - "stream": { - "XGROUP-HELP": [], - "XCLAIM": [ + ], + "HSETNX": [ { "begin_search": { "index": { @@ -2680,11 +2763,11 @@ } } ], - "XINFO-GROUPS": [ + "HSTRLEN": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -2696,28 +2779,29 @@ } } ], - "XREADGROUP": [ + "HVALS": [ { "begin_search": { - "keyword": { - "keyword": "STREAMS", - "startfrom": 4 + "index": { + "pos": 1 } }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, - "limit": 2 + "limit": 0 } } } - ], - "XINFO-CONSUMERS": [ + ] + }, + "hyperloglog": { + "PFADD": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -2729,27 +2813,27 @@ } } ], - "XGROUP-DELCONSUMER": [ + "PFCOUNT": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "XACK": [ + "PFDEBUG": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -2761,40 +2845,50 @@ } } ], - "XREAD": [ + "PFMERGE": [ { "begin_search": { - "keyword": { - "keyword": "STREAMS", - "startfrom": 1 + "index": { + "pos": 1 } }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, - "limit": 2 + "limit": 0 } } - } - ], - "XLEN": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "XTRIM": [ + "PFSELFTEST": [] + }, + "pubsub": { + "PSUBSCRIBE": [], + "PUBLISH": [], + "PUBSUB-CHANNELS": [], + "PUBSUB-HELP": [], + "PUBSUB-NUMPAT": [], + "PUBSUB-NUMSUB": [], + "PUBSUB-SHARDCHANNELS": [], + "PUBSUB-SHARDNUMSUB": [], + "PUBSUB": [], + "PUNSUBSCRIBE": [], + "SPUBLISH": [ { "begin_search": { "index": { @@ -2810,7 +2904,7 @@ } } ], - "XREVRANGE": [ + "SSUBSCRIBE": [ { "begin_search": { "index": { @@ -2819,32 +2913,34 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "XGROUP": [], - "XINFO": [], - "XGROUP-CREATE": [ + "SUBSCRIBE": [], + "SUNSUBSCRIBE": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "XDEL": [ + "UNSUBSCRIBE": [] + }, + "set": { + "SADD": [ { "begin_search": { "index": { @@ -2860,7 +2956,7 @@ } } ], - "XAUTOCLAIM": [ + "SCARD": [ { "begin_search": { "index": { @@ -2876,24 +2972,23 @@ } } ], - "XINFO-HELP": [], - "XGROUP-DESTROY": [ + "SDIFF": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "XADD": [ + "SDIFFSTORE": [ { "begin_search": { "index": { @@ -2907,25 +3002,23 @@ "limit": 0 } } - } - ], - "XSETID": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "XPENDING": [ + "SINTER": [ { "begin_search": { "index": { @@ -2934,34 +3027,34 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "XGROUP-SETID": [ + "SINTERCARD": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { - "range": { - "lastkey": 0, - "step": 1, - "limit": 0 + "keynum": { + "keynumidx": 0, + "firstkey": 1, + "step": 1 } } } ], - "XINFO-STREAM": [ + "SINTERSTORE": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -2971,9 +3064,7 @@ "limit": 0 } } - } - ], - "XGROUP-CREATECONSUMER": [ + }, { "begin_search": { "index": { @@ -2982,14 +3073,14 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "XRANGE": [ + "SISMEMBER": [ { "begin_search": { "index": { @@ -3004,13 +3095,8 @@ } } } - ] - }, - "transactions": { - "DISCARD": [], - "EXEC": [], - "MULTI": [], - "WATCH": [ + ], + "SMEMBERS": [ { "begin_search": { "index": { @@ -3019,17 +3105,14 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "UNWATCH": [] - }, - "hash": { - "HEXISTS": [ + "SMISMEMBER": [ { "begin_search": { "index": { @@ -3045,7 +3128,7 @@ } } ], - "HVALS": [ + "SMOVE": [ { "begin_search": { "index": { @@ -3059,13 +3142,11 @@ "limit": 0 } } - } - ], - "HMGET": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -3077,7 +3158,7 @@ } } ], - "HMSET": [ + "SPOP": [ { "begin_search": { "index": { @@ -3093,7 +3174,7 @@ } } ], - "HINCRBYFLOAT": [ + "SRANDMEMBER": [ { "begin_search": { "index": { @@ -3109,7 +3190,7 @@ } } ], - "HDEL": [ + "SREM": [ { "begin_search": { "index": { @@ -3125,7 +3206,7 @@ } } ], - "HGETALL": [ + "SSCAN": [ { "begin_search": { "index": { @@ -3141,7 +3222,7 @@ } } ], - "HSTRLEN": [ + "SUNION": [ { "begin_search": { "index": { @@ -3150,14 +3231,14 @@ }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } ], - "HKEYS": [ + "SUNIONSTORE": [ { "begin_search": { "index": { @@ -3171,25 +3252,49 @@ "limit": 0 } } - } - ], - "HRANDFIELD": [ + }, { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { "range": { - "lastkey": 0, + "lastkey": -1, "step": 1, "limit": 0 } } } - ], - "HLEN": [ + ] + }, + "sentinel": { + "SENTINEL-CKQUORUM": [], + "SENTINEL-CONFIG": [], + "SENTINEL-DEBUG": [], + "SENTINEL-FAILOVER": [], + "SENTINEL-FLUSHCONFIG": [], + "SENTINEL-GET-MASTER-ADDR-BY-NAME": [], + "SENTINEL-HELP": [], + "SENTINEL-INFO-CACHE": [], + "SENTINEL-IS-MASTER-DOWN-BY-ADDR": [], + "SENTINEL-MASTER": [], + "SENTINEL-MASTERS": [], + "SENTINEL-MONITOR": [], + "SENTINEL-MYID": [], + "SENTINEL-PENDING-SCRIPTS": [], + "SENTINEL-REMOVE": [], + "SENTINEL-REPLICAS": [], + "SENTINEL-RESET": [], + "SENTINEL-SENTINELS": [], + "SENTINEL-SET": [], + "SENTINEL-SIMULATE-FAILURE": [], + "SENTINEL-SLAVES": [], + "SENTINEL": [] + }, + "stream": { + "XACK": [ { "begin_search": { "index": { @@ -3205,7 +3310,7 @@ } } ], - "HGET": [ + "XADD": [ { "begin_search": { "index": { @@ -3221,7 +3326,7 @@ } } ], - "HSETNX": [ + "XAUTOCLAIM": [ { "begin_search": { "index": { @@ -3237,7 +3342,7 @@ } } ], - "HSET": [ + "XCLAIM": [ { "begin_search": { "index": { @@ -3253,7 +3358,7 @@ } } ], - "HINCRBY": [ + "XDEL": [ { "begin_search": { "index": { @@ -3269,11 +3374,11 @@ } } ], - "HSCAN": [ + "XGROUP-CREATE": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -3284,14 +3389,12 @@ } } } - ] - }, - "bitmap": { - "BITCOUNT": [ + ], + "XGROUP-CREATECONSUMER": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -3303,11 +3406,11 @@ } } ], - "SETBIT": [ + "XGROUP-DELCONSUMER": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -3319,7 +3422,7 @@ } } ], - "BITOP": [ + "XGROUP-DESTROY": [ { "begin_search": { "index": { @@ -3333,27 +3436,31 @@ "limit": 0 } } - }, + } + ], + "XGROUP-HELP": [], + "XGROUP-SETID": [ { "begin_search": { "index": { - "pos": 3 + "pos": 2 } }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "GETBIT": [ + "XGROUP": [], + "XINFO-CONSUMERS": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -3365,11 +3472,11 @@ } } ], - "BITFIELD_RO": [ + "XINFO-GROUPS": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -3381,11 +3488,12 @@ } } ], - "BITFIELD": [ + "XINFO-HELP": [], + "XINFO-STREAM": [ { "begin_search": { "index": { - "pos": 1 + "pos": 2 } }, "find_keys": { @@ -3397,7 +3505,8 @@ } } ], - "BITPOS": [ + "XINFO": [], + "XLEN": [ { "begin_search": { "index": { @@ -3412,13 +3521,8 @@ } } } - ] - }, - "pubsub": { - "PUBSUB-NUMSUB": [], - "PUBSUB": [], - "PUNSUBSCRIBE": [], - "SPUBLISH": [ + ], + "XPENDING": [ { "begin_search": { "index": { @@ -3434,15 +3538,7 @@ } } ], - "PUBSUB-SHARDNUMSUB": [], - "PUBSUB-SHARDCHANNELS": [], - "SUBSCRIBE": [], - "PUBSUB-HELP": [], - "PUBLISH": [], - "PUBSUB-NUMPAT": [], - "UNSUBSCRIBE": [], - "PSUBSCRIBE": [], - "SSUBSCRIBE": [ + "XRANGE": [ { "begin_search": { "index": { @@ -3451,75 +3547,48 @@ }, "find_keys": { "range": { - "lastkey": -1, + "lastkey": 0, "step": 1, "limit": 0 } } } ], - "PUBSUB-CHANNELS": [], - "SUNSUBSCRIBE": [ + "XREAD": [ { "begin_search": { - "index": { - "pos": 1 + "keyword": { + "keyword": "STREAMS", + "startfrom": 1 } }, "find_keys": { "range": { "lastkey": -1, "step": 1, - "limit": 0 + "limit": 2 } } } - ] - }, - "connection": { - "CLIENT-UNBLOCK": [], - "ECHO": [], - "CLIENT-ID": [], - "CLIENT-SETNAME": [], - "CLIENT-LIST": [], - "CLIENT-INFO": [], - "CLIENT-GETNAME": [], - "CLIENT-HELP": [], - "CLIENT-TRACKINGINFO": [], - "CLIENT-NO-EVICT": [], - "CLIENT-PAUSE": [], - "CLIENT-REPLY": [], - "HELLO": [], - "QUIT": [], - "CLIENT-KILL": [], - "CLIENT-CACHING": [], - "CLIENT-GETREDIR": [], - "AUTH": [], - "PING": [], - "CLIENT-UNPAUSE": [], - "CLIENT": [], - "RESET": [], - "CLIENT-TRACKING": [], - "SELECT": [] - }, - "hyperloglog": { - "PFCOUNT": [ + ], + "XREADGROUP": [ { "begin_search": { - "index": { - "pos": 1 + "keyword": { + "keyword": "STREAMS", + "startfrom": 4 } }, "find_keys": { "range": { "lastkey": -1, "step": 1, - "limit": 0 + "limit": 2 } } } ], - "PFMERGE": [ + "XREVRANGE": [ { "begin_search": { "index": { @@ -3533,23 +3602,9 @@ "limit": 0 } } - }, - { - "begin_search": { - "index": { - "pos": 2 - } - }, - "find_keys": { - "range": { - "lastkey": -1, - "step": 1, - "limit": 0 - } - } } ], - "PFADD": [ + "XSETID": [ { "begin_search": { "index": { @@ -3565,12 +3620,11 @@ } } ], - "PFSELFTEST": [], - "PFDEBUG": [ + "XTRIM": [ { "begin_search": { "index": { - "pos": 2 + "pos": 1 } }, "find_keys": { @@ -3585,21 +3639,21 @@ } }, "container": [ - "MODULE", + "ACL", + "CLIENT", "CLUSTER", - "XGROUP", "COMMAND", - "SLOWLOG", - "OBJECT", - "MEMORY", "CONFIG", "FUNCTION", "LATENCY", - "XINFO", - "CLIENT", - "SENTINEL", + "MEMORY", + "MODULE", + "OBJECT", "PUBSUB", "SCRIPT", - "ACL" + "SENTINEL", + "SLOWLOG", + "XGROUP", + "XINFO" ] } \ No newline at end of file diff --git a/shake.toml b/shake.toml new file mode 100644 index 00000000..b68dbf93 --- /dev/null +++ b/shake.toml @@ -0,0 +1,73 @@ +function = "" + + +[sync_reader] +cluster = false # set to true if source is a redis cluster +address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false +sync_rdb = true # set to false if you don't want to sync rdb +sync_aof = true # set to false if you don't want to sync aof + +# [scan_reader] +# cluster = false # set to true if source is a redis cluster +# address = "127.0.0.1:6379" # when cluster is true, set address to one of the cluster node +# username = "" # keep empty if not using ACL +# password = "" # keep empty if no authentication is required +# ksn = false # set to true to enabled Redis keyspace notifications (KSN) subscription +# tls = false + +# [rdb_reader] +# filepath = "/tmp/dump.rdb" + +# [aof_reader] +# filepath = "/tmp/.aof" +# timestamp = 0 # Unix timestamp (subsecond) + +[redis_writer] +cluster = false # set to true if target is a redis cluster +address = "127.0.0.1:6380" # when cluster is true, set address to one of the cluster node +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false + + +[advanced] +dir = "data" +ncpu = 0 # runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores +pprof_port = 0 # pprof port, 0 means disable +status_port = 0 # status port, 0 means disable + +# log +log_file = "shake.log" +log_level = "info" # debug, info or warn +log_interval = 5 # in seconds + +# redis-shake gets key and value from rdb file, and uses RESTORE command to +# create the key in target redis. Redis RESTORE will return a "Target key name +# is busy" error when key already exists. You can use this configuration item +# to change the default behavior of restore: +# panic: redis-shake will stop when meet "Target key name is busy" error. +# rewrite: redis-shake will replace the key with new value. +# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. +rdb_restore_command_behavior = "panic" # panic, rewrite or skip + +# redis-shake uses pipeline to improve sending performance. +# This item limits the maximum number of commands in a pipeline. +pipeline_count_limit = 1024 + +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default. This amount is normally 1gb. +target_redis_client_max_querybuf_len = 1024_000_000 + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. +target_redis_proto_max_bulk_len = 512_000_000 + +# If the source is Elasticache or MemoryDB, you can set this item. +aws_psync = "" # example: aws_psync = "10.0.0.1:6379@nmfu2sl5osync,10.0.0.1:6379@xhma21xfkssync" + +[module] +# The data format for BF.LOADCHUNK is not compatible in different versions. v2.6.3 <=> 20603 +target_mbbloom_version = 20603 diff --git a/sync.toml b/sync.toml deleted file mode 100644 index 71629e40..00000000 --- a/sync.toml +++ /dev/null @@ -1,56 +0,0 @@ -type = "sync" - -[source] -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -address = "127.0.0.1:6379" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false -elasticache_psync = "" # using when source is ElastiCache. ref: https://github.com/alibaba/RedisShake/issues/373 - -[target] -type = "standalone" # "standalone" or "cluster" -version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... -# When the target is a cluster, write the address of one of the nodes. -# redis-shake will obtain other nodes through the `cluster nodes` command. -address = "127.0.0.1:6380" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false - -[advanced] -dir = "data" - -# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores -ncpu = 4 - -# pprof port, 0 means disable -pprof_port = 0 - -# metric port, 0 means disable -metrics_port = 0 - -# log -log_file = "redis-shake.log" -log_level = "info" # debug, info or warn -log_interval = 5 # in seconds - -# redis-shake gets key and value from rdb file, and uses RESTORE command to -# create the key in target redis. Redis RESTORE will return a "Target key name -# is busy" error when key already exists. You can use this configuration item -# to change the default behavior of restore: -# panic: redis-shake will stop when meet "Target key name is busy" error. -# rewrite: redis-shake will replace the key with new value. -# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. -rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip - -# pipeline -pipeline_count_limit = 1024 - -# Client query buffers accumulate new commands. They are limited to a fixed -# amount by default. This amount is normally 1gb. -target_redis_client_max_querybuf_len = 1024_000_000 - -# In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited to 512 mb. -target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file diff --git a/test.sh b/test.sh index c41811b6..91039c66 100755 --- a/test.sh +++ b/test.sh @@ -1,7 +1,9 @@ #!/bin/bash set -e +# unit test go test ./... -v -cd test -python main.py \ No newline at end of file +# black box test +cd tests/ +pybbt cases --verbose --flags modules \ No newline at end of file diff --git a/test/assets/empty.toml b/test/assets/empty.toml deleted file mode 100644 index 6ac5ea65..00000000 --- a/test/assets/empty.toml +++ /dev/null @@ -1,54 +0,0 @@ -type = "sync" - -[source] -address = "127.0.0.1:6379" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false -elasticache_psync = "" # using when source is ElastiCache. ref: https://github.com/alibaba/RedisShake/issues/373 - -[target] -type = "cluster" # standalone or cluster -# When the target is a cluster, write the address of one of the nodes. -# redis-shake will obtain other nodes through the `cluster nodes` command. -address = "127.0.0.1:30001" -username = "" # keep empty if not using ACL -password = "" # keep empty if no authentication is required -tls = false - -[advanced] -dir = "data" - -# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores -ncpu = 3 - -# pprof port, 0 means disable -pprof_port = 0 - -# metric port, 0 means disable -metrics_port = 0 - -# log -log_file = "redis-shake.log" -log_level = "info" # debug, info or warn -log_interval = 5 # in seconds - -# redis-shake gets key and value from rdb file, and uses RESTORE command to -# create the key in target redis. Redis RESTORE will return a "Target key name -# is busy" error when key already exists. You can use this configuration item -# to change the default behavior of restore: -# panic: redis-shake will stop when meet "Target key name is busy" error. -# rewrite: redis-shake will replace the key with new value. -# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. -rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip - -# pipeline -pipeline_count_limit = 1024 - -# Client query buffers accumulate new commands. They are limited to a fixed -# amount by default. This amount is normally 1gb. -target_redis_client_max_querybuf_len = 1024_000_000 - -# In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited to 512 mb. -target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file diff --git a/test/cases/auth.py b/test/cases/auth.py deleted file mode 100644 index 1194b7f4..00000000 --- a/test/cases/auth.py +++ /dev/null @@ -1,27 +0,0 @@ -import time - -from utils import * - - -def main(): - r0 = Redis() - r0.client.config_set("requirepass", "password") - r0.client.execute_command("auth", "password") # for Redis 4.0 - r1 = Redis() - r1.client.config_set("requirepass", "password") - r1.client.execute_command("auth", "password") # for Redis 4.0 - - t = get_empty_config() - t["source"]["address"] = r0.get_address() - t["source"]["password"] = "password" - t["target"]["type"] = "standalone" - t["target"]["address"] = r1.get_address() - t["target"]["password"] = "password" - - rs = RedisShake() - rs.run(t) - - # wait sync need use http interface - r0.client.set("finished", "1") - time.sleep(2) - assert r1.client.get("finished") == b"1" diff --git a/test/cases/auth_acl.py b/test/cases/auth_acl.py deleted file mode 100644 index 66438c75..00000000 --- a/test/cases/auth_acl.py +++ /dev/null @@ -1,35 +0,0 @@ -import time - -from utils import * - - -def main(): - r0 = Redis() - try: - r0.client.acl_list() - except Exception: - return - r0.client.execute_command("acl", "setuser", "user0", ">password0", "~*", "+@all") - r0.client.execute_command("acl", "setuser", "user0", "on") - r0.client.execute_command("auth", "user0", "password0") # for Redis 4.0 - r1 = Redis() - r1.client.execute_command("acl", "setuser", "user1", ">password1", "~*", "+@all") - r1.client.execute_command("acl", "setuser", "user1", "on") - r1.client.execute_command("auth", "user1", "password1") # for Redis 4.0 - - t = get_empty_config() - t["source"]["address"] = r0.get_address() - t["source"]["username"] = "user0" - t["source"]["password"] = "password0" - t["target"]["type"] = "standalone" - t["target"]["address"] = r1.get_address() - t["target"]["username"] = "user1" - t["target"]["password"] = "password1" - - rs = RedisShake() - rs.run(t) - - # wait sync need use http interface - r0.client.set("finished", "1") - time.sleep(2) - assert r1.client.get("finished") == b"1" diff --git a/test/cases/cluster/__init__.py b/test/cases/cluster/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/cases/cluster/sync.py b/test/cases/cluster/sync.py deleted file mode 100644 index 4bb57e5a..00000000 --- a/test/cases/cluster/sync.py +++ /dev/null @@ -1,5 +0,0 @@ -from utils import * - - -def main(): - pass diff --git a/test/cases/example.py b/test/cases/example.py deleted file mode 100644 index 8cf3afaa..00000000 --- a/test/cases/example.py +++ /dev/null @@ -1,9 +0,0 @@ -from utils import * - - -def main(): - assert True - - -if __name__ == '__main__': - pass diff --git a/test/cases/issues/__init__.py b/test/cases/issues/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/cases/types/__init__.py b/test/cases/types/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/test/cases/types/type_hash.py b/test/cases/types/type_hash.py deleted file mode 100644 index a60f604a..00000000 --- a/test/cases/types/type_hash.py +++ /dev/null @@ -1,25 +0,0 @@ -import redis - -prefix = "hash" - - -def add_rdb_data(c: redis.Redis): - c.hset(f"{prefix}_rdb_k", "key0", "value0") - for i in range(10000): - c.hset(f"{prefix}_rdb_k_large", f"key{i}", f"value{i}") - - -def add_aof_data(c: redis.Redis): - c.hset(f"{prefix}_aof_k", "key0", "value0") - for i in range(10000): - c.hset(f"{prefix}_aof_k_large", f"key{i}", f"value{i}") - - -def check_data(c: redis.Redis): - assert c.hget(f"{prefix}_rdb_k", "key0") == b"value0" - assert c.hmget(f"{prefix}_rdb_k_large", *[f"key{i}" for i in range(10000)]) == [f"value{i}".encode() for i in - range(10000)] - - assert c.hget(f"{prefix}_aof_k", "key0") == b"value0" - assert c.hmget(f"{prefix}_aof_k_large", *[f"key{i}" for i in range(10000)]) == [f"value{i}".encode() for i in - range(10000)] diff --git a/test/cases/types/type_list.py b/test/cases/types/type_list.py deleted file mode 100644 index f823a958..00000000 --- a/test/cases/types/type_list.py +++ /dev/null @@ -1,23 +0,0 @@ -import redis - -prefix = "list" - -elements = [f"element_{i}" for i in range(10000)] - - -def add_rdb_data(c: redis.Redis): - c.rpush(f"{prefix}_rdb_k", 0, 1, 2, 3, 4, 5, 6, 7) - c.rpush(f"{prefix}_rdb_k0", *elements) - - -def add_aof_data(c: redis.Redis): - c.rpush(f"{prefix}_aof_k", 0, 1, 2, 3, 4, 5, 6, 7) - c.rpush(f"{prefix}_aof_k0", *elements) - - -def check_data(c: redis.Redis): - assert c.lrange(f"{prefix}_rdb_k", 0, -1) == [b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7"] - assert c.lrange(f"{prefix}_rdb_k0", 0, -1) == [f"element_{i}".encode() for i in range(10000)] - - assert c.lrange(f"{prefix}_aof_k", 0, -1) == [b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7"] - assert c.lrange(f"{prefix}_aof_k0", 0, -1) == [f"element_{i}".encode() for i in range(10000)] diff --git a/test/cases/types/type_set.py b/test/cases/types/type_set.py deleted file mode 100644 index 288d95ab..00000000 --- a/test/cases/types/type_set.py +++ /dev/null @@ -1,23 +0,0 @@ -import redis - -prefix = "set" - - -def add_rdb_data(c: redis.Redis): - c.sadd(f"{prefix}_rdb_k", 0, 1, 2, 3, 4, 5, 6, 7) - elements = [f"element_{i}" for i in range(10000)] - c.sadd(f"{prefix}_rdb_k0", *elements) - - -def add_aof_data(c: redis.Redis): - c.sadd(f"{prefix}_aof_k", 0, 1, 2, 3, 4, 5, 6, 7) - elements = [f"element_{i}" for i in range(10000)] - c.sadd(f"{prefix}_aof_k0", *elements) - - -def check_data(c: redis.Redis): - assert c.smembers(f"{prefix}_rdb_k") == {b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7"} - assert c.smembers(f"{prefix}_rdb_k0") == {f"element_{i}".encode() for i in range(10000)} - - assert c.smembers(f"{prefix}_aof_k") == {b"0", b"1", b"2", b"3", b"4", b"5", b"6", b"7"} - assert c.smembers(f"{prefix}_aof_k0") == {f"element_{i}".encode() for i in range(10000)} diff --git a/test/cases/types/type_stream.py b/test/cases/types/type_stream.py deleted file mode 100644 index d1f207e5..00000000 --- a/test/cases/types/type_stream.py +++ /dev/null @@ -1,34 +0,0 @@ -import redis - -prefix = "stream" - -fields = {f"field_{i}": f"value_{i}" for i in range(64)} -STREAM_LENGTH = 128 - - -def add_rdb_data(c: redis.Redis): - c.xadd(f"{prefix}_rdb_k", {"key0": "value0"}, "*") - for i in range(STREAM_LENGTH): - c.xadd(f"{prefix}_rdb_k_large", fields=fields, id="*") - - -def add_aof_data(c: redis.Redis): - c.xadd(f"{prefix}_aof_k", {"key0": "value0"}, "*") - for i in range(STREAM_LENGTH): - c.xadd(f"{prefix}_aof_k_large", fields=fields, id="*") - - -def check_data(c: redis.Redis): - ret = c.xread(streams={f"{prefix}_rdb_k": "0-0"}, count=1)[0][1] - assert ret[0][1] == {b"key0": b"value0"} - - ret = c.xread(streams={f"{prefix}_rdb_k_large": "0-0"}, count=STREAM_LENGTH)[0][1] - for i in range(STREAM_LENGTH): - assert ret[i][1] == {k.encode(): v.encode() for k, v in fields.items()} - - ret = c.xread(streams={f"{prefix}_aof_k": "0-0"}, count=1)[0][1] - assert ret[0][1] == {b"key0": b"value0"} - - ret = c.xread(streams={f"{prefix}_aof_k_large": "0-0"}, count=STREAM_LENGTH)[0][1] - for i in range(STREAM_LENGTH): - assert ret[i][1] == {k.encode(): v.encode() for k, v in fields.items()} diff --git a/test/cases/types/type_string.py b/test/cases/types/type_string.py deleted file mode 100644 index c3c00b15..00000000 --- a/test/cases/types/type_string.py +++ /dev/null @@ -1,29 +0,0 @@ -import redis - -prefix = "string" - - -def add_rdb_data(c: redis.Redis): - c.set(f"{prefix}_rdb_k", "v") - c.set(f"{prefix}_rdb_int", 0) - c.set(f"{prefix}_rdb_int0", -1) - c.set(f"{prefix}_rdb_int1", 123456789) - - -def add_aof_data(c: redis.Redis): - c.set(f"{prefix}_aof_k", "v") - c.set(f"{prefix}_aof_int", 0) - c.set(f"{prefix}_aof_int0", -1) - c.set(f"{prefix}_aof_int1", 123456789) - - -def check_data(c: redis.Redis): - assert c.get(f"{prefix}_rdb_k") == b"v" - assert c.get(f"{prefix}_rdb_int") == b'0' - assert c.get(f"{prefix}_rdb_int0") == b'-1' - assert c.get(f"{prefix}_rdb_int1") == b'123456789' - - assert c.get(f"{prefix}_aof_k") == b"v" - assert c.get(f"{prefix}_aof_int") == b'0' - assert c.get(f"{prefix}_aof_int0") == b'-1' - assert c.get(f"{prefix}_aof_int1") == b'123456789' diff --git a/test/cases/types/type_zset.py b/test/cases/types/type_zset.py deleted file mode 100644 index 6fba0410..00000000 --- a/test/cases/types/type_zset.py +++ /dev/null @@ -1,27 +0,0 @@ -import redis - -prefix = "zset" -float_maps = {"a": 1.1111, "b": 2.2222, "c": 3.3333} -maps = {str(item): item for item in range(10000)} - - -def add_rdb_data(c: redis.Redis): - c.zadd(f"{prefix}_rdb_k_float", float_maps) - c.zadd(f"{prefix}_rdb_k", maps) - - -def add_aof_data(c: redis.Redis): - c.zadd(f"{prefix}_aof_k_float", float_maps) - c.zadd(f"{prefix}_aof_k", maps) - - -def check_data(c: redis.Redis): - for k, v in c.zrange(f"{prefix}_rdb_k_float", 0, -1, withscores=True): - assert float_maps[k.decode()] == v - for k, v in c.zrange(f"{prefix}_rdb_k", 0, -1, withscores=True): - assert maps[k.decode()] == v - - for k, v in c.zrange(f"{prefix}_aof_k_float", 0, -1, withscores=True): - assert float_maps[k.decode()] == v - for k, v in c.zrange(f"{prefix}_aof_k", 0, -1, withscores=True): - assert maps[k.decode()] == v diff --git a/test/cases/types/types.py b/test/cases/types/types.py deleted file mode 100644 index 743ce3c5..00000000 --- a/test/cases/types/types.py +++ /dev/null @@ -1,62 +0,0 @@ -import time - -import jury -from utils import * - -from . import type_string, type_list, type_set, type_hash, type_zset, type_stream - - -def main(): - rs = RedisShake() - r0 = Redis() - r1 = Redis() - t = get_empty_config() - t["source"]["address"] = r0.get_address() - t["target"]["type"] = "standalone" - t["target"]["address"] = r1.get_address() - - timer = jury.Timer() - type_string.add_rdb_data(r0.client) - type_list.add_rdb_data(r0.client) - type_set.add_rdb_data(r0.client) - type_hash.add_rdb_data(r0.client) - type_zset.add_rdb_data(r0.client) - type_stream.add_rdb_data(r0.client) - jury.log(f"add_rdb_data: {timer.elapsed_time()}s") - - # run redis-shake - rs.run(t) - time.sleep(1) - - timer = jury.Timer() - type_string.add_aof_data(r0.client) - type_list.add_aof_data(r0.client) - type_set.add_aof_data(r0.client) - type_hash.add_aof_data(r0.client) - type_zset.add_aof_data(r0.client) - type_stream.add_aof_data(r0.client) - jury.log(f"add_aof_data: {timer.elapsed_time()}s") - - # wait sync need use http interface - timer = jury.Timer() - r0.client.set("finished", "1") - cnt = 0 - while r1.client.get("finished") != b"1": - time.sleep(0.5) - cnt += 1 - if cnt > 20: - raise Exception("sync timeout") - jury.log(f"sync time: {timer.elapsed_time()}s") - - timer = jury.Timer() - type_string.check_data(r1.client) - type_list.check_data(r1.client) - type_set.check_data(r1.client) - type_hash.check_data(r1.client) - type_zset.check_data(r1.client) - type_stream.check_data(r1.client) - jury.log(f"check_data: {timer.elapsed_time()}s") - - -if __name__ == '__main__': - main() diff --git a/test/cases/types/types_rewrite.py b/test/cases/types/types_rewrite.py deleted file mode 100644 index 8ffc8ec4..00000000 --- a/test/cases/types/types_rewrite.py +++ /dev/null @@ -1,63 +0,0 @@ -import time - -import jury -from utils import * - -from . import type_string, type_list, type_set, type_hash, type_zset, type_stream - - -def main(): - rs = RedisShake() - r0 = Redis() - r1 = Redis() - t = get_empty_config() - t["advanced"]["target_redis_proto_max_bulk_len"] = 0 - t["source"]["address"] = r0.get_address() - t["target"]["type"] = "standalone" - t["target"]["address"] = r1.get_address() - - timer = jury.Timer() - type_string.add_rdb_data(r0.client) - type_list.add_rdb_data(r0.client) - type_set.add_rdb_data(r0.client) - type_hash.add_rdb_data(r0.client) - type_zset.add_rdb_data(r0.client) - type_stream.add_rdb_data(r0.client) - jury.log(f"add_rdb_data: {timer.elapsed_time()}s") - - # run redis-shake - rs.run(t) - time.sleep(1) - - timer = jury.Timer() - type_string.add_aof_data(r0.client) - type_list.add_aof_data(r0.client) - type_set.add_aof_data(r0.client) - type_hash.add_aof_data(r0.client) - type_zset.add_aof_data(r0.client) - type_stream.add_aof_data(r0.client) - jury.log(f"add_aof_data: {timer.elapsed_time()}s") - - # wait sync need use http interface - timer = jury.Timer() - r0.client.set("finished", "1") - cnt = 0 - while r1.client.get("finished") != b"1": - time.sleep(0.5) - cnt += 1 - if cnt > 20: - raise Exception("sync timeout") - jury.log(f"sync time: {timer.elapsed_time()}s") - - timer = jury.Timer() - type_string.check_data(r1.client) - type_list.check_data(r1.client) - type_set.check_data(r1.client) - type_hash.check_data(r1.client) - type_zset.check_data(r1.client) - type_stream.check_data(r1.client) - jury.log(f"check_data: {timer.elapsed_time()}s") - - -if __name__ == '__main__': - main() diff --git a/test/main.py b/test/main.py deleted file mode 100644 index 95da2c8a..00000000 --- a/test/main.py +++ /dev/null @@ -1,19 +0,0 @@ -import jury - -cases = [ - "cases/example", - "cases/types/types", - "cases/types/types_rewrite", - "cases/cluster/sync", - "cases/auth", - "cases/auth_acl", -] - - -def main(): - j = jury.Jury(cases) - j.run() - - -if __name__ == '__main__': - main() diff --git a/test/requirements.txt b/test/requirements.txt deleted file mode 100644 index c2b8943a..00000000 --- a/test/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -jury-test==0.0.3 -redis==4.3.3 -toml==0.10.2 diff --git a/test/utils/__init__.py b/test/utils/__init__.py deleted file mode 100644 index 4eba8f97..00000000 --- a/test/utils/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .redis_ import Redis -from .cluster_ import Cluster -from .redis_shake import get_empty_config, RedisShake diff --git a/test/utils/cluster_.py b/test/utils/cluster_.py deleted file mode 100644 index 5b8a956e..00000000 --- a/test/utils/cluster_.py +++ /dev/null @@ -1,18 +0,0 @@ -import os - -from .redis_ import Redis -import jury - - -class Cluster: - def __init__(self, num=3): - self.nodes = [] - self.num = num - for i in range(num): - self.nodes.append(Redis(args=["--cluster-enabled", "yes"])) - host_port_list = [f"{node.host}:{node.port}" for node in self.nodes] - jury.log_yellow(f"Redis cluster created, {self.num} nodes. {host_port_list}") - os.system("redis-cli --cluster-yes --cluster create " + " ".join(host_port_list)) - - def push_table(self): - pass diff --git a/test/utils/constant.py b/test/utils/constant.py deleted file mode 100644 index 9598674b..00000000 --- a/test/utils/constant.py +++ /dev/null @@ -1,6 +0,0 @@ -from pathlib import Path - -BASE_PATH = f"{Path(__file__).parent.parent.parent.absolute()}" # project path - -PATH_REDIS_SHAKE = f"{BASE_PATH}/bin/redis-shake" -PATH_EMPTY_CONFIG_FILE = f"{BASE_PATH}/test/assets/empty.toml" diff --git a/test/utils/redis_.py b/test/utils/redis_.py deleted file mode 100644 index 2ac5ed92..00000000 --- a/test/utils/redis_.py +++ /dev/null @@ -1,42 +0,0 @@ -import time - -import jury -import redis - - -class Redis: - def __init__(self, args=None): - self.args = args - if args is None: - self.args = [] - - self.redis = None - self.host = "127.0.0.1" - self.port = jury.get_free_port() - self.dir = f"{jury.get_case_dir()}/redis_{self.port}" - self.server = jury.Launcher(args=["redis-server", "--port", str(self.port)] + self.args, work_dir=self.dir) - self.__wait_start() - jury.log_yellow(f"Redis started(pid={self.server.get_pid()}). redis-cli -p {self.port}") - - def get_address(self): - return f"127.0.0.1:{self.port}" - - def __wait_start(self, timeout=5): - timer = jury.Timer() - while True: - try: - self.__create_client() - self.client.ping() - break - except redis.exceptions.ConnectionError: - time.sleep(0.01) - except redis.exceptions.ResponseError as e: - if str(e) == "LOADING Redis is loading the dataset in memory": - time.sleep(0.01) - else: - raise e - if timer.elapsed_time() > timeout: - raise Exception(f"tair start timeout, {self.dir}") - - def __create_client(self): - self.client = redis.Redis(port=self.port, single_connection_client=True) diff --git a/test/utils/redis_shake.py b/test/utils/redis_shake.py deleted file mode 100644 index 5d7a581c..00000000 --- a/test/utils/redis_shake.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -from pathlib import Path - -import jury -import toml - -from .constant import PATH_REDIS_SHAKE, PATH_EMPTY_CONFIG_FILE - - -def get_empty_config(): - with open(PATH_EMPTY_CONFIG_FILE, "r") as f: - return toml.load(f) - - -class RedisShake: - def __init__(self): - self.server = None - self.redis = None - self.dir = f"{jury.get_case_dir()}/redis_shake" - if not os.path.exists(self.dir): - Path(self.dir).mkdir(parents=True, exist_ok=True) - - def run(self, toml_config): - with open(f"{self.dir}/redis-shake.toml", "w") as f: - toml.dump(toml_config, f) - self.server = jury.Launcher(args=[PATH_REDIS_SHAKE, "redis-shake.toml"], work_dir=self.dir) diff --git a/test/.gitignore b/tests/.gitignore similarity index 100% rename from test/.gitignore rename to tests/.gitignore diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..e9f6cc24 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,13 @@ +本项目使用 pybbt 工具进行黑盒测试。pybbt 是一个基于 Python 的工具，简化了软件的黑盒测试过程。要运行测试用例，请执行以下命令： + +```bash +pybbt cases --verbose --flags modules +``` +该命令将以详细日志记录的方式运行 cases 目录中的测试用例，并向测试用例传递 modules 标志。 + +如果本地没有安装 modules，可以不起用 modules 标志，这样就会跳过需要 modules 的测试用例: +```bash +pybbt cases --verbose +``` + +更多关于 pybbt 的信息、安装说明和用法示例，请参阅完整的文档: https://pypi.org/project/pybbt/ \ No newline at end of file diff --git a/tests/appendonlydir/appendonly.aof.1.base.rdb b/tests/appendonlydir/appendonly.aof.1.base.rdb new file mode 100644 index 00000000..a1807790 Binary files /dev/null and b/tests/appendonlydir/appendonly.aof.1.base.rdb differ diff --git a/tests/appendonlydir/appendonly.aof.1.incr.aof b/tests/appendonlydir/appendonly.aof.1.incr.aof new file mode 100644 index 00000000..2dc5b78c --- /dev/null +++ b/tests/appendonlydir/appendonly.aof.1.incr.aof @@ -0,0 +1,63 @@ +#TS:1697476300 +*2 +$6 +SELECT +$1 +0 +*3 +$3 +SET +$12 +string_0_str +$6 +string +*3 +$3 +SET +$12 +string_0_int +$1 +0 +*3 +$3 +SET +$13 +string_0_int0 +$2 +-1 +*3 +$3 +SET +$13 +string_0_int1 +$9 +123456789 +#TS:1697476303 +*3 +$3 +SET +$12 +string_1_str +$6 +string +*3 +$3 +SET +$12 +string_1_int +$1 +0 +*3 +$3 +SET +$13 +string_1_int0 +$2 +-1 +*3 +$3 +SET +$13 +string_1_int1 +$9 +123456789 diff --git a/tests/appendonlydir/appendonly.aof.manifest b/tests/appendonlydir/appendonly.aof.manifest new file mode 100644 index 00000000..7f8bb725 --- /dev/null +++ b/tests/appendonlydir/appendonly.aof.manifest @@ -0,0 +1,2 @@ +file appendonly.aof.1.base.rdb seq 1 type b +file appendonly.aof.1.incr.aof seq 1 type i diff --git a/tests/cases/aof.py b/tests/cases/aof.py new file mode 100644 index 00000000..91b50612 --- /dev/null +++ b/tests/cases/aof.py @@ -0,0 +1,159 @@ +import pybbt as p + +import helpers as h + +import os + +def get_aof_file_relative_path(): + if h.REDIS_SERVER_VERSION == 7.0: + aof_file = "/appendonlydir/appendonly.aof.manifest" + else: + aof_file = "/appendonly.aof" + return aof_file + +def test(src, dst): + + cross_slots_cmd = not (src.is_cluster() or dst.is_cluster()) + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_TRUE(src.do("save")) + + opts = h.ShakeOpts.create_aof_opts(f"{src.dir}{get_aof_file_relative_path()}", dst) + h.Shake.run_once(opts) + # check data + inserter.check_data(src, cross_slots_cmd=cross_slots_cmd) + inserter.check_data(dst, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_EQ(src.dbsize(), dst.dbsize()) + +def test_error(src, dst): + #set aof + ret = src.do("CONFIG SET", "appendonly", "yes") + p.log(f"aof_ret: {ret}") + cross_slots_cmd = not (src.is_cluster() or dst.is_cluster()) + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_TRUE(src.do("save")) + #destroy file + file_path = src.dir + get_aof_file_relative_path() + with open(file_path, "r+") as file: + destroy_data = "xxxxs" + file.seek(0, 0) + file.write(destroy_data) + + + opts = h.ShakeOpts.create_aof_opts(f"{src.dir}/appendonlydir/appendonly.aof.manifest", dst) + p.log(f"opts: {opts}") + h.Shake.run_once(opts) + + #cant restore + p.ASSERT_EQ( dst.dbsize(), 0) + + + +def test_rm_file(src, dst): + cross_slots_cmd = not (src.is_cluster() or dst.is_cluster()) + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_TRUE(src.do("save")) + #rm file + file_path = src.dir + "/appendonlydir/appendonly.aof.1.base.rdb" + os.remove(file_path) + opts = h.ShakeOpts.create_aof_opts(f"{src.dir}{get_aof_file_relative_path()}", dst) + h.Shake.run_once(opts) + #cant restore + p.ASSERT_EQ(dst.dbsize(), 0) + +def test_timestamp(dst): + current_directory = os.getcwd() + opts = h.ShakeOpts.create_aof_opts(f"{current_directory}{get_aof_file_relative_path()}", dst, 1697476302) + h.Shake.run_once(opts) + #checkout data + pip = dst.pipeline() + prefix = "string" + i = 0 + pip.get(f"{prefix}_{i}_str") + pip.get(f"{prefix}_{i}_int") + pip.get(f"{prefix}_{i}_int0") + pip.get(f"{prefix}_{i}_int1") + ret = pip.execute() + p.ASSERT_EQ(ret, [b"string", b"0", b"-1", b"123456789"]) + p.ASSERT_EQ(dst.dbsize(), 4) + +@p.subcase() +def aof_to_standalone(): + if h.REDIS_SERVER_VERSION < 7.0: + return + src = h.Redis() + #set aof + ret = src.do("CONFIG SET", "appendonly", "yes") + p.log(f"aof_ret: {ret}") + + ret = src.do("CONFIG SET", "aof-timestamp-enabled", "yes") + p.log(f"aof_ret: {ret}") + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def aof_to_standalone_rm_file(): + if h.REDIS_SERVER_VERSION < 7.0: + return + src = h.Redis() + #set aof + ret = src.do("CONFIG SET", "appendonly", "yes") + dst = h.Redis() + test_rm_file(src, dst) + +@p.subcase() +def aof_to_standalone_error(): + if h.REDIS_SERVER_VERSION < 7.0: + return + src = h.Redis() + #set aof + ret = src.do("CONFIG SET", "appendonly", "yes") + dst = h.Redis() + test_error(src, dst) + +@p.subcase() +def aof_to_cluster(): + if h.REDIS_SERVER_VERSION < 7.0: + return + src = h.Redis() + #set aof + ret = src.do("CONFIG SET", "appendonly", "yes") + p.log(f"aof_ret: {ret}") + dst = h.Cluster() + test(src, dst) + +@p.subcase() +def aof_to_standalone_single(): + if h.REDIS_SERVER_VERSION >= 7.0: + return + src = h.Redis() + #set preamble no + ret = src.do("CONFIG SET", "aof-use-rdb-preamble", "no") + p.log(f"aof_ret: {ret}") + #set aof + ret = src.do("CONFIG SET", "appendonly", "yes") + p.log(f"aof_ret: {ret}") + dst = h.Redis() + test(src, dst) + +@p.subcase() +def aof_to_standalone_timestamp(): + if h.REDIS_SERVER_VERSION < 7.0: + return + dst = h.Redis() + test_timestamp(dst) + +@p.case(tags=["sync"]) +def main(): + aof_to_standalone() + aof_to_standalone_single() + aof_to_standalone_error() + aof_to_standalone_rm_file() + aof_to_cluster() + aof_to_standalone_timestamp() +if __name__ == '__main__': + main() diff --git a/tests/cases/auth_acl.py b/tests/cases/auth_acl.py new file mode 100644 index 00000000..5601126d --- /dev/null +++ b/tests/cases/auth_acl.py @@ -0,0 +1,49 @@ +import pybbt as p + +import helpers as h + + +@p.subcase() +def acl(): + src = h.Redis() + dst = h.Redis() + + src.client.execute_command("acl", "setuser", "user0", ">password0", "~*", "+@all") + src.client.execute_command("acl", "setuser", "user0", "on") + src.client.execute_command("auth", "user0", "password0") # for Redis 4.0 + + dst.client.execute_command("acl", "setuser", "user1", ">password1", "~*", "+@all") + dst.client.execute_command("acl", "setuser", "user1", "on") + dst.client.execute_command("auth", "user1", "password1") # for Redis 4.0 + + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=True) + + opts = h.ShakeOpts.create_sync_opts(src, dst) + opts["sync_reader"]["username"] = "user0" + opts["sync_reader"]["password"] = "password0" + opts["redis_writer"]["username"] = "user1" + opts["redis_writer"]["password"] = "password1" + p.log(f"opts: {opts}") + shake = h.Shake(opts) + + # wait sync done + p.ASSERT_TRUE_TIMEOUT(lambda: shake.is_consistent()) + p.log(shake.get_status()) + + # check data + inserter.check_data(src, cross_slots_cmd=True) + inserter.check_data(dst, cross_slots_cmd=True) + p.ASSERT_EQ(src.dbsize(), dst.dbsize()) + + +@p.case(tags=["acl"]) +def main(): + if h.REDIS_SERVER_VERSION < 6.0: + return + + acl() + + +if __name__ == '__main__': + main() diff --git a/tests/cases/function.py b/tests/cases/function.py new file mode 100644 index 00000000..1279d2d7 --- /dev/null +++ b/tests/cases/function.py @@ -0,0 +1,70 @@ +import pybbt as p + +import helpers as h + + +@p.subcase() +def filter_db(): + src = h.Redis() + dst = h.Redis() + + opts = h.ShakeOpts.create_sync_opts(src, dst) + opts["function"] = """ + shake.log(DB) + if DB == 0 + then + return + end + shake.call(DB, ARGV) + """ + p.log(f"opts: {opts}") + shake = h.Shake(opts) + + for db in range(16): + src.do("select", db) + src.do("set", "key", "value") + + # wait sync done + p.ASSERT_TRUE_TIMEOUT(lambda: shake.is_consistent(), timeout=10) + + dst.do("select", 0) + p.ASSERT_EQ(dst.do("get", "key"), None) + for db in range(1, 16): + dst.do("select", db) + p.ASSERT_EQ(dst.do("get", "key"), b"value") + + +@p.subcase() +def split_mset_to_set(): + src = h.Redis() + dst = h.Redis() + opts = h.ShakeOpts.create_sync_opts(src, dst) + opts["function"] = """ + shake.log(KEYS) + if CMD == "MSET" + then + for i = 2, #ARGV, 2 -- MSET k1 v1 k2 v2 k3 v3 ... + do + shake.call(1, {"SET", ARGV[i], ARGV[i+1]}) -- move to db 1 + end + end + """ + p.log(f"opts: {opts}") + shake = h.Shake(opts) + src.do("mset", "k1", "v1", "k2", "v2", "k3", "v3") + # wait sync done + p.ASSERT_TRUE_TIMEOUT(lambda: shake.is_consistent(), timeout=10) + dst.do("select", 1) + p.ASSERT_EQ(dst.do("get", "k1"), b"v1") + p.ASSERT_EQ(dst.do("get", "k2"), b"v2") + p.ASSERT_EQ(dst.do("get", "k3"), b"v3") + + +@p.case(tags=["function"]) +def main(): + filter_db() + split_mset_to_set() + + +if __name__ == '__main__': + main() diff --git a/tests/cases/rdb.py b/tests/cases/rdb.py new file mode 100644 index 00000000..e020b939 --- /dev/null +++ b/tests/cases/rdb.py @@ -0,0 +1,45 @@ +import pybbt as p + +import helpers as h + + +def test(src, dst): + cross_slots_cmd = not (src.is_cluster() or dst.is_cluster()) + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_TRUE(src.do("save")) + + opts = h.ShakeOpts.create_rdb_opts(f"{src.dir}/dump.rdb", dst) + p.log(f"opts: {opts}") + h.Shake.run_once(opts) + + # check data + inserter.check_data(src, cross_slots_cmd=cross_slots_cmd) + inserter.check_data(dst, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_EQ(src.dbsize(), dst.dbsize()) + + +@p.subcase() +def rdb_to_standalone(): + src = h.Redis() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def rdb_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Redis() + dst = h.Cluster() + test(src, dst) + + +@p.case(tags=["sync"]) +def main(): + rdb_to_standalone() + rdb_to_cluster() + + +if __name__ == '__main__': + main() diff --git a/tests/cases/scan.py b/tests/cases/scan.py new file mode 100644 index 00000000..1023c703 --- /dev/null +++ b/tests/cases/scan.py @@ -0,0 +1,68 @@ +import pybbt as p + +import helpers as h + + +def test(src, dst): + cross_slots_cmd = not (src.is_cluster() or dst.is_cluster()) + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_TRUE(src.do("save")) + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) # add data again + + opts = h.ShakeOpts.create_scan_opts(src, dst) + p.log(f"opts: {opts}") + + # run shake + h.Shake.run_once(opts) + + # check data + inserter.check_data(src, cross_slots_cmd=cross_slots_cmd) + inserter.check_data(dst, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_EQ(src.dbsize(), dst.dbsize()) + + +@p.subcase() +def standalone_to_standalone(): + src = h.Redis() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def standalone_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Redis() + dst = h.Cluster() + test(src, dst) + + +@p.subcase() +def cluster_to_standalone(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Cluster() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def cluster_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Cluster() + dst = h.Cluster() + test(src, dst) + + +@p.case(tags=["scan"]) +def main(): + standalone_to_standalone() + standalone_to_cluster() + cluster_to_standalone() + cluster_to_cluster() + + +if __name__ == '__main__': + main() diff --git a/tests/cases/sync.py b/tests/cases/sync.py new file mode 100644 index 00000000..6d5df062 --- /dev/null +++ b/tests/cases/sync.py @@ -0,0 +1,79 @@ +import time + +import pybbt as p + +import helpers as h + + +def test(src, dst): + cross_slots_cmd = not (src.is_cluster() or dst.is_cluster()) + inserter = h.DataInserter() + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + + p.ASSERT_TRUE(src.do("save")) + + opts = h.ShakeOpts.create_sync_opts(src, dst) + p.log(f"opts: {opts}") + shake = h.Shake(opts) + + # wait sync done + p.ASSERT_TRUE_TIMEOUT(lambda: shake.is_consistent(), timeout=10) + + # add data again + inserter.add_data(src, cross_slots_cmd=cross_slots_cmd) + + # wait sync done + p.ASSERT_TRUE_TIMEOUT(lambda: shake.is_consistent()) + p.log(shake.get_status()) + time.sleep(5) + + # check data + inserter.check_data(src, cross_slots_cmd=cross_slots_cmd) + inserter.check_data(dst, cross_slots_cmd=cross_slots_cmd) + p.ASSERT_EQ(src.dbsize(), dst.dbsize()) + + +@p.subcase() +def standalone_to_standalone(): + src = h.Redis() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def standalone_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Redis() + dst = h.Cluster() + test(src, dst) + + +@p.subcase() +def cluster_to_standalone(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Cluster() + dst = h.Redis() + test(src, dst) + + +@p.subcase() +def cluster_to_cluster(): + if h.REDIS_SERVER_VERSION < 3.0: + return + src = h.Cluster() + dst = h.Cluster() + test(src, dst) + + +@p.case(tags=["sync"]) +def main(): + standalone_to_standalone() + standalone_to_cluster() + cluster_to_standalone() + cluster_to_cluster() + + +if __name__ == '__main__': + main() diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..01eae554 --- /dev/null +++ b/tests/helpers/__init__.py @@ -0,0 +1,5 @@ +from .cluster import Cluster +from .constant import REDIS_SERVER_VERSION +from .data_inserter import DataInserter +from .redis import Redis +from .shake import Shake, ShakeOpts diff --git a/tests/helpers/cluster.py b/tests/helpers/cluster.py new file mode 100644 index 00000000..a9b85608 --- /dev/null +++ b/tests/helpers/cluster.py @@ -0,0 +1,47 @@ +import pybbt as p +import redis +from redis.cluster import ClusterNode + +from helpers.redis import Redis + + +class Cluster: + def __init__(self): + self.num = 2 + self.nodes = [] + for i in range(self.num): + self.nodes.append(Redis(args=["--cluster-enabled", "yes"])) + p.ASSERT_EQ(self.nodes[0].do("cluster", "addslots", *range(0, 8192)), b"OK") + p.ASSERT_EQ(self.nodes[1].do("cluster", "addslots", *range(8192, 16384)), b"OK") + p.ASSERT_EQ(self.nodes[0].do("cluster", "meet", self.nodes[1].host, self.nodes[1].port), b"OK") + p.ASSERT_EQ(self.nodes[1].do("cluster", "meet", self.nodes[0].host, self.nodes[0].port), b"OK") + self.client = redis.RedisCluster(startup_nodes=[ + ClusterNode(self.nodes[0].host, self.nodes[0].port), + ClusterNode(self.nodes[1].host, self.nodes[1].port) + ], require_full_coverage=True) + p.ASSERT_EQ_TIMEOUT(lambda: self.client.cluster_info()["cluster_state"], "ok", 10) + p.log(f"cluster started at {self.nodes[0].get_address()}") + p.log(self.client.cluster_nodes()) + + def do(self, *args): + try: + ret = self.client.execute_command(*args) + except redis.exceptions.ResponseError as e: + return f"-{str(e)}" + return ret + + def pipeline(self): + return self.client.pipeline(transaction=False) + + def get_address(self): + return self.nodes[0].get_address() + + def dbsize(self): + size = 0 + for node in self.nodes: + size += node.dbsize() + return size + + @staticmethod + def is_cluster(): + return True diff --git a/tests/helpers/commands/__init__.py b/tests/helpers/commands/__init__.py new file mode 100644 index 00000000..b8a42486 --- /dev/null +++ b/tests/helpers/commands/__init__.py @@ -0,0 +1,5 @@ +from .select import SelectChecker +from .string import StringChecker +from .tair_string import TairStringChecker +from .tair_hash import TairHashChecker +from .tair_zset import TairZsetChecker \ No newline at end of file diff --git a/tests/helpers/commands/checker.py b/tests/helpers/commands/checker.py new file mode 100644 index 00000000..756d306b --- /dev/null +++ b/tests/helpers/commands/checker.py @@ -0,0 +1,9 @@ +from helpers.redis import Redis + + +class Checker: + def add_data(self, r: Redis, cross_slots_cmd: bool): + ... + + def check_data(self, r: Redis, cross_slots_cmd: bool): + ... diff --git a/tests/helpers/commands/list.py b/tests/helpers/commands/list.py new file mode 100644 index 00000000..de4d0c80 --- /dev/null +++ b/tests/helpers/commands/list.py @@ -0,0 +1,31 @@ +import pybbt + +from helpers.commands.checker import Checker +from helpers.redis import Redis + + +class ListChecker(Checker): + PREFIX = "list" + + def __init__(self): + self.cnt = 0 + + def add_data(self, r: Redis, cross_slots_cmd: bool): + p = r.pipeline() + p.lpush(f"{self.PREFIX}_{self.cnt}_list", 0) + p.lpush(f"{self.PREFIX}_{self.cnt}_list", 1) + p.lpush(f"{self.PREFIX}_{self.cnt}_list", 2) + p.lpush(f"{self.PREFIX}_{self.cnt}_list_str", "string0") + p.lpush(f"{self.PREFIX}_{self.cnt}_list_str", "string1") + p.lpush(f"{self.PREFIX}_{self.cnt}_list_str", "string2") + ret = p.execute() + pybbt.ASSERT_EQ(ret, [1, 2, 3, 1, 2, 3]) + self.cnt += 1 + + def check_data(self, r: Redis, cross_slots_cmd: bool): + for i in range(self.cnt): + p = r.pipeline() + p.lrange(f"{self.PREFIX}_{i}_list", 0, -1) + p.lrange(f"{self.PREFIX}_{i}_list_str", 0, -1) + ret = p.execute() + pybbt.ASSERT_EQ(ret, [[b"2", b"1", b"0"], [b"string2", b"string1", b"string0"]]) diff --git a/tests/helpers/commands/select.py b/tests/helpers/commands/select.py new file mode 100644 index 00000000..c39c3716 --- /dev/null +++ b/tests/helpers/commands/select.py @@ -0,0 +1,41 @@ +import pybbt + +from helpers.commands.checker import Checker +from helpers.redis import Redis + + +class SelectChecker(Checker): + PREFIX = "select" + + def __init__(self): + self.cnt = 0 + + def add_data(self, r: Redis, cross_slots_cmd: bool): + if not cross_slots_cmd: + return + p = r.pipeline() + p.select(1) + p.set(f"{self.PREFIX}_{self.cnt}_db1", "db1") + p.select(2) + p.set(f"{self.PREFIX}_{self.cnt}_db2", "db2") + p.select(3) + p.set(f"{self.PREFIX}_{self.cnt}_db3", "db3") + p.select(0) + ret = p.execute() + pybbt.ASSERT_EQ(ret, [True, True, True, True, True, True, True]) + self.cnt += 1 + + def check_data(self, r: Redis, cross_slots_cmd: bool): + if not cross_slots_cmd: + return + for i in range(self.cnt): + p = r.pipeline() + p.select(1) + p.get(f"{self.PREFIX}_{i}_db1") + p.select(2) + p.get(f"{self.PREFIX}_{i}_db2") + p.select(3) + p.get(f"{self.PREFIX}_{i}_db3") + p.select(0) + ret = p.execute() + pybbt.ASSERT_EQ(ret, [True, b'db1', True, b'db2', True, b'db3', True]) diff --git a/tests/helpers/commands/string.py b/tests/helpers/commands/string.py new file mode 100644 index 00000000..0e5f0395 --- /dev/null +++ b/tests/helpers/commands/string.py @@ -0,0 +1,31 @@ +import pybbt + +from helpers.commands.checker import Checker +from helpers.redis import Redis + + +class StringChecker(Checker): + PREFIX = "string" + + def __init__(self): + self.cnt = 0 + + def add_data(self, r: Redis, cross_slots_cmd: bool): + p = r.pipeline() + p.set(f"{self.PREFIX}_{self.cnt}_str", "string") + p.set(f"{self.PREFIX}_{self.cnt}_int", 0) + p.set(f"{self.PREFIX}_{self.cnt}_int0", -1) + p.set(f"{self.PREFIX}_{self.cnt}_int1", 123456789) + ret = p.execute() + pybbt.ASSERT_EQ(ret, [True, True, True, True]) + self.cnt += 1 + + def check_data(self, r: Redis, cross_slots_cmd: bool): + for i in range(self.cnt): + p = r.pipeline() + p.get(f"{self.PREFIX}_{i}_str") + p.get(f"{self.PREFIX}_{i}_int") + p.get(f"{self.PREFIX}_{i}_int0") + p.get(f"{self.PREFIX}_{i}_int1") + ret = p.execute() + pybbt.ASSERT_EQ(ret, [b"string", b"0", b"-1", b"123456789"]) diff --git a/tests/helpers/commands/tair_hash.py b/tests/helpers/commands/tair_hash.py new file mode 100644 index 00000000..99d88028 --- /dev/null +++ b/tests/helpers/commands/tair_hash.py @@ -0,0 +1,57 @@ +import pybbt + +from helpers.commands.checker import Checker +from helpers.constant import REDIS_SERVER_MODULES_ENABLED +from helpers.redis import Redis + + +class TairHashChecker(Checker): + PREFIX = "tairHash" + + def __init__(self): + self.cnt = 0 + + def add_data(self, r: Redis, cross_slots_cmd: bool): + if not REDIS_SERVER_MODULES_ENABLED: + return + + p = r.pipeline() + # different parameters type + p.execute_command("EXHSET", f"{self.PREFIX}_{self.cnt}", "field", "value") + p.execute_command("EXHSET", f"{self.PREFIX}_{self.cnt}_ABS", "field_abs", "value_abs", "ABS", 2) + p.execute_command("EXHSET", f"{self.PREFIX}_{self.cnt}_EX", "field_ex", "value_ex", "EX", 20000) + + # different key + # different field + p.execute_command("EXHSET", f"{self.PREFIX}_{self.cnt}_ALL_01", "field_all_01", "value_all_01", "EX", 20000, "ABS", 2) + p.execute_command("EXHSET", f"{self.PREFIX}_{self.cnt}_ALL_01", "field_all_02", "value_all_02", "EX", 20000, "ABS", 3) + + p.execute_command("EXHSET", f"{self.PREFIX}_{self.cnt}_ALL_02", "field_all_01", "value_all_01", "EX", 20000, "ABS", 2) + p.execute_command("EXHSET", f"{self.PREFIX}_{self.cnt}_ALL_02", "field_all_02", "value_all_02", "EX", 20000, "ABS", 3) + + ret = p.execute() + # pybbt.ASSERT_EQ(ret, [b"1", b"1", b"1", b"1",b"1", b"1",b"1"]) + pybbt.ASSERT_EQ(ret, [1, 1, 1, 1, 1, 1, 1]) + + self.cnt += 1 + + def check_data(self, r: Redis, cross_slots_cmd: bool): + if not REDIS_SERVER_MODULES_ENABLED: + return + + for i in range(self.cnt): + p = r.pipeline() + + p.execute_command("EXHGET", f"{self.PREFIX}_{i}", "field") + p.execute_command("EXHGET", f"{self.PREFIX}_{i}_ABS", "field_abs") + p.execute_command("EXHGET", f"{self.PREFIX}_{i}_EX", "field_ex") + + p.execute_command("EXHGET", f"{self.PREFIX}_{i}_ALL_01", "field_all_01") + p.execute_command("EXHGET", f"{self.PREFIX}_{i}_ALL_01", "field_all_02") + + p.execute_command("EXHGET", f"{self.PREFIX}_{i}_ALL_02", "field_all_01") + p.execute_command("EXHGET", f"{self.PREFIX}_{i}_ALL_02", "field_all_02") + + ret = p.execute() + # 需要确定一下如果一个命令返回多个值是如何封装的 + pybbt.ASSERT_EQ(ret, [b"value", b"value_abs", b"value_ex", b"value_all_01", b"value_all_02", b"value_all_01", b"value_all_02", ]) diff --git a/tests/helpers/commands/tair_string.py b/tests/helpers/commands/tair_string.py new file mode 100644 index 00000000..13a51632 --- /dev/null +++ b/tests/helpers/commands/tair_string.py @@ -0,0 +1,45 @@ +import pybbt + +from helpers.commands.checker import Checker +from helpers.constant import REDIS_SERVER_MODULES_ENABLED +from helpers.redis import Redis + + +class TairStringChecker(Checker): + PREFIX = "tairString" + + def __init__(self): + self.cnt = 0 + + def add_data(self, r: Redis, cross_slots_cmd: bool): + if not REDIS_SERVER_MODULES_ENABLED: + return + + p = r.pipeline() + # different parameters type + p.execute_command("EXSET", f"{self.PREFIX}_{self.cnt}_ABS", "abs_value", "VER", 2) + p.execute_command("EXSET", f"{self.PREFIX}_{self.cnt}_FLAGS", "flags_value", "FLAGS", 2) + p.execute_command("Exset", f"{self.PREFIX}_{self.cnt}_EX", "ex_value", "EX", 20000) + + # different key + p.execute_command("Exset", f"{self.PREFIX}_{self.cnt}_ALL_01", "all_value_01", "EX", 20000, "ABS", 3, "FLAGS", 4) + p.execute_command("Exset", f"{self.PREFIX}_{self.cnt}_ALL_02", "all_value_02", "EX", 20000, "ABS", 4, "FLAGS", 5) + ret = p.execute() + pybbt.ASSERT_EQ(ret, [b"OK", b"OK", b"OK", b"OK", b"OK"]) + self.cnt += 1 + + def check_data(self, r: Redis, cross_slots_cmd: bool): + if not REDIS_SERVER_MODULES_ENABLED: + return + + for i in range(self.cnt): + p = r.pipeline() + + p.execute_command("EXGET", f"{self.PREFIX}_{i}_ABS") + p.execute_command("EXGET", f"{self.PREFIX}_{i}_FLAGS", "WITHFLAGS") + p.execute_command("EXGET", f"{self.PREFIX}_{i}_EX") + p.execute_command("EXGET", f"{self.PREFIX}_{i}_ALL_01", "WITHFLAGS") + p.execute_command("EXGET", f"{self.PREFIX}_{i}_ALL_02", "WITHFLAGS") + + ret = p.execute() + pybbt.ASSERT_EQ(ret, [[b"abs_value", 1], [b"flags_value", 1, 2], [b"ex_value", 1], [b"all_value_01", 3, 4], [b"all_value_02", 4, 5]]) diff --git a/tests/helpers/commands/tair_zset.py b/tests/helpers/commands/tair_zset.py new file mode 100644 index 00000000..65858845 --- /dev/null +++ b/tests/helpers/commands/tair_zset.py @@ -0,0 +1,49 @@ +import pybbt + +from helpers.commands.checker import Checker +from helpers.constant import REDIS_SERVER_MODULES_ENABLED +from helpers.redis import Redis + + +class TairZsetChecker(Checker): + PREFIX = "tairZset" + + def __init__(self): + self.cnt = 0 + + def add_data(self, r: Redis, cross_slots_cmd: bool): + if not REDIS_SERVER_MODULES_ENABLED: + return + + p = r.pipeline() + + # different key + # int or float + p.execute_command("EXZADD", f"{self.PREFIX}_{self.cnt}_key01", "1.1#1.2", "mem01", "2.2#2.3", "mem02") + p.execute_command("EXZADD", f"{self.PREFIX}_{self.cnt}_key01", "3.3#3.4", "mem03", "4.4#4.5", "mem04") + p.execute_command("EXZADD", f"{self.PREFIX}_{self.cnt}_key02", "1.1#1.2", "mem01") + p.execute_command("EXZADD", f"{self.PREFIX}_{self.cnt}_key02", "2.2#2.3", "mem02") + ret = p.execute() + pybbt.ASSERT_EQ(ret, [2, 2, 1, 1]) + + self.cnt += 1 + + def check_data(self, r: Redis, cross_slots_cmd: bool): + if not REDIS_SERVER_MODULES_ENABLED: + return + + for i in range(self.cnt): + p = r.pipeline() + p.execute_command("EXZSCORE", f"{self.PREFIX}_{i}_key01", "mem01") + p.execute_command("EXZSCORE", f"{self.PREFIX}_{i}_key01", "mem02") + p.execute_command("EXZSCORE", f"{self.PREFIX}_{i}_key01", "mem03") + p.execute_command("EXZSCORE", f"{self.PREFIX}_{i}_key01", "mem04") + p.execute_command("EXZSCORE", f"{self.PREFIX}_{i}_key02", "mem01") + p.execute_command("EXZSCORE", f"{self.PREFIX}_{i}_key02", "mem02") + + ret = p.execute() + pybbt.ASSERT_EQ(ret, + [b'1.1000000000000001#1.2', + b'2.2000000000000002#2.2999999999999998', b'3.2999999999999998#3.3999999999999999', + b'4.4000000000000004#4.5', b'1.1000000000000001#1.2', + b'2.2000000000000002#2.2999999999999998']) diff --git a/tests/helpers/constant.py b/tests/helpers/constant.py new file mode 100644 index 00000000..3d529c45 --- /dev/null +++ b/tests/helpers/constant.py @@ -0,0 +1,21 @@ +import shutil +import subprocess +from pathlib import Path + +import pybbt + +BASE_PATH = f"{Path(__file__).parent.parent.parent.absolute()}" # project path +PATH_REDIS_SHAKE = f"{BASE_PATH}/bin/redis-shake" +PATH_REDIS_SERVER = shutil.which('redis-server') +output = subprocess.check_output(f"{PATH_REDIS_SERVER} --version", shell=True) +output_str = output.decode("utf-8") +REDIS_SERVER_VERSION = float(output_str.split("=")[1].split(" ")[0][:3]) + +# REDIS_SERVER_MODULES_ENABLED +REDIS_SERVER_MODULES_ENABLED = REDIS_SERVER_VERSION >= 5.0 and "modules" in pybbt.get_global_flags() + +if __name__ == '__main__': + print(BASE_PATH) + print(PATH_REDIS_SHAKE) + print(PATH_REDIS_SERVER) + print(REDIS_SERVER_VERSION) diff --git a/tests/helpers/data_inserter.py b/tests/helpers/data_inserter.py new file mode 100644 index 00000000..0bca3fc9 --- /dev/null +++ b/tests/helpers/data_inserter.py @@ -0,0 +1,21 @@ +from helpers.commands import SelectChecker, StringChecker, TairHashChecker, TairStringChecker, TairZsetChecker +from helpers.redis import Redis + + +class DataInserter: + def __init__(self, ): + self.checkers = [ + StringChecker(), + SelectChecker(), + TairStringChecker(), + TairHashChecker(), + TairZsetChecker(), + ] + + def add_data(self, r: Redis, cross_slots_cmd: bool): + for checker in self.checkers: + checker.add_data(r, cross_slots_cmd) + + def check_data(self, r: Redis, cross_slots_cmd: bool): + for checker in self.checkers: + checker.check_data(r, cross_slots_cmd) diff --git a/tests/helpers/redis.py b/tests/helpers/redis.py new file mode 100644 index 00000000..17e817b5 --- /dev/null +++ b/tests/helpers/redis.py @@ -0,0 +1,66 @@ +import time + +import pybbt +import redis + +from helpers.constant import PATH_REDIS_SERVER, REDIS_SERVER_MODULES_ENABLED, REDIS_SERVER_VERSION +from helpers.utils.network import get_free_port +from helpers.utils.timer import Timer + + +class Redis: + def __init__(self, args=None): + self.case_ctx = pybbt.get_case_context() + if args is None: + args = [] + self.host = "127.0.0.1" + self.port = get_free_port() + self.dir = f"{self.case_ctx.dir}/redis_{self.port}" + args.extend(["--port", str(self.port)]) + if REDIS_SERVER_MODULES_ENABLED: + args.extend(["--loadmodule", "tairstring_module.so"]) + args.extend(["--loadmodule", "tairhash_module.so"]) + args.extend(["--loadmodule", "tairzset_module.so"]) + self.server = pybbt.Launcher(args=[PATH_REDIS_SERVER] + args, work_dir=self.dir) + + self._wait_start() + self.client = redis.Redis(host=self.host, port=self.port) + self.case_ctx.add_exit_hook(lambda: self.server.stop()) + pybbt.log_yellow(f"redis server started at {self.host}:{self.port}, redis-cli -p {self.port}") + + def _wait_start(self, timeout=5): + timer = Timer() + while True: + try: + r = redis.Redis(host=self.host, port=self.port) + r.ping() + return + except redis.exceptions.ConnectionError: + time.sleep(0.1) + if timer.elapsed() > timeout: + stdout = f"{self.dir}/stdout" + with open(stdout, "r") as f: + for line in f.readlines(): + pybbt.log_red(line.strip()) + raise TimeoutError("redis server not started") + + def do(self, *args): + try: + ret = self.client.execute_command(*args) + except redis.exceptions.ResponseError as e: + return f"-{str(e)}" + return ret + + def pipeline(self): + return self.client.pipeline(transaction=False) + + def get_address(self): + return f"{self.host}:{self.port}" + + def is_cluster(self): + if REDIS_SERVER_VERSION < 3.0: + return False + return self.client.info()["cluster_enabled"] + + def dbsize(self): + return self.client.dbsize() diff --git a/tests/helpers/shake.py b/tests/helpers/shake.py new file mode 100644 index 00000000..7664b3f2 --- /dev/null +++ b/tests/helpers/shake.py @@ -0,0 +1,106 @@ +import typing + +import pybbt +import requests +import toml + +from helpers.constant import PATH_REDIS_SHAKE +from helpers.redis import Redis +from helpers.utils.filesystem import create_empty_dir +from helpers.utils.network import get_free_port +from helpers.utils.timer import Timer + + +class ShakeOpts: + @staticmethod + def create_sync_opts(src: Redis, dst: Redis) -> typing.Dict: + d = { + "sync_reader": { + "cluster": src.is_cluster(), + "address": src.get_address() + }, + "redis_writer": { + "cluster": dst.is_cluster(), + "address": dst.get_address() + } + } + return d + + @staticmethod + def create_scan_opts(src: Redis, dst: Redis) -> typing.Dict: + d = { + "scan_reader": { + "cluster": src.is_cluster(), + "address": src.get_address() + }, + "redis_writer": { + "cluster": dst.is_cluster(), + "address": dst.get_address() + } + } + return d + + @staticmethod + def create_rdb_opts(rdb_path: str, dts: Redis) -> typing.Dict: + d = { + "rdb_reader": {"filepath": rdb_path}, + "redis_writer": { + "cluster": dts.is_cluster(), + "address": dts.get_address() + } + } + return d + + @staticmethod + def create_aof_opts(aof_path: str, dts: Redis, timestamp: int = 0) -> typing.Dict: + d = { + "aof_reader": {"filepath": aof_path, "timestamp": timestamp}, + "redis_writer": { + "cluster": dts.is_cluster(), + "address": dts.get_address() + } + } + return d + +class Shake: + def __init__(self, opts: typing.Dict): + self.case_ctx = pybbt.get_case_context() + self.status_port = get_free_port() + self.status_url = f"http://localhost:{self.status_port}" + opts["advanced"] = {"status_port": self.status_port, "log_level": "debug"} + + self.dir = f"{self.case_ctx.dir}/shake{self.status_port}" + create_empty_dir(self.dir) + with open(f"{self.dir}/shake.toml", "w") as f: + toml.dump(opts, f) + self.server = pybbt.Launcher(args=[PATH_REDIS_SHAKE, "shake.toml"], work_dir=self.dir) + self.case_ctx.add_exit_hook(lambda: self.server.stop()) + self._wait_start() + + @staticmethod + def run_once(opts: typing.Dict): + status_port = get_free_port() + run_dir = f"{pybbt.get_case_context().dir}/shake{status_port}" + create_empty_dir(run_dir) + with open(f"{run_dir}/shake.toml", "w") as f: + toml.dump(opts, f) + server = pybbt.Launcher(args=[PATH_REDIS_SHAKE, "shake.toml"], work_dir=run_dir) + server.wait_stop() + + def get_status(self): + ret = requests.get(self.status_url) + return ret.json() + + def _wait_start(self, timeout=5): + timer = Timer() + while True: + try: + self.get_status() + return + except requests.exceptions.ConnectionError: + pass + if timer.elapsed() > timeout: + raise Exception(f"Shake server not started in {timeout} seconds") + + def is_consistent(self): + return self.get_status()["consistent"] diff --git a/test/cases/__init__.py b/tests/helpers/utils/__init__.py similarity index 100% rename from test/cases/__init__.py rename to tests/helpers/utils/__init__.py diff --git a/tests/helpers/utils/filesystem.py b/tests/helpers/utils/filesystem.py new file mode 100644 index 00000000..ba6067de --- /dev/null +++ b/tests/helpers/utils/filesystem.py @@ -0,0 +1,25 @@ +import os +import shutil + + +def file_size(file): + return os.stat(file).st_size + + +def file_truncate(file, size): + with open(file, "rb+") as f: + content = f.read() + f.truncate(size) + return content[size:] + + +def file_append(file, content): + with open(file, "ab") as f: + f.write(content) + + +def create_empty_dir(path): + if os.path.exists(path): + shutil.rmtree(path) + os.makedirs(path) + diff --git a/tests/helpers/utils/network.py b/tests/helpers/utils/network.py new file mode 100644 index 00000000..a66ce8c7 --- /dev/null +++ b/tests/helpers/utils/network.py @@ -0,0 +1,45 @@ +import random +import socket +import threading + + +def is_port_available(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(('localhost', port)) + s.close() + return True + except OSError: + return False + + +MIN_PORT = 20000 +MAX_PORT = 40000 + +port_cursor = random.choice(range(MIN_PORT, MAX_PORT, 10)) + +g_lock = threading.Lock() + + +def get_free_port(): + global port_cursor + global g_lock + with g_lock: + while True: + port_cursor += 1 + if port_cursor == MAX_PORT: + port_cursor = MIN_PORT + + if is_port_available(port_cursor): + return port_cursor + + +__all__ = [ + "is_port_available", + "get_free_port", +] + +if __name__ == '__main__': + # test + for i in range(10): + print(get_free_port()) diff --git a/tests/helpers/utils/rand.py b/tests/helpers/utils/rand.py new file mode 100644 index 00000000..4d598cfe --- /dev/null +++ b/tests/helpers/utils/rand.py @@ -0,0 +1,8 @@ +import random +import string + + +def random_string(length=8) -> str: + chars = string.ascii_letters + string.digits + random_str = ''.join(random.choices(chars, k=length)) + return random_str diff --git a/tests/helpers/utils/timer.py b/tests/helpers/utils/timer.py new file mode 100644 index 00000000..4cfc5ed2 --- /dev/null +++ b/tests/helpers/utils/timer.py @@ -0,0 +1,9 @@ +import time + + +class Timer: + def __init__(self): + self.start_time = time.perf_counter() + + def elapsed(self): + return time.perf_counter() - self.start_time diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..a7514188 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,4 @@ +requests>=2.31.0 +toml>=0.10.2 +pybbt>=1.0.1 +redis>=4.5.4