From 2e10736064aee02d6b95e59a45ba99b7c70950b9 Mon Sep 17 00:00:00 2001 From: hwy1314 <1426317404@qq.com> Date: Tue, 25 Jul 2023 10:01:20 +0800 Subject: [PATCH 1/8] Implemented the basic framework of aof.go. --- internal/aof/aof.go | 207 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 206 insertions(+), 1 deletion(-) diff --git a/internal/aof/aof.go b/internal/aof/aof.go index 0f1eb09f..bfa205b7 100644 --- a/internal/aof/aof.go +++ b/internal/aof/aof.go @@ -3,17 +3,21 @@ package aof import ( "bytes" "encoding/binary" + "fmt" "log" "strconv" "unicode" "github.com/alibaba/RedisShake/internal/entry" + "golang.org/x/tools/cmd/getgo/server" ) const ( AofManifestFileTypeBase = "b" /* Base file */ AofManifestTypeHist = "h" /* History file */ AofManifestTypeIncr = "i" /* INCR file */ + RDB_FORMAT_SUFFIX = ".rdb" + AOF_FORMAT_SUFFIX = ".aof" ) /* AOF manifest definition */ @@ -34,7 +38,13 @@ func aofInfoCreate() *aofInfo { return new(aofInfo) } -func StringNeedsRepr(s string) int { +var Aof_Info aofInfo = *aofInfoCreate() + +func (a *aofInfo) get_AofInfo_Name() string { + return a.fileName +} + +func stringNeedsRepr(s string) int { len := len(s) point := 0 for len > 0 { @@ -255,6 +265,11 @@ func aofInfoDup(orig *aofInfo) *aofInfo { return ai } +type listIter struct { + next *listNode + direction int +} + type lists struct { head, tail *listNode len uint64 @@ -273,6 +288,23 @@ func listCreate() *lists { lists.len = 0 return lists } +func listNext(iter *listIter) *listNode { + current := iter.next + + if current != nil { + if iter.direction == 0 { + iter.next = current.next + } else { + iter.next = current.prev + } + } + return current +} + +func (list *lists) ListsRewind(li *listIter) { + li.next = list.head + li.direction = 0 +} type aofManifest struct { baseAofInfo *aofInfo @@ -329,6 +361,52 @@ func (ld *Loader) ParseRDB() int { return 0 } +func stringcatprintf(s string, fmtStr string, args ...interface{}) string { + result := fmt.Sprintf(fmtStr, args...) + return s + result +} + +func stringcatrepr(s string, p string, length int) string { + s = s + string("\"") + for i := 0; i < length; i++ { + switch p[i] { + case '\\', '"': + s = stringcatprintf(s, "\\%c", p[i]) + case '\n': + s = s + "\\n" + case '\r': + s = s + "\\r" + case '\t': + s = s + "\\t" + case '\a': + s = s + "\\a" + case '\b': + s = s + "\\b" + default: + if strconv.IsPrint(rune(p[i])) { + s = s + string(p[i]) + } else { + s = s + "\\x%02x" + } + } + } + return s + "\"" +} + +func aofInfoFormat(buf string, ai *aofInfo) string { + var filenameRepr string + if stringNeedsRepr(ai.fileName) == 1 { + filenameRepr = stringcatrepr("", ai.fileName, len(ai.fileName)) + } + var ret string + if filenameRepr != "" { + ret = stringcatprintf(buf, "%s %s %s %d %s %c\n", AOF_MANIFEST_KEY_FILE_NAME, filenameRepr, AOF_MANIFEST_KEY_FILE_SEQ, ai.fileSeq, AOF_MANIFEST_KEY_FILE_TYPE, ai.aofFileType) + } else { + ret = stringcatprintf(buf, "%s %s %s %d %s %c\n", AOF_MANIFEST_KEY_FILE_NAME, ai.fileName, AOF_MANIFEST_KEY_FILE_SEQ, ai.fileSeq, AOF_MANIFEST_KEY_FILE_TYPE, ai.aofFileType) + } + return ret +} + func aofManifestcreate() *aofManifest { am := &aofManifest{ incrAofList: listCreate(), @@ -336,3 +414,130 @@ func aofManifestcreate() *aofManifest { } return am } + +func listDup(orig *lists) *lists { + var copy *lists + var iter listIter + var node *listNode + copy = listCreate() + if copy == nil { + return nil + } + copy.ListsRewind(&iter) + node = listNext(&iter) + var value interface{} + for node != nil { + value = node.value + } + + if listAddNodeTail(copy, value) == nil { + return nil + } + return copy +} + +func AOFManifestDup(orig *aofManifest) *aofManifest { + if orig == nil { + panic("orig is nil") + } + + am := &aofManifest{ + currBaseFileSeq: orig.currBaseFileSeq, + currIncrFIleSeq: orig.currIncrFIleSeq, + dirty: orig.dirty, + } + + if orig.baseAofInfo != nil { + am.baseAofInfo = aofInfoDup(orig.baseAofInfo) + } + + am.incrAofList = listDup(orig.incrAofList) + am.historyList = listDup(orig.historyList) + + if am.incrAofList == nil || am.historyList == nil { + panic("IncrAOFlist or HistoryAOFlist is nil") + } + return am +} + +func getAofManifestAsString(am *aofManifest) string { + if am == nil { + panic("am is nil") + } + var buf string + var ln *listNode + var li listIter + + if am.baseAofInfo != nil { + buf = aofInfoFormat(buf, am.baseAofInfo) + } + am.historyList.ListsRewind(&li) + ln = listNext(&li) + for ln != nil { + ai, ok := ln.value.(*aofInfo) + if ok { + buf = aofInfoFormat(buf, ai) + } + ln = listNext(&li) + } + + am.incrAofList.ListsRewind(&li) + ln = listNext(&li) + for ln != nil { + ai, ok := ln.value.(*aofInfo) + if ok { + buf = aofInfoFormat(buf, ai) + } + ln = listNext(&li) + } + + return buf + +} + +func getNewBaseFileNameAndMarkPreAsHistory(am *aofManifest) string { + if am == nil { + log.Fatal("aofManifest is nil") + } + if am.baseAofInfo != nil { + if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { + log.Fatal("base_aof_info has invalid file_type") + } + am.baseAofInfo.aofFileType = AofManifestTypeHist + } + var formatSuffix string + if server.aofUseRdbPreamble { + formatSuffix = RDB_FORMAT_SUFFIX + } else { + formatSuffix = AOF_FORMAT_SUFFIX + } + ai := aofInfoCreate() + ai.fileName = fmt.Sprintf("%s.%d%s%d", Aof_Info.get_AofInfo_Name(), am.currBaseFileSeq+1, BASE_FILE_SUFFIX, formatSuffix) + ai.fileSeq = am.currBaseFileSeq + 1 + ai.aofFileType = AofManifestFileTypeBase + am.baseAofInfo = ai + am.dirty = 1 + return am.baseAofInfo.fileName +} + +// server 未处理 +func aofLoadManifestFromDisk() { + aof_manifest := aofManifestcreate() + if !difExists(server.aof_dirname) { + fmt.Printf("The AOF directory %s doesn't exist\n", server.AofDirname) + return + } + + am_name := getAofManifestFileName() + am_filepath := makePath(server.aof_dirname, am_name) + if !fileExist(am_filepath) { + fmt.Printf("The AOF directory %s doesn't exist\n", server.AofDirname) + return + } + + am := aofLoadManifestFromFile(am_filepath) + if am != nil { + aofManifestFreeAndUpdate(am) + } + +} From 7b7a6b3821d1af843c1fc36243d52429beae2ef7 Mon Sep 17 00:00:00 2001 From: hwy1314 <1426317404@qq.com> Date: Sat, 29 Jul 2023 18:33:21 +0800 Subject: [PATCH 2/8] Implemented the basic framework of aofcheck.go. --- internal/aof/aof.go | 597 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 588 insertions(+), 9 deletions(-) diff --git a/internal/aof/aof.go b/internal/aof/aof.go index bfa205b7..10dde8c3 100644 --- a/internal/aof/aof.go +++ b/internal/aof/aof.go @@ -1,15 +1,20 @@ package aof import ( + "bufio" "bytes" "encoding/binary" "fmt" + "io" "log" + "os" "strconv" + "strings" + "time" "unicode" "github.com/alibaba/RedisShake/internal/entry" - "golang.org/x/tools/cmd/getgo/server" + "github.com/alibaba/RedisShake/internal/writer" ) const ( @@ -18,8 +23,28 @@ const ( AofManifestTypeIncr = "i" /* INCR file */ RDB_FORMAT_SUFFIX = ".rdb" AOF_FORMAT_SUFFIX = ".aof" + BASE_FILE_SUFFIX = ".base" + INCR_FILE_SUFFIX = ".incr" + TEMP_FILE_NAME_PREFIX = "temp-" + C_OK = 1 + C_ERR = -1 + EINTR = 4 + MANIFEST_NAME_SUFFIX = ".manifest" + AOF_NOT_EXIST = 1 + AOF_OPEN_ERR = 3 + AOF_OK = 0 + AOF_EMPTY = 2 + AOF_FAILED = 4 + AOF_TRUNCATED = 5 + SIZE_MAX = 128 ) +var rdbFileBeingLoaded string = "" + +func updateLoadingFileName(filename string) { + rdbFileBeingLoaded = filename +} + /* AOF manifest definition */ type aofInfo struct { fileName string @@ -27,6 +52,15 @@ type aofInfo struct { aofFileType string } +type server struct { + aof_dirname string + aofUseRdbPreamble int + aof_manifest *aofManifest + aof_filename string + aof_current_size int64 + aof_rewrite_base_size int64 +} + func IntToBytes(n int) []byte { data := int64(n) bytebuf := bytes.NewBuffer([]byte{}) @@ -34,6 +68,12 @@ func IntToBytes(n int) []byte { return bytebuf.Bytes() } +func ustime() int64 { + tv := time.Now() + ust := int64(tv.UnixNano()) / 1000 + return ust +} + func aofInfoCreate() *aofInfo { return new(aofInfo) } @@ -59,6 +99,24 @@ func stringNeedsRepr(s string) int { return 0 } +func dirExists(dname string) int { + _, err := os.Stat(dname) + if err != nil { + return 0 + } + + return 1 +} + +func fileExist(filename string) int { + _, err := os.Stat(filename) + if err != nil { + return 0 + } + + return 1 +} + /*如果字符串中包含要转义的字符，则返回一 *通过sdscatrer（），否则为零。 * @@ -435,6 +493,10 @@ func listDup(orig *lists) *lists { } return copy } +func ListsRewindTail(list *lists, li *listIter) { + li.next = list.tail + li.direction = 1 +} func AOFManifestDup(orig *aofManifest) *aofManifest { if orig == nil { @@ -506,13 +568,13 @@ func getNewBaseFileNameAndMarkPreAsHistory(am *aofManifest) string { am.baseAofInfo.aofFileType = AofManifestTypeHist } var formatSuffix string - if server.aofUseRdbPreamble { + if server.aofUseRdbPreamble == 1 { formatSuffix = RDB_FORMAT_SUFFIX } else { formatSuffix = AOF_FORMAT_SUFFIX } ai := aofInfoCreate() - ai.fileName = fmt.Sprintf("%s.%d%s%d", Aof_Info.get_AofInfo_Name(), am.currBaseFileSeq+1, BASE_FILE_SUFFIX, formatSuffix) + ai.fileName = stringcatprintf("%s.%d%s%d", Aof_Info.get_AofInfo_Name(), am.currBaseFileSeq+1, BASE_FILE_SUFFIX, formatSuffix) ai.fileSeq = am.currBaseFileSeq + 1 ai.aofFileType = AofManifestFileTypeBase am.baseAofInfo = ai @@ -522,22 +584,539 @@ func getNewBaseFileNameAndMarkPreAsHistory(am *aofManifest) string { // server 未处理 func aofLoadManifestFromDisk() { - aof_manifest := aofManifestcreate() - if !difExists(server.aof_dirname) { - fmt.Printf("The AOF directory %s doesn't exist\n", server.AofDirname) + server.aof_manifest = aofManifestcreate() + if !dirExists(server.aof_dirname) { + fmt.Printf("The AOF directory %s doesn't exist\n", server.aof_dirname) return } am_name := getAofManifestFileName() am_filepath := makePath(server.aof_dirname, am_name) - if !fileExist(am_filepath) { - fmt.Printf("The AOF directory %s doesn't exist\n", server.AofDirname) + if fileExist(am_filepath) == 0 { + fmt.Printf("The AOF directory %s doesn't exist\n", server.aof_dirname) return } am := aofLoadManifestFromFile(am_filepath) if am != nil { - aofManifestFreeAndUpdate(am) + server.aof_manifest = &am + } + +} + +func getNewIncrAofName(am *aofManifest) string { + ai := aofInfoCreate() + ai.aofFileType = AofManifestTypeIncr + ai.fileName = stringcatprintf("", "%s.%d%s%s", server.aof_filename, am.currIncrFIleSeq+1, INCR_FILE_SUFFIX, AOF_FORMAT_SUFFIX) + ai.fileSeq = am.currIncrFIleSeq + 1 + listAddNodeTail(am.incrAofList, ai) + am.dirty = 1 + return ai.fileName +} + +func getTempIncrAofNanme() string { + return stringcatprintf("", "%s%s%s", TEMP_FILE_NAME_PREFIX, server.aof_filename, INCR_FILE_SUFFIX) +} + +func listIndex(list *lists, index int64) *listNode { + var n *listNode + + if index < 0 { + index = (-index) - 1 + n = list.tail + for ; index > 0 && n != nil; index-- { + n = n.prev + } + } else { + n = list.head + for ; index > 0 && n != nil; index-- { + n = n.next + } + } + return n +} + +func listLinkNodeHead(list *lists, node *listNode) { + if list.len == 0 { + list.head = node + list.tail = node + node.prev = nil + node.next = nil + } else { + node.prev = nil + node.next = list.head + list.head.prev = node + list.head = node + } + list.len++ +} + +func listAddNodeHead(list *lists, value interface{}) *lists { + node := &listNode{ + value: value, + } + listLinkNodeHead(list, node) + + return list +} + +func listUnlinkNode(list *lists, node *listNode) { + if node.prev != nil { + node.prev.next = node.next + } else { + list.head = node.next + } + if node.next != nil { + node.next.prev = node.prev + } else { + list.tail = node.prev + } + node.next = nil + node.prev = nil + + list.len-- +} +func listDelNode(list *lists, node *listNode) { + listUnlinkNode(list, node) + +} + +func getLastIncrAofName(am *aofManifest) string { + if am == nil { + log.Fatal(("aofManifest is nil")) + } + + if am.incrAofList.len == 0 { + return getNewIncrAofName(am) + } + + lastnode := listIndex(am.incrAofList, -1) + + ai, ok := lastnode.value.(aofInfo) + if !ok { + log.Fatal("Failed to convert lastnode.value to aofInfo") + } + return ai.fileName +} + +/*func markRewrittenIncrAofAsHistory(am *aofManifest) { + if am == nil { + log.Fatal("aofManifest is nil") + } + if am.incrAofList.len == 0 { + return + } + var ln *listNode + var li listIter + + ListsRewindTail(am.incrAofList, &li) + + // "server.aof_fd != -1" means AOF enabled, then we must skip the + // last AOF, because this file is our currently writing. + if server.aof_fd != -1 { + ln = listNext(&li) + if ln == nil { + log.Fatal("List element is nil") + } + } + + // Move aofInfo from 'incr_aof_list' to 'history_aof_list'. + for ln != nil { + ai, ok := ln.value.(*aofInfo) + if !ok { + log.Fatal("Failed to convert ln.Value to *aofInfo") + } + if ai.aofFileType != AofManifestTypeIncr { + log.Fatal("Unexpected file type") + } + + hai := aofInfoDup(ai) + hai.aofFileType = AofManifestTypeHist + listAddNodeHead(am.historyList, hai) + listDelNode(am.incrAofList, ln) + } + + am.dirty = 1 +}*/ + +func getAofManifestFileName() string { + return stringcatprintf("", "%s%s", server.aof_filename, MANIFEST_NAME_SUFFIX) +} + +func getTempAofManifestFileName() string { + return stringcatprintf("", "%s%s%s", TEMP_FILE_NAME_PREFIX, server.aof_filename, MANIFEST_NAME_SUFFIX) +} + +//serverLog 未处理 +/* +func writeAofManifestFile(buf []byte)int{ + var ret =C_OK + + amName:=getAofManifestFileName() + amFilePath:=makePath(server.aofDirname,amName) + tmpAmName:=getTempAofManifestFileName() + tmpAmFilePath:=makePath(server.aofDirname,tmpAmName) + + fd,err:=os.OpenFile(tmpAmFilePath,os.O_WRONLY|os.O_TRUNC|os.O_CREATE,0644) + + if err!=nil{ + fmt.Sprintf("Can't open the AOF manifest file %s: %s", tmpAmName, err) + fd.Close() + ret=C_ERR + return ret + } + _, err = fd.Write(buf) + if err != nil { + fmt.Printf("Error trying to write the temporary AOF manifest file %s: %s\n", tmpAmName, err.Error()) + fd.Close() + ret=C_ERR + return ret + } + + + err=fd.Sync() + if err!=nil{ + fmt.Sprintf("Fail to sync the temp AOF file %s: %s.",tmpAmName,err) + fd.Close() + ret=C_ERR + return ret + } + + if(rename(tmpamfilepath,amFilePath)!=0){ + fmt.Sprintf("Fail to fsync AOF directory %s:%s.",amFilePath,err) + ret=C_ERR + fd.Close() + return ret + } + + err=fsyncFileDir(amFilePath) + if err!=nil{ + fmt.Sprintf("Fail to fsync AOF directory %s:%s.",amFilePath,err) + fd.Close() + ret=C_ERR + return ret + } + + return ret + +}*/ +func loadSingleAppendOnlyFile(filename string) int { + var fargc int + loops := 0 + ret := AOF_OK + aof_filepath := makePath(server.aof_dirname, filename) + fp, err := os.Open(aof_filepath) + if err != nil { + if os.IsNotExist(err) { + if _, err := os.Stat(aof_filepath); err == nil || !os.IsNotExist(err) { + fmt.Println("Fatal error: can't open the append log file %s for reading: %s", filename, err.Error()) + return AOF_OPEN_ERR + } else { + fmt.Println("The append log file %s doesn't exist: %s", filename, err.Error()) + return AOF_NOT_EXIST + } + fmt.Println("Fatal error: can't open the append log file %s for reading: %s", filename, err.Error()) + return AOF_OPEN_ERR + } + defer fp.Close() + + stat, _ := fp.Stat() + if stat.Size() == 0 { + return AOF_EMPTY + } + } + sig := make([]byte, 5) + if n, err := fp.Read(sig); err != nil || n != 5 || !bytes.Equal(sig, []byte("REDIS")) { + if _, err := fp.Seek(0, 0); err != nil { + fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + ret = AOF_FAILED + return ret + } + } else { + //RDB + + } + reader := bufio.NewReader(fp) + for { //serve + if loops%1024 == 0 { + //一些与事件处理和模块加载进度相关的操作，具体实现可能涉及更多的代码。例如，processEventsWhileBlocked 可能是处理在阻塞期间积累的事件的函数调用，而 processModuleLoadingProgressEvent 则是处理模块加载进度事件的函数调用。 + } + + line, err := reader.ReadString('\n') + { + if err != nil { + if err == io.EOF { + break + } + } else { + _, errs := fp.Seek(0, os.SEEK_CUR) + if errs == nil { + fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + ret = AOF_FAILED + return ret + } + } + if line[0] == '#' { + continue + } + if line[0] != '*' { + fmt.Println("Bad file format reading the append only file %s:", + "make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + } + argc, _ := strconv.Atoi(string(line[1:])) + if argc < 1 { + fmt.Println("Bad file format reading the append only file %s:", + "make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + } + if argc > int(SIZE_MAX) { + fmt.Println("Bad file format reading the append only file %s:", + "make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + } + e := entry.NewEntry() + argv := []string{} + + for j := 0; j < argc; j++ { + line, err := reader.ReadString('\n') + if err != nil || line[0] != '$' { + if err == io.EOF { + fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + ret = AOF_FAILED + return ret + } else { + fmt.Println("Bad file format reading the append only file %s:", + "make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + } + } + len, _ := strconv.ParseInt(line[1:], 10, 64) + + argstring := make([]byte, len) + _, err = io.ReadFull(fp, argstring) + if err != nil { + fargc = j + fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + ret = AOF_FAILED + return ret + } + //argv[j] = createObject(OBJ_STRING, argsds) //这里没写 + argv = append(argv, string(argstring)) + CRLF := make([]byte, 2) + _, err = io.ReadFull(fp, CRLF) + if err != nil { + fargc = j + 1 // Free up to j. + fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + ret = AOF_FAILED + return ret + } + } + for _, value := range argv { + ok := commands.lookupCommand(value) //包未导出 而且键值对判定问题 + if ok == 0 { + fmt.Println("unknown command. argv=%v", argv) + ret = AOF_FAILED + return ret + } + } + for _, value := range argv { + e.Argv = append(e.Argv, value) + } + rw := writer.NewRedisWriter(address, username, password, isTls) + rw.Write(e) //是否go携程 + + } + + } +} +func aofFileExist(filename string) int { + filepath := makePath(server.aof_dirname, filename) + ret := fileExist(filepath) + return ret +} + +// 这里的time没写完· +func getAppendOnlyFileSize(filename string, status *int) int64 { + var size int64 + + aofFilepath := makePath(server.aof_dirname, filename) + start := time.Now() + + stat, err := os.Stat(aofFilepath) + if err != nil { + if status != nil { + if os.IsNotExist(err) { + *status = AOF_NOT_EXIST + } else { + *status = AOF_OPEN_ERR + } + } + log.Fatal("Unable to obtain the AOF file %s length. stat: %s", filename, err.Error()) + size = 0 + } else { + if status != nil { + *status = AOF_OK + } + size = stat.Size() + } + + latency := time.Since(start).Milliseconds() + latencyAddSampleIfNeeded("aof-fstat", latency) //延迟监控 + /*可以看到，条件部分包括两个判断：首先检查 server.latency_monitor_threshold 是否为非零值（即已配置阈值）， + 然后判断 (var) 是否大于等于 server.latency_monitor_threshold。只有当这两个条件都为真时，才会调用 latencyAddSample 函数。 + + 这段代码的目的是确保只有当给定的 var 值超过了配置的阈值时，才会将样本添加到延迟监控中。*/ + + return size +} + +func getBaseAndIncrAppendOnlyFilesSize(am *aofManifest, status *int) int64 { + var size int64 + var ln *listNode + var li *listIter + if am.baseAofInfo != nil { + if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { + log.Fatal("File type must be base.") + } + size += getAppendOnlyFileSize(am.baseAofInfo.fileName, status) + if *status != AOF_OK { + return 0 + } + } + + am.incrAofList.ListsRewind(li) + ln = listNext(li) + for ln != nil { + ai := ln.value.(*aofInfo) + if ai.aofFileType != AofManifestTypeIncr { + log.Fatal("File type must be Incr") + } + size += getAppendOnlyFileSize(ai.fileName, status) + if *status != AOF_OK { + return 0 + } + } + return size +} + +func getBaseAndIncrAppendOnlyFilesNum(am *aofManifest) int { + num := 0 + if am.baseAofInfo != nil { + num++ + } + if am.incrAofList != nil { + num += int(am.incrAofList.len) + } + return num +} + +func loadAppendOnlyFile(am *aofManifest) int { + if am == nil { + log.Fatalf("aofManifest is null") + } + status := AOF_OK + ret := AOF_OK + var start int64 + var totalSize int64 = 0 + var baseSize int64 = 0 + var aofName string + var totalNum, aofNum, lastFile int + + if aofFileExist(server.aof_filename) == 1 { + if dirExists(server.aof_dirname) == 0 || + (am.baseAofInfo == nil && am.incrAofList.len == 0) || + (am.baseAofInfo != nil && am.incrAofList.len == 0 && + strings.Compare(am.baseAofInfo.fileName, server.aof_filename) == 0 && aofFileExist(server.aof_filename) == 0) { + log.Fatalf("This is an old version of the AOF file") //原本这里是要升级 + } + } + + if am.baseAofInfo == nil && am.incrAofList == nil { + return AOF_NOT_EXIST + } + + totalNum = getBaseAndIncrAppendOnlyFilesNum(am) + if totalNum <= 0 { + log.Fatalf("Assertion failed: IncrAppendOnlyFilestotalNum > 0") + } + + totalSize = getBaseAndIncrAppendOnlyFilesSize(am, &status) + if status != AOF_OK { + if status == AOF_NOT_EXIST { + status = AOF_FAILED + } + return status + } else if totalSize == 0 { + return AOF_EMPTY + } + + startLoading(totalSize, RDBFLAGS_AOF_PREAMBLE, 0) //这个嗲放有问题 + //这段代码是一个函数 startLoading 的实现，用于在全局状态中标记正在进行加载，并设置用于提供加载统计信息的字段。 + + if am.baseAofInfo != nil { + if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { + aofName = string(am.baseAofInfo.fileName) + updateLoadingFileName(aofName) + baseSize = getAppendOnlyFileSize(aofName, nil) + lastFile = totalNum + start = ustime() + ret = loadSingleAppendOnlyFile(aofName) + if ret == AOF_OK || (ret == AOF_TRUNCATED && lastFile == 1) { + fmt.Println("DB loaded from base file %s: %.3f seconds", aofName, float64(ustime()-start)/1000000) + } + + if ret == AOF_EMPTY { + ret = AOF_OK + } + + if ret == AOF_TRUNCATED && lastFile == 0 { + ret = AOF_FAILED + fmt.Println("Fatal error: the truncated file is not the last file") + } + + if ret == AOF_OPEN_ERR || ret == AOF_FAILED { + stopLoading(ret == AOF_OK || ret == AOF_TRUNCATED) + return ret + } + } + } + + if am.incrAofList.len > 0 { + var ln *listNode + var li listIter + + am.incrAofList.ListsRewind(&li) + ln = listNext(&li) + for ln != nil { + ai := ln.value.(*aofInfo) + if ai.aofFileType != AofManifestTypeIncr { + log.Fatalf("The manifestType must be Incr") + } + aofName = ai.fileName + updateLoadingFileName(aofName) + lastFile = totalNum + aofNum++ + start = ustime() + ret = loadSingleAppendOnlyFile(aofName) + if ret == AOF_OK || (ret == AOF_TRUNCATED && lastFile == 1) { + fmt.Println("DB loaded from incr file %s: %.3f seconds", aofName, float64(ustime()-start)/1000000) + } + + if ret == AOF_EMPTY { + ret = AOF_OK + } + + if ret == AOF_TRUNCATED && lastFile == 0 { + ret = AOF_FAILED + fmt.Println("Fatal error: the truncated file is not the last file") + } + + if ret == AOF_OPEN_ERR || ret == AOF_FAILED { + stopLoading(ret == AOF_OK || ret == AOF_TRUNCATED) /*总体而言，这段代码的目的是在加载过程结束时，更新全局状态中的加载相关字段，并触发加载结束事件以通知相关模块。 + 具体这些字段和事件的含义和功能可能需要参考完整代码和相关函数的定义才能理解清楚*/ + return ret + } + } + } + server.aof_current_size = totalSize + server.aof_rewrite_base_size = baseSize } From 1c6e5fd2b25451a82ed4222b5daaecc3ffacbbf5 Mon Sep 17 00:00:00 2001 From: hwy1314 <1426317404@qq.com> Date: Tue, 8 Aug 2023 18:35:26 +0800 Subject: [PATCH 3/8] Further improved the functions of aof.go and checkaof.go. --- internal/aof/aof.go | 9 +- internal/aof/aof_check.go | 2 +- internal/testing/appendonly.aof.manifest | 2 + internal/testing/mai.go | 137 +++++++++++++++++++++++ internal/testing/mai_test.go | 43 +++++++ 5 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 internal/testing/appendonly.aof.manifest create mode 100644 internal/testing/mai.go create mode 100644 internal/testing/mai_test.go diff --git a/internal/aof/aof.go b/internal/aof/aof.go index 10dde8c3..cf44decb 100644 --- a/internal/aof/aof.go +++ b/internal/aof/aof.go @@ -13,6 +13,7 @@ import ( "time" "unicode" + "github.com/alibaba/RedisShake/internal/commands" "github.com/alibaba/RedisShake/internal/entry" "github.com/alibaba/RedisShake/internal/writer" ) @@ -909,7 +910,7 @@ func loadSingleAppendOnlyFile(filename string) int { } } for _, value := range argv { - ok := commands.lookupCommand(value) //包未导出 而且键值对判定问题 + ok := commands.LookupCommand(value) //包未导出 而且键值对判定问题 if ok == 0 { fmt.Println("unknown command. argv=%v", argv) ret = AOF_FAILED @@ -919,7 +920,7 @@ func loadSingleAppendOnlyFile(filename string) int { for _, value := range argv { e.Argv = append(e.Argv, value) } - rw := writer.NewRedisWriter(address, username, password, isTls) + rw := theWriter.NewRedisWriter(address, username, password, isTls) rw.Write(e) //是否go携程 } @@ -1109,8 +1110,8 @@ func loadAppendOnlyFile(am *aofManifest) int { } if ret == AOF_OPEN_ERR || ret == AOF_FAILED { - stopLoading(ret == AOF_OK || ret == AOF_TRUNCATED) /*总体而言，这段代码的目的是在加载过程结束时，更新全局状态中的加载相关字段，并触发加载结束事件以通知相关模块。 - 具体这些字段和事件的含义和功能可能需要参考完整代码和相关函数的定义才能理解清楚*/ + //to do stopLoading(ret == AOF_OK || ret == AOF_TRUNCATED) /*总体而言，这段代码的目的是在加载过程结束时，更新全局状态中的加载相关字段，并触发加载结束事件以通知相关模块。 + //具体这些字段和事件的含义和功能可能需要参考完整代码和相关函数的定义才能理解清楚*/ return ret } } diff --git a/internal/aof/aof_check.go b/internal/aof/aof_check.go index 2a86be5e..8484a795 100644 --- a/internal/aof/aof_check.go +++ b/internal/aof/aof_check.go @@ -66,7 +66,7 @@ func getInputAofFileType(aofFilepath string) AofFileType { return "AOF_RESP" } } - +//test ok func filelsManifest(aofFilepath string) bool { var is_manifest bool = false fp, err := os.Open(aofFilepath) diff --git a/internal/testing/appendonly.aof.manifest b/internal/testing/appendonly.aof.manifest new file mode 100644 index 00000000..eca380fc --- /dev/null +++ b/internal/testing/appendonly.aof.manifest @@ -0,0 +1,2 @@ +file appendonly.aof.2.base.rdb seq 2 type b +file appendonly.aof.2.incr.aof seq 2 type i diff --git a/internal/testing/mai.go b/internal/testing/mai.go new file mode 100644 index 00000000..90fe71ec --- /dev/null +++ b/internal/testing/mai.go @@ -0,0 +1,137 @@ +package main + +import ( + "bufio" + "fmt" + "io" + "math" + "os" + + "strconv" +) + +func ReadBytes(fp *os.File, target *[]byte, length int64) int { + var real int64 + _, _ = fp.Seek(0, io.SeekCurrent) + n, err := fp.Read(*target) + + real = int64(n) + if err != nil || real != length { + fmt.Printf("Expected to read %d bytes, got %d bytes\n", length, real) + return 0 + } + return 1 +} +func readLong(fp *os.File, prefix byte, target *int64) int { + + var err error + _, err = fp.Seek(0, io.SeekCurrent) + if err != nil { + fmt.Printf("Failed to get current position, aborting...\n") + os.Exit(1) + } + reader := bufio.NewReader(fp) + + buf, err := reader.ReadBytes('\n') + + println(buf) + if err != nil { + fmt.Println("Failed to read line from file") + return 0 + } + if buf[0] != prefix { + fmt.Printf("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) + return 0 + } + value, err := strconv.ParseInt(string(buf[1:len(buf)-2]), 10, 64) //去除了换行符/r/n + if err != nil { + fmt.Println("Failed to parse long value") + return 0 + } + *target = value + + return 1 + +} +func fileIsRDB(aofFilepath string) bool { + fp, err := os.Open(aofFilepath) + if err != nil { + fmt.Printf("Cannot open file %s:%s\n", aofFilepath, err.Error()) + os.Exit(1) + } + sb, err := os.Stat(aofFilepath) + if err != nil { + fmt.Printf("cannot stat file: %s\n", aofFilepath) + os.Exit(1) + } + size := sb.Size() + if size == 0 { + fp.Close() + return false + } + if size >= 8 { + sig := make([]byte, 5) + _, err := fp.Read(sig) + if err == nil && string(sig) == "REDIS" { + fp.Close() + return true + } + } + fp.Close() + return false +} + +// test ok +func readBytes(fp *os.File, target *[]byte, length int64) int { + var real int64 + _, _ = fp.Seek(0, io.SeekCurrent) + n, err := fp.Read(*target) + real = int64(n) + if err != nil || real != length { + fmt.Printf("Expected to read %d bytes, got %d bytes\n", length, real) + return 0 + } + return 1 +} + +// testok +func consumeNewline(buf []byte) int { + if buf[0] != '\r' || buf[1] != '\n' { + fmt.Printf("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) + return 0 + } + return 1 +} +func readString(fp *os.File, target *string) int { + var len int64 + *target = "" + if readLong(fp, '$', &len) == 0 { + return 0 + } + println("readlong ok\n") + println(len) + if len < 0 || len > math.MaxInt64-2 { + fmt.Printf("Expected to read string of %d bytes, which is not in the suitable range\n", len) + return 0 + } + + // Increase length to also consume \r\n + len += 2 + data := make([]byte, len) + if readBytes(fp, &data, len) == 0 { + return 0 + } + + if consumeNewline(data[len-2:]) == 0 { + return 0 + } + + *target = string(data[:len-2]) + return 1 +} + +func main() { + + isManifest := fileIsRDB("appendonly.aof.2.base.rdb") + fmt.Println("Is manifest:", isManifest) +} diff --git a/internal/testing/mai_test.go b/internal/testing/mai_test.go new file mode 100644 index 00000000..8a82f49a --- /dev/null +++ b/internal/testing/mai_test.go @@ -0,0 +1,43 @@ +package main + +import ( + //"fmt" + //"io" + //"math" + "os" + "testing" +) + +func TestReadString(t *testing.T) { + // Create a temporary file and write some data into it + tempFile, err := os.CreateTemp("", "testfile") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tempFile.Name()) + data := []byte("$5\r\nHello\r\n") + _, err = tempFile.Write(data) + if err != nil { + t.Fatal(err) + } + + // Open the temporary file for reading + file, err := os.Open(tempFile.Name()) + if err != nil { + t.Fatal(err) + } + defer file.Close() + + // Test reading a valid string + var target string + expectedResult := 1 + result := readString(file, &target) + if result != expectedResult { + t.Errorf("1Expected %d, but got %d", expectedResult, result) + } + expectedValue := "Hello" + if target != expectedValue { + t.Errorf("2Expected value '%s', but got '%s'", expectedValue, target) + } + +} From b1f8511871066ebc6bdf6b7f070d079cb08f4ad6 Mon Sep 17 00:00:00 2001 From: hwy1314 <1426317404@qq.com> Date: Mon, 21 Aug 2023 22:55:29 +0800 Subject: [PATCH 4/8] Implemented the reading function of most AOF files. But the cluster commands have not been filtered yet. AOF files cannot be restored at a point in time. --- go.mod | 3 +- go.sum | 3 + internal/aof/aof.go | 627 ++++++++++++++++------------- internal/aof/aof_check.go | 473 ++++++++++++---------- internal/aof/aof_check_test.go | 27 ++ internal/commands/keys.go | 14 +- internal/commands/table.go | 3 +- internal/config/config.go | 14 +- internal/rdb/rdb.go | 9 +- internal/rdb/rdb_check.go | 208 ++++++++++ internal/rdb/structure/float.go | 27 +- internal/rdb/structure/intset.go | 24 ++ internal/rdb/structure/length.go | 58 ++- internal/rdb/structure/listpack.go | 20 +- internal/rdb/structure/string.go | 37 +- internal/rdb/structure/ziplist.go | 34 +- internal/rdb/types/hash.go | 59 ++- internal/rdb/types/interface.go | 38 +- internal/rdb/types/list.go | 63 ++- internal/rdb/types/module2.go | 35 +- internal/rdb/types/set.go | 28 +- internal/rdb/types/stream.go | 207 +++++++++- internal/rdb/types/string.go | 10 +- internal/rdb/types/zset.go | 76 +++- internal/reader/aof_reader.go | 61 ++- internal/statistics/statistics.go | 15 +- internal/testing/go.mod | 3 + 27 files changed, 1652 insertions(+), 524 deletions(-) create mode 100644 internal/aof/aof_check_test.go create mode 100644 internal/rdb/rdb_check.go create mode 100644 internal/testing/go.mod diff --git a/go.mod b/go.mod index 4dc27cf2..db88342c 100644 --- a/go.mod +++ b/go.mod @@ -12,5 +12,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/tools v0.10.0 ) diff --git a/go.sum b/go.sum index 55959bc7..a7f1240b 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/aof/aof.go b/internal/aof/aof.go index cf44decb..9e705127 100644 --- a/internal/aof/aof.go +++ b/internal/aof/aof.go @@ -3,10 +3,9 @@ package aof import ( "bufio" "bytes" - "encoding/binary" "fmt" "io" - "log" + "os" "strconv" "strings" @@ -14,8 +13,13 @@ import ( "unicode" "github.com/alibaba/RedisShake/internal/commands" + "github.com/alibaba/RedisShake/internal/config" + "github.com/alibaba/RedisShake/internal/entry" - "github.com/alibaba/RedisShake/internal/writer" + "github.com/alibaba/RedisShake/internal/log" + + "github.com/alibaba/RedisShake/internal/rdb" + "github.com/alibaba/RedisShake/internal/statistics" ) const ( @@ -38,11 +42,12 @@ const ( AOF_FAILED = 4 AOF_TRUNCATED = 5 SIZE_MAX = 128 + RDBFLAGS_AOF_PREAMBLE = 1 << 0 ) var rdbFileBeingLoaded string = "" -func updateLoadingFileName(filename string) { +func UpdateLoadingFileName(filename string) { rdbFileBeingLoaded = filename } @@ -53,7 +58,7 @@ type aofInfo struct { aofFileType string } -type server struct { +type INFO struct { aof_dirname string aofUseRdbPreamble int aof_manifest *aofManifest @@ -62,30 +67,85 @@ type server struct { aof_rewrite_base_size int64 } -func IntToBytes(n int) []byte { - data := int64(n) - bytebuf := bytes.NewBuffer([]byte{}) - binary.Write(bytebuf, binary.BigEndian, data) - return bytebuf.Bytes() +var AOFINFO INFO = *NewAOFINFO() + +func (a *INFO) GetAofdirName() string { + return a.aof_dirname } -func ustime() int64 { +func (a *INFO) SetAofDirName(dirname string) { + a.aof_dirname = dirname +} + +func (a *INFO) GetAofUseRdbPreamble() int { + return a.aofUseRdbPreamble +} + +func (a *INFO) SetAofUseRdbPreamble(useRdbPreamble int) { + a.aofUseRdbPreamble = useRdbPreamble +} + +func (a *INFO) GetAofManifest() *aofManifest { + return a.aof_manifest +} + +func (a *INFO) SetAofManifest(manifest *aofManifest) { + a.aof_manifest = manifest +} + +func (a *INFO) GetAofFilename() string { + return a.aof_filename +} + +func (a *INFO) SetAofFilename(filename string) { + a.aof_filename = filename +} + +func (a *INFO) GetAofCurrentSize() int64 { + return a.aof_current_size +} + +func (a *INFO) SetAofCurrentSize(size int64) { + a.aof_current_size = size +} + +func (a *INFO) GetAofRewriteBaseSize() int64 { + return a.aof_rewrite_base_size +} + +func (a *INFO) SetAofRewriteBaseSize(size int64) { + a.aof_rewrite_base_size = size +} +func NewAOFINFO() *INFO { + return &INFO{ + aof_dirname: config.Config.Source.AofDirName, + aofUseRdbPreamble: 0, + aof_manifest: nil, + aof_filename: config.Config.Source.AofFileName, + aof_current_size: 0, + aof_rewrite_base_size: 0, + } +} + +func Ustime() int64 { tv := time.Now() ust := int64(tv.UnixNano()) / 1000 return ust + } -func aofInfoCreate() *aofInfo { +func AofInfoCreate() *aofInfo { return new(aofInfo) } -var Aof_Info aofInfo = *aofInfoCreate() +var Aof_Info aofInfo = *AofInfoCreate() -func (a *aofInfo) get_AofInfo_Name() string { +func (a *aofInfo) GetAofInfoName() string { return a.fileName } -func stringNeedsRepr(s string) int { +// test ok +func StringNeedsRepr(s string) int { len := len(s) point := 0 for len > 0 { @@ -100,7 +160,8 @@ func stringNeedsRepr(s string) int { return 0 } -func dirExists(dname string) int { +// test ok +func DirExists(dname string) int { _, err := os.Stat(dname) if err != nil { return 0 @@ -109,7 +170,8 @@ func dirExists(dname string) int { return 1 } -func fileExist(filename string) int { +// test ok +func FileExist(filename string) int { _, err := os.Stat(filename) if err != nil { return 0 @@ -118,6 +180,12 @@ func fileExist(filename string) int { return 1 } +// test ok +func IsHexDigit(c byte) bool { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F') +} + /*如果字符串中包含要转义的字符，则返回一 *通过sdscatrer（），否则为零。 * @@ -148,7 +216,8 @@ func fileExist(filename string) int { } return s }*/ -func hexDigitToInt(c byte) int { +//test ok +func HexDigitToInt(c byte) int { switch c { case '0': return 0 @@ -187,7 +256,8 @@ func hexDigitToInt(c byte) int { } } -func splitArgs(line string) ([]string, int) { +// testok +func SplitArgs(line string) ([]string, int) { var p string p = line var current string @@ -205,20 +275,20 @@ func splitArgs(line string) ([]string, int) { insq := false // Set to true if we are in 'single quotes' done := false - if current == "" { - current = "" - } for !done { if inq { - _, err1 := strconv.ParseInt(string(p[i+2]), 16, 64) - _, err2 := strconv.ParseInt(string(p[i+3]), 16, 64) - if p[i] == '\\' && (p[i+1]) == 'x' && err1 == nil && err2 == nil { - int16 := (hexDigitToInt((p[i+2])) * 16) + hexDigitToInt(p[i+3]) - //var bytes []byte - //bytes=IntToBytes(int16) - //current = stringcatlen(current,bytes,1) - current += string(int16) - i += 3 + + if p[i] == '\\' && (p[i+1]) == 'x' && IsHexDigit(p[i+2]) && IsHexDigit(p[i+3]) { + _, err1 := strconv.ParseInt(string(p[i+2]), 16, 64) + _, err2 := strconv.ParseInt(string(p[i+3]), 16, 64) + if err1 == nil && err2 == nil { + int16 := (HexDigitToInt((p[i+2])) * 16) + HexDigitToInt(p[i+3]) + //var bytes []byte + //bytes=IntToBytes(int16) + //current = stringcatlen(current,bytes,1) + current = current + fmt.Sprint(int16) + i += 3 + } } else if p[i] == '\\' && i+1 < lens { c := p[i] @@ -239,7 +309,7 @@ func splitArgs(line string) ([]string, int) { } current += string(c) } else if p[i] == '"' { - if i+1 < lens && unicode.IsSpace((rune(p[i+1]))) { + if i+1 < lens && !unicode.IsSpace((rune(p[i+1]))) { return nil, 0 } done = true @@ -265,7 +335,7 @@ func splitArgs(line string) ([]string, int) { } else { switch p[i] { - case ' ', '\n', '\r', '\t': + case ' ', '\n', '\r', '\t', '\000': done = true break case '"': @@ -298,7 +368,8 @@ func splitArgs(line string) ([]string, int) { } } -func stringcatlen(s string, t []byte, lent int) string { +// test ok +func Stringcatlen(s string, t []byte, lent int) string { curlen := len(s) if curlen == 0 { @@ -313,11 +384,11 @@ func stringcatlen(s string, t []byte, lent int) string { return string(buf) } -func aofInfoDup(orig *aofInfo) *aofInfo { +func AofInfoDup(orig *aofInfo) *aofInfo { if orig == nil { - log.Fatal("Assertion failed: orig != nil") + log.Panicf("Assertion failed: orig != nil") } - ai := aofInfoCreate() + ai := AofInfoCreate() ai.fileName = orig.fileName ai.fileSeq = orig.fileSeq ai.aofFileType = orig.aofFileType @@ -340,14 +411,14 @@ type listNode struct { value interface{} } -func listCreate() *lists { +func ListCreate() *lists { lists := &lists{} lists.head = nil lists.tail = nil lists.len = 0 return lists } -func listNext(iter *listIter) *listNode { +func ListNext(iter *listIter) *listNode { current := iter.next if current != nil { @@ -380,17 +451,17 @@ type Loader struct { ch chan *entry.Entry } -func listAddNodeTail(lists *lists, value interface{}) *lists { +func ListAddNodeTail(lists *lists, value interface{}) *lists { node := &listNode{ value: value, prev: nil, next: nil, } - listLinkNodeTail(lists, node) + ListLinkNodeTail(lists, node) return lists } -func listLinkNodeTail(lists *lists, node *listNode) { +func ListLinkNodeTail(lists *lists, node *listNode) { if lists.len == 0 { lists.head = node lists.tail = node @@ -411,26 +482,22 @@ func NewLoader(filPath string, ch chan *entry.Entry) *Loader { return ld } -// TODO：完成checAofMain后写单测进行测试 -func (ld *Loader) ParseRDB() int { - // 加载aof目录 - // 进行check_aof， aof - CheckAofMain(ld.filPath) - // TODO：执行加载 - return 0 -} - -func stringcatprintf(s string, fmtStr string, args ...interface{}) string { +// testok +func Stringcatprintf(s string, fmtStr string, args ...interface{}) string { result := fmt.Sprintf(fmtStr, args...) - return s + result + if s == "" { + return result + } else { + return s + result + } } -func stringcatrepr(s string, p string, length int) string { +func Stringcatrepr(s string, p string, length int) string { s = s + string("\"") for i := 0; i < length; i++ { switch p[i] { case '\\', '"': - s = stringcatprintf(s, "\\%c", p[i]) + s = Stringcatprintf(s, "\\%c", p[i]) case '\n': s = s + "\\n" case '\r': @@ -452,44 +519,44 @@ func stringcatrepr(s string, p string, length int) string { return s + "\"" } -func aofInfoFormat(buf string, ai *aofInfo) string { +func AofInfoFormat(buf string, ai *aofInfo) string { var filenameRepr string - if stringNeedsRepr(ai.fileName) == 1 { - filenameRepr = stringcatrepr("", ai.fileName, len(ai.fileName)) + if StringNeedsRepr(ai.fileName) == 1 { + filenameRepr = Stringcatrepr("", ai.fileName, len(ai.fileName)) } var ret string if filenameRepr != "" { - ret = stringcatprintf(buf, "%s %s %s %d %s %c\n", AOF_MANIFEST_KEY_FILE_NAME, filenameRepr, AOF_MANIFEST_KEY_FILE_SEQ, ai.fileSeq, AOF_MANIFEST_KEY_FILE_TYPE, ai.aofFileType) + ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOF_MANIFEST_KEY_FILE_NAME, filenameRepr, AOF_MANIFEST_KEY_FILE_SEQ, ai.fileSeq, AOF_MANIFEST_KEY_FILE_TYPE, ai.aofFileType) } else { - ret = stringcatprintf(buf, "%s %s %s %d %s %c\n", AOF_MANIFEST_KEY_FILE_NAME, ai.fileName, AOF_MANIFEST_KEY_FILE_SEQ, ai.fileSeq, AOF_MANIFEST_KEY_FILE_TYPE, ai.aofFileType) + ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOF_MANIFEST_KEY_FILE_NAME, ai.fileName, AOF_MANIFEST_KEY_FILE_SEQ, ai.fileSeq, AOF_MANIFEST_KEY_FILE_TYPE, ai.aofFileType) } return ret } -func aofManifestcreate() *aofManifest { +func AofManifestcreate() *aofManifest { am := &aofManifest{ - incrAofList: listCreate(), - historyList: listCreate(), + incrAofList: ListCreate(), + historyList: ListCreate(), } return am } -func listDup(orig *lists) *lists { +func ListDup(orig *lists) *lists { var copy *lists var iter listIter var node *listNode - copy = listCreate() + copy = ListCreate() if copy == nil { return nil } copy.ListsRewind(&iter) - node = listNext(&iter) + node = ListNext(&iter) var value interface{} for node != nil { value = node.value } - if listAddNodeTail(copy, value) == nil { + if ListAddNodeTail(copy, value) == nil { return nil } return copy @@ -511,19 +578,20 @@ func AOFManifestDup(orig *aofManifest) *aofManifest { } if orig.baseAofInfo != nil { - am.baseAofInfo = aofInfoDup(orig.baseAofInfo) + am.baseAofInfo = AofInfoDup(orig.baseAofInfo) } - am.incrAofList = listDup(orig.incrAofList) - am.historyList = listDup(orig.historyList) + am.incrAofList = ListDup(orig.incrAofList) + am.historyList = ListDup(orig.historyList) if am.incrAofList == nil || am.historyList == nil { - panic("IncrAOFlist or HistoryAOFlist is nil") + fmt.Printf("IncrAOFlist or HistoryAOFlist is nil") + log.Panicf("IncrAOFlist or HistoryAOFlist is nil") } return am } -func getAofManifestAsString(am *aofManifest) string { +func GetAofManifestAsString(am *aofManifest) string { if am == nil { panic("am is nil") } @@ -532,50 +600,50 @@ func getAofManifestAsString(am *aofManifest) string { var li listIter if am.baseAofInfo != nil { - buf = aofInfoFormat(buf, am.baseAofInfo) + buf = AofInfoFormat(buf, am.baseAofInfo) } am.historyList.ListsRewind(&li) - ln = listNext(&li) + ln = ListNext(&li) for ln != nil { ai, ok := ln.value.(*aofInfo) if ok { - buf = aofInfoFormat(buf, ai) + buf = AofInfoFormat(buf, ai) } - ln = listNext(&li) + ln = ListNext(&li) } am.incrAofList.ListsRewind(&li) - ln = listNext(&li) + ln = ListNext(&li) for ln != nil { ai, ok := ln.value.(*aofInfo) if ok { - buf = aofInfoFormat(buf, ai) + buf = AofInfoFormat(buf, ai) } - ln = listNext(&li) + ln = ListNext(&li) } return buf } -func getNewBaseFileNameAndMarkPreAsHistory(am *aofManifest) string { +func GetNewBaseFileNameAndMarkPreAsHistory(am *aofManifest) string { if am == nil { - log.Fatal("aofManifest is nil") + log.Panicf("aofManifest is nil") } if am.baseAofInfo != nil { if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { - log.Fatal("base_aof_info has invalid file_type") + log.Panicf("base_aof_info has invalid file_type") } am.baseAofInfo.aofFileType = AofManifestTypeHist } var formatSuffix string - if server.aofUseRdbPreamble == 1 { + if AOFINFO.aofUseRdbPreamble == 1 { formatSuffix = RDB_FORMAT_SUFFIX } else { formatSuffix = AOF_FORMAT_SUFFIX } - ai := aofInfoCreate() - ai.fileName = stringcatprintf("%s.%d%s%d", Aof_Info.get_AofInfo_Name(), am.currBaseFileSeq+1, BASE_FILE_SUFFIX, formatSuffix) + ai := AofInfoCreate() + ai.fileName = Stringcatprintf("%s.%d%s%d", Aof_Info.GetAofInfoName(), am.currBaseFileSeq+1, BASE_FILE_SUFFIX, formatSuffix) ai.fileSeq = am.currBaseFileSeq + 1 ai.aofFileType = AofManifestFileTypeBase am.baseAofInfo = ai @@ -583,43 +651,43 @@ func getNewBaseFileNameAndMarkPreAsHistory(am *aofManifest) string { return am.baseAofInfo.fileName } -// server 未处理 -func aofLoadManifestFromDisk() { - server.aof_manifest = aofManifestcreate() - if !dirExists(server.aof_dirname) { - fmt.Printf("The AOF directory %s doesn't exist\n", server.aof_dirname) +// server 未处理 testok +func AofLoadManifestFromDisk() { + AOFINFO.aof_manifest = AofManifestcreate() + if DirExists(AOFINFO.aof_dirname) == 0 { + log.Infof("The AOF directory %v doesn't exist\n", AOFINFO.aof_dirname) return } - am_name := getAofManifestFileName() - am_filepath := makePath(server.aof_dirname, am_name) - if fileExist(am_filepath) == 0 { - fmt.Printf("The AOF directory %s doesn't exist\n", server.aof_dirname) + am_name := GetAofManifestFileName() + am_filepath := MakePath(AOFINFO.aof_dirname, am_name) + if FileExist(am_filepath) == 0 { + log.Infof("The AOF directory %v doesn't exist\n", AOFINFO.aof_dirname) return } - am := aofLoadManifestFromFile(am_filepath) + am := AofLoadManifestFromFile(am_filepath) if am != nil { - server.aof_manifest = &am + AOFINFO.aof_manifest = am } } -func getNewIncrAofName(am *aofManifest) string { - ai := aofInfoCreate() +func GetNewIncrAofName(am *aofManifest) string { + ai := AofInfoCreate() ai.aofFileType = AofManifestTypeIncr - ai.fileName = stringcatprintf("", "%s.%d%s%s", server.aof_filename, am.currIncrFIleSeq+1, INCR_FILE_SUFFIX, AOF_FORMAT_SUFFIX) + ai.fileName = Stringcatprintf("", "%s.%d%s%s", AOFINFO.aof_filename, am.currIncrFIleSeq+1, INCR_FILE_SUFFIX, AOF_FORMAT_SUFFIX) ai.fileSeq = am.currIncrFIleSeq + 1 - listAddNodeTail(am.incrAofList, ai) + ListAddNodeTail(am.incrAofList, ai) am.dirty = 1 return ai.fileName } -func getTempIncrAofNanme() string { - return stringcatprintf("", "%s%s%s", TEMP_FILE_NAME_PREFIX, server.aof_filename, INCR_FILE_SUFFIX) +func GetTempIncrAofNanme() string { + return Stringcatprintf("", "%s%s%s", TEMP_FILE_NAME_PREFIX, AOFINFO.aof_filename, INCR_FILE_SUFFIX) } -func listIndex(list *lists, index int64) *listNode { +func ListIndex(list *lists, index int64) *listNode { var n *listNode if index < 0 { @@ -637,7 +705,7 @@ func listIndex(list *lists, index int64) *listNode { return n } -func listLinkNodeHead(list *lists, node *listNode) { +func ListLinkNodeHead(list *lists, node *listNode) { if list.len == 0 { list.head = node list.tail = node @@ -652,16 +720,16 @@ func listLinkNodeHead(list *lists, node *listNode) { list.len++ } -func listAddNodeHead(list *lists, value interface{}) *lists { +func ListAddNodeHead(list *lists, value interface{}) *lists { node := &listNode{ value: value, } - listLinkNodeHead(list, node) + ListLinkNodeHead(list, node) return list } -func listUnlinkNode(list *lists, node *listNode) { +func ListUnlinkNode(list *lists, node *listNode) { if node.prev != nil { node.prev.next = node.next } else { @@ -677,147 +745,82 @@ func listUnlinkNode(list *lists, node *listNode) { list.len-- } -func listDelNode(list *lists, node *listNode) { - listUnlinkNode(list, node) +func ListDelNode(list *lists, node *listNode) { + ListUnlinkNode(list, node) } -func getLastIncrAofName(am *aofManifest) string { +func GetLastIncrAofName(am *aofManifest) string { if am == nil { - log.Fatal(("aofManifest is nil")) + log.Panicf(("aofManifest is nil")) } if am.incrAofList.len == 0 { - return getNewIncrAofName(am) + return GetNewIncrAofName(am) } - lastnode := listIndex(am.incrAofList, -1) + lastnode := ListIndex(am.incrAofList, -1) ai, ok := lastnode.value.(aofInfo) if !ok { - log.Fatal("Failed to convert lastnode.value to aofInfo") + fmt.Printf("Failed to convert lastnode.value to aofInfo") + log.Panicf("Failed to convert lastnode.value to aofInfo") } return ai.fileName } -/*func markRewrittenIncrAofAsHistory(am *aofManifest) { - if am == nil { - log.Fatal("aofManifest is nil") - } - if am.incrAofList.len == 0 { - return - } - var ln *listNode - var li listIter - - ListsRewindTail(am.incrAofList, &li) - - // "server.aof_fd != -1" means AOF enabled, then we must skip the - // last AOF, because this file is our currently writing. - if server.aof_fd != -1 { - ln = listNext(&li) - if ln == nil { - log.Fatal("List element is nil") - } - } - - // Move aofInfo from 'incr_aof_list' to 'history_aof_list'. - for ln != nil { - ai, ok := ln.value.(*aofInfo) - if !ok { - log.Fatal("Failed to convert ln.Value to *aofInfo") - } - if ai.aofFileType != AofManifestTypeIncr { - log.Fatal("Unexpected file type") - } - - hai := aofInfoDup(ai) - hai.aofFileType = AofManifestTypeHist - listAddNodeHead(am.historyList, hai) - listDelNode(am.incrAofList, ln) - } - - am.dirty = 1 -}*/ - -func getAofManifestFileName() string { - return stringcatprintf("", "%s%s", server.aof_filename, MANIFEST_NAME_SUFFIX) +func GetAofManifestFileName() string { + return Stringcatprintf("", "%s%s", AOFINFO.aof_filename, MANIFEST_NAME_SUFFIX) } -func getTempAofManifestFileName() string { - return stringcatprintf("", "%s%s%s", TEMP_FILE_NAME_PREFIX, server.aof_filename, MANIFEST_NAME_SUFFIX) +func GetTempAofManifestFileName() string { + return Stringcatprintf("", "%s%s%s", TEMP_FILE_NAME_PREFIX, AOFINFO.aof_filename, MANIFEST_NAME_SUFFIX) } -//serverLog 未处理 -/* -func writeAofManifestFile(buf []byte)int{ - var ret =C_OK - - amName:=getAofManifestFileName() - amFilePath:=makePath(server.aofDirname,amName) - tmpAmName:=getTempAofManifestFileName() - tmpAmFilePath:=makePath(server.aofDirname,tmpAmName) - - fd,err:=os.OpenFile(tmpAmFilePath,os.O_WRONLY|os.O_TRUNC|os.O_CREATE,0644) - - if err!=nil{ - fmt.Sprintf("Can't open the AOF manifest file %s: %s", tmpAmName, err) - fd.Close() - ret=C_ERR - return ret - } - _, err = fd.Write(buf) - if err != nil { - fmt.Printf("Error trying to write the temporary AOF manifest file %s: %s\n", tmpAmName, err.Error()) - fd.Close() - ret=C_ERR - return ret - } - - - err=fd.Sync() - if err!=nil{ - fmt.Sprintf("Fail to sync the temp AOF file %s: %s.",tmpAmName,err) - fd.Close() - ret=C_ERR - return ret - } - - if(rename(tmpamfilepath,amFilePath)!=0){ - fmt.Sprintf("Fail to fsync AOF directory %s:%s.",amFilePath,err) - ret=C_ERR - fd.Close() - return ret - } - - err=fsyncFileDir(amFilePath) - if err!=nil{ - fmt.Sprintf("Fail to fsync AOF directory %s:%s.",amFilePath,err) - fd.Close() - ret=C_ERR - return ret +func StartLoading(size int64, rdbflags int, async int) { + /* Load the DB */ + statistics.Metrics.Loading = true + if async == 1 { + statistics.Metrics.AsyncLoading = true + } + statistics.Metrics.LoadingStartTime = time.Now().Unix() + statistics.Metrics.LoadingLoadedBytes = 0 + statistics.Metrics.LoadingTotalBytes = size + fmt.Printf("The AOF file starts loading.\n") + log.Infof("The AOF file starts loading.\n") +} +func StopLoading(ret int) { + statistics.Metrics.Loading = false + statistics.Metrics.AsyncLoading = false + if ret == AOF_OK || ret == AOF_TRUNCATED { + fmt.Printf("The aof file was successfully loaded\n") + log.Infof("The aof file was successfully loaded\n") + } else { + fmt.Printf("There was an error opening the AOF file.\n") + log.Infof("There was an error opening the AOF file.\n") } +} - return ret - -}*/ -func loadSingleAppendOnlyFile(filename string) int { - var fargc int +// test ok +func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry) int { + //没释放命令 loops := 0 ret := AOF_OK - aof_filepath := makePath(server.aof_dirname, filename) - fp, err := os.Open(aof_filepath) + AofFilepath := MakePath(AOFINFO.aof_dirname, filename) + var sizes int64 = 0 + fp, err := os.Open(AofFilepath) if err != nil { if os.IsNotExist(err) { - if _, err := os.Stat(aof_filepath); err == nil || !os.IsNotExist(err) { - fmt.Println("Fatal error: can't open the append log file %s for reading: %s", filename, err.Error()) + if _, err := os.Stat(AofFilepath); err == nil || !os.IsNotExist(err) { + fmt.Printf("Fatal error: can't open the append log file %v for reading: %v", filename, err.Error()) + log.Infof("Fatal error: can't open the append log file %v for reading: %v", filename, err.Error()) return AOF_OPEN_ERR } else { - fmt.Println("The append log file %s doesn't exist: %s", filename, err.Error()) + fmt.Printf("The append log file %v doesn't exist: %v", filename, err.Error()) + log.Infof("The append log file %v doesn't exist: %v", filename, err.Error()) return AOF_NOT_EXIST } - fmt.Println("Fatal error: can't open the append log file %s for reading: %s", filename, err.Error()) - return AOF_OPEN_ERR + } defer fp.Close() @@ -829,14 +832,20 @@ func loadSingleAppendOnlyFile(filename string) int { sig := make([]byte, 5) if n, err := fp.Read(sig); err != nil || n != 5 || !bytes.Equal(sig, []byte("REDIS")) { if _, err := fp.Seek(0, 0); err != nil { - fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) + log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) ret = AOF_FAILED return ret } } else { - //RDB + fmt.Printf("Reading RDB base file on AOF loading...") + log.Infof("Reading RDB base file on AOF loading...") + ldRDB := rdb.NewLoader(AofFilepath, ch) + ldRDB.ParseRDB() + //RDB } + sizes += 5 reader := bufio.NewReader(fp) for { //serve if loops%1024 == 0 { @@ -852,26 +861,28 @@ func loadSingleAppendOnlyFile(filename string) int { } else { _, errs := fp.Seek(0, os.SEEK_CUR) if errs == nil { - fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) + log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) ret = AOF_FAILED return ret } } + sizes += int64(len(line)) if line[0] == '#' { continue } if line[0] != '*' { - fmt.Println("Bad file format reading the append only file %s:", - "make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + fmt.Printf("825") + log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) } argc, _ := strconv.Atoi(string(line[1:])) if argc < 1 { - fmt.Println("Bad file format reading the append only file %s:", - "make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + fmt.Printf("830") + log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) } if argc > int(SIZE_MAX) { - fmt.Println("Bad file format reading the append only file %s:", - "make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + fmt.Printf("834") + log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) } e := entry.NewEntry() argv := []string{} @@ -880,39 +891,44 @@ func loadSingleAppendOnlyFile(filename string) int { line, err := reader.ReadString('\n') if err != nil || line[0] != '$' { if err == io.EOF { - fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) + log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) ret = AOF_FAILED return ret } else { - fmt.Println("Bad file format reading the append only file %s:", - "make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + fmt.Printf("849") + log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) } } + sizes += int64(len(line)) len, _ := strconv.ParseInt(line[1:], 10, 64) argstring := make([]byte, len) - _, err = io.ReadFull(fp, argstring) + _, err = reader.Read(argstring) if err != nil { - fargc = j - fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) + log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) ret = AOF_FAILED return ret } //argv[j] = createObject(OBJ_STRING, argsds) //这里没写 argv = append(argv, string(argstring)) CRLF := make([]byte, 2) - _, err = io.ReadFull(fp, CRLF) + _, err = reader.Read(CRLF) if err != nil { - fargc = j + 1 // Free up to j. - fmt.Println("Unrecoverable error reading the append only file %s: %s", filename, err) + //fargc = j + 1 // Free up to j. + fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) + log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) ret = AOF_FAILED return ret } + sizes += len + 2 } for _, value := range argv { ok := commands.LookupCommand(value) //包未导出 而且键值对判定问题 if ok == 0 { - fmt.Println("unknown command. argv=%v", argv) + fmt.Printf("unknown command. argv=%v", argv) + log.Infof("unknown command. argv=%v", argv) ret = AOF_FAILED return ret } @@ -920,25 +936,30 @@ func loadSingleAppendOnlyFile(filename string) int { for _, value := range argv { e.Argv = append(e.Argv, value) } - rw := theWriter.NewRedisWriter(address, username, password, isTls) - rw.Write(e) //是否go携程 + ld.ch <- e + /*rw := writer.NewRedisWriter(config.Config.Source.Address, config.Config.Source.Username, config.Config.Source.Password, config.Config.Target.IsTLS) + rw.Write(e) //是否go携程*/ } } + statistics.Metrics.LoadingLoadedBytes = sizes + return ret } -func aofFileExist(filename string) int { - filepath := makePath(server.aof_dirname, filename) - ret := fileExist(filepath) + +// test ok +func AofFileExist(filename string) int { + filepath := MakePath(AOFINFO.aof_dirname, filename) + ret := FileExist(filepath) return ret } -// 这里的time没写完· -func getAppendOnlyFileSize(filename string, status *int) int64 { +// 这里的time没写完· testok +func GetAppendOnlyFileSize(filename string, status *int) int64 { var size int64 - aofFilepath := makePath(server.aof_dirname, filename) - start := time.Now() + aofFilepath := MakePath(AOFINFO.aof_dirname, filename) + //start := time.Now() stat, err := os.Stat(aofFilepath) if err != nil { @@ -949,7 +970,8 @@ func getAppendOnlyFileSize(filename string, status *int) int64 { *status = AOF_OPEN_ERR } } - log.Fatal("Unable to obtain the AOF file %s length. stat: %s", filename, err.Error()) + fmt.Printf("Unable to obtain the AOF file %v length. stat: %v", filename, err.Error()) + log.Panicf("Unable to obtain the AOF file %v length. stat: %v", filename, err.Error()) size = 0 } else { if status != nil { @@ -958,8 +980,8 @@ func getAppendOnlyFileSize(filename string, status *int) int64 { size = stat.Size() } - latency := time.Since(start).Milliseconds() - latencyAddSampleIfNeeded("aof-fstat", latency) //延迟监控 + //latency := time.Since(start).Milliseconds() + //latencyAddSampleIfNeeded("aof-fstat", latency) //延迟监控 /*可以看到，条件部分包括两个判断：首先检查 server.latency_monitor_threshold 是否为非零值（即已配置阈值）， 然后判断 (var) 是否大于等于 server.latency_monitor_threshold。只有当这两个条件都为真时，才会调用 latencyAddSample 函数。 @@ -968,36 +990,41 @@ func getAppendOnlyFileSize(filename string, status *int) int64 { return size } -func getBaseAndIncrAppendOnlyFilesSize(am *aofManifest, status *int) int64 { +// testok +func GetBaseAndIncrAppendOnlyFilesSize(am *aofManifest, status *int) int64 { var size int64 - var ln *listNode - var li *listIter + var ln *listNode = new(listNode) + var li *listIter = new(listIter) if am.baseAofInfo != nil { if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { - log.Fatal("File type must be base.") + fmt.Printf("File type must be base.") + log.Panicf("File type must be base.") } - size += getAppendOnlyFileSize(am.baseAofInfo.fileName, status) + size += GetAppendOnlyFileSize(am.baseAofInfo.fileName, status) if *status != AOF_OK { return 0 } } am.incrAofList.ListsRewind(li) - ln = listNext(li) + ln = ListNext(li) for ln != nil { ai := ln.value.(*aofInfo) if ai.aofFileType != AofManifestTypeIncr { - log.Fatal("File type must be Incr") + fmt.Printf("File type must be Incr") + log.Panicf("File type must be Incr") } - size += getAppendOnlyFileSize(ai.fileName, status) + size += GetAppendOnlyFileSize(ai.fileName, status) if *status != AOF_OK { return 0 } + ln = ListNext(li) } return size } -func getBaseAndIncrAppendOnlyFilesNum(am *aofManifest) int { +// test ok +func GetBaseAndIncrAppendOnlyFilesNum(am *aofManifest) int { num := 0 if am.baseAofInfo != nil { num++ @@ -1008,9 +1035,10 @@ func getBaseAndIncrAppendOnlyFilesNum(am *aofManifest) int { return num } -func loadAppendOnlyFile(am *aofManifest) int { +func (ld *Loader) LoadAppendOnlyFile(am *aofManifest, ch chan *entry.Entry) int { if am == nil { - log.Fatalf("aofManifest is null") + fmt.Printf("aofManifest is null") + log.Panicf("aofManifest is null") } status := AOF_OK ret := AOF_OK @@ -1020,12 +1048,13 @@ func loadAppendOnlyFile(am *aofManifest) int { var aofName string var totalNum, aofNum, lastFile int - if aofFileExist(server.aof_filename) == 1 { - if dirExists(server.aof_dirname) == 0 || + if AofFileExist(AOFINFO.aof_filename) == 1 { + if DirExists(AOFINFO.aof_dirname) == 0 || (am.baseAofInfo == nil && am.incrAofList.len == 0) || (am.baseAofInfo != nil && am.incrAofList.len == 0 && - strings.Compare(am.baseAofInfo.fileName, server.aof_filename) == 0 && aofFileExist(server.aof_filename) == 0) { - log.Fatalf("This is an old version of the AOF file") //原本这里是要升级 + strings.Compare(am.baseAofInfo.fileName, AOFINFO.aof_filename) == 0 && AofFileExist(AOFINFO.aof_filename) == 0) { + fmt.Printf("This is an old version of the AOF file") //原本这里是要升级 + log.Panicf("This is an old version of the AOF file") //原本这里是要升级 } } @@ -1033,12 +1062,13 @@ func loadAppendOnlyFile(am *aofManifest) int { return AOF_NOT_EXIST } - totalNum = getBaseAndIncrAppendOnlyFilesNum(am) + totalNum = GetBaseAndIncrAppendOnlyFilesNum(am) if totalNum <= 0 { - log.Fatalf("Assertion failed: IncrAppendOnlyFilestotalNum > 0") + fmt.Printf("Assertion failed: IncrAppendOnlyFilestotalNum > 0") + log.Panicf("Assertion failed: IncrAppendOnlyFilestotalNum > 0") } - totalSize = getBaseAndIncrAppendOnlyFilesSize(am, &status) + totalSize = GetBaseAndIncrAppendOnlyFilesSize(am, &status) if status != AOF_OK { if status == AOF_NOT_EXIST { status = AOF_FAILED @@ -1048,19 +1078,20 @@ func loadAppendOnlyFile(am *aofManifest) int { return AOF_EMPTY } - startLoading(totalSize, RDBFLAGS_AOF_PREAMBLE, 0) //这个嗲放有问题 + StartLoading(totalSize, RDBFLAGS_AOF_PREAMBLE, 0) //这个嗲放有问题 //这段代码是一个函数 startLoading 的实现，用于在全局状态中标记正在进行加载，并设置用于提供加载统计信息的字段。 if am.baseAofInfo != nil { if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { aofName = string(am.baseAofInfo.fileName) - updateLoadingFileName(aofName) - baseSize = getAppendOnlyFileSize(aofName, nil) + UpdateLoadingFileName(aofName) + baseSize = GetAppendOnlyFileSize(aofName, nil) lastFile = totalNum - start = ustime() - ret = loadSingleAppendOnlyFile(aofName) + start = Ustime() + ret = ld.LoadSingleAppendOnlyFile(aofName, ch) if ret == AOF_OK || (ret == AOF_TRUNCATED && lastFile == 1) { - fmt.Println("DB loaded from base file %s: %.3f seconds", aofName, float64(ustime()-start)/1000000) + fmt.Printf("DB loaded from base file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) + log.Infof("DB loaded from base file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) } if ret == AOF_EMPTY { @@ -1069,35 +1100,50 @@ func loadAppendOnlyFile(am *aofManifest) int { if ret == AOF_TRUNCATED && lastFile == 0 { ret = AOF_FAILED - fmt.Println("Fatal error: the truncated file is not the last file") + fmt.Printf("Fatal error: the truncated file is not the last file") + log.Infof("Fatal error: the truncated file is not the last file") } if ret == AOF_OPEN_ERR || ret == AOF_FAILED { - stopLoading(ret == AOF_OK || ret == AOF_TRUNCATED) + if ret == AOF_OK || ret == AOF_TRUNCATED { + fmt.Printf("The aof file was successfully loaded\n") + log.Infof("The aof file was successfully loaded\n") + } else { + if ret == AOF_OPEN_ERR { + fmt.Printf("There was an error opening the AOF file.\n") + log.Infof("There was an error opening the AOF file.\n") + } else { + fmt.Printf("Failed to open AOF file.\n") + log.Infof("Failed to open AOF file.\n") + } + } return ret } } } if am.incrAofList.len > 0 { - var ln *listNode + var ln *listNode = new(listNode) var li listIter am.incrAofList.ListsRewind(&li) - ln = listNext(&li) + ln = ListNext(&li) for ln != nil { ai := ln.value.(*aofInfo) if ai.aofFileType != AofManifestTypeIncr { - log.Fatalf("The manifestType must be Incr") + fmt.Printf("The manifestType must be Incr") + + log.Panicf("The manifestType must be Incr") } aofName = ai.fileName - updateLoadingFileName(aofName) + UpdateLoadingFileName(aofName) lastFile = totalNum aofNum++ - start = ustime() - ret = loadSingleAppendOnlyFile(aofName) + start = Ustime() + ret = ld.LoadSingleAppendOnlyFile(aofName, ch) if ret == AOF_OK || (ret == AOF_TRUNCATED && lastFile == 1) { - fmt.Println("DB loaded from incr file %s: %.3f seconds", aofName, float64(ustime()-start)/1000000) + fmt.Printf("DB loaded from incr file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) + log.Infof("DB loaded from incr file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) } if ret == AOF_EMPTY { @@ -1106,18 +1152,29 @@ func loadAppendOnlyFile(am *aofManifest) int { if ret == AOF_TRUNCATED && lastFile == 0 { ret = AOF_FAILED - fmt.Println("Fatal error: the truncated file is not the last file") + fmt.Printf("Fatal error: the truncated file is not the last file\n") + log.Infof("Fatal error: the truncated file is not the last file\n") } if ret == AOF_OPEN_ERR || ret == AOF_FAILED { //to do stopLoading(ret == AOF_OK || ret == AOF_TRUNCATED) /*总体而言，这段代码的目的是在加载过程结束时，更新全局状态中的加载相关字段，并触发加载结束事件以通知相关模块。 //具体这些字段和事件的含义和功能可能需要参考完整代码和相关函数的定义才能理解清楚*/ + if ret == AOF_OPEN_ERR { + fmt.Printf("There was an error opening the AOF file.\n") + log.Infof("There was an error opening the AOF file.\n") + } else { + fmt.Printf("Failed to open AOF file.\n") + log.Infof("Failed to open AOF file.\n") + } return ret } + ln = ListNext(&li) } } - server.aof_current_size = totalSize - server.aof_rewrite_base_size = baseSize + AOFINFO.aof_current_size = totalSize + AOFINFO.aof_rewrite_base_size = baseSize + return ret + } diff --git a/internal/aof/aof_check.go b/internal/aof/aof_check.go index 8484a795..50d76bbc 100644 --- a/internal/aof/aof_check.go +++ b/internal/aof/aof_check.go @@ -4,12 +4,15 @@ import ( "bufio" "fmt" "io" - "log" "math" "os" "path" + "path/filepath" "strconv" "strings" + + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/rdb" //"time" ) @@ -18,6 +21,8 @@ type AofFileType string var errors [1044]byte var line int64 = 1 var epos int64 +var fp *os.File +var pos int64 const ( aofResp AofFileType = "AOF_RESP" @@ -38,45 +43,52 @@ const ( // check 里面的主函数 func CheckAofMain(aofFilePath string) (checkResult bool, fileType AofFileType, err error) { - /* - getAofType := getInputAofFileTye(filePath) // 获取aof文件的类型 - switch getAOFType { - case AOF_MULTI_PART: - checkResult, err = checkMultiPartAof(dirpath, filepath, fix) - return checkResult, aofMultiPart, nil - case AOF_RESP: - checkResult, err := checkOldStyleAof(filepath) - return checkResult, aofResp, nil - case AOF_RDB_PREAMBLE: - checkResult, err := checkOldStyleAof(filepath) - return checkResult, aofRdbPreamble, nil - } - return result, err - */ - //TODO: mock result + var filepaths string + var tempFilepath [1025]byte + var dirpath string + fix := 1 + filepaths = aofFilePath + + copy(tempFilepath[:], filepaths) + dirpath = filepath.Dir(string(tempFilepath[:])) + + fileType = GetInputAofFileType(filepaths) + switch fileType { + case "AOF_MULTI_PART": + CheckMultipartAof(dirpath, filepaths, fix) + break + case "AOF_RESP": + CheckOldStyleAof(filepaths, fix, false) + break + case "AOF_RDB_PREAMBLE": + CheckOldStyleAof(filepaths, fix, true) + break + } return true, aofMultiPart, nil } -func getInputAofFileType(aofFilepath string) AofFileType { - if filelsManifest(aofFilepath) { +func GetInputAofFileType(aofFilepath string) AofFileType { + if FilelsManifest(aofFilepath) { return "AOF_MULTI_PART" - } else if fileIsRDB(aofFilepath) { + } else if FileIsRDB(aofFilepath) { return "AOF_RDB_PREAMBLE" } else { return "AOF_RESP" } } -//test ok -func filelsManifest(aofFilepath string) bool { + +// test ok +func FilelsManifest(aofFilepath string) bool { var is_manifest bool = false fp, err := os.Open(aofFilepath) if err != nil { - fmt.Printf("Cannot open file %s:%s\n", aofFilepath, err.Error()) + fmt.Printf("Cannot open file %v:%v\n", aofFilepath, err.Error()) + log.Infof("Cannot open file %v:%v\n", aofFilepath, err.Error()) os.Exit(1) } sb, err := os.Stat(aofFilepath) if err != nil { - fmt.Printf("cannot stat file: %s\n", aofFilepath) + fmt.Printf("cannot stat file: %v\n", aofFilepath) os.Exit(1) } size := sb.Size() @@ -91,7 +103,7 @@ func filelsManifest(aofFilepath string) bool { if err == io.EOF { break } else { - fmt.Printf("cannot read file: %s\n", aofFilepath) + fmt.Printf("cannot read file: %v\n", aofFilepath) os.Exit(1) } } @@ -105,15 +117,16 @@ func filelsManifest(aofFilepath string) bool { return is_manifest } -func fileIsRDB(aofFilepath string) bool { +// test ok +func FileIsRDB(aofFilepath string) bool { fp, err := os.Open(aofFilepath) if err != nil { - fmt.Printf("Cannot open file %s:%s\n", aofFilepath, err.Error()) + fmt.Printf("Cannot open file %v:%v\n", aofFilepath, err.Error()) os.Exit(1) } sb, err := os.Stat(aofFilepath) if err != nil { - fmt.Printf("cannot stat file: %s\n", aofFilepath) + fmt.Printf("cannot stat file: %v\n", aofFilepath) os.Exit(1) } size := sb.Size() @@ -133,117 +146,145 @@ func fileIsRDB(aofFilepath string) bool { return false } -func printAofStyle(ret int, aofFileName string, aofType string) { +func PrintAofStyle(ret int, aofFileName string, aofType string) { switch ret { case AOF_CHECK_OK: - fmt.Printf("%s %s is valid\n", aofType, aofFileName) + fmt.Printf("%v %v is valid\n", aofType, aofFileName) + log.Infof("%v %v is valid\n", aofType, aofFileName) case AOF_CHECK_EMPTY: - fmt.Printf("%s %s is empty\n", aofType, aofFileName) + fmt.Printf("%v %v is empty\n", aofType, aofFileName) + log.Infof("%v %v is empty\n", aofType, aofFileName) case AOF_CHECK_TIMESTAMP_TRUNCATED: - fmt.Printf("Successfully truncated AOF %s to timestamp %d\n", aofFileName, toTimestamp) + fmt.Printf("Successfully truncated AOF %v to timestamp %d\n", aofFileName, toTimestamp) + log.Infof("Successfully truncated AOF %v to timestamp %d\n", aofFileName, toTimestamp) case AOF_CHECK_TRUNCATED: - fmt.Printf("Successfully truncated AOF %s\n", aofFileName) + fmt.Printf("Successfully truncated AOF %v\n", aofFileName) + log.Infof("Successfully truncated AOF %v\n", aofFileName) } } -func makePath(paths string, filename string) string { +// test ok +func MakePath(paths string, filename string) string { return path.Join(paths, filename) } -func pathIsBaseName(path string) bool { +// test ok +func PathIsBaseName(path string) bool { return strings.IndexByte(path, '/') == -1 && strings.IndexByte(path, '\\') == -1 } -func readArgc(fp *os.File, target *int64) int { - return readLong(fp, '*', target) +// test ok +func ReadArgc(rd *bufio.Reader, target *int64) int { + return ReadLong(rd, ' ', target) } -func readString(fp *os.File, target *string) int { +// test ok +func ReadString(rd *bufio.Reader, target *string) int { var len int64 *target = "" - if readLong(fp, '$', &len) == 0 { + if ReadLong(rd, '$', &len) == 0 { return 0 } if len < 0 || len > math.MaxInt64-2 { fmt.Printf("Expected to read string of %d bytes, which is not in the suitable range\n", len) + log.Infof("Expected to read string of %d bytes, which is not in the suitable range\n", len) return 0 } // Increase length to also consume \r\n len += 2 data := make([]byte, len) - if readBytes(fp, &data, len) == 0 { + if ReadBytes(rd, &data, len) == 0 { return 0 } - if consumeNewline(data[len-2:]) == 0 { + if ConsumeNewline(data[len-2:]) == 0 { return 0 } *target = string(data[:len-2]) + //pos += 2 readbytes已经处理 return 1 } -func readBytes(fp *os.File, target *[]byte, length int64) int { +// test ok +func ReadBytes(rd *bufio.Reader, target *[]byte, length int64) int { var real int64 - epos, _ = fp.Seek(0, io.SeekCurrent) - n, err := fp.Read(*target) + n, err := rd.Read(*target) real = int64(n) if err != nil || real != length { fmt.Printf("Expected to read %d bytes, got %d bytes\n", length, real) + log.Infof("Expected to read %d bytes, got %d bytes\n", length, real) return 0 } + pos += real return 1 } -func consumeNewline(buf []byte) int { +// testok +func ConsumeNewline(buf []byte) int { if buf[0] != '\r' || buf[1] != '\n' { fmt.Printf("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) + log.Infof("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) return 0 } line += 1 return 1 } -func readLong(fp *os.File, prefix byte, target *int64) int { - buf := make([]byte, 128) +// test ok +func ReadLong(rd *bufio.Reader, prefix byte, target *int64) int { + var err error - epos, err = fp.Seek(0, io.SeekCurrent) - if err != nil { + var value int64 + /*if err != nil { fmt.Printf("Failed to get current position, aborting...\n") - os.Exit(1) - } - reader := bufio.NewReader(fp) - if _, err := reader.ReadBytes('\n'); err != nil { - return 0 - } - buf, err = reader.ReadBytes('\n') + log.Panicf("Failed to get current position, aborting...\n") + }*/ + + buf, err := rd.ReadBytes('\n') if err != nil { - fmt.Println("Failed to read line from file") - return 0 - } - if buf[0] != prefix { - fmt.Printf("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) + fmt.Printf("Failed to read line from file\n") + log.Infof("Failed to read line from file") return 0 } - value, err := strconv.ParseInt(string(buf[1:len(buf)-1]), 10, 64) //去除了换行符*8* - if err != nil { - fmt.Println("Failed to parse long value") - return 0 + pos += int64(len(buf)) + if prefix != ' ' { + if buf[0] != prefix { + fmt.Printf("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) + log.Infof("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) + return 0 + } + value, err = strconv.ParseInt(string(buf[1:len(buf)-2]), 10, 64) //去除了换行符/r/n + if err != nil { + fmt.Printf("Failed to parse long value") + log.Infof("Failed to parse long value") + return 0 + } + } else { + value, err = strconv.ParseInt(string(buf[0:len(buf)-2]), 10, 64) //去除了换行符/r/n + if err != nil { + fmt.Printf("Failed to parse long value") + log.Infof("Failed to parse long value") + return 0 + } } *target = value line += 1 return 1 } -func aofLoadManifestFromFile(am_filepath string) *aofManifest { + +// test ok +func AofLoadManifestFromFile(am_filepath string) *aofManifest { var maxseq int64 - am := aofManifestcreate() + am := AofManifestcreate() fp, err := os.Open(am_filepath) if err != nil { - log.Fatalf("Fatal error:can't open the AOF manifest %s for reading: %s", am_filepath, err) + fmt.Printf("Fatal error:can't open the AOF manifest %v for reading: %v", am_filepath, err) + log.Panicf("Fatal error:can't open the AOF manifest %v for reading: %v", am_filepath, err) } var argv []string var ai *aofInfo @@ -255,38 +296,45 @@ func aofLoadManifestFromFile(am_filepath string) *aofManifest { if err != nil { if err == io.EOF { if linenum == 0 { - log.Fatalf("Found an empty AOF manifest") + fmt.Printf("Found an empty AOF manifest\n") + log.Panicf("Found an empty AOF manifest") } else { break } } else { - log.Fatalf("Read AOF manifest failed") + fmt.Printf("Read AOF manifest failed\n") + log.Panicf("Read AOF manifest failed") } } + epos += int64(len(buf)) linenum++ if buf[0] == '#' { continue } if !strings.Contains(buf, "\n") { - log.Fatalf("The AOF manifest file contains too long line") + fmt.Printf("The AOF manifest file contains too long line\n") + log.Panicf("The AOF manifest file contains too long line") } line = strings.Trim(buf, " \t\r\n") if len(line) == 0 { - log.Fatalf("Invalid AOF manifest file format") + fmt.Printf("Invalid AOF manifest file format\n") + log.Panicf("Invalid AOF manifest file format") } argc := 0 - argv, argc = splitArgs(line) + argv, argc = SplitArgs(line) if argc < 6 || argc%2 != 0 { - log.Fatalf("Invalid AOF manifest file format") + fmt.Printf("Invalid AOF manifest file format\n") + log.Panicf("Invalid AOF manifest file format") } - ai = aofInfoCreate() + ai = AofInfoCreate() for i := 0; i < argc; i += 2 { if strings.EqualFold(argv[i], AOF_MANIFEST_KEY_FILE_NAME) { ai.fileName = string(argv[i+1]) - if !pathIsBaseName(string(ai.fileName)) { - log.Fatalf("File can't be a path, just a filename") + if !PathIsBaseName(string(ai.fileName)) { + fmt.Printf("File can't be a path, just a filename\n") + log.Panicf("File can't be a path, just a filename") } } else if strings.EqualFold(argv[i], AOF_MANIFEST_KEY_FILE_SEQ) { ai.fileSeq, _ = strconv.ParseInt(argv[i+1], 10, 64) @@ -295,26 +343,30 @@ func aofLoadManifestFromFile(am_filepath string) *aofManifest { } } if ai.fileName == "" || ai.fileSeq == 0 || ai.aofFileType == "" { - log.Fatalf("Invalid AOF manifest file format") + fmt.Printf("Invalid AOF manifest file format") + log.Panicf("Invalid AOF manifest file format") } //==nil if ai.aofFileType == AofManifestFileTypeBase { if am.baseAofInfo != nil { - log.Fatalf("Found duplicate base file information") + fmt.Printf("Found duplicate base file information\n") + log.Panicf("Found duplicate base file information") } am.baseAofInfo = ai am.currBaseFileSeq = ai.fileSeq } else if ai.aofFileType == AofManifestTypeHist { - am.historyList = listAddNodeTail(am.historyList, ai) + am.historyList = ListAddNodeTail(am.historyList, ai) } else if ai.aofFileType == AofManifestTypeIncr { if ai.fileSeq <= maxseq { - log.Fatalf("Found a non-monotonic sequence number") + fmt.Printf("Found a non-monotonic sequence number\n") + log.Panicf("Found a non-monotonic sequence number") } - am.incrAofList = listAddNodeTail(am.historyList, ai) + am.incrAofList = ListAddNodeTail(am.historyList, ai) am.currIncrFIleSeq = ai.fileSeq maxseq = ai.fileSeq } else { - log.Fatalf("Unknown AOF file type") + fmt.Printf("Unknown AOF file type\n") + log.Panicf("Unknown AOF file type") } line = " " ai = nil @@ -323,30 +375,38 @@ func aofLoadManifestFromFile(am_filepath string) *aofManifest { return am } -func processRESP(fp *os.File, filename string, outMulti *int) int { +// testok? +func ProcessRESP(rd *bufio.Reader, filename string, outMulti *int) int { var argc int64 var str string - - if readArgc(fp, &argc) == 0 { + var err error + /*epos, err := fp.Seek(0, io.SeekCurrent) + pos = epos*/ + if err != nil { + fmt.Printf("Failed to get current position, aborting...\n") + fmt.Println(err) + os.Exit(1) + } + if ReadArgc(rd, &argc) == 0 { return 0 } for i := int64(0); i < argc; i++ { - if readString(fp, &str) == 0 { + if ReadString(rd, &str) == 0 { return 0 } if i == 0 { if strings.EqualFold(str, "multi") { - (*outMulti)++ - if (*outMulti) > 0 { - err := fmt.Errorf("Unexpected MULTI in AOF %s", filename) + if (*outMulti) != 0 { + err := fmt.Errorf("Unexpected MULTI in AOF %v", filename) fmt.Println(err.Error()) return 0 } + (*outMulti)++ } else if strings.EqualFold(str, "exec") { (*outMulti)-- if (*outMulti) != 0 { - err := fmt.Errorf("Unexpected EXEC in AOF %s", filename) + err := fmt.Errorf("Unexpected EXEC in AOF %v", filename) fmt.Println(err.Error()) return 0 } @@ -357,92 +417,63 @@ func processRESP(fp *os.File, filename string, outMulti *int) int { return 1 } +// test ok // 截断可能有问题 -func processAnnotations(fp *os.File, filename string, lastFile bool) int { - - epos, err := fp.Seek(0, io.SeekCurrent) - if err != nil { - fmt.Printf("Failed to get current position, aborting...\n") - os.Exit(1) - } - reader := bufio.NewReader(fp) - buf, _, err := reader.ReadLine() +func ProcessAnnotations(rd *bufio.Reader, filename string, lastFile bool) int { + /* var err error + if err != nil { + fmt.Printf("Failed to get current position, aborting...\n") + fmt.Println(err) + os.Exit(1) + } + */ + buf, _, err := rd.ReadLine() if err != nil { - fmt.Printf("Failed to read annotations from AOF %s, aborting...\n", filename) + fmt.Printf("Failed to read annotations from AOF %v, aborting...\n", filename) os.Exit(1) } + pos += int64(len(buf)) + 2 - if toTimestamp != 0 && strings.HasPrefix(string(buf), "#TS:") { + if toTimestamp != 0 && strings.HasPrefix(string(buf), "TS:") { var ts int64 - ts, err = strconv.ParseInt(strings.TrimPrefix(string(buf), "#TS:"), 10, 64) + ts, err = strconv.ParseInt(strings.TrimPrefix(string(buf), "TS:"), 10, 64) if err != nil { fmt.Println("Invalid timestamp annotation") os.Exit(1) } + if ts <= toTimestamp { return 1 } - if epos == 0 { - fmt.Printf("AOF %s has nothing before timestamp %ld, aborting...\n", filename, toTimestamp) - os.Exit(1) + + if pos == 0 { + fmt.Printf("AOF %v has nothing before timestamp %d, aborting...\n", filename, toTimestamp) + log.Panicf("AOF %v has nothing before timestamp %d, aborting...\n", filename, toTimestamp) } - if lastFile == false { - fmt.Printf("Failed to truncate AOF %s to timestamp %ld to offset %ld because it is not the last file.\n", filename, toTimestamp, epos) - fmt.Println("If you insist, please delete all files after this file according to the manifest file and delete the corresponding records in manifest file manually. Then re-run redis-check-aof.") - os.Exit(1) + + if !lastFile { + fmt.Printf("Failed to truncate AOF %v to timestamp %d to offset %d because it is not the last file.\n", filename, toTimestamp, epos) + log.Infof("Failed to truncate AOF %v to timestamp %d to offset %d because it is not the last file.\n", filename, toTimestamp, epos) + log.Panicf("If you insist, please delete all files after this file according to the manifest file and delete the corresponding records in manifest file manually. Then re-run redis-check-aof.") } + // Truncate remaining AOF if exceeding 'toTimestamp' - if err := fp.Truncate(epos); err != nil { - fmt.Printf("Failed to truncate AOF %s to timestamp %ld\n", filename, toTimestamp) - os.Exit(1) + if err := fp.Truncate(pos); err != nil { + log.Panicf("Failed to truncate AOF %v to timestamp %d\n", filename, toTimestamp) } else { + return 0 } } + return 1 } -/* -func checkRdbMain(argc int, argv *[]string, fp *os.File) int { - - if argc != 2 && fp == nil { - fmt.Fprintf(os.Stderr, "Usage: %s \n", argv[0]) - os.Exit(1) - } else if argv[1] == "-v" || argv[1] == "--version" { - version := checkRdbVersion() - fmt.Printf("redis-check-rdb %s\n", version) - os.Exit(0) - } - - tv := time.Now() - sec := int64(tv.Unix()) - usec := int64(tv.Nanosecond()) / 1000 - pid := int64(os.Getpid()) - seed := ((sec * 1000000) + usec) ^ pid - init_genrand64(seed) - rdbCheckMode = 1 - rdbCheckInfo("Checking RDB file %s", args[1]) - rdbCheckSetupSignals() - retval := redis_Check_RDB(args[1], fp) - if retval == 0 { - rdbCheckInfo("\\o/ RDB looks OK! \\o/") - rdbShowGenericInfo() - } - if fp != nil { - if retval == 0 { - return 0 - } else { - return 1 - } - } - os.Exit(retval) - } -*/ -func checkMultipartAof(dirpath string, manifestFilepath string, fix int) { +func CheckMultipartAof(dirpath string, manifestFilepath string, fix int) { totalNum := 0 aofNum := 0 var ret int - am := aofLoadManifestFromFile(manifestFilepath) + am := AofLoadManifestFromFile(manifestFilepath) if am.baseAofInfo != nil { totalNum++ } @@ -451,130 +482,167 @@ func checkMultipartAof(dirpath string, manifestFilepath string, fix int) { } if am.baseAofInfo != nil { aofFilename := am.baseAofInfo.fileName - aofFilepath := makePath(dirpath, aofFilename) + aofFilepath := MakePath(dirpath, aofFilename) lastFile := (aofNum + 1) == totalNum - aofPreable := fileIsRDB(aofFilepath) + aofPreable := FileIsRDB(aofFilepath) if aofPreable { fmt.Printf("Start to check BASE AOF (RDB format).\n") } else { fmt.Printf("Start to check BASE AOF (AOF format).\n") } - ret = checkSingleAof(aofFilename, aofFilepath, lastFile, fix, aofPreable) - printAofStyle(ret, aofFilename, "BASE AOF") + ret = CheckSingleAof(aofFilename, aofFilepath, lastFile, fix, aofPreable) + PrintAofStyle(ret, aofFilename, "BASE AOF") } if am.incrAofList.len != 0 { - fmt.Printf("start to check INCR INCR files.") + log.Infof("start to check INCR INCR files.") var ln *listNode ln = am.incrAofList.head for ln != nil { ai := ln.value.(*aofInfo) aofFilename := ai.fileName - aofFilepath := makePath(dirpath, aofFilename) + aofFilepath := MakePath(dirpath, aofFilename) lastFile := (aofNum + 1) == totalNum - ret = checkSingleAof(aofFilename, aofFilepath, lastFile, fix, false) - printAofStyle(ret, aofFilename, "INCR AOF") + ret = CheckSingleAof(aofFilename, aofFilepath, lastFile, fix, false) + PrintAofStyle(ret, aofFilename, "INCR AOF") //stringfree(aofFilepath) ln = ln.next } } //aofManifestFree(am) - fmt.Println("All AOF files and manifest are vaild") + log.Infof("All AOF files and manifest are vaild") } -func checkOldStyleAof(aofFilepath string, fix int, preamble bool) { +func CheckOldStyleAof(aofFilepath string, fix int, preamble bool) { fmt.Printf("Start checking Old-Style AOF\n") - var ret = checkSingleAof(aofFilepath, aofFilepath, true, fix, preamble) - printAofStyle(ret, aofFilepath, "AOF") + var ret = CheckSingleAof(aofFilepath, aofFilepath, true, fix, preamble) + PrintAofStyle(ret, aofFilepath, "AOF") } - -func checkSingleAof(aofFilename, aofFilepath string, lastFile bool, fix int, preamble bool) int { - var pos, diff int64 +func CheckSingleAof(aofFilename, aofFilepath string, lastFile bool, fix int, preamble bool) int { + var rdbpos int64 = 0 multi := 0 - buf := make([]byte, 2) - - fp, err := os.OpenFile(aofFilepath, os.O_RDWR, 0666) + epos = 0 + pos = epos + buf := make([]byte, 1) + var err error + fp, err = os.OpenFile(aofFilepath, os.O_RDWR, 0666) if err != nil { - log.Fatalf("Cannot open file %s:%s,aborting...\n", aofFilepath, err) + fmt.Printf("Cannot open file %v:%v,aborting...\n", aofFilepath, err) + log.Panicf("Cannot open file %v:%v,aborting...\n", aofFilepath, err) } sb, err := fp.Stat() if err != nil { - log.Fatalf("Cannot stat file: %s,aborting...\n", aofFilename) + fmt.Printf("Cannot stat file: %v,aborting...\n", aofFilename) + log.Panicf("Cannot stat file: %v,aborting...\n", aofFilename) } size := sb.Size() if size == 0 { return AOF_CHECK_EMPTY } - if preamble != false { - argv := []string{aofFilepath} - err := checkRdbMain(2, argv, fp) - if err == -1 { - log.Fatal("RDB preamble of AOF file is not sane, aborting.") + rd := bufio.NewReader(fp) + if preamble { + + rdbpos = rdb.RedisCheckRDBMain(aofFilepath, fp) + if rdbpos == -1 { + fmt.Printf("RDB preamble of AOF file is not sane, aborting.\n") + log.Panicf("RDB preamble of AOF file is not sane, aborting.") } else { fmt.Println("RDB preamble is OK, proceeding with AOF tail...") + _, err = fp.Seek(rdbpos, io.SeekStart) + if err != nil { + fmt.Printf(("Failed to seek in AOF %v: %v\n"), aofFilename, err) + log.Panicf(("Failed to seek in AOF %v: %v"), aofFilename, err) + } + pos = rdbpos } } + for { - if multi != 0 { + /* if multi == 0 { var err error - pos, err = fp.Seek(0, io.SeekCurrent) + epos, err = fp.Seek(pos, io.SeekStart) if err != nil { - log.Fatalf(("Failed to seek in AOF %s: %s"), aofFilename, err) + fmt.Printf(("Failed to seek in AOF %v: %v\n"), aofFilename, err) + log.Panicf(("Failed to seek in AOF %v: %v"), aofFilename, err) } - } - if _, err := fp.Read(buf); err != nil { + if epos == 0 && preamble { + epos, err = fp.Seek(rdbpos, io.SeekCurrent) + if err != nil { + fmt.Printf(("Failed to seek in AOF %v: %v\n"), aofFilename, err) + log.Panicf(("Failed to seek in AOF %v: %v"), aofFilename, err) + } + } + + pos = epos + if err != nil { + fmt.Printf(("Failed to seek in AOF %v: %v\n"), aofFilename, err) + log.Panicf(("Failed to seek in AOF %v: %v"), aofFilename, err) + } + }*/ + println("568:", pos) + if _, err := rd.Read(buf); err != nil { + if err == io.EOF { + break } - log.Fatalf("Failed to read from AOF %s, aborting...\n", aofFilename) - } - if _, err := fp.Seek(-1, io.SeekCurrent); err != nil { - log.Fatalf("Failed to fseek in AOF %s: %s", aofFilename, err) + fmt.Printf("Failed to read from AOF %v, aborting...\n", aofFilename) + log.Panicf("Failed to read from AOF %v, aborting...\n", aofFilename) } + pos += int64(len(buf)) + /*if _, err := fp.Seek(-1, io.SeekCurrent); err != nil { + fmt.Printf("Failed to fseek in AOF %v: %v\n", aofFilename, err) + log.Panicf("Failed to fseek in AOF %v: %v", aofFilename, err) + }*/ + fmt.Printf("%s\n", buf) switch buf[0] { case '#': - if processAnnotations(fp, aofFilepath, lastFile) == 0 { + if ProcessAnnotations(rd, aofFilepath, lastFile) == 0 { fp.Close() + return AOF_CHECK_TIMESTAMP_TRUNCATED } + break case '*': - if processRESP(fp, aofFilepath, &multi) == 0 { + if ProcessRESP(rd, aofFilepath, &multi) == 0 { + break } default: - fmt.Printf("AOF %s format error\n", aofFilename) + fmt.Printf("AOF %v format error\n", aofFilename) + log.Infof("AOF %v format error\n", aofFilename) break } } - if _, err := fp.Stat(); err == nil && multi == 1 && len(errors) == 0 { + /*if _, err := fp.Stat(); err == nil && multi == 1 { if _, err := fp.Seek(0, io.SeekEnd); err != nil { if _, err := fp.Seek(0, io.SeekCurrent); err == io.EOF { - fmt.Println("Reached EOF before reading EXEC for MULTI") + fmt.Printf("Reached EOF before reading EXEC for MULTI\n") + log.Infof("Reached EOF before reading EXEC for MULTI\n") } } - } + }*/ - if len(errors) > 0 { - fmt.Println(errors) - } + diff := size - pos - diff = size - pos if diff == 0 && toTimestamp == 1 { - fmt.Printf("Truncate nothing in AOF %s to timestamp %d\n", aofFilename, toTimestamp) + fmt.Printf("Truncate nothing in AOF %v to timestamp %d\n", aofFilename, toTimestamp) + log.Infof("Truncate nothing in AOF %v to timestamp %d\n", aofFilename, toTimestamp) return AOF_CHECK_OK } - fmt.Printf("AOF analyzed: filename=%s, size=%d, ok_up_to=%d, ok_up_to_line=%d, diff=%d\n", aofFilename, size, pos, line, diff) + log.Infof("AOF analyzed: filename=%v, size=%d, ok_up_to=%d, ok_up_to_line=%d, diff=%d\n", aofFilename, size, epos, line, diff) if diff > 0 { if fix == 1 { - if lastFile == false { - fmt.Printf("Failed to truncate AOF %s because it is not the last file\n", aofFilename) + if !lastFile { + fmt.Printf("Failed to truncate AOF %v because it is not the last file\n", aofFilename) + log.Panicf("Failed to truncate AOF %v because it is not the last file\n", aofFilename) os.Exit(1) } - fmt.Printf("this will shrink the AOF %s from %d bytes,with %d bytes,to %d bytes\n", aofFilename, size, diff, pos) + fmt.Printf("this will shrink the AOF %v from %d bytes,with %d bytes,to %d bytes\n", aofFilename, size, diff, epos) fmt.Print("Continue? [y/N]: ") reader := bufio.NewReader(os.Stdin) input, err := reader.ReadString('\n') @@ -584,16 +652,17 @@ func checkSingleAof(aofFilename, aofFilepath string, lastFile bool, fix int, pre } if err := fp.Truncate(pos); err != nil { - fmt.Printf("Failed to truncate AOF %s\n", aofFilename) + fmt.Printf("Failed to truncate AOF %v\n", aofFilename) os.Exit(1) } else { return AOF_CHECK_TRUNCATED } } else { - fmt.Printf("AOF %s is not valid.Use the --fix potion to try fixing it.\n", aofFilename) + fmt.Printf("AOF %v is not valid.Use the --fix potion to try fixing it.\n", aofFilename) os.Exit(1) } } fp.Close() + return AOF_CHECK_OK } diff --git a/internal/aof/aof_check_test.go b/internal/aof/aof_check_test.go new file mode 100644 index 00000000..b1bf73ff --- /dev/null +++ b/internal/aof/aof_check_test.go @@ -0,0 +1,27 @@ +package aof + +import ( + "testing" +) + +func TestCheckAofMain(t *testing.T) { + + aofFilePath := "D:/BaiduNetdiskDownload/sa/appendonly.aof.manifest" + + checkResult, fileType, err := CheckAofMain(aofFilePath) + + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + expectedCheckResult := true + if checkResult != expectedCheckResult { + t.Errorf("Unexpected check result. Got: %v, Expected: %v", checkResult, expectedCheckResult) + } + + expectedFileType := "AOF_MULTI_PART" + if string(fileType) != expectedFileType { + t.Errorf("Unexpected file type. Got: %v, Expected: %v", fileType, expectedFileType) + } + +} diff --git a/internal/commands/keys.go b/internal/commands/keys.go index 0899e055..f8045493 100644 --- a/internal/commands/keys.go +++ b/internal/commands/keys.go @@ -2,11 +2,12 @@ package commands import ( "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/utils" "math" "strconv" "strings" + + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/utils" ) // CalcKeys https://redis.io/docs/reference/key-specs/ @@ -114,3 +115,12 @@ findHashTag: } return utils.Crc16(key) & 0x3FFF } + +func LookupCommand(cmd string) int { + _, ok := redisCommands[cmd] + if !ok { + log.Warnf("unknown command. argv=%v", cmd) + return 0 + } + return 1 +} diff --git a/internal/commands/table.go b/internal/commands/table.go index 686e9ade..3f4feb13 100644 --- a/internal/commands/table.go +++ b/internal/commands/table.go @@ -18,6 +18,7 @@ var containers = map[string]bool{ "SENTINEL": true, "SLOWLOG": true, } + var redisCommands = map[string]redisCommand{ "LLEN": { "LIST", @@ -33,7 +34,7 @@ var redisCommands = map[string]redisCommand{ 0, 0, 0, - 0, + 0, }, }, }, diff --git a/internal/config/config.go b/internal/config/config.go index e4b4fbea..32ad6918 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,10 +3,11 @@ package config import ( "bytes" "fmt" - "github.com/pelletier/go-toml/v2" "io/ioutil" "os" "runtime" + + "github.com/pelletier/go-toml/v2" ) type tomlSource struct { @@ -19,8 +20,11 @@ type tomlSource struct { ElastiCachePSync string `toml:"elasticache_psync"` // restore mode - RDBFilePath string `toml:"rdb_file_path"` - AOFFilePath string `toml:"aof_file_path"` // add the aof path + RDBFilePath string `toml:"rdb_file_path"` + AofDirName string `toml:"aof_dirname"` + AofFileName string `toml:"aof_filename"` + AOFFilePath string `toml:"aof_file_path"` // add the aof path + TruncateToTimestamp int64 `toml:"truncate-to-timestamp"` //When reading an AOF file, truncate the file by timestamp. } type tomlTarget struct { @@ -76,7 +80,9 @@ func init() { // restore Config.Source.RDBFilePath = "" Config.Source.AOFFilePath = "" - + Config.Source.TruncateToTimestamp = 0 + Config.Source.AofDirName = "" + Config.Source.AofFileName = "" // target Config.Target.Type = "standalone" Config.Target.Version = 5.0 diff --git a/internal/rdb/rdb.go b/internal/rdb/rdb.go index 45bd426b..fd1172f4 100644 --- a/internal/rdb/rdb.go +++ b/internal/rdb/rdb.go @@ -4,6 +4,11 @@ import ( "bufio" "bytes" "encoding/binary" + "io" + "os" + "strconv" + "time" + "github.com/alibaba/RedisShake/internal/config" "github.com/alibaba/RedisShake/internal/entry" "github.com/alibaba/RedisShake/internal/log" @@ -11,10 +16,6 @@ import ( "github.com/alibaba/RedisShake/internal/rdb/types" "github.com/alibaba/RedisShake/internal/statistics" "github.com/alibaba/RedisShake/internal/utils" - "io" - "os" - "strconv" - "time" ) const ( diff --git a/internal/rdb/rdb_check.go b/internal/rdb/rdb_check.go new file mode 100644 index 00000000..6969b1ab --- /dev/null +++ b/internal/rdb/rdb_check.go @@ -0,0 +1,208 @@ +package rdb + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "strconv" + "time" + + "github.com/alibaba/RedisShake/internal/config" + "github.com/alibaba/RedisShake/internal/entry" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/rdb/structure" + "github.com/alibaba/RedisShake/internal/rdb/types" +) + +func RedisCheckRDBMain(filepath string, fp *os.File) int64 { + if fp == nil { + fmt.Fprintf(os.Stderr, "The file: %s fp is nil", filepath) + os.Exit(1) + } + fmt.Printf("Checking RDB file %s", filepath) + log.Infof("Checking RDB file %s", filepath) + ld := NewLoader(filepath, nil) + + RDBPos := ld.CheckParseRDB() + + if RDBPos > 0 { + fmt.Printf("\\o/ RDB looks OK! \\o/") + log.Infof("\\o/ RDB looks OK! \\o/") + } else { + return -1 + } + return RDBPos +} +func (ld *Loader) CheckParseRDB() int64 { + var err error + ld.fp, err = os.OpenFile(ld.filPath, os.O_RDONLY, 0666) + if err != nil { + log.Panicf("open file failed. file_path=[%s], error=[%s]", ld.filPath, err) + } + defer func() { + err = ld.fp.Close() + if err != nil { + log.Panicf("close file failed. file_path=[%s], error=[%s]", ld.filPath, err) + } + }() + rd := bufio.NewReader(ld.fp) + //magic + version + buf := make([]byte, 9) + _, err = io.ReadFull(rd, buf) + if err != nil { + log.PanicError(err) + } + if !bytes.Equal(buf[:5], []byte("REDIS")) { + log.Panicf("verify magic string, invalid file format. bytes=[%v]", buf[:5]) + } + version, err := strconv.Atoi(string(buf[5:])) + if err != nil { + log.PanicError(err) + } + log.Infof("RDB version: %d", version) + + // read entries + rdbpos := ld.CheckparseRDBEntry(rd) + + return rdbpos +} + +func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { + // for stat + var RDBPos int64 + var rdbsize int64 = 9 + UpdateRDBSize := func() { + var err error + RDBPos, err = ld.fp.Seek(0, io.SeekCurrent) + //println(RDBPos) + if err != nil { + log.PanicError(err) + } + + } + defer UpdateRDBSize() + + // read one entry + tick := time.Tick(time.Second * 1) + for true { + typeByte := structure.ReadByte(rd) + rdbsize += 1 + switch typeByte { + case kFlagIdle: + tempidle, tempOffset := structure.ReadLengthWithOffset(rd) + ld.idle = int64(tempidle) + rdbsize += tempOffset + case kFlagFreq: + ld.freq = int64(structure.ReadByte(rd)) + rdbsize += 1 + case kFlagAUX: + key, tempOffset := structure.ReadStringWithOffset(rd) + rdbsize += tempOffset + value, tempOffset := structure.ReadStringWithOffset(rd) + rdbsize += tempOffset + + if key == "repl-stream-db" { + var err error + ld.replStreamDbId, err = strconv.Atoi(value) + if err != nil { + log.PanicError(err) + } + log.Infof("RDB repl-stream-db: %d position: %d", ld.replStreamDbId, RDBPos) + } else if key == "lua" { + e := entry.NewEntry() + e.Argv = []string{"script", "load", value} + e.IsBase = true + + log.Infof("LUA script: [%s]", value) + } else { + log.Infof("RDB AUX fields. key=[%s], value=[%s]", key, value) + } + case kFlagResizeDB: + dbSize, dbsizeoffset := structure.ReadLengthWithOffset(rd) + expireSize, expiresizeoffset := structure.ReadLengthWithOffset(rd) + rdbsize += dbsizeoffset + expiresizeoffset + log.Infof("RDB resize db. db_size=[%d], expire_size=[%d]", dbSize, expireSize) + case kFlagExpireMs: + ld.expireMs = int64(structure.ReadUint64(rd)) - time.Now().UnixMilli() + rdbsize += 8 + if ld.expireMs < 0 { + ld.expireMs = 1 + } + case kFlagExpire: + ld.expireMs = int64(structure.ReadUint32(rd))*1000 - time.Now().UnixMilli() + rdbsize += 4 + if ld.expireMs < 0 { + ld.expireMs = 1 + } + case kFlagSelect: + DBId, DBIDoffset := structure.ReadLengthWithOffset(rd) + ld.nowDBId = int(DBId) + rdbsize += DBIDoffset + case kEOF: + UpdateRDBSize() + println("rdbsize", rdbsize) + + return rdbsize + default: + key, keyoffset := structure.ReadStringWithOffset(rd) + rdbsize += keyoffset + var value bytes.Buffer + anotherReader := io.TeeReader(rd, &value) + o, tempOffsets := types.ParseObjectWithOffset(anotherReader, typeByte, key) + + rdbsize += tempOffsets + //rdbsize -= tempOffsets + //rdbsize += int64(value.Len()) + if uint64(value.Len()) > config.Config.Advanced.TargetRedisProtoMaxBulkLen { + cmds := o.Rewrite() + for _, cmd := range cmds { + e := entry.NewEntry() + e.IsBase = true + e.DbId = ld.nowDBId + e.Argv = cmd + + } + if ld.expireMs != 0 { + e := entry.NewEntry() + e.IsBase = true + e.DbId = ld.nowDBId + e.Argv = []string{"PEXPIRE", key, strconv.FormatInt(ld.expireMs, 10)} + } + } else { + e := entry.NewEntry() + e.IsBase = true + e.DbId = ld.nowDBId + + v := ld.createValueDump(typeByte, value.Bytes()) + + //value 口口口 + e.Argv = []string{"restore", key, strconv.FormatInt(ld.expireMs, 10), v} + if config.Config.Advanced.RDBRestoreCommandBehavior == "rewrite" { + if config.Config.Target.Version < 3.0 { + log.Panicf("RDB restore command behavior is rewrite, but target redis version is %f, not support REPLACE modifier,position: %d", config.Config.Target.Version, RDBPos) + } + e.Argv = append(e.Argv, "replace") + } + if ld.idle != 0 && config.Config.Target.Version >= 5.0 { + e.Argv = append(e.Argv, "idletime", strconv.FormatInt(ld.idle, 10)) + } + if ld.freq != 0 && config.Config.Target.Version >= 5.0 { + e.Argv = append(e.Argv, "freq", strconv.FormatInt(ld.freq, 10)) + } + + } + ld.expireMs = 0 + ld.idle = 0 + ld.freq = 0 + } + select { + case <-tick: + UpdateRDBSize() + default: + } + UpdateRDBSize() + } + return RDBPos +} diff --git a/internal/rdb/structure/float.go b/internal/rdb/structure/float.go index 8351d96a..580af530 100644 --- a/internal/rdb/structure/float.go +++ b/internal/rdb/structure/float.go @@ -2,10 +2,11 @@ package structure import ( "encoding/binary" - "github.com/alibaba/RedisShake/internal/log" "io" "math" "strconv" + + "github.com/alibaba/RedisShake/internal/log" ) func ReadFloat(rd io.Reader) float64 { @@ -33,6 +34,30 @@ func ReadFloat(rd io.Reader) float64 { } } +func ReadFloatWithOffset(rd io.Reader) (float64, int64) { + u := ReadUint8(rd) + switch u { + case 253: + return math.NaN(), 1 + case 254: + return math.Inf(0), 1 + case 255: + return math.Inf(-1), 1 + default: + buf := make([]byte, u) + _, err := io.ReadFull(rd, buf) + if err != nil { + return 0, 2 + } + + v, err := strconv.ParseFloat(string(buf), 64) + if err != nil { + log.PanicError(err) + } + return v, 2 + } +} + func ReadDouble(rd io.Reader) float64 { var buf = make([]byte, 8) _, err := io.ReadFull(rd, buf) diff --git a/internal/rdb/structure/intset.go b/internal/rdb/structure/intset.go index a829cd60..e440287e 100644 --- a/internal/rdb/structure/intset.go +++ b/internal/rdb/structure/intset.go @@ -30,3 +30,27 @@ func ReadIntset(rd io.Reader) []string { } return elements } + +func ReadIntsetWithOffset(rd io.Reader) ([]string, int64) { + tempString, offset := ReadStringWithOffset(rd) + rd = bufio.NewReader(strings.NewReader(tempString)) + + encodingType := int(ReadUint32(rd)) + size := int(ReadUint32(rd)) + elements := make([]string, size) + + for i := 0; i < size; i++ { + intBytes := ReadBytes(rd, encodingType) + var intString string + switch encodingType { + case 2: + intString = strconv.FormatInt(int64(int16(binary.LittleEndian.Uint16(intBytes))), 10) + case 4: + intString = strconv.FormatInt(int64(int32(binary.LittleEndian.Uint32(intBytes))), 10) + case 8: + intString = strconv.FormatInt(int64(int64(binary.LittleEndian.Uint64(intBytes))), 10) + } + elements[i] = intString + } + return elements, offset +} diff --git a/internal/rdb/structure/length.go b/internal/rdb/structure/length.go index e62543fe..4e9ecce3 100644 --- a/internal/rdb/structure/length.go +++ b/internal/rdb/structure/length.go @@ -3,8 +3,9 @@ package structure import ( "encoding/binary" "fmt" - "github.com/alibaba/RedisShake/internal/log" "io" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -29,8 +30,9 @@ func ReadLength(rd io.Reader) uint64 { func readEncodedLength(rd io.Reader) (length uint64, special bool, err error) { var lengthBuffer = make([]byte, 8) - + var offset int64 = 0 firstByte := ReadByte(rd) + offset = 1 first2bits := (firstByte & 0xc0) >> 6 // first 2 bits of encoding switch first2bits { case RDB6ByteLen: @@ -38,6 +40,7 @@ func readEncodedLength(rd io.Reader) (length uint64, special bool, err error) { case RDB14ByteLen: nextByte := ReadByte(rd) length = (uint64(firstByte)&0x3f)<<8 | uint64(nextByte) + offset += 1 case len32or64Bit: if firstByte == RDB32ByteLen { _, err = io.ReadFull(rd, lengthBuffer[0:4]) @@ -45,12 +48,14 @@ func readEncodedLength(rd io.Reader) (length uint64, special bool, err error) { return 0, false, fmt.Errorf("read len32Bit failed: %s", err.Error()) } length = uint64(binary.BigEndian.Uint32(lengthBuffer)) + offset += 4 } else if firstByte == RDB64ByteLen { _, err = io.ReadFull(rd, lengthBuffer) if err != nil { return 0, false, fmt.Errorf("read len64Bit failed: %s", err.Error()) } length = binary.BigEndian.Uint64(lengthBuffer) + offset += 8 } else { return 0, false, fmt.Errorf("illegal length encoding: %x", firstByte) } @@ -60,3 +65,52 @@ func readEncodedLength(rd io.Reader) (length uint64, special bool, err error) { } return length, special, nil } + +func ReadLengthWithOffset(rd io.Reader) (uint64, int64) { + length, special, offset, err := readEncodedLengthWithOffset(rd) + if special { + log.Panicf("illegal length special=true, encoding: %d", length) + } + if err != nil { + log.PanicError(err) + } + return length, offset +} +func readEncodedLengthWithOffset(rd io.Reader) (length uint64, special bool, offsets int64, err error) { + var lengthBuffer = make([]byte, 8) + var offset int64 = 0 + firstByte := ReadByte(rd) + offset = 1 + first2bits := (firstByte & 0xc0) >> 6 // first 2 bits of encoding + switch first2bits { + case RDB6ByteLen: + length = uint64(firstByte) & 0x3f + case RDB14ByteLen: + nextByte := ReadByte(rd) + length = (uint64(firstByte)&0x3f)<<8 | uint64(nextByte) + offset += 1 + case len32or64Bit: + if firstByte == RDB32ByteLen { + _, err = io.ReadFull(rd, lengthBuffer[0:4]) + offset += 4 + if err != nil { + return 0, false, offset, fmt.Errorf("read len32Bit failed: %s", err.Error()) + } + length = uint64(binary.BigEndian.Uint32(lengthBuffer)) + + } else if firstByte == RDB64ByteLen { + _, err = io.ReadFull(rd, lengthBuffer) + offset += 8 + if err != nil { + return 0, false, offset, fmt.Errorf("read len64Bit failed: %s", err.Error()) + } + length = binary.BigEndian.Uint64(lengthBuffer) + } else { + return 0, false, offset, fmt.Errorf("illegal length encoding: %x", firstByte) + } + case lenSpecial: + special = true + length = uint64(firstByte) & 0x3f + } + return length, special, offset, nil +} diff --git a/internal/rdb/structure/listpack.go b/internal/rdb/structure/listpack.go index 2d30374a..65e93d94 100644 --- a/internal/rdb/structure/listpack.go +++ b/internal/rdb/structure/listpack.go @@ -2,11 +2,12 @@ package structure import ( "bufio" - "github.com/alibaba/RedisShake/internal/log" "io" "math" "strconv" "strings" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -54,6 +55,23 @@ func ReadListpack(rd io.Reader) []string { } return elements } +func ReadListpackWithOffset(rd io.Reader) ([]string, int64) { + tempstring, offset := ReadStringWithOffset(rd) + rd = bufio.NewReader(strings.NewReader(tempstring)) + + _ = ReadUint32(rd) // bytes + size := int(ReadUint16(rd)) + var elements []string + for i := 0; i < size; i++ { + ele := readListpackEntry(rd) + elements = append(elements, ele) + } + lastByte := ReadByte(rd) + if lastByte != 0xFF { + log.Panicf("ReadListpack: last byte is not 0xFF, but [%d]", lastByte) + } + return elements, offset +} // redis/src/Listpack.c lpGet() func readListpackEntry(rd io.Reader) string { diff --git a/internal/rdb/structure/string.go b/internal/rdb/structure/string.go index f53559e3..3c973dd5 100644 --- a/internal/rdb/structure/string.go +++ b/internal/rdb/structure/string.go @@ -1,9 +1,10 @@ package structure import ( - "github.com/alibaba/RedisShake/internal/log" "io" "strconv" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -41,6 +42,40 @@ func ReadString(rd io.Reader) string { } return string(ReadBytes(rd, int(length))) } +func ReadStringWithOffset(rd io.Reader) (string, int64) { + length, special, offset, err := readEncodedLengthWithOffset(rd) + if err != nil { + log.PanicError(err) + } + if special { + switch length { + case RDBEncInt8: + b := ReadInt8(rd) + offset += 1 + return strconv.Itoa(int(b)), offset + case RDBEncInt16: + b := ReadInt16(rd) + offset += 2 + return strconv.Itoa(int(b)), offset + case RDBEncInt32: + b := ReadInt32(rd) + offset += 4 + return strconv.Itoa(int(b)), offset + case RDBEncLZF: + inLen, inlenoffset := ReadLengthWithOffset(rd) + offset += inlenoffset + outLen, outLenoffset := ReadLengthWithOffset(rd) + offset += outLenoffset + in := ReadBytes(rd, int(inLen)) + offset += int64(inLen) + return lzfDecompress(in, int(outLen)), offset + default: + log.Panicf("Unknown string encode type %d", length) + } + } + offset += int64(length) + return string(ReadBytes(rd, int(length))), offset +} func lzfDecompress(in []byte, outLen int) string { out := make([]byte, outLen) diff --git a/internal/rdb/structure/ziplist.go b/internal/rdb/structure/ziplist.go index 141ea026..3a7b370e 100644 --- a/internal/rdb/structure/ziplist.go +++ b/internal/rdb/structure/ziplist.go @@ -3,10 +3,11 @@ package structure import ( "bufio" "encoding/binary" - "github.com/alibaba/RedisShake/internal/log" "io" "strconv" "strings" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -24,6 +25,7 @@ const ( ) func ReadZipList(rd io.Reader) []string { + rd = bufio.NewReader(strings.NewReader(ReadString(rd))) // The general layout of the ziplist is as follows: @@ -51,6 +53,36 @@ func ReadZipList(rd io.Reader) []string { return elements } +func ReadZipListWithOffset(rd io.Reader) ([]string, int64) { + + Strings, offset := ReadStringWithOffset(rd) + rd = bufio.NewReader(strings.NewReader(Strings)) + + // The general layout of the ziplist is as follows: + // ... + _ = ReadUint32(rd) // zlbytes + _ = ReadUint32(rd) // zltail + + size := int(ReadUint16(rd)) + var elements []string + if size == 65535 { // 2^16-1, we need to traverse the entire list to know how many items it holds. + for firstByte := ReadByte(rd); firstByte != 0xFE; firstByte = ReadByte(rd) { + ele := readZipListEntry(rd, firstByte) + elements = append(elements, ele) + } + } else { + for i := 0; i < size; i++ { + firstByte := ReadByte(rd) + ele := readZipListEntry(rd, firstByte) + elements = append(elements, ele) + } + if lastByte := ReadByte(rd); lastByte != 0xFF { + log.Panicf("invalid zipList lastByte encoding: %d", lastByte) + } + } + return elements, offset +} + /* * So practically an entry is encoded in the following way: * diff --git a/internal/rdb/types/hash.go b/internal/rdb/types/hash.go index e27b67e8..5a9785e9 100644 --- a/internal/rdb/types/hash.go +++ b/internal/rdb/types/hash.go @@ -1,9 +1,10 @@ package types import ( + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) type HashObject struct { @@ -28,6 +29,27 @@ func (o *HashObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { } } +func (o *HashObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + o.key = key + o.value = make(map[string]string) + switch typeByte { + case rdbTypeHash: + offset := o.readHashWithOffset(rd) + return offset + case rdbTypeHashZipmap: + o.readHashZipmap(rd) + case rdbTypeHashZiplist: + offset := o.readHashZiplistWithOffset(rd) + return offset + case rdbTypeHashListpack: + offset := o.readHashListpackWithOffset(rd) + return offset + default: + log.Panicf("unknown hash type. typeByte=[%d]", typeByte) + return 0 + } + return 0 +} func (o *HashObject) readHash(rd io.Reader) { size := int(structure.ReadLength(rd)) for i := 0; i < size; i++ { @@ -37,6 +59,18 @@ func (o *HashObject) readHash(rd io.Reader) { } } +func (o *HashObject) readHashWithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + for i := 0; i < int(size); i++ { + key, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + value, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + o.value[key] = value + } + return offset +} + func (o *HashObject) readHashZipmap(rd io.Reader) { log.Panicf("not implemented rdbTypeZipmap") } @@ -51,6 +85,17 @@ func (o *HashObject) readHashZiplist(rd io.Reader) { } } +func (o *HashObject) readHashZiplistWithOffset(rd io.Reader) int64 { + list, offset := structure.ReadZipListWithOffset(rd) + size := len(list) + for i := 0; i < size; i += 2 { + key := list[i] + value := list[i+1] + o.value[key] = value + } + return offset +} + func (o *HashObject) readHashListpack(rd io.Reader) { list := structure.ReadListpack(rd) size := len(list) @@ -59,6 +104,18 @@ func (o *HashObject) readHashListpack(rd io.Reader) { value := list[i+1] o.value[key] = value } + +} + +func (o *HashObject) readHashListpackWithOffset(rd io.Reader) int64 { + list, offset := structure.ReadListpackWithOffset(rd) + size := len(list) + for i := 0; i < size; i += 2 { + key := list[i] + value := list[i+1] + o.value[key] = value + } + return offset } func (o *HashObject) Rewrite() []RedisCmd { diff --git a/internal/rdb/types/interface.go b/internal/rdb/types/interface.go index 7d9fb3d5..b72d331a 100644 --- a/internal/rdb/types/interface.go +++ b/internal/rdb/types/interface.go @@ -1,8 +1,9 @@ package types import ( - "github.com/alibaba/RedisShake/internal/log" "io" + + "github.com/alibaba/RedisShake/internal/log" ) const ( @@ -99,6 +100,41 @@ func ParseObject(rd io.Reader, typeByte byte, key string) RedisObject { return nil } +func ParseObjectWithOffset(rd io.Reader, typeByte byte, key string) (RedisObject, int64) { + switch typeByte { + case rdbTypeString: // string + o := new(StringObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeList, rdbTypeListZiplist, rdbTypeListQuicklist, rdbTypeListQuicklist2: // list + o := new(ListObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeSet, rdbTypeSetIntset: // set + o := new(SetObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeZSet, rdbTypeZSet2, rdbTypeZSetZiplist, rdbTypeZSetListpack: // zset + o := new(ZsetObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeHash, rdbTypeHashZipmap, rdbTypeHashZiplist, rdbTypeHashListpack: // hash + o := new(HashObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeStreamListpacks, rdbTypeStreamListpacks2: // stream + o := new(StreamObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + case rdbTypeModule, rdbTypeModule2: // module + o := new(ModuleObject) + offset := o.LoadFromBufferWithOffset(rd, key, typeByte) + return o, offset + } + log.Panicf("unknown type byte: %d", typeByte) + return nil, 0 +} + func moduleTypeNameByID(moduleId uint64) string { nameList := make([]byte, 9) moduleId >>= 10 diff --git a/internal/rdb/types/list.go b/internal/rdb/types/list.go index 57e23c08..73fae7d5 100644 --- a/internal/rdb/types/list.go +++ b/internal/rdb/types/list.go @@ -1,9 +1,10 @@ package types import ( + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) // quicklist node container formats @@ -33,6 +34,27 @@ func (o *ListObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { log.Panicf("unknown list type %d", typeByte) } } +func (o *ListObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + var offset int64 = 0 + o.key = key + switch typeByte { + case rdbTypeList: + offset = o.readListWithOffset(rd) + return offset + case rdbTypeListZiplist: + o.elements, offset = structure.ReadZipListWithOffset(rd) + return offset + case rdbTypeListQuicklist: + offset = o.readQuickListWithOffset(rd) + return offset + case rdbTypeListQuicklist2: + offset = o.readQuickList2WithOffset(rd) + return offset + default: + log.Panicf("unknown list type %d", typeByte) + } + return offset +} func (o *ListObject) Rewrite() []RedisCmd { cmds := make([]RedisCmd, len(o.elements)) @@ -50,6 +72,16 @@ func (o *ListObject) readList(rd io.Reader) { o.elements = append(o.elements, ele) } } +func (o *ListObject) readListWithOffset(rd io.Reader) int64 { + sizes, offset := structure.ReadLengthWithOffset(rd) + size := int(sizes) + for i := 0; i < size; i++ { + ele, offsets := structure.ReadStringWithOffset(rd) + offset += offsets + o.elements = append(o.elements, ele) + } + return offset +} func (o *ListObject) readQuickList(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -58,6 +90,15 @@ func (o *ListObject) readQuickList(rd io.Reader) { o.elements = append(o.elements, ziplistElements...) } } +func (o *ListObject) readQuickListWithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + for i := 0; i < int(size); i++ { + ziplistElements, offsets := structure.ReadZipListWithOffset(rd) + offset += offsets + o.elements = append(o.elements, ziplistElements...) + } + return offset +} func (o *ListObject) readQuickList2(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -74,3 +115,23 @@ func (o *ListObject) readQuickList2(rd io.Reader) { } } } + +func (o *ListObject) readQuickList2WithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + for i := 0; i < int(size); i++ { + container, offsets := structure.ReadLengthWithOffset(rd) + offset += offsets + if container == quicklistNodeContainerPlain { + ele, offsets := structure.ReadStringWithOffset(rd) + offset += offsets + o.elements = append(o.elements, ele) + } else if container == quicklistNodeContainerPacked { + listpackElements, TempOffsets := structure.ReadListpackWithOffset(rd) + offset += TempOffsets + o.elements = append(o.elements, listpackElements...) + } else { + log.Panicf("unknown quicklist container %d", container) + } + } + return offset +} diff --git a/internal/rdb/types/module2.go b/internal/rdb/types/module2.go index 26ab46e0..a6c4492a 100644 --- a/internal/rdb/types/module2.go +++ b/internal/rdb/types/module2.go @@ -1,9 +1,10 @@ package types import ( + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) type ModuleObject struct { @@ -34,6 +35,38 @@ func (o *ModuleObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { } } +func (o *ModuleObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + if typeByte == rdbTypeModule { + log.Panicf("module type with version 1 is not supported, key=[%s]", key) + } + moduleId, offset := structure.ReadLengthWithOffset(rd) + moduleName := moduleTypeNameByID(moduleId) + opcode := structure.ReadByte(rd) + offset += 1 + for opcode != rdbModuleOpcodeEOF { + switch opcode { + case rdbModuleOpcodeSINT: + case rdbModuleOpcodeUINT: + _, tempOffset := structure.ReadLengthWithOffset(rd) + offset += tempOffset + case rdbModuleOpcodeFLOAT: + _, tempOffset := structure.ReadFloatWithOffset(rd) + offset += tempOffset + case rdbModuleOpcodeDOUBLE: + structure.ReadDouble(rd) + offset += 8 + case rdbModuleOpcodeSTRING: + _, tempOffset := structure.ReadStringWithOffset(rd) + offset += tempOffset + default: + log.Panicf("unknown module opcode=[%d], module name=[%s]", opcode, moduleName) + } + opcode = structure.ReadByte(rd) + offset += 1 + } + return offset +} + func (o *ModuleObject) Rewrite() []RedisCmd { log.Panicf("module Rewrite not implemented") return nil diff --git a/internal/rdb/types/set.go b/internal/rdb/types/set.go index ac02f378..807cd173 100644 --- a/internal/rdb/types/set.go +++ b/internal/rdb/types/set.go @@ -1,9 +1,10 @@ package types import ( + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) type SetObject struct { @@ -22,6 +23,21 @@ func (o *SetObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { log.Panicf("unknown set type. typeByte=[%d]", typeByte) } } +func (o *SetObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + o.key = key + switch typeByte { + case rdbTypeSet: + tempOffset := o.readSetWithOffset(rd) + return tempOffset + case rdbTypeSetIntset: + var tempOffsets int64 + o.elements, tempOffsets = structure.ReadIntsetWithOffset(rd) + return tempOffsets + default: + log.Panicf("unknown set type. typeByte=[%d]", typeByte) + } + return 0 +} func (o *SetObject) readSet(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -31,6 +47,16 @@ func (o *SetObject) readSet(rd io.Reader) { o.elements[i] = val } } +func (o *SetObject) readSetWithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + o.elements = make([]string, size) + for i := 0; i < int(size); i++ { + val, TempOffsets := structure.ReadStringWithOffset(rd) + offset += TempOffsets + o.elements[i] = val + } + return offset +} func (o *SetObject) Rewrite() []RedisCmd { cmds := make([]RedisCmd, len(o.elements)) diff --git a/internal/rdb/types/stream.go b/internal/rdb/types/stream.go index 9b0a6b03..d7df7016 100644 --- a/internal/rdb/types/stream.go +++ b/internal/rdb/types/stream.go @@ -3,10 +3,11 @@ package types import ( "encoding/binary" "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb/structure" "io" "strconv" + + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/rdb/structure" ) /* @@ -56,6 +57,22 @@ func (o *StreamObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { } } +func (o *StreamObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + o.key = key + switch typeByte { + case rdbTypeStreamListpacks: + offset := o.readStreamWithOffset(rd, key, typeByte) + return offset + case rdbTypeStreamListpacks2: + offset := o.readStreamWithOffset(rd, key, typeByte) + return offset + default: + log.Panicf("unknown hash type. typeByte=[%d]", typeByte) + return 0 + } + +} + // see redis rewriteStreamObject() func (o *StreamObject) readStream(rd io.Reader, masterKey string, typeByte byte) { @@ -227,6 +244,192 @@ func (o *StreamObject) readStream(rd io.Reader, masterKey string, typeByte byte) } } +func (o *StreamObject) readStreamWithOffset(rd io.Reader, masterKey string, typeByte byte) int64 { + // 1. length(number of listpack), k1, v1, k2, v2, ..., number, ms, seq + + /* Load the number of Listpack. */ + nListpack, offset := structure.ReadLengthWithOffset(rd) + for i := 0; i < int(nListpack); i++ { + /* Load key */ + key, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + /* key is streamId, like: 1612181627287-0 */ + masterMs := int64(binary.BigEndian.Uint64([]byte(key[:8]))) + masterSeq := int64(binary.BigEndian.Uint64([]byte(key[8:]))) + + /* value is a listpack */ + elements, tempOffsets := structure.ReadListpackWithOffset(rd) + offset += tempOffsets + inx := 0 + + /* The front of stream listpack is master entry */ + /* Parse the master entry */ + count := nextInteger(&inx, elements) // count + deleted := nextInteger(&inx, elements) // deleted + numFields := int(nextInteger(&inx, elements)) // num-fields + + fields := elements[3 : 3+numFields] // fields + inx = 3 + numFields + + // master entry end by zero + lastEntry := nextString(&inx, elements) + if lastEntry != "0" { + log.Panicf("master entry not ends by zero. lastEntry=[%s]", lastEntry) + } + + /* Parse entries */ + for count != 0 || deleted != 0 { + flags := nextInteger(&inx, elements) // [is_same_fields|is_deleted] + entryMs := nextInteger(&inx, elements) + entrySeq := nextInteger(&inx, elements) + + args := []string{"xadd", masterKey, fmt.Sprintf("%v-%v", entryMs+masterMs, entrySeq+masterSeq)} + + if flags&2 == 2 { // same fields, get field from master entry. + for j := 0; j < numFields; j++ { + args = append(args, fields[j], nextString(&inx, elements)) + } + } else { // get field by lp.Next() + num := int(nextInteger(&inx, elements)) + args = append(args, elements[inx:inx+num*2]...) + inx += num * 2 + } + + _ = nextString(&inx, elements) // lp_count + + if flags&1 == 1 { // is_deleted + deleted -= 1 + } else { + count -= 1 + o.cmds = append(o.cmds, args) + } + } + } + + /* Load total number of items inside the stream. */ + _, tempOffsets := structure.ReadLengthWithOffset(rd) // number + offset += tempOffsets + /* Load the last entry ID. */ + lastMs, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + lastSeq, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + lastid := fmt.Sprintf("%v-%v", lastMs, lastSeq) + if nListpack == 0 { + /* Use the XADD MAXLEN 0 trick to generate an empty stream if + * the key we are serializing is an empty string, which is possible + * for the Stream type. */ + args := []string{"xadd", masterKey, "MAXLEN", "0", lastid, "x", "y"} + o.cmds = append(o.cmds, args) + } + + /* Append XSETID after XADD, make sure lastid is correct, + * in case of XDEL lastid. */ + o.cmds = append(o.cmds, []string{"xsetid", masterKey, lastid}) + + if typeByte == rdbTypeStreamListpacks2 { + /* Load the first entry ID. */ + _, tempOffsets = structure.ReadLengthWithOffset(rd) // first_ms + offset += tempOffsets + _, tempOffsets = structure.ReadLengthWithOffset(rd) // first_seq + offset += tempOffsets + /* Load the maximal deleted entry ID. */ + _, tempOffsets = structure.ReadLengthWithOffset(rd) // max_deleted_ms + offset += tempOffsets + _, tempOffsets = structure.ReadLengthWithOffset(rd) // max_deleted_seq + offset += tempOffsets + + /* Load the offset. */ + _, tempOffsets = structure.ReadLengthWithOffset(rd) // offset + offset += tempOffsets + } + + /* 2. nConsumerGroup, groupName, ms, seq, PEL, Consumers */ + + /* Load the number of groups. */ + nConsumerGroup, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + for i := 0; i < int(nConsumerGroup); i++ { + /* Load groupName */ + groupName, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + /* Load the last ID */ + lastMs, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + lastSeq, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + lastid := fmt.Sprintf("%v-%v", lastMs, lastSeq) + + /* Create Group */ + o.cmds = append(o.cmds, []string{"CREATE", masterKey, groupName, lastid}) + + /* Load group offset. */ + if typeByte == rdbTypeStreamListpacks2 { + _, tempOffsets = structure.ReadLengthWithOffset(rd) // offset + offset += tempOffsets + } + + /* Load the global PEL */ + nPel, tempOffsets := structure.ReadLengthWithOffset(rd) + mapId2Time := make(map[string]uint64) + mapId2Count := make(map[string]uint64) + + for j := 0; j < int(nPel); j++ { + /* Load streamId */ + tmpBytes := structure.ReadBytes(rd, 16) + offset += 16 + ms := binary.BigEndian.Uint64(tmpBytes[:8]) + seq := binary.BigEndian.Uint64(tmpBytes[8:]) + streamId := fmt.Sprintf("%v-%v", ms, seq) + + /* Load deliveryTime */ + deliveryTime := structure.ReadUint64(rd) + offset += 8 + /* Load deliveryCount */ + deliveryCount, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + /* Save deliveryTime and deliveryCount */ + mapId2Time[streamId] = deliveryTime + mapId2Count[streamId] = deliveryCount + } + + /* Generate XCLAIMs for each consumer that happens to + * have pending entries. Empty consumers are discarded. */ + nConsumer, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + for j := 0; j < int(nConsumer); j++ { + /* Load consumerName */ + consumerName, tempOffsets := structure.ReadStringWithOffset(rd) + offset += tempOffsets + + /* Load lastSeenTime */ + _ = structure.ReadUint64(rd) + offset += 8 + /* Consumer PEL */ + nPEL, tempOffsets := structure.ReadLengthWithOffset(rd) + offset += tempOffsets + for i := 0; i < int(nPEL); i++ { + + /* Load streamId */ + tmpBytes := structure.ReadBytes(rd, 16) + offset += 16 + ms := binary.BigEndian.Uint64(tmpBytes[:8]) + seq := binary.BigEndian.Uint64(tmpBytes[8:]) + streamId := fmt.Sprintf("%v-%v", ms, seq) + + /* Send */ + args := []string{ + "xclaim", masterKey, groupName, consumerName, "0", streamId, + "TIME", strconv.FormatUint(mapId2Time[streamId], 10), + "RETRYCOUNT", strconv.FormatUint(mapId2Count[streamId], 10), + "JUSTID", "FORCE"} + o.cmds = append(o.cmds, args) + } + } + } + return offset +} + func nextInteger(inx *int, elements []string) int64 { ele := elements[*inx] *inx++ diff --git a/internal/rdb/types/string.go b/internal/rdb/types/string.go index 2adf51c6..6cb07b06 100644 --- a/internal/rdb/types/string.go +++ b/internal/rdb/types/string.go @@ -1,8 +1,9 @@ package types import ( - "github.com/alibaba/RedisShake/internal/rdb/structure" "io" + + "github.com/alibaba/RedisShake/internal/rdb/structure" ) type StringObject struct { @@ -14,7 +15,12 @@ func (o *StringObject) LoadFromBuffer(rd io.Reader, key string, _ byte) { o.key = key o.value = structure.ReadString(rd) } - +func (o *StringObject) LoadFromBufferWithOffset(rd io.Reader, key string, _ byte) int64 { + var offsets int64 + o.key = key + o.value, offsets = structure.ReadStringWithOffset(rd) + return offsets +} func (o *StringObject) Rewrite() []RedisCmd { cmd := RedisCmd{} cmd = append(cmd, "set", o.key, o.value) diff --git a/internal/rdb/types/zset.go b/internal/rdb/types/zset.go index 34f99b37..fc2b0611 100644 --- a/internal/rdb/types/zset.go +++ b/internal/rdb/types/zset.go @@ -2,9 +2,10 @@ package types import ( "fmt" + "io" + "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb/structure" - "io" ) type ZSetEntry struct { @@ -32,6 +33,26 @@ func (o *ZsetObject) LoadFromBuffer(rd io.Reader, key string, typeByte byte) { log.Panicf("unknown zset type. typeByte=[%d]", typeByte) } } +func (o *ZsetObject) LoadFromBufferWithOffset(rd io.Reader, key string, typeByte byte) int64 { + o.key = key + switch typeByte { + case rdbTypeZSet: + offset := o.readZsetWithOffset(rd) + return offset + case rdbTypeZSet2: + offset := o.readZset2WithOffset(rd) + return offset + case rdbTypeZSetZiplist: + offset := o.readZsetZiplistWithOffset(rd) + return offset + case rdbTypeZSetListpack: + offset := o.readZsetListpackWithOffset(rd) + return offset + default: + log.Panicf("unknown zset type. typeByte=[%d]", typeByte) + } + return 0 +} func (o *ZsetObject) readZset(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -42,6 +63,19 @@ func (o *ZsetObject) readZset(rd io.Reader) { o.elements[i].Score = fmt.Sprintf("%f", score) } } +func (o *ZsetObject) readZsetWithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + o.elements = make([]ZSetEntry, size) + var tempOffsets int64 = 0 + for i := 0; i < int(size); i++ { + o.elements[i].Member, tempOffsets = structure.ReadStringWithOffset(rd) + offset += tempOffsets + score, tempOffsets := structure.ReadFloatWithOffset(rd) + offset += tempOffsets + o.elements[i].Score = fmt.Sprintf("%f", score) + } + return offset +} func (o *ZsetObject) readZset2(rd io.Reader) { size := int(structure.ReadLength(rd)) @@ -52,6 +86,19 @@ func (o *ZsetObject) readZset2(rd io.Reader) { o.elements[i].Score = fmt.Sprintf("%f", score) } } +func (o *ZsetObject) readZset2WithOffset(rd io.Reader) int64 { + size, offset := structure.ReadLengthWithOffset(rd) + o.elements = make([]ZSetEntry, size) + var tempOffsets int64 + for i := 0; i < int(size); i++ { + o.elements[i].Member, tempOffsets = structure.ReadStringWithOffset(rd) + offset += tempOffsets + score := structure.ReadDouble(rd) + offset += 8 + o.elements[i].Score = fmt.Sprintf("%f", score) + } + return offset +} func (o *ZsetObject) readZsetZiplist(rd io.Reader) { list := structure.ReadZipList(rd) @@ -66,6 +113,19 @@ func (o *ZsetObject) readZsetZiplist(rd io.Reader) { } } +func (o *ZsetObject) readZsetZiplistWithOffset(rd io.Reader) int64 { + list, offset := structure.ReadZipListWithOffset(rd) + size := len(list) + if size%2 != 0 { + log.Panicf("zset listpack size is not even. size=[%d]", size) + } + o.elements = make([]ZSetEntry, size/2) + for i := 0; i < size; i += 2 { + o.elements[i/2].Member = list[i] + o.elements[i/2].Score = list[i+1] + } + return offset +} func (o *ZsetObject) readZsetListpack(rd io.Reader) { list := structure.ReadListpack(rd) size := len(list) @@ -79,6 +139,20 @@ func (o *ZsetObject) readZsetListpack(rd io.Reader) { } } +func (o *ZsetObject) readZsetListpackWithOffset(rd io.Reader) int64 { + list, offset := structure.ReadListpackWithOffset(rd) + size := len(list) + if size%2 != 0 { + log.Panicf("zset listpack size is not even. size=[%d]", size) + } + o.elements = make([]ZSetEntry, size/2) + for i := 0; i < size; i += 2 { + o.elements[i/2].Member = list[i] + o.elements[i/2].Score = list[i+1] + } + return offset +} + func (o *ZsetObject) Rewrite() []RedisCmd { cmds := make([]RedisCmd, len(o.elements)) for inx, ele := range o.elements { diff --git a/internal/reader/aof_reader.go b/internal/reader/aof_reader.go index c4fedcf4..2748adf8 100644 --- a/internal/reader/aof_reader.go +++ b/internal/reader/aof_reader.go @@ -3,7 +3,14 @@ package reader // this file references rdb_reader.go import ( + "os" + "path" + "path/filepath" + + "github.com/alibaba/RedisShake/internal/aof" "github.com/alibaba/RedisShake/internal/entry" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/statistics" ) type aofReader struct { @@ -13,13 +20,55 @@ type aofReader struct { // TODO:待完善参考rdb reader func NewAOFReader(path string) Reader { - - return nil + log.Infof("NewAOFReader: path=[%s]", path) + absolutePath, err := filepath.Abs(path) + if err != nil { + log.Panicf("NewAOFReader: filepath.Abs error: %s", err.Error()) + } + log.Infof("NewAOFReader: absolute path=[%s]", absolutePath) + r := &aofReader{ + path: absolutePath, + ch: make(chan *entry.Entry), + } + return r } func (r *aofReader) StartRead() chan *entry.Entry { - //调用 aof中的函数 - // aof.NewLoader() - // aof.ParseRDB() - return nil + r.ch = make(chan *entry.Entry, 1024) + + go func() { + // 开始解析 AOF 文件 + + aof.AofLoadManifestFromDisk() + am := aof.AOFINFO.GetAofManifest() + + if am == nil { + log.Infof("start send AOF。path=[%s]", r.path) + fi, err := os.Stat(r.path) + if err != nil { + log.Panicf("NewAOFReader: os.Stat error：%s", err.Error()) + } + statistics.Metrics.AofFileSize = uint64(fi.Size()) + statistics.Metrics.AofReceivedSize = uint64(fi.Size()) + aofLoader := aof.NewLoader(r.path, r.ch) + paths := path.Join(aof.AOFINFO.GetAofdirName(), aof.AOFINFO.GetAofFilename()) + _ = aofLoader.LoadSingleAppendOnlyFile(paths, r.ch) + log.Infof("Send AOF finished. path=[%s]", r.path) + close(r.ch) + } else { + log.Infof("start send AOF。path=[%s]", r.path) + fi, err := os.Stat(r.path) + if err != nil { + log.Panicf("NewAOFReader: os.Stat error：%s", err.Error()) + } + statistics.Metrics.AofFileSize = uint64(fi.Size()) + statistics.Metrics.AofReceivedSize = uint64(fi.Size()) + aofLoader := aof.NewLoader(r.path, r.ch) + _ = aofLoader.LoadAppendOnlyFile(aof.AOFINFO.GetAofManifest(), r.ch) + log.Infof("Send AOF finished. path=[%s]", r.path) + close(r.ch) + } + }() + + return r.ch } diff --git a/internal/statistics/statistics.go b/internal/statistics/statistics.go index 9af801d5..e6d6367b 100644 --- a/internal/statistics/statistics.go +++ b/internal/statistics/statistics.go @@ -3,12 +3,13 @@ package statistics import ( "encoding/json" "fmt" - "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/log" "math/bits" "net/http" "strings" "time" + + "github.com/alibaba/RedisShake/internal/config" + "github.com/alibaba/RedisShake/internal/log" ) type metrics struct { @@ -26,10 +27,18 @@ type metrics struct { RdbReceivedSize uint64 `json:"rdb_received_size"` RdbSendSize uint64 `json:"rdb_send_size"` + //loading aof + Loading bool `json:"loading"` + AsyncLoading bool `json:"async_loading"` + LoadingStartTime int64 `json:"loading_start_time"` + LoadingLoadedBytes int64 `json:"loading_loaded_bytes"` + LoadingTotalBytes int64 `json:"loading_total_bytes"` + // aof AofReceivedOffset uint64 `json:"aof_received_offset"` AofAppliedOffset uint64 `json:"aof_applied_offset"` - + AofFileSize uint64 `json:"aof_file_size"` + AofReceivedSize uint64 `json:"aof_received_size"` // for performance debug InQueueEntriesCount uint64 `json:"in_queue_entries_count"` UnansweredBytesCount uint64 `json:"unanswered_bytes_count"` diff --git a/internal/testing/go.mod b/internal/testing/go.mod new file mode 100644 index 00000000..c42afaf0 --- /dev/null +++ b/internal/testing/go.mod @@ -0,0 +1,3 @@ +module example.com/my-module + +go 1.20 From 3d321a1a4f603767ae902c36d562dc069191526c Mon Sep 17 00:00:00 2001 From: hwy1314 <1426317404@qq.com> Date: Wed, 23 Aug 2023 20:38:57 +0800 Subject: [PATCH 5/8] Fixed some bugs in reading AOF and changed the format of some code. --- cmd/redis-shake/main.go | 9 +- internal/aof/aof.go | 933 ++++++++++------------- internal/aof/aof_check.go | 237 ++---- internal/aof/aof_check_test.go | 27 - internal/config/config.go | 10 +- internal/rdb/rdb.go | 2 +- internal/rdb/rdb_check.go | 19 +- internal/rdb/structure/length.go | 6 +- internal/rdb/structure/string.go | 8 +- internal/reader/aof_reader.go | 6 +- internal/testing/appendonly.aof.manifest | 2 - internal/testing/go.mod | 3 - internal/testing/mai.go | 137 ---- internal/testing/mai_test.go | 43 -- restore.toml | 7 +- 15 files changed, 491 insertions(+), 958 deletions(-) delete mode 100644 internal/aof/aof_check_test.go delete mode 100644 internal/testing/appendonly.aof.manifest delete mode 100644 internal/testing/go.mod delete mode 100644 internal/testing/mai.go delete mode 100644 internal/testing/mai_test.go diff --git a/cmd/redis-shake/main.go b/cmd/redis-shake/main.go index 70f5fad7..07c3aa23 100644 --- a/cmd/redis-shake/main.go +++ b/cmd/redis-shake/main.go @@ -2,6 +2,11 @@ package main import ( "fmt" + "net/http" + _ "net/http/pprof" + "os" + "runtime" + "github.com/alibaba/RedisShake/internal/commands" "github.com/alibaba/RedisShake/internal/config" "github.com/alibaba/RedisShake/internal/filter" @@ -9,10 +14,6 @@ import ( "github.com/alibaba/RedisShake/internal/reader" "github.com/alibaba/RedisShake/internal/statistics" "github.com/alibaba/RedisShake/internal/writer" - "net/http" - _ "net/http/pprof" - "os" - "runtime" ) func main() { diff --git a/internal/aof/aof.go b/internal/aof/aof.go index 9e705127..a808f4a9 100644 --- a/internal/aof/aof.go +++ b/internal/aof/aof.go @@ -5,19 +5,15 @@ import ( "bytes" "fmt" "io" - "os" "strconv" "strings" "time" "unicode" - "github.com/alibaba/RedisShake/internal/commands" "github.com/alibaba/RedisShake/internal/config" - "github.com/alibaba/RedisShake/internal/entry" "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/rdb" "github.com/alibaba/RedisShake/internal/statistics" ) @@ -26,107 +22,27 @@ const ( AofManifestFileTypeBase = "b" /* Base file */ AofManifestTypeHist = "h" /* History file */ AofManifestTypeIncr = "i" /* INCR file */ - RDB_FORMAT_SUFFIX = ".rdb" - AOF_FORMAT_SUFFIX = ".aof" - BASE_FILE_SUFFIX = ".base" - INCR_FILE_SUFFIX = ".incr" - TEMP_FILE_NAME_PREFIX = "temp-" - C_OK = 1 - C_ERR = -1 + RdbFormatSuffix = ".rdb" + AofFormatSuffix = ".aof" + BaseFileSuffix = ".base" + IncrFileSuffix = ".incr" + TempFileNamePrefix = "temp-" + COK = 1 + CERR = -1 EINTR = 4 - MANIFEST_NAME_SUFFIX = ".manifest" - AOF_NOT_EXIST = 1 - AOF_OPEN_ERR = 3 - AOF_OK = 0 - AOF_EMPTY = 2 - AOF_FAILED = 4 - AOF_TRUNCATED = 5 - SIZE_MAX = 128 - RDBFLAGS_AOF_PREAMBLE = 1 << 0 + ManifestNameSuffix = ".manifest" + AofNotExist = 1 + AofOpenErr = 3 + AofOK = 0 + AofEmpty = 2 + AofFailed = 4 + AofTruncated = 5 + SizeMax = 128 + RdbFlagsAofPreamble = 1 << 0 ) var rdbFileBeingLoaded string = "" -func UpdateLoadingFileName(filename string) { - rdbFileBeingLoaded = filename -} - -/* AOF manifest definition */ -type aofInfo struct { - fileName string - fileSeq int64 - aofFileType string -} - -type INFO struct { - aof_dirname string - aofUseRdbPreamble int - aof_manifest *aofManifest - aof_filename string - aof_current_size int64 - aof_rewrite_base_size int64 -} - -var AOFINFO INFO = *NewAOFINFO() - -func (a *INFO) GetAofdirName() string { - return a.aof_dirname -} - -func (a *INFO) SetAofDirName(dirname string) { - a.aof_dirname = dirname -} - -func (a *INFO) GetAofUseRdbPreamble() int { - return a.aofUseRdbPreamble -} - -func (a *INFO) SetAofUseRdbPreamble(useRdbPreamble int) { - a.aofUseRdbPreamble = useRdbPreamble -} - -func (a *INFO) GetAofManifest() *aofManifest { - return a.aof_manifest -} - -func (a *INFO) SetAofManifest(manifest *aofManifest) { - a.aof_manifest = manifest -} - -func (a *INFO) GetAofFilename() string { - return a.aof_filename -} - -func (a *INFO) SetAofFilename(filename string) { - a.aof_filename = filename -} - -func (a *INFO) GetAofCurrentSize() int64 { - return a.aof_current_size -} - -func (a *INFO) SetAofCurrentSize(size int64) { - a.aof_current_size = size -} - -func (a *INFO) GetAofRewriteBaseSize() int64 { - return a.aof_rewrite_base_size -} - -func (a *INFO) SetAofRewriteBaseSize(size int64) { - a.aof_rewrite_base_size = size -} -func NewAOFINFO() *INFO { - return &INFO{ - aof_dirname: config.Config.Source.AofDirName, - aofUseRdbPreamble: 0, - aof_manifest: nil, - aof_filename: config.Config.Source.AofFileName, - aof_current_size: 0, - aof_rewrite_base_size: 0, - } -} - func Ustime() int64 { tv := time.Now() ust := int64(tv.UnixNano()) / 1000 @@ -134,17 +50,6 @@ func Ustime() int64 { } -func AofInfoCreate() *aofInfo { - return new(aofInfo) -} - -var Aof_Info aofInfo = *AofInfoCreate() - -func (a *aofInfo) GetAofInfoName() string { - return a.fileName -} - -// test ok func StringNeedsRepr(s string) int { len := len(s) point := 0 @@ -160,7 +65,6 @@ func StringNeedsRepr(s string) int { return 0 } -// test ok func DirExists(dname string) int { _, err := os.Stat(dname) if err != nil { @@ -170,7 +74,6 @@ func DirExists(dname string) int { return 1 } -// test ok func FileExist(filename string) int { _, err := os.Stat(filename) if err != nil { @@ -180,43 +83,11 @@ func FileExist(filename string) int { return 1 } -// test ok func IsHexDigit(c byte) bool { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') } -/*如果字符串中包含要转义的字符，则返回一 - *通过sdscatrer（），否则为零。 - * - *通常，这应该用于以某种方式帮助保护聚合字符串 - *与sdsspliargs（）兼容。因此，空间也将 - *被视为需要逃跑。 - */ -/*func stringTrim(s string,cset string)string{ - slen:=len(s) - sp:=0 - ep:=slen-1 - - for sp<=ep&&strings.ContainsRune(cset,rune(s[sp])){ - sp++ - } - - for ep>sp&&strings.ContainsRune(cset,rune(s[sp])){ - ep-- - } - trimmed:=s[sp:ep+1] - trimmedlen:=len(trimmed) - if sp<0||ep 0 && n != nil; index-- { + n = n.prev + } + } else { + n = list.head + for ; index > 0 && n != nil; index-- { + n = n.next + } + } + return n +} + +func ListLinkNodeHead(list *lists, node *listNode) { + if list.len == 0 { + list.head = node + list.tail = node node.prev = nil node.next = nil } else { - node.prev = lists.tail - node.next = nil - lists.tail.next = node - lists.tail = node + node.prev = nil + node.next = list.head + list.head.prev = node + list.head = node } - lists.len++ + list.len++ } -func NewLoader(filPath string, ch chan *entry.Entry) *Loader { - ld := new(Loader) - ld.ch = ch - ld.filPath = filPath - return ld + +func ListAddNodeHead(list *lists, value interface{}) *lists { + node := &listNode{ + value: value, + } + ListLinkNodeHead(list, node) + + return list } -// testok -func Stringcatprintf(s string, fmtStr string, args ...interface{}) string { - result := fmt.Sprintf(fmtStr, args...) - if s == "" { - return result +func ListUnlinkNode(list *lists, node *listNode) { + if node.prev != nil { + node.prev.next = node.next } else { - return s + result + list.head = node.next } + if node.next != nil { + node.next.prev = node.prev + } else { + list.tail = node.prev + } + node.next = nil + node.prev = nil + + list.len-- } +func ListDelNode(list *lists, node *listNode) { + ListUnlinkNode(list, node) -func Stringcatrepr(s string, p string, length int) string { - s = s + string("\"") - for i := 0; i < length; i++ { - switch p[i] { - case '\\', '"': - s = Stringcatprintf(s, "\\%c", p[i]) - case '\n': - s = s + "\\n" - case '\r': - s = s + "\\r" - case '\t': - s = s + "\\t" - case '\a': - s = s + "\\a" - case '\b': - s = s + "\\b" - default: - if strconv.IsPrint(rune(p[i])) { - s = s + string(p[i]) - } else { - s = s + "\\x%02x" - } - } - } - return s + "\"" } -func AofInfoFormat(buf string, ai *aofInfo) string { - var filenameRepr string - if StringNeedsRepr(ai.fileName) == 1 { - filenameRepr = Stringcatrepr("", ai.fileName, len(ai.fileName)) - } - var ret string - if filenameRepr != "" { - ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOF_MANIFEST_KEY_FILE_NAME, filenameRepr, AOF_MANIFEST_KEY_FILE_SEQ, ai.fileSeq, AOF_MANIFEST_KEY_FILE_TYPE, ai.aofFileType) - } else { - ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOF_MANIFEST_KEY_FILE_NAME, ai.fileName, AOF_MANIFEST_KEY_FILE_SEQ, ai.fileSeq, AOF_MANIFEST_KEY_FILE_TYPE, ai.aofFileType) - } - return ret +type Loader struct { + filPath string + ch chan *entry.Entry +} + +func NewLoader(filPath string, ch chan *entry.Entry) *Loader { + ld := new(Loader) + ld.ch = ch + ld.filPath = filPath + return ld +} + +type aofManifest struct { + baseAofInfo *aofInfo + incrAofList *lists + historyList *lists + currBaseFileSeq int64 + currIncrFIleSeq int64 + dirty int64 } func AofManifestcreate() *aofManifest { @@ -541,31 +575,6 @@ func AofManifestcreate() *aofManifest { return am } -func ListDup(orig *lists) *lists { - var copy *lists - var iter listIter - var node *listNode - copy = ListCreate() - if copy == nil { - return nil - } - copy.ListsRewind(&iter) - node = ListNext(&iter) - var value interface{} - for node != nil { - value = node.value - } - - if ListAddNodeTail(copy, value) == nil { - return nil - } - return copy -} -func ListsRewindTail(list *lists, li *listIter) { - li.next = list.tail - li.direction = 1 -} - func AOFManifestDup(orig *aofManifest) *aofManifest { if orig == nil { panic("orig is nil") @@ -585,7 +594,6 @@ func AOFManifestDup(orig *aofManifest) *aofManifest { am.historyList = ListDup(orig.historyList) if am.incrAofList == nil || am.historyList == nil { - fmt.Printf("IncrAOFlist or HistoryAOFlist is nil") log.Panicf("IncrAOFlist or HistoryAOFlist is nil") } return am @@ -637,13 +645,13 @@ func GetNewBaseFileNameAndMarkPreAsHistory(am *aofManifest) string { am.baseAofInfo.aofFileType = AofManifestTypeHist } var formatSuffix string - if AOFINFO.aofUseRdbPreamble == 1 { - formatSuffix = RDB_FORMAT_SUFFIX + if AOFINFO.AofUseRdbPreamble == 1 { + formatSuffix = RdbFormatSuffix } else { - formatSuffix = AOF_FORMAT_SUFFIX + formatSuffix = AofFormatSuffix } ai := AofInfoCreate() - ai.fileName = Stringcatprintf("%s.%d%s%d", Aof_Info.GetAofInfoName(), am.currBaseFileSeq+1, BASE_FILE_SUFFIX, formatSuffix) + ai.fileName = Stringcatprintf("%s.%d%s%d", Aof_Info.GetAofInfoName(), am.currBaseFileSeq+1, BaseFileSuffix, formatSuffix) ai.fileSeq = am.currBaseFileSeq + 1 ai.aofFileType = AofManifestFileTypeBase am.baseAofInfo = ai @@ -651,24 +659,23 @@ func GetNewBaseFileNameAndMarkPreAsHistory(am *aofManifest) string { return am.baseAofInfo.fileName } -// server 未处理 testok func AofLoadManifestFromDisk() { - AOFINFO.aof_manifest = AofManifestcreate() - if DirExists(AOFINFO.aof_dirname) == 0 { - log.Infof("The AOF directory %v doesn't exist\n", AOFINFO.aof_dirname) + AOFINFO.AofManifest = AofManifestcreate() + if DirExists(AOFINFO.AofDirname) == 0 { + log.Infof("The AOF directory %v doesn't exist\n", AOFINFO.AofDirname) return } am_name := GetAofManifestFileName() - am_filepath := MakePath(AOFINFO.aof_dirname, am_name) + am_filepath := MakePath(AOFINFO.AofDirname, am_name) if FileExist(am_filepath) == 0 { - log.Infof("The AOF directory %v doesn't exist\n", AOFINFO.aof_dirname) + log.Infof("The AOF directory %v doesn't exist\n", AOFINFO.AofDirname) return } am := AofLoadManifestFromFile(am_filepath) if am != nil { - AOFINFO.aof_manifest = am + AOFINFO.AofManifest = am } } @@ -676,7 +683,7 @@ func AofLoadManifestFromDisk() { func GetNewIncrAofName(am *aofManifest) string { ai := AofInfoCreate() ai.aofFileType = AofManifestTypeIncr - ai.fileName = Stringcatprintf("", "%s.%d%s%s", AOFINFO.aof_filename, am.currIncrFIleSeq+1, INCR_FILE_SUFFIX, AOF_FORMAT_SUFFIX) + ai.fileName = Stringcatprintf("", "%s.%d%s%s", AOFINFO.AofFilename, am.currIncrFIleSeq+1, IncrFileSuffix, AofFormatSuffix) ai.fileSeq = am.currIncrFIleSeq + 1 ListAddNodeTail(am.incrAofList, ai) am.dirty = 1 @@ -684,70 +691,7 @@ func GetNewIncrAofName(am *aofManifest) string { } func GetTempIncrAofNanme() string { - return Stringcatprintf("", "%s%s%s", TEMP_FILE_NAME_PREFIX, AOFINFO.aof_filename, INCR_FILE_SUFFIX) -} - -func ListIndex(list *lists, index int64) *listNode { - var n *listNode - - if index < 0 { - index = (-index) - 1 - n = list.tail - for ; index > 0 && n != nil; index-- { - n = n.prev - } - } else { - n = list.head - for ; index > 0 && n != nil; index-- { - n = n.next - } - } - return n -} - -func ListLinkNodeHead(list *lists, node *listNode) { - if list.len == 0 { - list.head = node - list.tail = node - node.prev = nil - node.next = nil - } else { - node.prev = nil - node.next = list.head - list.head.prev = node - list.head = node - } - list.len++ -} - -func ListAddNodeHead(list *lists, value interface{}) *lists { - node := &listNode{ - value: value, - } - ListLinkNodeHead(list, node) - - return list -} - -func ListUnlinkNode(list *lists, node *listNode) { - if node.prev != nil { - node.prev.next = node.next - } else { - list.head = node.next - } - if node.next != nil { - node.next.prev = node.prev - } else { - list.tail = node.prev - } - node.next = nil - node.prev = nil - - list.len-- -} -func ListDelNode(list *lists, node *listNode) { - ListUnlinkNode(list, node) - + return Stringcatprintf("", "%s%s%s", TempFileNamePrefix, AOFINFO.AofFilename, IncrFileSuffix) } func GetLastIncrAofName(am *aofManifest) string { @@ -770,11 +714,11 @@ func GetLastIncrAofName(am *aofManifest) string { } func GetAofManifestFileName() string { - return Stringcatprintf("", "%s%s", AOFINFO.aof_filename, MANIFEST_NAME_SUFFIX) + return Stringcatprintf("", "%s%s", AOFINFO.AofFilename, ManifestNameSuffix) } func GetTempAofManifestFileName() string { - return Stringcatprintf("", "%s%s%s", TEMP_FILE_NAME_PREFIX, AOFINFO.aof_filename, MANIFEST_NAME_SUFFIX) + return Stringcatprintf("", "%s%s%s", TempFileNamePrefix, AOFINFO.AofFilename, ManifestNameSuffix) } func StartLoading(size int64, rdbflags int, async int) { @@ -786,39 +730,103 @@ func StartLoading(size int64, rdbflags int, async int) { statistics.Metrics.LoadingStartTime = time.Now().Unix() statistics.Metrics.LoadingLoadedBytes = 0 statistics.Metrics.LoadingTotalBytes = size - fmt.Printf("The AOF file starts loading.\n") log.Infof("The AOF file starts loading.\n") } func StopLoading(ret int) { statistics.Metrics.Loading = false statistics.Metrics.AsyncLoading = false - if ret == AOF_OK || ret == AOF_TRUNCATED { - fmt.Printf("The aof file was successfully loaded\n") + if ret == AofOK || ret == AofTruncated { log.Infof("The aof file was successfully loaded\n") } else { - fmt.Printf("There was an error opening the AOF file.\n") log.Infof("There was an error opening the AOF file.\n") } } -// test ok +func AofFileExist(filename string) int { + filepath := MakePath(AOFINFO.AofDirname, filename) + ret := FileExist(filepath) + return ret +} + +func GetAppendOnlyFileSize(filename string, status *int) int64 { + var size int64 + + aofFilepath := MakePath(AOFINFO.AofDirname, filename) + + stat, err := os.Stat(aofFilepath) + if err != nil { + if status != nil { + if os.IsNotExist(err) { + *status = AofNotExist + } else { + *status = AofOpenErr + } + } + log.Panicf("Unable to obtain the AOF file %v length. stat: %v", filename, err.Error()) + size = 0 + } else { + if status != nil { + *status = AofOK + } + size = stat.Size() + } + return size +} + +func GetBaseAndIncrAppendOnlyFilesSize(am *aofManifest, status *int) int64 { + var size int64 + var ln *listNode = new(listNode) + var li *listIter = new(listIter) + if am.baseAofInfo != nil { + if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { + log.Panicf("File type must be base.") + } + size += GetAppendOnlyFileSize(am.baseAofInfo.fileName, status) + if *status != AofOK { + return 0 + } + } + + am.incrAofList.ListsRewind(li) + ln = ListNext(li) + for ln != nil { + ai := ln.value.(*aofInfo) + if ai.aofFileType != AofManifestTypeIncr { + log.Panicf("File type must be Incr") + } + size += GetAppendOnlyFileSize(ai.fileName, status) + if *status != AofOK { + return 0 + } + ln = ListNext(li) + } + return size +} + +func GetBaseAndIncrAppendOnlyFilesNum(am *aofManifest) int { + num := 0 + if am.baseAofInfo != nil { + num++ + } + if am.incrAofList != nil { + num += int(am.incrAofList.len) + } + return num +} + func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry) int { - //没释放命令 - loops := 0 - ret := AOF_OK - AofFilepath := MakePath(AOFINFO.aof_dirname, filename) + ret := AofOK + AofFilepath := MakePath(AOFINFO.AofDirname, filename) var sizes int64 = 0 fp, err := os.Open(AofFilepath) if err != nil { if os.IsNotExist(err) { if _, err := os.Stat(AofFilepath); err == nil || !os.IsNotExist(err) { - fmt.Printf("Fatal error: can't open the append log file %v for reading: %v", filename, err.Error()) log.Infof("Fatal error: can't open the append log file %v for reading: %v", filename, err.Error()) - return AOF_OPEN_ERR + return AofOpenErr } else { - fmt.Printf("The append log file %v doesn't exist: %v", filename, err.Error()) log.Infof("The append log file %v doesn't exist: %v", filename, err.Error()) - return AOF_NOT_EXIST + return AofNotExist } } @@ -826,119 +834,94 @@ func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry stat, _ := fp.Stat() if stat.Size() == 0 { - return AOF_EMPTY + return AofEmpty } } sig := make([]byte, 5) if n, err := fp.Read(sig); err != nil || n != 5 || !bytes.Equal(sig, []byte("REDIS")) { if _, err := fp.Seek(0, 0); err != nil { - fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AOF_FAILED + ret = AofFailed return ret } } else { - fmt.Printf("Reading RDB base file on AOF loading...") log.Infof("Reading RDB base file on AOF loading...") ldRDB := rdb.NewLoader(AofFilepath, ch) ldRDB.ParseRDB() - - //RDB + return AofOK + //Skipped RDB checksum and has not been processed yet. } sizes += 5 reader := bufio.NewReader(fp) - for { //serve - if loops%1024 == 0 { - //一些与事件处理和模块加载进度相关的操作，具体实现可能涉及更多的代码。例如，processEventsWhileBlocked 可能是处理在阻塞期间积累的事件的函数调用，而 processModuleLoadingProgressEvent 则是处理模块加载进度事件的函数调用。 - } + for { - line, err := reader.ReadString('\n') + line, err := reader.ReadBytes('\n') { if err != nil { if err == io.EOF { break } } else { - _, errs := fp.Seek(0, os.SEEK_CUR) - if errs == nil { - fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) + _, errs := fp.Seek(0, io.SeekCurrent) + if errs != nil { log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AOF_FAILED + ret = AofFailed return ret } } sizes += int64(len(line)) + if line[0] == '#' { continue } if line[0] != '*' { - fmt.Printf("825") log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) } - argc, _ := strconv.Atoi(string(line[1:])) + argc, _ := strconv.ParseInt(string(line[1:len(line)-2]), 10, 64) if argc < 1 { - fmt.Printf("830") log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) } - if argc > int(SIZE_MAX) { - fmt.Printf("834") + if argc > int64(SizeMax) { log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) } e := entry.NewEntry() argv := []string{} - for j := 0; j < argc; j++ { + for j := 0; j < int(argc); j++ { line, err := reader.ReadString('\n') if err != nil || line[0] != '$' { if err == io.EOF { - fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AOF_FAILED + ret = AofFailed return ret } else { - fmt.Printf("849") log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) } } sizes += int64(len(line)) - len, _ := strconv.ParseInt(line[1:], 10, 64) + len, _ := strconv.ParseInt(string(line[1:len(line)-2]), 10, 64) argstring := make([]byte, len) _, err = reader.Read(argstring) if err != nil { - fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AOF_FAILED + ret = AofFailed return ret } - //argv[j] = createObject(OBJ_STRING, argsds) //这里没写 argv = append(argv, string(argstring)) CRLF := make([]byte, 2) _, err = reader.Read(CRLF) if err != nil { - //fargc = j + 1 // Free up to j. - fmt.Printf("Unrecoverable error reading the append only file %v: %v", filename, err) log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AOF_FAILED + ret = AofFailed return ret } sizes += len + 2 } - for _, value := range argv { - ok := commands.LookupCommand(value) //包未导出 而且键值对判定问题 - if ok == 0 { - fmt.Printf("unknown command. argv=%v", argv) - log.Infof("unknown command. argv=%v", argv) - ret = AOF_FAILED - return ret - } - } for _, value := range argv { e.Argv = append(e.Argv, value) } ld.ch <- e - /*rw := writer.NewRedisWriter(config.Config.Source.Address, config.Config.Source.Username, config.Config.Source.Password, config.Config.Target.IsTLS) - rw.Write(e) //是否go携程*/ } @@ -947,173 +930,75 @@ func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry return ret } -// test ok -func AofFileExist(filename string) int { - filepath := MakePath(AOFINFO.aof_dirname, filename) - ret := FileExist(filepath) - return ret -} - -// 这里的time没写完· testok -func GetAppendOnlyFileSize(filename string, status *int) int64 { - var size int64 - - aofFilepath := MakePath(AOFINFO.aof_dirname, filename) - //start := time.Now() - - stat, err := os.Stat(aofFilepath) - if err != nil { - if status != nil { - if os.IsNotExist(err) { - *status = AOF_NOT_EXIST - } else { - *status = AOF_OPEN_ERR - } - } - fmt.Printf("Unable to obtain the AOF file %v length. stat: %v", filename, err.Error()) - log.Panicf("Unable to obtain the AOF file %v length. stat: %v", filename, err.Error()) - size = 0 - } else { - if status != nil { - *status = AOF_OK - } - size = stat.Size() - } - - //latency := time.Since(start).Milliseconds() - //latencyAddSampleIfNeeded("aof-fstat", latency) //延迟监控 - /*可以看到，条件部分包括两个判断：首先检查 server.latency_monitor_threshold 是否为非零值（即已配置阈值）， - 然后判断 (var) 是否大于等于 server.latency_monitor_threshold。只有当这两个条件都为真时，才会调用 latencyAddSample 函数。 - - 这段代码的目的是确保只有当给定的 var 值超过了配置的阈值时，才会将样本添加到延迟监控中。*/ - - return size -} - -// testok -func GetBaseAndIncrAppendOnlyFilesSize(am *aofManifest, status *int) int64 { - var size int64 - var ln *listNode = new(listNode) - var li *listIter = new(listIter) - if am.baseAofInfo != nil { - if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { - fmt.Printf("File type must be base.") - log.Panicf("File type must be base.") - } - size += GetAppendOnlyFileSize(am.baseAofInfo.fileName, status) - if *status != AOF_OK { - return 0 - } - } - - am.incrAofList.ListsRewind(li) - ln = ListNext(li) - for ln != nil { - ai := ln.value.(*aofInfo) - if ai.aofFileType != AofManifestTypeIncr { - fmt.Printf("File type must be Incr") - log.Panicf("File type must be Incr") - } - size += GetAppendOnlyFileSize(ai.fileName, status) - if *status != AOF_OK { - return 0 - } - ln = ListNext(li) - } - return size -} - -// test ok -func GetBaseAndIncrAppendOnlyFilesNum(am *aofManifest) int { - num := 0 - if am.baseAofInfo != nil { - num++ - } - if am.incrAofList != nil { - num += int(am.incrAofList.len) - } - return num -} - func (ld *Loader) LoadAppendOnlyFile(am *aofManifest, ch chan *entry.Entry) int { if am == nil { - fmt.Printf("aofManifest is null") log.Panicf("aofManifest is null") } - status := AOF_OK - ret := AOF_OK + status := AofOK + ret := AofOK var start int64 var totalSize int64 = 0 var baseSize int64 = 0 var aofName string var totalNum, aofNum, lastFile int - if AofFileExist(AOFINFO.aof_filename) == 1 { - if DirExists(AOFINFO.aof_dirname) == 0 || + if AofFileExist(AOFINFO.AofFilename) == 1 { + if DirExists(AOFINFO.AofDirname) == 0 || (am.baseAofInfo == nil && am.incrAofList.len == 0) || (am.baseAofInfo != nil && am.incrAofList.len == 0 && - strings.Compare(am.baseAofInfo.fileName, AOFINFO.aof_filename) == 0 && AofFileExist(AOFINFO.aof_filename) == 0) { - fmt.Printf("This is an old version of the AOF file") //原本这里是要升级 - log.Panicf("This is an old version of the AOF file") //原本这里是要升级 + strings.Compare(am.baseAofInfo.fileName, AOFINFO.AofFilename) == 0 && AofFileExist(AOFINFO.AofFilename) == 0) { + log.Panicf("This is an old version of the AOF file") } } if am.baseAofInfo == nil && am.incrAofList == nil { - return AOF_NOT_EXIST + return AofNotExist } totalNum = GetBaseAndIncrAppendOnlyFilesNum(am) if totalNum <= 0 { - fmt.Printf("Assertion failed: IncrAppendOnlyFilestotalNum > 0") log.Panicf("Assertion failed: IncrAppendOnlyFilestotalNum > 0") } totalSize = GetBaseAndIncrAppendOnlyFilesSize(am, &status) - if status != AOF_OK { - if status == AOF_NOT_EXIST { - status = AOF_FAILED + if status != AofOK { + if status == AofNotExist { + status = AofFailed } return status } else if totalSize == 0 { - return AOF_EMPTY + return AofEmpty } - StartLoading(totalSize, RDBFLAGS_AOF_PREAMBLE, 0) //这个嗲放有问题 - //这段代码是一个函数 startLoading 的实现，用于在全局状态中标记正在进行加载，并设置用于提供加载统计信息的字段。 - + StartLoading(totalSize, RdbFlagsAofPreamble, 0) if am.baseAofInfo != nil { - if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { + if am.baseAofInfo.aofFileType == AofManifestFileTypeBase { aofName = string(am.baseAofInfo.fileName) UpdateLoadingFileName(aofName) baseSize = GetAppendOnlyFileSize(aofName, nil) lastFile = totalNum start = Ustime() ret = ld.LoadSingleAppendOnlyFile(aofName, ch) - if ret == AOF_OK || (ret == AOF_TRUNCATED && lastFile == 1) { - fmt.Printf("DB loaded from base file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) + if ret == AofOK || (ret == AofTruncated && lastFile == 1) { log.Infof("DB loaded from base file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) } - if ret == AOF_EMPTY { - ret = AOF_OK + if ret == AofEmpty { + ret = AofOK } - if ret == AOF_TRUNCATED && lastFile == 0 { - ret = AOF_FAILED - fmt.Printf("Fatal error: the truncated file is not the last file") + if ret == AofTruncated && lastFile == 0 { + ret = AofFailed log.Infof("Fatal error: the truncated file is not the last file") } - if ret == AOF_OPEN_ERR || ret == AOF_FAILED { - if ret == AOF_OK || ret == AOF_TRUNCATED { - fmt.Printf("The aof file was successfully loaded\n") + if ret == AofOpenErr || ret == AofFailed { + if ret == AofOK || ret == AofTruncated { log.Infof("The aof file was successfully loaded\n") } else { - if ret == AOF_OPEN_ERR { - fmt.Printf("There was an error opening the AOF file.\n") + if ret == AofOpenErr { log.Infof("There was an error opening the AOF file.\n") } else { - fmt.Printf("Failed to open AOF file.\n") log.Infof("Failed to open AOF file.\n") } } @@ -1131,8 +1016,6 @@ func (ld *Loader) LoadAppendOnlyFile(am *aofManifest, ch chan *entry.Entry) int for ln != nil { ai := ln.value.(*aofInfo) if ai.aofFileType != AofManifestTypeIncr { - fmt.Printf("The manifestType must be Incr") - log.Panicf("The manifestType must be Incr") } aofName = ai.fileName @@ -1141,29 +1024,23 @@ func (ld *Loader) LoadAppendOnlyFile(am *aofManifest, ch chan *entry.Entry) int aofNum++ start = Ustime() ret = ld.LoadSingleAppendOnlyFile(aofName, ch) - if ret == AOF_OK || (ret == AOF_TRUNCATED && lastFile == 1) { - fmt.Printf("DB loaded from incr file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) + if ret == AofOK || (ret == AofTruncated && lastFile == 1) { log.Infof("DB loaded from incr file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) } - if ret == AOF_EMPTY { - ret = AOF_OK + if ret == AofEmpty { + ret = AofOK } - if ret == AOF_TRUNCATED && lastFile == 0 { - ret = AOF_FAILED - fmt.Printf("Fatal error: the truncated file is not the last file\n") + if ret == AofTruncated && lastFile == 0 { + ret = AofFailed log.Infof("Fatal error: the truncated file is not the last file\n") } - if ret == AOF_OPEN_ERR || ret == AOF_FAILED { - //to do stopLoading(ret == AOF_OK || ret == AOF_TRUNCATED) /*总体而言，这段代码的目的是在加载过程结束时，更新全局状态中的加载相关字段，并触发加载结束事件以通知相关模块。 - //具体这些字段和事件的含义和功能可能需要参考完整代码和相关函数的定义才能理解清楚*/ - if ret == AOF_OPEN_ERR { - fmt.Printf("There was an error opening the AOF file.\n") + if ret == AofOpenErr || ret == AofFailed { + if ret == AofOpenErr { log.Infof("There was an error opening the AOF file.\n") } else { - fmt.Printf("Failed to open AOF file.\n") log.Infof("Failed to open AOF file.\n") } return ret @@ -1173,8 +1050,8 @@ func (ld *Loader) LoadAppendOnlyFile(am *aofManifest, ch chan *entry.Entry) int } - AOFINFO.aof_current_size = totalSize - AOFINFO.aof_rewrite_base_size = baseSize + AOFINFO.AofCurrentSize = totalSize + AOFINFO.AofRewriteBaseSize = baseSize return ret } diff --git a/internal/aof/aof_check.go b/internal/aof/aof_check.go index 50d76bbc..6baa47df 100644 --- a/internal/aof/aof_check.go +++ b/internal/aof/aof_check.go @@ -13,32 +13,29 @@ import ( "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb" - //"time" ) type AofFileType string -var errors [1044]byte var line int64 = 1 -var epos int64 var fp *os.File var pos int64 const ( - aofResp AofFileType = "AOF_RESP" - aofRdbPreamble AofFileType = "AOF_RDB_PREAMBLE" - aofMultiPart AofFileType = "AOF_MULTI_PART" - rdbCheckMode = 1 - MANIFEST_MAX_LINE = 1024 - AOF_CHECK_OK = 0 - AOF_CHECK_EMPTY = 1 - AOF_CHECK_TRUNCATED = 2 - AOF_CHECK_TIMESTAMP_TRUNCATED = 3 - toTimestamp = 0 - AOF_MANIFEST_KEY_FILE_NAME = "file" - AOF_MANIFEST_KEY_FILE_SEQ = "seq" - AOF_MANIFEST_KEY_FILE_TYPE = "type" - AOF_ANNOTATION_LINE_MAX_LEN = 1024 + aofResp AofFileType = "AOF_RESP" + aofRdbPreamble AofFileType = "AOF_RDB_PREAMBLE" + aofMultiPart AofFileType = "AOF_MULTI_PART" + rdbCheckMode = 1 + ManifestMaxLine = 1024 + AofCheckOk = 0 + AofCheckEmpty = 1 + AofCheckTruncated = 2 + AofCheckTimeStampTruncated = 3 + toTimestamp = 0 + AofManifestKeyFileName = "file" + AofManifestKeyFileSeq = "seq" + AofManifestKeyFileType = "type" + AofAnnoTationLineMaxLen = 1024 ) // check 里面的主函数 @@ -56,13 +53,10 @@ func CheckAofMain(aofFilePath string) (checkResult bool, fileType AofFileType, e switch fileType { case "AOF_MULTI_PART": CheckMultipartAof(dirpath, filepaths, fix) - break case "AOF_RESP": CheckOldStyleAof(filepaths, fix, false) - break case "AOF_RDB_PREAMBLE": CheckOldStyleAof(filepaths, fix, true) - break } return true, aofMultiPart, nil } @@ -77,18 +71,16 @@ func GetInputAofFileType(aofFilepath string) AofFileType { } } -// test ok func FilelsManifest(aofFilepath string) bool { var is_manifest bool = false fp, err := os.Open(aofFilepath) if err != nil { - fmt.Printf("Cannot open file %v:%v\n", aofFilepath, err.Error()) log.Infof("Cannot open file %v:%v\n", aofFilepath, err.Error()) os.Exit(1) } sb, err := os.Stat(aofFilepath) if err != nil { - fmt.Printf("cannot stat file: %v\n", aofFilepath) + log.Infof("cannot stat file: %v\n", aofFilepath) os.Exit(1) } size := sb.Size() @@ -103,8 +95,7 @@ func FilelsManifest(aofFilepath string) bool { if err == io.EOF { break } else { - fmt.Printf("cannot read file: %v\n", aofFilepath) - os.Exit(1) + log.Panicf("cannot read file: %v\n", aofFilepath) } } if lines[0] == '#' { @@ -117,17 +108,14 @@ func FilelsManifest(aofFilepath string) bool { return is_manifest } -// test ok func FileIsRDB(aofFilepath string) bool { fp, err := os.Open(aofFilepath) if err != nil { - fmt.Printf("Cannot open file %v:%v\n", aofFilepath, err.Error()) - os.Exit(1) + log.Panicf("Cannot open file %v:%v\n", aofFilepath, err.Error()) } sb, err := os.Stat(aofFilepath) if err != nil { - fmt.Printf("cannot stat file: %v\n", aofFilepath) - os.Exit(1) + log.Panicf("cannot stat file: %v\n", aofFilepath) } size := sb.Size() if size == 0 { @@ -148,38 +136,30 @@ func FileIsRDB(aofFilepath string) bool { func PrintAofStyle(ret int, aofFileName string, aofType string) { switch ret { - case AOF_CHECK_OK: - fmt.Printf("%v %v is valid\n", aofType, aofFileName) + case AofCheckOk: log.Infof("%v %v is valid\n", aofType, aofFileName) - case AOF_CHECK_EMPTY: - fmt.Printf("%v %v is empty\n", aofType, aofFileName) + case AofCheckEmpty: log.Infof("%v %v is empty\n", aofType, aofFileName) - case AOF_CHECK_TIMESTAMP_TRUNCATED: - fmt.Printf("Successfully truncated AOF %v to timestamp %d\n", aofFileName, toTimestamp) + case AofCheckTimeStampTruncated: log.Infof("Successfully truncated AOF %v to timestamp %d\n", aofFileName, toTimestamp) - case AOF_CHECK_TRUNCATED: - fmt.Printf("Successfully truncated AOF %v\n", aofFileName) + case AofCheckTruncated: log.Infof("Successfully truncated AOF %v\n", aofFileName) } } -// test ok func MakePath(paths string, filename string) string { return path.Join(paths, filename) } -// test ok func PathIsBaseName(path string) bool { return strings.IndexByte(path, '/') == -1 && strings.IndexByte(path, '\\') == -1 } -// test ok func ReadArgc(rd *bufio.Reader, target *int64) int { return ReadLong(rd, ' ', target) } -// test ok func ReadString(rd *bufio.Reader, target *string) int { var len int64 *target = "" @@ -188,7 +168,6 @@ func ReadString(rd *bufio.Reader, target *string) int { } if len < 0 || len > math.MaxInt64-2 { - fmt.Printf("Expected to read string of %d bytes, which is not in the suitable range\n", len) log.Infof("Expected to read string of %d bytes, which is not in the suitable range\n", len) return 0 } @@ -205,17 +184,14 @@ func ReadString(rd *bufio.Reader, target *string) int { } *target = string(data[:len-2]) - //pos += 2 readbytes已经处理 return 1 } -// test ok func ReadBytes(rd *bufio.Reader, target *[]byte, length int64) int { var real int64 n, err := rd.Read(*target) real = int64(n) if err != nil || real != length { - fmt.Printf("Expected to read %d bytes, got %d bytes\n", length, real) log.Infof("Expected to read %d bytes, got %d bytes\n", length, real) return 0 } @@ -223,10 +199,8 @@ func ReadBytes(rd *bufio.Reader, target *[]byte, length int64) int { return 1 } -// testok func ConsumeNewline(buf []byte) int { if buf[0] != '\r' || buf[1] != '\n' { - fmt.Printf("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) log.Infof("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) return 0 } @@ -234,39 +208,29 @@ func ConsumeNewline(buf []byte) int { return 1 } -// test ok func ReadLong(rd *bufio.Reader, prefix byte, target *int64) int { var err error var value int64 - /*if err != nil { - fmt.Printf("Failed to get current position, aborting...\n") - log.Panicf("Failed to get current position, aborting...\n") - }*/ - buf, err := rd.ReadBytes('\n') if err != nil { - fmt.Printf("Failed to read line from file\n") log.Infof("Failed to read line from file") return 0 } pos += int64(len(buf)) if prefix != ' ' { if buf[0] != prefix { - fmt.Printf("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) log.Infof("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) return 0 } - value, err = strconv.ParseInt(string(buf[1:len(buf)-2]), 10, 64) //去除了换行符/r/n + value, err = strconv.ParseInt(string(buf[1:len(buf)-2]), 10, 64) //Removed line breaks\r\n if err != nil { - fmt.Printf("Failed to parse long value") log.Infof("Failed to parse long value") return 0 } } else { - value, err = strconv.ParseInt(string(buf[0:len(buf)-2]), 10, 64) //去除了换行符/r/n + value, err = strconv.ParseInt(string(buf[0:len(buf)-2]), 10, 64) //Removed line breaks\r\n if err != nil { - fmt.Printf("Failed to parse long value") log.Infof("Failed to parse long value") return 0 } @@ -277,13 +241,11 @@ func ReadLong(rd *bufio.Reader, prefix byte, target *int64) int { } -// test ok func AofLoadManifestFromFile(am_filepath string) *aofManifest { var maxseq int64 am := AofManifestcreate() fp, err := os.Open(am_filepath) if err != nil { - fmt.Printf("Fatal error:can't open the AOF manifest %v for reading: %v", am_filepath, err) log.Panicf("Fatal error:can't open the AOF manifest %v for reading: %v", am_filepath, err) } var argv []string @@ -296,60 +258,51 @@ func AofLoadManifestFromFile(am_filepath string) *aofManifest { if err != nil { if err == io.EOF { if linenum == 0 { - fmt.Printf("Found an empty AOF manifest\n") log.Panicf("Found an empty AOF manifest") } else { break } } else { - fmt.Printf("Read AOF manifest failed\n") log.Panicf("Read AOF manifest failed") } } - epos += int64(len(buf)) + linenum++ if buf[0] == '#' { continue } if !strings.Contains(buf, "\n") { - fmt.Printf("The AOF manifest file contains too long line\n") log.Panicf("The AOF manifest file contains too long line") } line = strings.Trim(buf, " \t\r\n") if len(line) == 0 { - fmt.Printf("Invalid AOF manifest file format\n") log.Panicf("Invalid AOF manifest file format") } argc := 0 argv, argc = SplitArgs(line) if argc < 6 || argc%2 != 0 { - fmt.Printf("Invalid AOF manifest file format\n") log.Panicf("Invalid AOF manifest file format") } ai = AofInfoCreate() for i := 0; i < argc; i += 2 { - if strings.EqualFold(argv[i], AOF_MANIFEST_KEY_FILE_NAME) { + if strings.EqualFold(argv[i], AofManifestKeyFileName) { ai.fileName = string(argv[i+1]) if !PathIsBaseName(string(ai.fileName)) { - fmt.Printf("File can't be a path, just a filename\n") log.Panicf("File can't be a path, just a filename") } - } else if strings.EqualFold(argv[i], AOF_MANIFEST_KEY_FILE_SEQ) { + } else if strings.EqualFold(argv[i], AofManifestKeyFileSeq) { ai.fileSeq, _ = strconv.ParseInt(argv[i+1], 10, 64) - } else if strings.EqualFold(argv[i], AOF_MANIFEST_KEY_FILE_TYPE) { + } else if strings.EqualFold(argv[i], AofManifestKeyFileType) { ai.aofFileType = string(argv[i+1][0]) } } if ai.fileName == "" || ai.fileSeq == 0 || ai.aofFileType == "" { - fmt.Printf("Invalid AOF manifest file format") log.Panicf("Invalid AOF manifest file format") } - //==nil if ai.aofFileType == AofManifestFileTypeBase { if am.baseAofInfo != nil { - fmt.Printf("Found duplicate base file information\n") log.Panicf("Found duplicate base file information") } am.baseAofInfo = ai @@ -358,14 +311,12 @@ func AofLoadManifestFromFile(am_filepath string) *aofManifest { am.historyList = ListAddNodeTail(am.historyList, ai) } else if ai.aofFileType == AofManifestTypeIncr { if ai.fileSeq <= maxseq { - fmt.Printf("Found a non-monotonic sequence number\n") log.Panicf("Found a non-monotonic sequence number") } am.incrAofList = ListAddNodeTail(am.historyList, ai) am.currIncrFIleSeq = ai.fileSeq maxseq = ai.fileSeq } else { - fmt.Printf("Unknown AOF file type\n") log.Panicf("Unknown AOF file type") } line = " " @@ -375,18 +326,10 @@ func AofLoadManifestFromFile(am_filepath string) *aofManifest { return am } -// testok? func ProcessRESP(rd *bufio.Reader, filename string, outMulti *int) int { var argc int64 var str string - var err error - /*epos, err := fp.Seek(0, io.SeekCurrent) - pos = epos*/ - if err != nil { - fmt.Printf("Failed to get current position, aborting...\n") - fmt.Println(err) - os.Exit(1) - } + if ReadArgc(rd, &argc) == 0 { return 0 } @@ -398,16 +341,14 @@ func ProcessRESP(rd *bufio.Reader, filename string, outMulti *int) int { if i == 0 { if strings.EqualFold(str, "multi") { if (*outMulti) != 0 { - err := fmt.Errorf("Unexpected MULTI in AOF %v", filename) - fmt.Println(err.Error()) + log.Infof("Unexpected MULTI in AOF %v", filename) return 0 } (*outMulti)++ } else if strings.EqualFold(str, "exec") { (*outMulti)-- if (*outMulti) != 0 { - err := fmt.Errorf("Unexpected EXEC in AOF %v", filename) - fmt.Println(err.Error()) + log.Infof("Unexpected EXEC in AOF %v", filename) return 0 } } @@ -417,20 +358,10 @@ func ProcessRESP(rd *bufio.Reader, filename string, outMulti *int) int { return 1 } -// test ok -// 截断可能有问题 func ProcessAnnotations(rd *bufio.Reader, filename string, lastFile bool) int { - /* var err error - if err != nil { - fmt.Printf("Failed to get current position, aborting...\n") - fmt.Println(err) - os.Exit(1) - } - */ buf, _, err := rd.ReadLine() if err != nil { - fmt.Printf("Failed to read annotations from AOF %v, aborting...\n", filename) - os.Exit(1) + log.Panicf("Failed to read annotations from AOF %v, aborting...\n", filename) } pos += int64(len(buf)) + 2 @@ -438,8 +369,7 @@ func ProcessAnnotations(rd *bufio.Reader, filename string, lastFile bool) int { var ts int64 ts, err = strconv.ParseInt(strings.TrimPrefix(string(buf), "TS:"), 10, 64) if err != nil { - fmt.Println("Invalid timestamp annotation") - os.Exit(1) + log.Panicf("Invalid timestamp annotation") } if ts <= toTimestamp { @@ -447,13 +377,11 @@ func ProcessAnnotations(rd *bufio.Reader, filename string, lastFile bool) int { } if pos == 0 { - fmt.Printf("AOF %v has nothing before timestamp %d, aborting...\n", filename, toTimestamp) log.Panicf("AOF %v has nothing before timestamp %d, aborting...\n", filename, toTimestamp) } if !lastFile { - fmt.Printf("Failed to truncate AOF %v to timestamp %d to offset %d because it is not the last file.\n", filename, toTimestamp, epos) - log.Infof("Failed to truncate AOF %v to timestamp %d to offset %d because it is not the last file.\n", filename, toTimestamp, epos) + log.Infof("Failed to truncate AOF %v to timestamp %d to offset %d because it is not the last file.\n", filename, toTimestamp, pos) log.Panicf("If you insist, please delete all files after this file according to the manifest file and delete the corresponding records in manifest file manually. Then re-run redis-check-aof.") } @@ -486,16 +414,16 @@ func CheckMultipartAof(dirpath string, manifestFilepath string, fix int) { lastFile := (aofNum + 1) == totalNum aofPreable := FileIsRDB(aofFilepath) if aofPreable { - fmt.Printf("Start to check BASE AOF (RDB format).\n") + log.Infof("Start to check BASE AOF (RDB format).\n") } else { - fmt.Printf("Start to check BASE AOF (AOF format).\n") + log.Infof("Start to check BASE AOF (AOF format).\n") } ret = CheckSingleAof(aofFilename, aofFilepath, lastFile, fix, aofPreable) PrintAofStyle(ret, aofFilename, "BASE AOF") } if am.incrAofList.len != 0 { - log.Infof("start to check INCR INCR files.") + log.Infof("start to check INCR INCR files.\n") var ln *listNode ln = am.incrAofList.head for ln != nil { @@ -505,17 +433,15 @@ func CheckMultipartAof(dirpath string, manifestFilepath string, fix int) { lastFile := (aofNum + 1) == totalNum ret = CheckSingleAof(aofFilename, aofFilepath, lastFile, fix, false) PrintAofStyle(ret, aofFilename, "INCR AOF") - //stringfree(aofFilepath) ln = ln.next } } - //aofManifestFree(am) log.Infof("All AOF files and manifest are vaild") } func CheckOldStyleAof(aofFilepath string, fix int, preamble bool) { - fmt.Printf("Start checking Old-Style AOF\n") + log.Infof("Start checking Old-Style AOF\n") var ret = CheckSingleAof(aofFilepath, aofFilepath, true, fix, preamble) PrintAofStyle(ret, aofFilepath, "AOF") @@ -523,36 +449,32 @@ func CheckOldStyleAof(aofFilepath string, fix int, preamble bool) { func CheckSingleAof(aofFilename, aofFilepath string, lastFile bool, fix int, preamble bool) int { var rdbpos int64 = 0 multi := 0 - epos = 0 - pos = epos + pos = 0 buf := make([]byte, 1) var err error fp, err = os.OpenFile(aofFilepath, os.O_RDWR, 0666) if err != nil { - fmt.Printf("Cannot open file %v:%v,aborting...\n", aofFilepath, err) log.Panicf("Cannot open file %v:%v,aborting...\n", aofFilepath, err) } sb, err := fp.Stat() if err != nil { - fmt.Printf("Cannot stat file: %v,aborting...\n", aofFilename) log.Panicf("Cannot stat file: %v,aborting...\n", aofFilename) } size := sb.Size() if size == 0 { - return AOF_CHECK_EMPTY + return AofCheckEmpty } rd := bufio.NewReader(fp) if preamble { rdbpos = rdb.RedisCheckRDBMain(aofFilepath, fp) if rdbpos == -1 { - fmt.Printf("RDB preamble of AOF file is not sane, aborting.\n") - log.Panicf("RDB preamble of AOF file is not sane, aborting.") + log.Panicf("RDB preamble of AOF file is not sane, aborting.\n") } else { - fmt.Println("RDB preamble is OK, proceeding with AOF tail...") + log.Infof("RDB preamble is OK, proceeding with AOF tail...\n") _, err = fp.Seek(rdbpos, io.SeekStart) if err != nil { - fmt.Printf(("Failed to seek in AOF %v: %v\n"), aofFilename, err) + log.Panicf(("Failed to seek in AOF %v: %v"), aofFilename, err) } pos = rdbpos @@ -560,109 +482,64 @@ func CheckSingleAof(aofFilename, aofFilepath string, lastFile bool, fix int, pre } for { - /* if multi == 0 { - var err error - epos, err = fp.Seek(pos, io.SeekStart) - if err != nil { - fmt.Printf(("Failed to seek in AOF %v: %v\n"), aofFilename, err) - log.Panicf(("Failed to seek in AOF %v: %v"), aofFilename, err) - } - if epos == 0 && preamble { - epos, err = fp.Seek(rdbpos, io.SeekCurrent) - if err != nil { - fmt.Printf(("Failed to seek in AOF %v: %v\n"), aofFilename, err) - log.Panicf(("Failed to seek in AOF %v: %v"), aofFilename, err) - } - } - pos = epos - if err != nil { - fmt.Printf(("Failed to seek in AOF %v: %v\n"), aofFilename, err) - log.Panicf(("Failed to seek in AOF %v: %v"), aofFilename, err) - } - }*/ - println("568:", pos) if _, err := rd.Read(buf); err != nil { if err == io.EOF { break } - fmt.Printf("Failed to read from AOF %v, aborting...\n", aofFilename) log.Panicf("Failed to read from AOF %v, aborting...\n", aofFilename) } - pos += int64(len(buf)) - /*if _, err := fp.Seek(-1, io.SeekCurrent); err != nil { - fmt.Printf("Failed to fseek in AOF %v: %v\n", aofFilename, err) - log.Panicf("Failed to fseek in AOF %v: %v", aofFilename, err) - }*/ - fmt.Printf("%s\n", buf) - switch buf[0] { - case '#': + if buf[0] == '#' { if ProcessAnnotations(rd, aofFilepath, lastFile) == 0 { fp.Close() - - return AOF_CHECK_TIMESTAMP_TRUNCATED + return AofCheckTimeStampTruncated } - break - case '*': + } else if buf[0] == '*' { if ProcessRESP(rd, aofFilepath, &multi) == 0 { - break } - default: - fmt.Printf("AOF %v format error\n", aofFilename) + } else { log.Infof("AOF %v format error\n", aofFilename) break } } - /*if _, err := fp.Stat(); err == nil && multi == 1 { - if _, err := fp.Seek(0, io.SeekEnd); err != nil { - if _, err := fp.Seek(0, io.SeekCurrent); err == io.EOF { - fmt.Printf("Reached EOF before reading EXEC for MULTI\n") - log.Infof("Reached EOF before reading EXEC for MULTI\n") - } - } - }*/ diff := size - pos - if diff == 0 && toTimestamp == 1 { - fmt.Printf("Truncate nothing in AOF %v to timestamp %d\n", aofFilename, toTimestamp) log.Infof("Truncate nothing in AOF %v to timestamp %d\n", aofFilename, toTimestamp) - return AOF_CHECK_OK + return AofCheckOk } - log.Infof("AOF analyzed: filename=%v, size=%d, ok_up_to=%d, ok_up_to_line=%d, diff=%d\n", aofFilename, size, epos, line, diff) + log.Infof("AOF analyzed: filename=%v, size=%d, ok_up_to=%d, ok_up_to_line=%d, diff=%d\n", aofFilename, size, pos, line, diff) if diff > 0 { if fix == 1 { if !lastFile { - fmt.Printf("Failed to truncate AOF %v because it is not the last file\n", aofFilename) log.Panicf("Failed to truncate AOF %v because it is not the last file\n", aofFilename) os.Exit(1) } - fmt.Printf("this will shrink the AOF %v from %d bytes,with %d bytes,to %d bytes\n", aofFilename, size, diff, epos) + fmt.Printf("this will shrink the AOF %v from %d bytes,with %d bytes,to %d bytes\n", aofFilename, size, diff, pos) fmt.Print("Continue? [y/N]: ") reader := bufio.NewReader(os.Stdin) input, err := reader.ReadString('\n') if err != nil || strings.ToLower(string(input[0])) != "y" { - fmt.Println("Aborting...") - os.Exit(1) + log.Panicf("Aborting...") + } if err := fp.Truncate(pos); err != nil { - fmt.Printf("Failed to truncate AOF %v\n", aofFilename) - os.Exit(1) + log.Panicf("Failed to truncate AOF %v\n", aofFilename) + } else { - return AOF_CHECK_TRUNCATED + return AofCheckTruncated } } else { - fmt.Printf("AOF %v is not valid.Use the --fix potion to try fixing it.\n", aofFilename) - os.Exit(1) + log.Panicf("AOF %v is not valid.Use the --fix potion to try fixing it.\n", aofFilename) } } fp.Close() - return AOF_CHECK_OK + return AofCheckOk } diff --git a/internal/aof/aof_check_test.go b/internal/aof/aof_check_test.go deleted file mode 100644 index b1bf73ff..00000000 --- a/internal/aof/aof_check_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package aof - -import ( - "testing" -) - -func TestCheckAofMain(t *testing.T) { - - aofFilePath := "D:/BaiduNetdiskDownload/sa/appendonly.aof.manifest" - - checkResult, fileType, err := CheckAofMain(aofFilePath) - - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - expectedCheckResult := true - if checkResult != expectedCheckResult { - t.Errorf("Unexpected check result. Got: %v, Expected: %v", checkResult, expectedCheckResult) - } - - expectedFileType := "AOF_MULTI_PART" - if string(fileType) != expectedFileType { - t.Errorf("Unexpected file type. Got: %v, Expected: %v", fileType, expectedFileType) - } - -} diff --git a/internal/config/config.go b/internal/config/config.go index 32ad6918..1d304d9b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -68,7 +68,7 @@ type tomlShakeConfig struct { var Config tomlShakeConfig func init() { - Config.Type = "sync" + Config.Type = "restore" // source Config.Source.Version = 5.0 @@ -79,14 +79,14 @@ func init() { Config.Source.ElastiCachePSync = "" // restore Config.Source.RDBFilePath = "" - Config.Source.AOFFilePath = "" + Config.Source.AOFFilePath = "/home/hwy/appendonlydir" Config.Source.TruncateToTimestamp = 0 - Config.Source.AofDirName = "" - Config.Source.AofFileName = "" + Config.Source.AofDirName = "/home/hwy/appendonlydir" + Config.Source.AofFileName = "appendonly.aof" // target Config.Target.Type = "standalone" Config.Target.Version = 5.0 - Config.Target.Address = "" + Config.Target.Address = "localhos:6379" Config.Target.Username = "" Config.Target.Password = "" Config.Target.IsTLS = false diff --git a/internal/rdb/rdb.go b/internal/rdb/rdb.go index fd1172f4..935707ed 100644 --- a/internal/rdb/rdb.go +++ b/internal/rdb/rdb.go @@ -106,7 +106,7 @@ func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { defer UpdateRDBSentSize() // read one entry tick := time.Tick(time.Second * 1) - for true { + for { typeByte := structure.ReadByte(rd) switch typeByte { case kFlagIdle: diff --git a/internal/rdb/rdb_check.go b/internal/rdb/rdb_check.go index 6969b1ab..231ff0c7 100644 --- a/internal/rdb/rdb_check.go +++ b/internal/rdb/rdb_check.go @@ -21,14 +21,12 @@ func RedisCheckRDBMain(filepath string, fp *os.File) int64 { fmt.Fprintf(os.Stderr, "The file: %s fp is nil", filepath) os.Exit(1) } - fmt.Printf("Checking RDB file %s", filepath) log.Infof("Checking RDB file %s", filepath) ld := NewLoader(filepath, nil) RDBPos := ld.CheckParseRDB() if RDBPos > 0 { - fmt.Printf("\\o/ RDB looks OK! \\o/") log.Infof("\\o/ RDB looks OK! \\o/") } else { return -1 @@ -39,12 +37,12 @@ func (ld *Loader) CheckParseRDB() int64 { var err error ld.fp, err = os.OpenFile(ld.filPath, os.O_RDONLY, 0666) if err != nil { - log.Panicf("open file failed. file_path=[%s], error=[%s]", ld.filPath, err) + log.Panicf("open file failed. filepath=[%s], error=[%s]", ld.filPath, err) } defer func() { err = ld.fp.Close() if err != nil { - log.Panicf("close file failed. file_path=[%s], error=[%s]", ld.filPath, err) + log.Panicf("close file failed. filepath=[%s], error=[%s]", ld.filPath, err) } }() rd := bufio.NewReader(ld.fp) @@ -76,7 +74,6 @@ func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { UpdateRDBSize := func() { var err error RDBPos, err = ld.fp.Seek(0, io.SeekCurrent) - //println(RDBPos) if err != nil { log.PanicError(err) } @@ -86,7 +83,7 @@ func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { // read one entry tick := time.Tick(time.Second * 1) - for true { + for { typeByte := structure.ReadByte(rd) rdbsize += 1 switch typeByte { @@ -123,7 +120,7 @@ func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { dbSize, dbsizeoffset := structure.ReadLengthWithOffset(rd) expireSize, expiresizeoffset := structure.ReadLengthWithOffset(rd) rdbsize += dbsizeoffset + expiresizeoffset - log.Infof("RDB resize db. db_size=[%d], expire_size=[%d]", dbSize, expireSize) + log.Infof("RDB resize db. dbsize=[%d], expiresize=[%d]", dbSize, expireSize) case kFlagExpireMs: ld.expireMs = int64(structure.ReadUint64(rd)) - time.Now().UnixMilli() rdbsize += 8 @@ -142,19 +139,15 @@ func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { rdbsize += DBIDoffset case kEOF: UpdateRDBSize() - println("rdbsize", rdbsize) - return rdbsize default: - key, keyoffset := structure.ReadStringWithOffset(rd) - rdbsize += keyoffset + key, tempoffset := structure.ReadStringWithOffset(rd) + rdbsize += tempoffset var value bytes.Buffer anotherReader := io.TeeReader(rd, &value) o, tempOffsets := types.ParseObjectWithOffset(anotherReader, typeByte, key) rdbsize += tempOffsets - //rdbsize -= tempOffsets - //rdbsize += int64(value.Len()) if uint64(value.Len()) > config.Config.Advanced.TargetRedisProtoMaxBulkLen { cmds := o.Rewrite() for _, cmd := range cmds { diff --git a/internal/rdb/structure/length.go b/internal/rdb/structure/length.go index 4e9ecce3..bd82c903 100644 --- a/internal/rdb/structure/length.go +++ b/internal/rdb/structure/length.go @@ -30,9 +30,8 @@ func ReadLength(rd io.Reader) uint64 { func readEncodedLength(rd io.Reader) (length uint64, special bool, err error) { var lengthBuffer = make([]byte, 8) - var offset int64 = 0 firstByte := ReadByte(rd) - offset = 1 + var offset int64 = 1 first2bits := (firstByte & 0xc0) >> 6 // first 2 bits of encoding switch first2bits { case RDB6ByteLen: @@ -78,9 +77,8 @@ func ReadLengthWithOffset(rd io.Reader) (uint64, int64) { } func readEncodedLengthWithOffset(rd io.Reader) (length uint64, special bool, offsets int64, err error) { var lengthBuffer = make([]byte, 8) - var offset int64 = 0 firstByte := ReadByte(rd) - offset = 1 + var offset int64 = 1 first2bits := (firstByte & 0xc0) >> 6 // first 2 bits of encoding switch first2bits { case RDB6ByteLen: diff --git a/internal/rdb/structure/string.go b/internal/rdb/structure/string.go index 3c973dd5..ebcbd63c 100644 --- a/internal/rdb/structure/string.go +++ b/internal/rdb/structure/string.go @@ -62,10 +62,10 @@ func ReadStringWithOffset(rd io.Reader) (string, int64) { offset += 4 return strconv.Itoa(int(b)), offset case RDBEncLZF: - inLen, inlenoffset := ReadLengthWithOffset(rd) - offset += inlenoffset - outLen, outLenoffset := ReadLengthWithOffset(rd) - offset += outLenoffset + inLen, tempoffset := ReadLengthWithOffset(rd) + offset += tempoffset + outLen, tempoffset := ReadLengthWithOffset(rd) + offset += tempoffset in := ReadBytes(rd, int(inLen)) offset += int64(inLen) return lzfDecompress(in, int(outLen)), offset diff --git a/internal/reader/aof_reader.go b/internal/reader/aof_reader.go index 2748adf8..a688edbe 100644 --- a/internal/reader/aof_reader.go +++ b/internal/reader/aof_reader.go @@ -1,7 +1,5 @@ package reader -// this file references rdb_reader.go - import ( "os" "path" @@ -18,7 +16,6 @@ type aofReader struct { ch chan *entry.Entry } -// TODO:待完善参考rdb reader func NewAOFReader(path string) Reader { log.Infof("NewAOFReader: path=[%s]", path) absolutePath, err := filepath.Abs(path) @@ -37,8 +34,6 @@ func (r *aofReader) StartRead() chan *entry.Entry { r.ch = make(chan *entry.Entry, 1024) go func() { - // 开始解析 AOF 文件 - aof.AofLoadManifestFromDisk() am := aof.AOFINFO.GetAofManifest() @@ -56,6 +51,7 @@ func (r *aofReader) StartRead() chan *entry.Entry { log.Infof("Send AOF finished. path=[%s]", r.path) close(r.ch) } else { + log.Infof("start send AOF。path=[%s]", r.path) fi, err := os.Stat(r.path) if err != nil { diff --git a/internal/testing/appendonly.aof.manifest b/internal/testing/appendonly.aof.manifest deleted file mode 100644 index eca380fc..00000000 --- a/internal/testing/appendonly.aof.manifest +++ /dev/null @@ -1,2 +0,0 @@ -file appendonly.aof.2.base.rdb seq 2 type b -file appendonly.aof.2.incr.aof seq 2 type i diff --git a/internal/testing/go.mod b/internal/testing/go.mod deleted file mode 100644 index c42afaf0..00000000 --- a/internal/testing/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module example.com/my-module - -go 1.20 diff --git a/internal/testing/mai.go b/internal/testing/mai.go deleted file mode 100644 index 90fe71ec..00000000 --- a/internal/testing/mai.go +++ /dev/null @@ -1,137 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "io" - "math" - "os" - - "strconv" -) - -func ReadBytes(fp *os.File, target *[]byte, length int64) int { - var real int64 - _, _ = fp.Seek(0, io.SeekCurrent) - n, err := fp.Read(*target) - - real = int64(n) - if err != nil || real != length { - fmt.Printf("Expected to read %d bytes, got %d bytes\n", length, real) - return 0 - } - return 1 -} -func readLong(fp *os.File, prefix byte, target *int64) int { - - var err error - _, err = fp.Seek(0, io.SeekCurrent) - if err != nil { - fmt.Printf("Failed to get current position, aborting...\n") - os.Exit(1) - } - reader := bufio.NewReader(fp) - - buf, err := reader.ReadBytes('\n') - - println(buf) - if err != nil { - fmt.Println("Failed to read line from file") - return 0 - } - if buf[0] != prefix { - fmt.Printf("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) - return 0 - } - value, err := strconv.ParseInt(string(buf[1:len(buf)-2]), 10, 64) //去除了换行符/r/n - if err != nil { - fmt.Println("Failed to parse long value") - return 0 - } - *target = value - - return 1 - -} -func fileIsRDB(aofFilepath string) bool { - fp, err := os.Open(aofFilepath) - if err != nil { - fmt.Printf("Cannot open file %s:%s\n", aofFilepath, err.Error()) - os.Exit(1) - } - sb, err := os.Stat(aofFilepath) - if err != nil { - fmt.Printf("cannot stat file: %s\n", aofFilepath) - os.Exit(1) - } - size := sb.Size() - if size == 0 { - fp.Close() - return false - } - if size >= 8 { - sig := make([]byte, 5) - _, err := fp.Read(sig) - if err == nil && string(sig) == "REDIS" { - fp.Close() - return true - } - } - fp.Close() - return false -} - -// test ok -func readBytes(fp *os.File, target *[]byte, length int64) int { - var real int64 - _, _ = fp.Seek(0, io.SeekCurrent) - n, err := fp.Read(*target) - real = int64(n) - if err != nil || real != length { - fmt.Printf("Expected to read %d bytes, got %d bytes\n", length, real) - return 0 - } - return 1 -} - -// testok -func consumeNewline(buf []byte) int { - if buf[0] != '\r' || buf[1] != '\n' { - fmt.Printf("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) - return 0 - } - return 1 -} -func readString(fp *os.File, target *string) int { - var len int64 - *target = "" - if readLong(fp, '$', &len) == 0 { - return 0 - } - println("readlong ok\n") - println(len) - if len < 0 || len > math.MaxInt64-2 { - fmt.Printf("Expected to read string of %d bytes, which is not in the suitable range\n", len) - return 0 - } - - // Increase length to also consume \r\n - len += 2 - data := make([]byte, len) - if readBytes(fp, &data, len) == 0 { - return 0 - } - - if consumeNewline(data[len-2:]) == 0 { - return 0 - } - - *target = string(data[:len-2]) - return 1 -} - -func main() { - - isManifest := fileIsRDB("appendonly.aof.2.base.rdb") - fmt.Println("Is manifest:", isManifest) -} diff --git a/internal/testing/mai_test.go b/internal/testing/mai_test.go deleted file mode 100644 index 8a82f49a..00000000 --- a/internal/testing/mai_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package main - -import ( - //"fmt" - //"io" - //"math" - "os" - "testing" -) - -func TestReadString(t *testing.T) { - // Create a temporary file and write some data into it - tempFile, err := os.CreateTemp("", "testfile") - if err != nil { - t.Fatal(err) - } - defer os.Remove(tempFile.Name()) - data := []byte("$5\r\nHello\r\n") - _, err = tempFile.Write(data) - if err != nil { - t.Fatal(err) - } - - // Open the temporary file for reading - file, err := os.Open(tempFile.Name()) - if err != nil { - t.Fatal(err) - } - defer file.Close() - - // Test reading a valid string - var target string - expectedResult := 1 - result := readString(file, &target) - if result != expectedResult { - t.Errorf("1Expected %d, but got %d", expectedResult, result) - } - expectedValue := "Hello" - if target != expectedValue { - t.Errorf("2Expected value '%s', but got '%s'", expectedValue, target) - } - -} diff --git a/restore.toml b/restore.toml index 6ed1d1ae..787e19b1 100644 --- a/restore.toml +++ b/restore.toml @@ -4,8 +4,11 @@ type = "restore" version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... # Path to the dump.rdb file. Absolute path or relative path. Note # that relative paths are relative to the dir directory. -rdb_file_path = "dump.rdb" - + rdb_file_path = "" + aof_dirname="/home/hwy/appendonlydir" + aof_filename="appendonly.aof" + aof_file_path="/home/hwy/appendonlydir" + truncate-to-timestamp=0 [target] type = "standalone" # standalone or cluster # When the target is a cluster, write the address of one of the nodes. From f68eaa868ad750441e346fc52e6ce95bffdd4a9b Mon Sep 17 00:00:00 2001 From: hwy1314 <1426317404@qq.com> Date: Sat, 26 Aug 2023 19:21:42 +0800 Subject: [PATCH 6/8] Fixed some errors related to reading AOF files, removed some unnecessary code, and slightly adjusted the code format, adding a testing folder appendonlydir and a unit testing main_ fest.go. --- appendonlydir/appendonly.aof.manifest | 2 + cmd/redis-shake/main_test.go | 63 +++ internal/aof/aof.go | 594 +++++++++++++------------- internal/aof/aof_check.go | 356 +++++++-------- internal/commands/table.go | 2 +- internal/config/config.go | 19 +- internal/rdb/rdb.go | 2 +- internal/reader/aof_reader.go | 14 +- restore.toml | 4 +- 9 files changed, 564 insertions(+), 492 deletions(-) create mode 100644 appendonlydir/appendonly.aof.manifest create mode 100644 cmd/redis-shake/main_test.go diff --git a/appendonlydir/appendonly.aof.manifest b/appendonlydir/appendonly.aof.manifest new file mode 100644 index 00000000..eca380fc --- /dev/null +++ b/appendonlydir/appendonly.aof.manifest @@ -0,0 +1,2 @@ +file appendonly.aof.2.base.rdb seq 2 type b +file appendonly.aof.2.incr.aof seq 2 type i diff --git a/cmd/redis-shake/main_test.go b/cmd/redis-shake/main_test.go new file mode 100644 index 00000000..b0d38dce --- /dev/null +++ b/cmd/redis-shake/main_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/go-redis/redis" +) + +func TestMainFunction(t *testing.T) { + + os.Args = []string{"redis-shake", "/home/hwy/kaiyuan/restore.toml"} + main() + + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", + DB: 0, + }) + + pong, err := client.Ping().Result() + if err != nil { + t.Fatalf("Failed to connect to Redis: %v", err) + } + fmt.Println("Connected to Redis:", pong) + + expected := map[string]string{ + "kl": "kl", + "key0": "2022-03-29 17:25:54.592593", + "key1": "2022-03-29 17:25:54.876326", + "key2": "2022-03-29 17:25:52.871918", + "key3": "2022-03-29 17:25:53.034060", + "key4": "2022-03-29 17:25:53.196913", + "key5": "2022-03-29 17:25:53.356234", + "key6": "2022-03-29 17:25:53.513544", + "key7": "2022-03-29 17:25:53.671556", + "key8": "2022-03-29 17:25:53.861237", + "key9": "2022-03-29 17:25:54.020518", + "key10": "2022-03-29 17:25:54.177881", + "key11": "2022-03-29 17:25:54.337640", + } + for key, value := range expected { + result, err := client.Get(key).Result() + if err != nil { + t.Fatalf("Failed to read key %s from Redis: %v", key, err) + } + + if result != value { + t.Errorf("Value for key %s is incorrect. Expected: %s, Got: %s", key, value, result) + } + } + + result, err := client.SMembers("superpowers").Result() + if err != nil { + t.Fatalf("Failed to read set from Redis: %v", err) + } + strings := result[0] + if strings != "reflexes" { + t.Errorf("read set wrong") + } + +} diff --git a/internal/aof/aof.go b/internal/aof/aof.go index a808f4a9..0c8215c9 100644 --- a/internal/aof/aof.go +++ b/internal/aof/aof.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strconv" "strings" "time" @@ -19,29 +20,29 @@ import ( ) const ( - AofManifestFileTypeBase = "b" /* Base file */ - AofManifestTypeHist = "h" /* History file */ - AofManifestTypeIncr = "i" /* INCR file */ - RdbFormatSuffix = ".rdb" - AofFormatSuffix = ".aof" - BaseFileSuffix = ".base" + AOFManifestFileTypeBase = "b" /* Base File */ + AOFManifestTypeHist = "h" /* History File */ + AOFManifestTypeIncr = "i" /* INCR File */ + RDBFormatSuffix = ".RDB" + AOFFormatSuffix = ".AOF" + BaseFileSuffix = ".Base" IncrFileSuffix = ".incr" TempFileNamePrefix = "temp-" COK = 1 CERR = -1 EINTR = 4 ManifestNameSuffix = ".manifest" - AofNotExist = 1 - AofOpenErr = 3 - AofOK = 0 - AofEmpty = 2 - AofFailed = 4 - AofTruncated = 5 + AOFNotExist = 1 + AOFOpenErr = 3 + AOFOK = 0 + AOFEmpty = 2 + AOFFailed = 4 + AOFTruncated = 5 SizeMax = 128 - RdbFlagsAofPreamble = 1 << 0 + RDBFlagsAOFPreamble = 1 << 0 ) -var rdbFileBeingLoaded string = "" +var RDBFileBeingLoaded string = "" func Ustime() int64 { tv := time.Now() @@ -65,8 +66,8 @@ func StringNeedsRepr(s string) int { return 0 } -func DirExists(dname string) int { - _, err := os.Stat(dname) +func DirExists(dName string) int { + _, err := os.Stat(dName) if err != nil { return 0 } @@ -74,8 +75,8 @@ func DirExists(dname string) int { return 1 } -func FileExist(filename string) int { - _, err := os.Stat(filename) +func FileExist(FileName string) int { + _, err := os.Stat(FileName) if err != nil { return 0 } @@ -129,7 +130,7 @@ func HexDigitToInt(c byte) int { func SplitArgs(line string) ([]string, int) { var p string = line - var current string + var Current string var vector []string argc := 0 i := 0 @@ -151,7 +152,7 @@ func SplitArgs(line string) ([]string, int) { _, err2 := strconv.ParseInt(string(p[i+3]), 16, 64) if err1 == nil && err2 == nil { int16 := (HexDigitToInt((p[i+2])) * 16) + HexDigitToInt(p[i+3]) - current = current + fmt.Sprint(int16) + Current = Current + fmt.Sprint(int16) i += 3 } @@ -168,7 +169,7 @@ func SplitArgs(line string) ([]string, int) { default: c = p[i] } - current += string(c) + Current += string(c) } else if p[i] == '"' { if i+1 < lens && !unicode.IsSpace((rune(p[i+1]))) { return nil, 0 @@ -177,12 +178,12 @@ func SplitArgs(line string) ([]string, int) { } else if i >= lens { return nil, 0 } else { - current += string(p[i]) + Current += string(p[i]) } } else if insq { if p[i] == '\\' && p[i+1] == '\'' { i++ - current += "'" + Current += "'" } else if p[i] == '\'' { if i+1 < lens && !unicode.IsSpace((rune(p[i+1]))) { return nil, 0 @@ -191,7 +192,7 @@ func SplitArgs(line string) ([]string, int) { } else if i >= lens { return nil, 0 } else { - current += string(p[i]) + Current += string(p[i]) } } else { @@ -203,7 +204,7 @@ func SplitArgs(line string) ([]string, int) { case '\'': insq = true default: - current += string(p[i]) + Current += string(p[i]) } } if i < lens { @@ -214,9 +215,9 @@ func SplitArgs(line string) ([]string, int) { } } - vector = append(vector, current) + vector = append(vector, Current) argc++ - current = "" + Current = "" } else { return vector, argc @@ -275,125 +276,124 @@ func Stringcatrepr(s string, p string, length int) string { return s + "\"" } -func UpdateLoadingFileName(filename string) { - rdbFileBeingLoaded = filename +func UpdateLoadingFileName(FileName string) { + RDBFileBeingLoaded = FileName } /* AOF manifest definition */ -type aofInfo struct { - fileName string - fileSeq int64 - aofFileType string +type AOFInfo struct { + FileName string + FileSeq int64 + AOFFileType string } -func AofInfoCreate() *aofInfo { - return new(aofInfo) +func AOFInfoCreate() *AOFInfo { + return new(AOFInfo) } -var Aof_Info aofInfo = *AofInfoCreate() +var AOF_Info AOFInfo = *AOFInfoCreate() -func (a *aofInfo) GetAofInfoName() string { - return a.fileName +func (a *AOFInfo) GetAOFInfoName() string { + return a.FileName } -func AofInfoDup(orig *aofInfo) *aofInfo { +func AOFInfoDup(orig *AOFInfo) *AOFInfo { if orig == nil { log.Panicf("Assertion failed: orig != nil") } - ai := AofInfoCreate() - ai.fileName = orig.fileName - ai.fileSeq = orig.fileSeq - ai.aofFileType = orig.aofFileType + ai := AOFInfoCreate() + ai.FileName = orig.FileName + ai.FileSeq = orig.FileSeq + ai.AOFFileType = orig.AOFFileType return ai } -func AofInfoFormat(buf string, ai *aofInfo) string { - var filenameRepr string - if StringNeedsRepr(ai.fileName) == 1 { - filenameRepr = Stringcatrepr("", ai.fileName, len(ai.fileName)) +func AOFInfoFormat(buf string, ai *AOFInfo) string { + var FileNameRepr string + if StringNeedsRepr(ai.FileName) == 1 { + FileNameRepr = Stringcatrepr("", ai.FileName, len(ai.FileName)) } var ret string - if filenameRepr != "" { - ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AofManifestKeyFileName, filenameRepr, AofManifestKeyFileSeq, ai.fileSeq, AofManifestKeyFileType, ai.aofFileType) + if FileNameRepr != "" { + ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, FileNameRepr, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) } else { - ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AofManifestKeyFileName, ai.fileName, AofManifestKeyFileSeq, ai.fileSeq, AofManifestKeyFileType, ai.aofFileType) + ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, ai.FileName, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) } return ret } type INFO struct { - AofDirname string - AofUseRdbPreamble int - AofManifest *aofManifest - AofFilename string - AofCurrentSize int64 - AofRewriteBaseSize int64 + AOFDirName string + AOFUseRDBPreamble int + AOFManifest *AOFManifest + AOFFileName string + AOFCurrentSize int64 + AOFRewriteBaseSize int64 } -var AOFINFO INFO = *NewAOFINFO() +var AOFFileInfo INFO -func (a *INFO) GetAofdirName() string { - return a.AofDirname +func (a *INFO) GetAOFDirName() string { + return a.AOFDirName } -func NewAOFINFO() *INFO { +func NewAOFFileInfo() *INFO { return &INFO{ - AofDirname: config.Config.Source.AofDirName, - AofUseRdbPreamble: 0, - AofManifest: nil, - AofFilename: config.Config.Source.AofFileName, - AofCurrentSize: 0, - AofRewriteBaseSize: 0, + AOFDirName: filepath.Dir(config.Config.Source.AOFFilePath), + AOFUseRDBPreamble: 0, + AOFManifest: nil, + AOFFileName: filepath.Base(config.Config.Source.AOFFilePath), + AOFCurrentSize: 0, + AOFRewriteBaseSize: 0, } } -func (a *INFO) SetAofDirName(dirname string) { - a.AofDirname = dirname +func (a *INFO) SetAOFDirName(DirName string) { + a.AOFDirName = DirName } -func (a *INFO) GetAofUseRdbPreamble() int { - return a.AofUseRdbPreamble +func (a *INFO) GetAOFUseRDBPreamble() int { + return a.AOFUseRDBPreamble } -func (a *INFO) SetAofUseRdbPreamble(useRdbPreamble int) { - a.AofUseRdbPreamble = useRdbPreamble +func (a *INFO) SetAOFUseRDBPreamble(useRDBPreamble int) { + a.AOFUseRDBPreamble = useRDBPreamble } -func (a *INFO) GetAofManifest() *aofManifest { - return a.AofManifest +func (a *INFO) GetAOFManifest() *AOFManifest { + return a.AOFManifest } - -func (a *INFO) SetAofManifest(manifest *aofManifest) { - a.AofManifest = manifest +func (a *INFO) SetAOFManifest(manifest *AOFManifest) { + a.AOFManifest = manifest } -func (a *INFO) GetAofFilename() string { - return a.AofFilename +func (a *INFO) GetAOFFileName() string { + return a.AOFFileName } -func (a *INFO) SetAofFilename(filename string) { - a.AofFilename = filename +func (a *INFO) SetAOFFileName(FileName string) { + a.AOFFileName = FileName } -func (a *INFO) GetAofCurrentSize() int64 { - return a.AofCurrentSize +func (a *INFO) GetAOFCurrentSize() int64 { + return a.AOFCurrentSize } -func (a *INFO) SetAofCurrentSize(size int64) { - a.AofCurrentSize = size +func (a *INFO) SetAOFCurrentSize(size int64) { + a.AOFCurrentSize = size } -func (a *INFO) GetAofRewriteBaseSize() int64 { - return a.AofRewriteBaseSize +func (a *INFO) GetAOFRewriteBaseSize() int64 { + return a.AOFRewriteBaseSize } -func (a *INFO) SetAofRewriteBaseSize(size int64) { - a.AofRewriteBaseSize = size +func (a *INFO) SetAOFRewriteBaseSize(size int64) { + a.AOFRewriteBaseSize = size } type listIter struct { next *listNode - direction int + Direction int } type lists struct { @@ -415,21 +415,21 @@ func ListCreate() *lists { return lists } func ListNext(iter *listIter) *listNode { - current := iter.next + Current := iter.next - if current != nil { - if iter.direction == 0 { - iter.next = current.next + if Current != nil { + if iter.Direction == 0 { + iter.next = Current.next } else { - iter.next = current.prev + iter.next = Current.prev } } - return current + return Current } func (list *lists) ListsRewind(li *listIter) { li.next = list.head - li.direction = 0 + li.Direction = 0 } func ListLinkNodeTail(lists *lists, node *listNode) { @@ -459,7 +459,7 @@ func ListAddNodeTail(lists *lists, value interface{}) *lists { func ListsRewindTail(list *lists, li *listIter) { li.next = list.tail - li.direction = 1 + li.Direction = 1 } func ListDup(orig *lists) *lists { @@ -558,48 +558,48 @@ func NewLoader(filPath string, ch chan *entry.Entry) *Loader { return ld } -type aofManifest struct { - baseAofInfo *aofInfo - incrAofList *lists - historyList *lists - currBaseFileSeq int64 - currIncrFIleSeq int64 - dirty int64 +type AOFManifest struct { + BaseAOFInfo *AOFInfo + incrAOFList *lists + HistoryList *lists + CurrBaseFileSeq int64 + CurrIncrFileSeq int64 + Dirty int64 } -func AofManifestcreate() *aofManifest { - am := &aofManifest{ - incrAofList: ListCreate(), - historyList: ListCreate(), +func AOFManifestcreate() *AOFManifest { + am := &AOFManifest{ + incrAOFList: ListCreate(), + HistoryList: ListCreate(), } return am } -func AOFManifestDup(orig *aofManifest) *aofManifest { +func AOFManifestDup(orig *AOFManifest) *AOFManifest { if orig == nil { panic("orig is nil") } - am := &aofManifest{ - currBaseFileSeq: orig.currBaseFileSeq, - currIncrFIleSeq: orig.currIncrFIleSeq, - dirty: orig.dirty, + am := &AOFManifest{ + CurrBaseFileSeq: orig.CurrBaseFileSeq, + CurrIncrFileSeq: orig.CurrIncrFileSeq, + Dirty: orig.Dirty, } - if orig.baseAofInfo != nil { - am.baseAofInfo = AofInfoDup(orig.baseAofInfo) + if orig.BaseAOFInfo != nil { + am.BaseAOFInfo = AOFInfoDup(orig.BaseAOFInfo) } - am.incrAofList = ListDup(orig.incrAofList) - am.historyList = ListDup(orig.historyList) + am.incrAOFList = ListDup(orig.incrAOFList) + am.HistoryList = ListDup(orig.HistoryList) - if am.incrAofList == nil || am.historyList == nil { + if am.incrAOFList == nil || am.HistoryList == nil { log.Panicf("IncrAOFlist or HistoryAOFlist is nil") } return am } -func GetAofManifestAsString(am *aofManifest) string { +func GetAOFManifestAsString(am *AOFManifest) string { if am == nil { panic("am is nil") } @@ -607,25 +607,25 @@ func GetAofManifestAsString(am *aofManifest) string { var ln *listNode var li listIter - if am.baseAofInfo != nil { - buf = AofInfoFormat(buf, am.baseAofInfo) + if am.BaseAOFInfo != nil { + buf = AOFInfoFormat(buf, am.BaseAOFInfo) } - am.historyList.ListsRewind(&li) + am.HistoryList.ListsRewind(&li) ln = ListNext(&li) for ln != nil { - ai, ok := ln.value.(*aofInfo) + ai, ok := ln.value.(*AOFInfo) if ok { - buf = AofInfoFormat(buf, ai) + buf = AOFInfoFormat(buf, ai) } ln = ListNext(&li) } - am.incrAofList.ListsRewind(&li) + am.incrAOFList.ListsRewind(&li) ln = ListNext(&li) for ln != nil { - ai, ok := ln.value.(*aofInfo) + ai, ok := ln.value.(*AOFInfo) if ok { - buf = AofInfoFormat(buf, ai) + buf = AOFInfoFormat(buf, ai) } ln = ListNext(&li) } @@ -634,94 +634,94 @@ func GetAofManifestAsString(am *aofManifest) string { } -func GetNewBaseFileNameAndMarkPreAsHistory(am *aofManifest) string { +func GetNewBaseFileNameAndMarkPreAsHistory(am *AOFManifest) string { if am == nil { - log.Panicf("aofManifest is nil") + log.Panicf("AOFManifest is nil") } - if am.baseAofInfo != nil { - if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { - log.Panicf("base_aof_info has invalid file_type") + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType != AOFManifestFileTypeBase { + log.Panicf("Base_AOF_info has invalid File_type") } - am.baseAofInfo.aofFileType = AofManifestTypeHist + am.BaseAOFInfo.AOFFileType = AOFManifestTypeHist } var formatSuffix string - if AOFINFO.AofUseRdbPreamble == 1 { - formatSuffix = RdbFormatSuffix + if AOFFileInfo.AOFUseRDBPreamble == 1 { + formatSuffix = RDBFormatSuffix } else { - formatSuffix = AofFormatSuffix + formatSuffix = AOFFormatSuffix } - ai := AofInfoCreate() - ai.fileName = Stringcatprintf("%s.%d%s%d", Aof_Info.GetAofInfoName(), am.currBaseFileSeq+1, BaseFileSuffix, formatSuffix) - ai.fileSeq = am.currBaseFileSeq + 1 - ai.aofFileType = AofManifestFileTypeBase - am.baseAofInfo = ai - am.dirty = 1 - return am.baseAofInfo.fileName -} - -func AofLoadManifestFromDisk() { - AOFINFO.AofManifest = AofManifestcreate() - if DirExists(AOFINFO.AofDirname) == 0 { - log.Infof("The AOF directory %v doesn't exist\n", AOFINFO.AofDirname) + ai := AOFInfoCreate() + ai.FileName = Stringcatprintf("%s.%d%s%d", AOF_Info.GetAOFInfoName(), am.CurrBaseFileSeq+1, BaseFileSuffix, formatSuffix) + ai.FileSeq = am.CurrBaseFileSeq + 1 + ai.AOFFileType = AOFManifestFileTypeBase + am.BaseAOFInfo = ai + am.Dirty = 1 + return am.BaseAOFInfo.FileName +} + +func AOFLoadManifestFromDisk() { + AOFFileInfo.AOFManifest = AOFManifestcreate() + if DirExists(AOFFileInfo.AOFDirName) == 0 { + log.Infof("The AOF Directory %v doesn't exist\n", AOFFileInfo.AOFDirName) return } - am_name := GetAofManifestFileName() - am_filepath := MakePath(AOFINFO.AofDirname, am_name) - if FileExist(am_filepath) == 0 { - log.Infof("The AOF directory %v doesn't exist\n", AOFINFO.AofDirname) + am_Name := GetAOFManifestFileName() + am_Filepath := MakePath(AOFFileInfo.AOFDirName, am_Name) + if FileExist(am_Filepath) == 0 { + log.Infof("The AOF Directory %v doesn't exist\n", AOFFileInfo.AOFDirName) return } - am := AofLoadManifestFromFile(am_filepath) + am := AOFLoadManifestFromFile(am_Filepath) if am != nil { - AOFINFO.AofManifest = am + AOFFileInfo.AOFManifest = am } } -func GetNewIncrAofName(am *aofManifest) string { - ai := AofInfoCreate() - ai.aofFileType = AofManifestTypeIncr - ai.fileName = Stringcatprintf("", "%s.%d%s%s", AOFINFO.AofFilename, am.currIncrFIleSeq+1, IncrFileSuffix, AofFormatSuffix) - ai.fileSeq = am.currIncrFIleSeq + 1 - ListAddNodeTail(am.incrAofList, ai) - am.dirty = 1 - return ai.fileName +func GetNewIncrAOFName(am *AOFManifest) string { + ai := AOFInfoCreate() + ai.AOFFileType = AOFManifestTypeIncr + ai.FileName = Stringcatprintf("", "%s.%d%s%s", AOFFileInfo.AOFFileName, am.CurrIncrFileSeq+1, IncrFileSuffix, AOFFormatSuffix) + ai.FileSeq = am.CurrIncrFileSeq + 1 + ListAddNodeTail(am.incrAOFList, ai) + am.Dirty = 1 + return ai.FileName } -func GetTempIncrAofNanme() string { - return Stringcatprintf("", "%s%s%s", TempFileNamePrefix, AOFINFO.AofFilename, IncrFileSuffix) +func GetTempIncrAOFNanme() string { + return Stringcatprintf("", "%s%s%s", TempFileNamePrefix, AOFFileInfo.AOFFileName, IncrFileSuffix) } -func GetLastIncrAofName(am *aofManifest) string { +func GetLastIncrAOFName(am *AOFManifest) string { if am == nil { - log.Panicf(("aofManifest is nil")) + log.Panicf(("AOFManifest is nil")) } - if am.incrAofList.len == 0 { - return GetNewIncrAofName(am) + if am.incrAOFList.len == 0 { + return GetNewIncrAOFName(am) } - lastnode := ListIndex(am.incrAofList, -1) + lastnode := ListIndex(am.incrAOFList, -1) - ai, ok := lastnode.value.(aofInfo) + ai, ok := lastnode.value.(AOFInfo) if !ok { - fmt.Printf("Failed to convert lastnode.value to aofInfo") - log.Panicf("Failed to convert lastnode.value to aofInfo") + fmt.Printf("Failed to convert lastnode.value to AOFInfo") + log.Panicf("Failed to convert lastnode.value to AOFInfo") } - return ai.fileName + return ai.FileName } -func GetAofManifestFileName() string { - return Stringcatprintf("", "%s%s", AOFINFO.AofFilename, ManifestNameSuffix) +func GetAOFManifestFileName() string { + return AOFFileInfo.AOFFileName } -func GetTempAofManifestFileName() string { - return Stringcatprintf("", "%s%s%s", TempFileNamePrefix, AOFINFO.AofFilename, ManifestNameSuffix) +func GetTempAOFManifestFileName() string { + return Stringcatprintf("", "%s%s", TempFileNamePrefix, AOFFileInfo.AOFFileName) } -func StartLoading(size int64, rdbflags int, async int) { +func StartLoading(size int64, RDBflags int, async int) { /* Load the DB */ statistics.Metrics.Loading = true if async == 1 { @@ -730,72 +730,72 @@ func StartLoading(size int64, rdbflags int, async int) { statistics.Metrics.LoadingStartTime = time.Now().Unix() statistics.Metrics.LoadingLoadedBytes = 0 statistics.Metrics.LoadingTotalBytes = size - log.Infof("The AOF file starts loading.\n") + log.Infof("The AOF File starts loading.\n") } func StopLoading(ret int) { statistics.Metrics.Loading = false statistics.Metrics.AsyncLoading = false - if ret == AofOK || ret == AofTruncated { - log.Infof("The aof file was successfully loaded\n") + if ret == AOFOK || ret == AOFTruncated { + log.Infof("The AOF File was successfully loaded\n") } else { - log.Infof("There was an error opening the AOF file.\n") + log.Infof("There was an error opening the AOF File.\n") } } -func AofFileExist(filename string) int { - filepath := MakePath(AOFINFO.AofDirname, filename) - ret := FileExist(filepath) +func AOFFileExist(FileName string) int { + Filepath := MakePath(AOFFileInfo.AOFDirName, FileName) + ret := FileExist(Filepath) return ret } -func GetAppendOnlyFileSize(filename string, status *int) int64 { +func GetAppendOnlyFileSize(FileName string, status *int) int64 { var size int64 - aofFilepath := MakePath(AOFINFO.AofDirname, filename) + AOFFilePath := MakePath(AOFFileInfo.AOFDirName, FileName) - stat, err := os.Stat(aofFilepath) + stat, err := os.Stat(AOFFilePath) if err != nil { if status != nil { if os.IsNotExist(err) { - *status = AofNotExist + *status = AOFNotExist } else { - *status = AofOpenErr + *status = AOFOpenErr } } - log.Panicf("Unable to obtain the AOF file %v length. stat: %v", filename, err.Error()) + log.Panicf("Unable to obtain the AOF File %v length. stat: %v", FileName, err.Error()) size = 0 } else { if status != nil { - *status = AofOK + *status = AOFOK } size = stat.Size() } return size } -func GetBaseAndIncrAppendOnlyFilesSize(am *aofManifest, status *int) int64 { +func GetBaseAndIncrAppendOnlyFilesSize(am *AOFManifest, status *int) int64 { var size int64 var ln *listNode = new(listNode) var li *listIter = new(listIter) - if am.baseAofInfo != nil { - if am.baseAofInfo.aofFileType != AofManifestFileTypeBase { - log.Panicf("File type must be base.") + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType != AOFManifestFileTypeBase { + log.Panicf("File type must be Base.") } - size += GetAppendOnlyFileSize(am.baseAofInfo.fileName, status) - if *status != AofOK { + size += GetAppendOnlyFileSize(am.BaseAOFInfo.FileName, status) + if *status != AOFOK { return 0 } } - am.incrAofList.ListsRewind(li) + am.incrAOFList.ListsRewind(li) ln = ListNext(li) for ln != nil { - ai := ln.value.(*aofInfo) - if ai.aofFileType != AofManifestTypeIncr { + ai := ln.value.(*AOFInfo) + if ai.AOFFileType != AOFManifestTypeIncr { log.Panicf("File type must be Incr") } - size += GetAppendOnlyFileSize(ai.fileName, status) - if *status != AofOK { + size += GetAppendOnlyFileSize(ai.FileName, status) + if *status != AOFOK { return 0 } ln = ListNext(li) @@ -803,30 +803,30 @@ func GetBaseAndIncrAppendOnlyFilesSize(am *aofManifest, status *int) int64 { return size } -func GetBaseAndIncrAppendOnlyFilesNum(am *aofManifest) int { +func GetBaseAndIncrAppendOnlyFilesNum(am *AOFManifest) int { num := 0 - if am.baseAofInfo != nil { + if am.BaseAOFInfo != nil { num++ } - if am.incrAofList != nil { - num += int(am.incrAofList.len) + if am.incrAOFList != nil { + num += int(am.incrAOFList.len) } return num } -func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry) int { - ret := AofOK - AofFilepath := MakePath(AOFINFO.AofDirname, filename) +func (ld *Loader) LoadSingleAppendOnlyFile(FileName string, ch chan *entry.Entry) int { + ret := AOFOK + AOFFilepath := MakePath(AOFFileInfo.AOFDirName, FileName) var sizes int64 = 0 - fp, err := os.Open(AofFilepath) + fp, err := os.Open(AOFFilepath) if err != nil { if os.IsNotExist(err) { - if _, err := os.Stat(AofFilepath); err == nil || !os.IsNotExist(err) { - log.Infof("Fatal error: can't open the append log file %v for reading: %v", filename, err.Error()) - return AofOpenErr + if _, err := os.Stat(AOFFilepath); err == nil || !os.IsNotExist(err) { + log.Infof("Fatal error: can't open the append log File %v for reading: %v", FileName, err.Error()) + return AOFOpenErr } else { - log.Infof("The append log file %v doesn't exist: %v", filename, err.Error()) - return AofNotExist + log.Infof("The append log File %v doesn't exist: %v", FileName, err.Error()) + return AOFNotExist } } @@ -834,21 +834,21 @@ func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry stat, _ := fp.Stat() if stat.Size() == 0 { - return AofEmpty + return AOFEmpty } } sig := make([]byte, 5) if n, err := fp.Read(sig); err != nil || n != 5 || !bytes.Equal(sig, []byte("REDIS")) { if _, err := fp.Seek(0, 0); err != nil { - log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AofFailed + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed return ret } } else { - log.Infof("Reading RDB base file on AOF loading...") - ldRDB := rdb.NewLoader(AofFilepath, ch) + log.Infof("Reading RDB Base File on AOF loading...") + ldRDB := rdb.NewLoader(AOFFilepath, ch) ldRDB.ParseRDB() - return AofOK + return AOFOK //Skipped RDB checksum and has not been processed yet. } sizes += 5 @@ -864,8 +864,8 @@ func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry } else { _, errs := fp.Seek(0, io.SeekCurrent) if errs != nil { - log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AofFailed + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed return ret } } @@ -875,14 +875,14 @@ func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry continue } if line[0] != '*' { - log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) } argc, _ := strconv.ParseInt(string(line[1:len(line)-2]), 10, 64) if argc < 1 { - log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) } if argc > int64(SizeMax) { - log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) } e := entry.NewEntry() argv := []string{} @@ -891,11 +891,11 @@ func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry line, err := reader.ReadString('\n') if err != nil || line[0] != '$' { if err == io.EOF { - log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AofFailed + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed return ret } else { - log.Infof("Bad file format reading the append only file %v:make a backup of your AOF file, then use ./redis-check-aof --fix ", filename) + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) } } sizes += int64(len(line)) @@ -904,16 +904,16 @@ func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry argstring := make([]byte, len) _, err = reader.Read(argstring) if err != nil { - log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AofFailed + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed return ret } argv = append(argv, string(argstring)) CRLF := make([]byte, 2) _, err = reader.Read(CRLF) if err != nil { - log.Infof("Unrecoverable error reading the append only file %v: %v", filename, err) - ret = AofFailed + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed return ret } sizes += len + 2 @@ -930,29 +930,29 @@ func (ld *Loader) LoadSingleAppendOnlyFile(filename string, ch chan *entry.Entry return ret } -func (ld *Loader) LoadAppendOnlyFile(am *aofManifest, ch chan *entry.Entry) int { +func (ld *Loader) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Entry) int { if am == nil { - log.Panicf("aofManifest is null") + log.Panicf("AOFManifest is null") } - status := AofOK - ret := AofOK + status := AOFOK + ret := AOFOK var start int64 var totalSize int64 = 0 - var baseSize int64 = 0 - var aofName string - var totalNum, aofNum, lastFile int - - if AofFileExist(AOFINFO.AofFilename) == 1 { - if DirExists(AOFINFO.AofDirname) == 0 || - (am.baseAofInfo == nil && am.incrAofList.len == 0) || - (am.baseAofInfo != nil && am.incrAofList.len == 0 && - strings.Compare(am.baseAofInfo.fileName, AOFINFO.AofFilename) == 0 && AofFileExist(AOFINFO.AofFilename) == 0) { - log.Panicf("This is an old version of the AOF file") + var BaseSize int64 = 0 + var AOFName string + var totalNum, AOFNum, lastFile int + + if AOFFileExist(AOFFileInfo.AOFFileName) == 1 { + if DirExists(AOFFileInfo.AOFDirName) == 0 || + (am.BaseAOFInfo == nil && am.incrAOFList.len == 0) || + (am.BaseAOFInfo != nil && am.incrAOFList.len == 0 && + strings.Compare(am.BaseAOFInfo.FileName, AOFFileInfo.AOFFileName) == 0 && AOFFileExist(AOFFileInfo.AOFFileName) == 0) { + log.Panicf("This is an old version of the AOF File") } } - if am.baseAofInfo == nil && am.incrAofList == nil { - return AofNotExist + if am.BaseAOFInfo == nil && am.incrAOFList == nil { + return AOFNotExist } totalNum = GetBaseAndIncrAppendOnlyFilesNum(am) @@ -961,45 +961,45 @@ func (ld *Loader) LoadAppendOnlyFile(am *aofManifest, ch chan *entry.Entry) int } totalSize = GetBaseAndIncrAppendOnlyFilesSize(am, &status) - if status != AofOK { - if status == AofNotExist { - status = AofFailed + if status != AOFOK { + if status == AOFNotExist { + status = AOFFailed } return status } else if totalSize == 0 { - return AofEmpty + return AOFEmpty } - StartLoading(totalSize, RdbFlagsAofPreamble, 0) - if am.baseAofInfo != nil { - if am.baseAofInfo.aofFileType == AofManifestFileTypeBase { - aofName = string(am.baseAofInfo.fileName) - UpdateLoadingFileName(aofName) - baseSize = GetAppendOnlyFileSize(aofName, nil) + StartLoading(totalSize, RDBFlagsAOFPreamble, 0) + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType == AOFManifestFileTypeBase { + AOFName = string(am.BaseAOFInfo.FileName) + UpdateLoadingFileName(AOFName) + BaseSize = GetAppendOnlyFileSize(AOFName, nil) lastFile = totalNum start = Ustime() - ret = ld.LoadSingleAppendOnlyFile(aofName, ch) - if ret == AofOK || (ret == AofTruncated && lastFile == 1) { - log.Infof("DB loaded from base file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch) + if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) { + log.Infof("DB loaded from Base File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) } - if ret == AofEmpty { - ret = AofOK + if ret == AOFEmpty { + ret = AOFOK } - if ret == AofTruncated && lastFile == 0 { - ret = AofFailed - log.Infof("Fatal error: the truncated file is not the last file") + if ret == AOFTruncated && lastFile == 0 { + ret = AOFFailed + log.Infof("Fatal error: the truncated File is not the last File") } - if ret == AofOpenErr || ret == AofFailed { - if ret == AofOK || ret == AofTruncated { - log.Infof("The aof file was successfully loaded\n") + if ret == AOFOpenErr || ret == AOFFailed { + if ret == AOFOK || ret == AOFTruncated { + log.Infof("The AOF File was successfully loaded\n") } else { - if ret == AofOpenErr { - log.Infof("There was an error opening the AOF file.\n") + if ret == AOFOpenErr { + log.Panicf("There was an error opening the AOF File.\n") } else { - log.Infof("Failed to open AOF file.\n") + log.Panicf("Failed to open AOF File.\n") } } return ret @@ -1007,41 +1007,41 @@ func (ld *Loader) LoadAppendOnlyFile(am *aofManifest, ch chan *entry.Entry) int } } - if am.incrAofList.len > 0 { + if am.incrAOFList.len > 0 { var ln *listNode = new(listNode) var li listIter - am.incrAofList.ListsRewind(&li) + am.incrAOFList.ListsRewind(&li) ln = ListNext(&li) for ln != nil { - ai := ln.value.(*aofInfo) - if ai.aofFileType != AofManifestTypeIncr { + ai := ln.value.(*AOFInfo) + if ai.AOFFileType != AOFManifestTypeIncr { log.Panicf("The manifestType must be Incr") } - aofName = ai.fileName - UpdateLoadingFileName(aofName) + AOFName = ai.FileName + UpdateLoadingFileName(AOFName) lastFile = totalNum - aofNum++ + AOFNum++ start = Ustime() - ret = ld.LoadSingleAppendOnlyFile(aofName, ch) - if ret == AofOK || (ret == AofTruncated && lastFile == 1) { - log.Infof("DB loaded from incr file %v: %.3f seconds", aofName, float64(Ustime()-start)/1000000) + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch) + if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) { + log.Infof("DB loaded from incr File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) } - if ret == AofEmpty { - ret = AofOK + if ret == AOFEmpty { + ret = AOFOK } - if ret == AofTruncated && lastFile == 0 { - ret = AofFailed - log.Infof("Fatal error: the truncated file is not the last file\n") + if ret == AOFTruncated && lastFile == 0 { + ret = AOFFailed + log.Infof("Fatal error: the truncated File is not the last File\n") } - if ret == AofOpenErr || ret == AofFailed { - if ret == AofOpenErr { - log.Infof("There was an error opening the AOF file.\n") + if ret == AOFOpenErr || ret == AOFFailed { + if ret == AOFOpenErr { + log.Infof("There was an error opening the AOF File.\n") } else { - log.Infof("Failed to open AOF file.\n") + log.Infof("Failed to open AOF File.\n") } return ret } @@ -1050,8 +1050,8 @@ func (ld *Loader) LoadAppendOnlyFile(am *aofManifest, ch chan *entry.Entry) int } - AOFINFO.AofCurrentSize = totalSize - AOFINFO.AofRewriteBaseSize = baseSize + AOFFileInfo.AOFCurrentSize = totalSize + AOFFileInfo.AOFRewriteBaseSize = BaseSize return ret } diff --git a/internal/aof/aof_check.go b/internal/aof/aof_check.go index 6baa47df..1bd8ea25 100644 --- a/internal/aof/aof_check.go +++ b/internal/aof/aof_check.go @@ -11,77 +11,86 @@ import ( "strconv" "strings" + "github.com/alibaba/RedisShake/internal/config" "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb" ) -type AofFileType string - -var line int64 = 1 -var fp *os.File -var pos int64 +type AOFFileType string const ( - aofResp AofFileType = "AOF_RESP" - aofRdbPreamble AofFileType = "AOF_RDB_PREAMBLE" - aofMultiPart AofFileType = "AOF_MULTI_PART" + AOFResp AOFFileType = "AOF_RESP" + AOFRdbPreamble AOFFileType = "AOF_RDB_PREAMBLE" + AOFMultiPart AOFFileType = "AOF_MULTI_PART" rdbCheckMode = 1 ManifestMaxLine = 1024 - AofCheckOk = 0 - AofCheckEmpty = 1 - AofCheckTruncated = 2 - AofCheckTimeStampTruncated = 3 - toTimestamp = 0 - AofManifestKeyFileName = "file" - AofManifestKeyFileSeq = "seq" - AofManifestKeyFileType = "type" - AofAnnoTationLineMaxLen = 1024 + AOFCheckOk = 0 + AOFCheckEmpty = 1 + AOFCheckTruncated = 2 + AOFCheckTimeStampTruncated = 3 + AOFManifestKeyFileName = "File" + AOFManifestKeyFileSeq = "seq" + AOFManifestKeyFileType = "type" + AOFAnnoTationLineMaxLen = 1024 ) +type CheckAOFINFOF struct { + line int64 + fp *os.File + pos int64 + toTimestamp int64 +} + +func NewCheckAOFINFOF() *CheckAOFINFOF { + return &CheckAOFINFOF{ + line: 0, + fp: nil, + pos: 0, + toTimestamp: 0, + } +} + +var CheckAOFInfof = NewCheckAOFINFOF() + // check 里面的主函数 -func CheckAofMain(aofFilePath string) (checkResult bool, fileType AofFileType, err error) { - var filepaths string - var tempFilepath [1025]byte +func CheckAOFMain(AOFFilePath string) (checkResult bool, FileType AOFFileType, err error) { + var Filepaths string var dirpath string fix := 1 - filepaths = aofFilePath - - copy(tempFilepath[:], filepaths) - dirpath = filepath.Dir(string(tempFilepath[:])) + Filepaths = AOFFilePath + dirpath = filepath.Dir(string(Filepaths)) - fileType = GetInputAofFileType(filepaths) - switch fileType { + FileType = GetInputAOFFileType(Filepaths) + switch FileType { case "AOF_MULTI_PART": - CheckMultipartAof(dirpath, filepaths, fix) + CheckMultipartAOF(dirpath, Filepaths, fix) case "AOF_RESP": - CheckOldStyleAof(filepaths, fix, false) + CheckOldStyleAOF(Filepaths, fix, false) case "AOF_RDB_PREAMBLE": - CheckOldStyleAof(filepaths, fix, true) + CheckOldStyleAOF(Filepaths, fix, true) } - return true, aofMultiPart, nil + return true, AOFMultiPart, nil } -func GetInputAofFileType(aofFilepath string) AofFileType { - if FilelsManifest(aofFilepath) { +func GetInputAOFFileType(AOFFilePath string) AOFFileType { + if FilelsManifest(AOFFilePath) { return "AOF_MULTI_PART" - } else if FileIsRDB(aofFilepath) { + } else if FileIsRDB(AOFFilePath) { return "AOF_RDB_PREAMBLE" } else { return "AOF_RESP" } } -func FilelsManifest(aofFilepath string) bool { +func FilelsManifest(AOFFilePath string) bool { var is_manifest bool = false - fp, err := os.Open(aofFilepath) + fp, err := os.Open(AOFFilePath) if err != nil { - log.Infof("Cannot open file %v:%v\n", aofFilepath, err.Error()) - os.Exit(1) + log.Panicf("Cannot open File %v:%v\n", AOFFilePath, err.Error()) } - sb, err := os.Stat(aofFilepath) + sb, err := os.Stat(AOFFilePath) if err != nil { - log.Infof("cannot stat file: %v\n", aofFilepath) - os.Exit(1) + log.Panicf("cannot stat File: %v\n", AOFFilePath) } size := sb.Size() if size == 0 { @@ -95,7 +104,7 @@ func FilelsManifest(aofFilepath string) bool { if err == io.EOF { break } else { - log.Panicf("cannot read file: %v\n", aofFilepath) + log.Panicf("cannot read File: %v\n", AOFFilePath) } } if lines[0] == '#' { @@ -108,14 +117,14 @@ func FilelsManifest(aofFilepath string) bool { return is_manifest } -func FileIsRDB(aofFilepath string) bool { - fp, err := os.Open(aofFilepath) +func FileIsRDB(AOFFilePath string) bool { + fp, err := os.Open(AOFFilePath) if err != nil { - log.Panicf("Cannot open file %v:%v\n", aofFilepath, err.Error()) + log.Panicf("Cannot open File %v:%v\n", AOFFilePath, err.Error()) } - sb, err := os.Stat(aofFilepath) + sb, err := os.Stat(AOFFilePath) if err != nil { - log.Panicf("cannot stat file: %v\n", aofFilepath) + log.Panicf("cannot stat File: %v\n", AOFFilePath) } size := sb.Size() if size == 0 { @@ -134,26 +143,26 @@ func FileIsRDB(aofFilepath string) bool { return false } -func PrintAofStyle(ret int, aofFileName string, aofType string) { +func OutPutAOFStyle(ret int, AOFFileName string, AOFType string) { switch ret { - case AofCheckOk: - log.Infof("%v %v is valid\n", aofType, aofFileName) - case AofCheckEmpty: - log.Infof("%v %v is empty\n", aofType, aofFileName) - case AofCheckTimeStampTruncated: - log.Infof("Successfully truncated AOF %v to timestamp %d\n", aofFileName, toTimestamp) - case AofCheckTruncated: - log.Infof("Successfully truncated AOF %v\n", aofFileName) + case AOFCheckOk: + log.Infof("%v %v is valid\n", AOFType, AOFFileName) + case AOFCheckEmpty: + log.Infof("%v %v is empty\n", AOFType, AOFFileName) + case AOFCheckTimeStampTruncated: + log.Infof("Successfully truncated AOF %v to timestamp %d\n", AOFFileName, CheckAOFInfof.toTimestamp) + case AOFCheckTruncated: + log.Infof("Successfully truncated AOF %v\n", AOFFileName) } } -func MakePath(paths string, filename string) string { - return path.Join(paths, filename) +func MakePath(Paths string, FileName string) string { + return path.Join(Paths, FileName) } -func PathIsBaseName(path string) bool { - return strings.IndexByte(path, '/') == -1 && strings.IndexByte(path, '\\') == -1 +func PathIsBaseName(Path string) bool { + return strings.IndexByte(Path, '/') == -1 && strings.IndexByte(Path, '\\') == -1 } func ReadArgc(rd *bufio.Reader, target *int64) int { @@ -195,7 +204,7 @@ func ReadBytes(rd *bufio.Reader, target *[]byte, length int64) int { log.Infof("Expected to read %d bytes, got %d bytes\n", length, real) return 0 } - pos += real + CheckAOFInfof.pos += real return 1 } @@ -204,7 +213,7 @@ func ConsumeNewline(buf []byte) int { log.Infof("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) return 0 } - line += 1 + CheckAOFInfof.line += 1 return 1 } @@ -214,10 +223,10 @@ func ReadLong(rd *bufio.Reader, prefix byte, target *int64) int { var value int64 buf, err := rd.ReadBytes('\n') if err != nil { - log.Infof("Failed to read line from file") + log.Infof("Failed to read line from File") return 0 } - pos += int64(len(buf)) + CheckAOFInfof.pos += int64(len(buf)) if prefix != ' ' { if buf[0] != prefix { log.Infof("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) @@ -236,20 +245,20 @@ func ReadLong(rd *bufio.Reader, prefix byte, target *int64) int { } } *target = value - line += 1 + CheckAOFInfof.line += 1 return 1 } -func AofLoadManifestFromFile(am_filepath string) *aofManifest { +func AOFLoadManifestFromFile(am_Filepath string) *AOFManifest { var maxseq int64 - am := AofManifestcreate() - fp, err := os.Open(am_filepath) + am := AOFManifestcreate() + fp, err := os.Open(am_Filepath) if err != nil { - log.Panicf("Fatal error:can't open the AOF manifest %v for reading: %v", am_filepath, err) + log.Panicf("Fatal error:can't open the AOF manifest %v for reading: %v", am_Filepath, err) } var argv []string - var ai *aofInfo + var ai *AOFInfo var line string linenum := 0 reader := bufio.NewReader(fp) @@ -273,51 +282,51 @@ func AofLoadManifestFromFile(am_filepath string) *aofManifest { continue } if !strings.Contains(buf, "\n") { - log.Panicf("The AOF manifest file contains too long line") + log.Panicf("The AOF manifest File contains too long line") } line = strings.Trim(buf, " \t\r\n") if len(line) == 0 { - log.Panicf("Invalid AOF manifest file format") + log.Panicf("Invalid AOF manifest File format") } argc := 0 argv, argc = SplitArgs(line) if argc < 6 || argc%2 != 0 { - log.Panicf("Invalid AOF manifest file format") + log.Panicf("Invalid AOF manifest File format") } - ai = AofInfoCreate() + ai = AOFInfoCreate() for i := 0; i < argc; i += 2 { - if strings.EqualFold(argv[i], AofManifestKeyFileName) { - ai.fileName = string(argv[i+1]) - if !PathIsBaseName(string(ai.fileName)) { - log.Panicf("File can't be a path, just a filename") + if strings.EqualFold(argv[i], AOFManifestKeyFileName) { + ai.FileName = string(argv[i+1]) + if !PathIsBaseName(string(ai.FileName)) { + log.Panicf("File can't be a path, just a Filename") } - } else if strings.EqualFold(argv[i], AofManifestKeyFileSeq) { - ai.fileSeq, _ = strconv.ParseInt(argv[i+1], 10, 64) - } else if strings.EqualFold(argv[i], AofManifestKeyFileType) { - ai.aofFileType = string(argv[i+1][0]) + } else if strings.EqualFold(argv[i], AOFManifestKeyFileSeq) { + ai.FileSeq, _ = strconv.ParseInt(argv[i+1], 10, 64) + } else if strings.EqualFold(argv[i], AOFManifestKeyFileType) { + ai.AOFFileType = string(argv[i+1][0]) } } - if ai.fileName == "" || ai.fileSeq == 0 || ai.aofFileType == "" { - log.Panicf("Invalid AOF manifest file format") + if ai.FileName == "" || ai.FileSeq == 0 || ai.AOFFileType == "" { + log.Panicf("Invalid AOF manifest File format") } - if ai.aofFileType == AofManifestFileTypeBase { - if am.baseAofInfo != nil { - log.Panicf("Found duplicate base file information") + if ai.AOFFileType == AOFManifestFileTypeBase { + if am.BaseAOFInfo != nil { + log.Panicf("Found duplicate Base File information") } - am.baseAofInfo = ai - am.currBaseFileSeq = ai.fileSeq - } else if ai.aofFileType == AofManifestTypeHist { - am.historyList = ListAddNodeTail(am.historyList, ai) - } else if ai.aofFileType == AofManifestTypeIncr { - if ai.fileSeq <= maxseq { + am.BaseAOFInfo = ai + am.CurrBaseFileSeq = ai.FileSeq + } else if ai.AOFFileType == AOFManifestTypeHist { + am.HistoryList = ListAddNodeTail(am.HistoryList, ai) + } else if ai.AOFFileType == AOFManifestTypeIncr { + if ai.FileSeq <= maxseq { log.Panicf("Found a non-monotonic sequence number") } - am.incrAofList = ListAddNodeTail(am.historyList, ai) - am.currIncrFIleSeq = ai.fileSeq - maxseq = ai.fileSeq + am.incrAOFList = ListAddNodeTail(am.HistoryList, ai) + am.CurrIncrFileSeq = ai.FileSeq + maxseq = ai.FileSeq } else { - log.Panicf("Unknown AOF file type") + log.Panicf("Unknown AOF File type") } line = " " ai = nil @@ -326,7 +335,7 @@ func AofLoadManifestFromFile(am_filepath string) *aofManifest { return am } -func ProcessRESP(rd *bufio.Reader, filename string, outMulti *int) int { +func ProcessRESP(rd *bufio.Reader, Filename string, outMulti *int) int { var argc int64 var str string @@ -341,14 +350,14 @@ func ProcessRESP(rd *bufio.Reader, filename string, outMulti *int) int { if i == 0 { if strings.EqualFold(str, "multi") { if (*outMulti) != 0 { - log.Infof("Unexpected MULTI in AOF %v", filename) + log.Infof("Unexpected MULTI in AOF %v", Filename) return 0 } (*outMulti)++ } else if strings.EqualFold(str, "exec") { (*outMulti)-- if (*outMulti) != 0 { - log.Infof("Unexpected EXEC in AOF %v", filename) + log.Infof("Unexpected EXEC in AOF %v", Filename) return 0 } } @@ -358,36 +367,36 @@ func ProcessRESP(rd *bufio.Reader, filename string, outMulti *int) int { return 1 } -func ProcessAnnotations(rd *bufio.Reader, filename string, lastFile bool) int { +func ProcessAnnotations(rd *bufio.Reader, Filename string, lastFile bool) int { buf, _, err := rd.ReadLine() if err != nil { - log.Panicf("Failed to read annotations from AOF %v, aborting...\n", filename) + log.Panicf("Failed to read annotations from AOF %v, aborting...\n", Filename) } - pos += int64(len(buf)) + 2 - - if toTimestamp != 0 && strings.HasPrefix(string(buf), "TS:") { + CheckAOFInfof.pos += int64(len(buf)) + 2 + CheckAOFInfof.toTimestamp = config.Config.Source.AOFTruncateToTimestamp + if CheckAOFInfof.toTimestamp != 0 && strings.HasPrefix(string(buf), "TS:") { var ts int64 ts, err = strconv.ParseInt(strings.TrimPrefix(string(buf), "TS:"), 10, 64) if err != nil { log.Panicf("Invalid timestamp annotation") } - if ts <= toTimestamp { + if ts <= CheckAOFInfof.toTimestamp { return 1 } - if pos == 0 { - log.Panicf("AOF %v has nothing before timestamp %d, aborting...\n", filename, toTimestamp) + if CheckAOFInfof.pos == 0 { + log.Panicf("AOF %v has nothing before timestamp %d, aborting...\n", Filename, CheckAOFInfof.toTimestamp) } if !lastFile { - log.Infof("Failed to truncate AOF %v to timestamp %d to offset %d because it is not the last file.\n", filename, toTimestamp, pos) - log.Panicf("If you insist, please delete all files after this file according to the manifest file and delete the corresponding records in manifest file manually. Then re-run redis-check-aof.") + log.Infof("Failed to truncate AOF %v to timestamp %d to offset %d because it is not the last File.\n", Filename, CheckAOFInfof.toTimestamp, CheckAOFInfof.pos) + log.Panicf("If you insist, please delete all Files after this File according to the manifest File and delete the corresponding records in manifest File manually. Then re-run redis-check-AOF.") } // Truncate remaining AOF if exceeding 'toTimestamp' - if err := fp.Truncate(pos); err != nil { - log.Panicf("Failed to truncate AOF %v to timestamp %d\n", filename, toTimestamp) + if err := CheckAOFInfof.fp.Truncate(CheckAOFInfof.pos); err != nil { + log.Panicf("Failed to truncate AOF %v to timestamp %d\n", Filename, CheckAOFInfof.toTimestamp) } else { return 0 @@ -397,87 +406,88 @@ func ProcessAnnotations(rd *bufio.Reader, filename string, lastFile bool) int { return 1 } -func CheckMultipartAof(dirpath string, manifestFilepath string, fix int) { +func CheckMultipartAOF(DirPath string, ManifestFilePath string, fix int) { totalNum := 0 - aofNum := 0 + AOFNum := 0 var ret int - am := AofLoadManifestFromFile(manifestFilepath) - if am.baseAofInfo != nil { + am := AOFLoadManifestFromFile(ManifestFilePath) + if am.BaseAOFInfo != nil { totalNum++ } - if am.incrAofList != nil { - totalNum += int(am.incrAofList.len) + if am.incrAOFList != nil { + totalNum += int(am.incrAOFList.len) } - if am.baseAofInfo != nil { - aofFilename := am.baseAofInfo.fileName - aofFilepath := MakePath(dirpath, aofFilename) - lastFile := (aofNum + 1) == totalNum - aofPreable := FileIsRDB(aofFilepath) - if aofPreable { - log.Infof("Start to check BASE AOF (RDB format).\n") + if am.BaseAOFInfo != nil { + AOFFileName := am.BaseAOFInfo.FileName + AOFFilePath := MakePath(DirPath, AOFFileName) + lastFile := (AOFNum + 1) == totalNum + AOFPreable := FileIsRDB(AOFFilePath) + if AOFPreable { + log.Infof("Start to check Base AOF (RDB format).\n") } else { - log.Infof("Start to check BASE AOF (AOF format).\n") + log.Infof("Start to check Base AOF (AOF format).\n") } - ret = CheckSingleAof(aofFilename, aofFilepath, lastFile, fix, aofPreable) - PrintAofStyle(ret, aofFilename, "BASE AOF") + ret = CheckSingleAOF(AOFFileName, AOFFilePath, lastFile, fix, AOFPreable) + OutPutAOFStyle(ret, AOFFileName, "Base AOF") } - if am.incrAofList.len != 0 { - log.Infof("start to check INCR INCR files.\n") + if am.incrAOFList.len != 0 { + log.Infof("start to check INCR INCR Files.\n") var ln *listNode - ln = am.incrAofList.head + ln = am.incrAOFList.head for ln != nil { - ai := ln.value.(*aofInfo) - aofFilename := ai.fileName - aofFilepath := MakePath(dirpath, aofFilename) - lastFile := (aofNum + 1) == totalNum - ret = CheckSingleAof(aofFilename, aofFilepath, lastFile, fix, false) - PrintAofStyle(ret, aofFilename, "INCR AOF") + ai := ln.value.(*AOFInfo) + AOFFileName := ai.FileName + AOFFilePath := MakePath(DirPath, AOFFileName) + lastFile := (AOFNum + 1) == totalNum + ret = CheckSingleAOF(AOFFileName, AOFFilePath, lastFile, fix, false) + OutPutAOFStyle(ret, AOFFileName, "INCR AOF") ln = ln.next } } - log.Infof("All AOF files and manifest are vaild") + log.Infof("All AOF Files and manifest are vaild") } -func CheckOldStyleAof(aofFilepath string, fix int, preamble bool) { +func CheckOldStyleAOF(AOFFilePath string, fix int, preamble bool) { log.Infof("Start checking Old-Style AOF\n") - var ret = CheckSingleAof(aofFilepath, aofFilepath, true, fix, preamble) - PrintAofStyle(ret, aofFilepath, "AOF") + var ret = CheckSingleAOF(AOFFilePath, AOFFilePath, true, fix, preamble) + OutPutAOFStyle(ret, AOFFilePath, "AOF") } -func CheckSingleAof(aofFilename, aofFilepath string, lastFile bool, fix int, preamble bool) int { +func CheckSingleAOF(AOFFileName, AOFFilePath string, lastFile bool, fix int, preamble bool) int { var rdbpos int64 = 0 multi := 0 - pos = 0 + CheckAOFInfof.pos = 0 buf := make([]byte, 1) var err error - fp, err = os.OpenFile(aofFilepath, os.O_RDWR, 0666) + CheckAOFInfof.fp, err = os.OpenFile(AOFFilePath, os.O_RDWR, 0666) if err != nil { - log.Panicf("Cannot open file %v:%v,aborting...\n", aofFilepath, err) + log.Panicf("Cannot open File %v:%v,aborting...\n", AOFFilePath, err) } - sb, err := fp.Stat() + sb, err := CheckAOFInfof.fp.Stat() if err != nil { - log.Panicf("Cannot stat file: %v,aborting...\n", aofFilename) + log.Panicf("Cannot stat File: %v,aborting...\n", AOFFileName) } size := sb.Size() if size == 0 { - return AofCheckEmpty + return AOFCheckEmpty } - rd := bufio.NewReader(fp) + rd := bufio.NewReader(CheckAOFInfof.fp) if preamble { - rdbpos = rdb.RedisCheckRDBMain(aofFilepath, fp) + rdbpos = rdb.RedisCheckRDBMain(AOFFilePath, CheckAOFInfof.fp) + rdbpos += 8 //The RDB checksum has not been processed yet. if rdbpos == -1 { - log.Panicf("RDB preamble of AOF file is not sane, aborting.\n") + log.Panicf("RDB preamble of AOF File is not sane, aborting.\n") } else { log.Infof("RDB preamble is OK, proceeding with AOF tail...\n") - _, err = fp.Seek(rdbpos, io.SeekStart) + _, err = CheckAOFInfof.fp.Seek(rdbpos, io.SeekStart) if err != nil { - log.Panicf(("Failed to seek in AOF %v: %v"), aofFilename, err) + log.Panicf(("Failed to seek in AOF %v: %v"), AOFFileName, err) } - pos = rdbpos + CheckAOFInfof.pos = rdbpos } } @@ -489,38 +499,38 @@ func CheckSingleAof(aofFilename, aofFilepath string, lastFile bool, fix int, pre break } - log.Panicf("Failed to read from AOF %v, aborting...\n", aofFilename) + log.Panicf("Failed to read from AOF %v, aborting...\n", AOFFileName) } - pos += int64(len(buf)) + CheckAOFInfof.pos += int64(len(buf)) if buf[0] == '#' { - if ProcessAnnotations(rd, aofFilepath, lastFile) == 0 { - fp.Close() - return AofCheckTimeStampTruncated + if ProcessAnnotations(rd, AOFFilePath, lastFile) == 0 { + CheckAOFInfof.fp.Close() + return AOFCheckTimeStampTruncated } } else if buf[0] == '*' { - if ProcessRESP(rd, aofFilepath, &multi) == 0 { + if ProcessRESP(rd, AOFFilePath, &multi) == 0 { break } } else { - log.Infof("AOF %v format error\n", aofFilename) + log.Infof("AOF %v format error\n", AOFFileName) break } } - diff := size - pos - if diff == 0 && toTimestamp == 1 { - log.Infof("Truncate nothing in AOF %v to timestamp %d\n", aofFilename, toTimestamp) - return AofCheckOk + diff := size - CheckAOFInfof.pos + if diff == 0 && CheckAOFInfof.toTimestamp == 1 { + log.Infof("Truncate nothing in AOF %v to timestamp %d\n", AOFFileName, CheckAOFInfof.toTimestamp) + return AOFCheckOk } - log.Infof("AOF analyzed: filename=%v, size=%d, ok_up_to=%d, ok_up_to_line=%d, diff=%d\n", aofFilename, size, pos, line, diff) + log.Infof("AOF analyzed: Filename=%v, size=%d, ok_up_to=%d, ok_up_to_line=%d, diff=%d\n", AOFFileName, size, CheckAOFInfof.pos, CheckAOFInfof.line, diff) if diff > 0 { if fix == 1 { if !lastFile { - log.Panicf("Failed to truncate AOF %v because it is not the last file\n", aofFilename) + log.Panicf("Failed to truncate AOF %v because it is not the last File\n", AOFFileName) os.Exit(1) } - fmt.Printf("this will shrink the AOF %v from %d bytes,with %d bytes,to %d bytes\n", aofFilename, size, diff, pos) + fmt.Printf("this will shrink the AOF %v from %d bytes,with %d bytes,to %d bytes\n", AOFFileName, size, diff, CheckAOFInfof.pos) fmt.Print("Continue? [y/N]: ") reader := bufio.NewReader(os.Stdin) input, err := reader.ReadString('\n') @@ -529,17 +539,17 @@ func CheckSingleAof(aofFilename, aofFilepath string, lastFile bool, fix int, pre } - if err := fp.Truncate(pos); err != nil { - log.Panicf("Failed to truncate AOF %v\n", aofFilename) + if err := CheckAOFInfof.fp.Truncate(CheckAOFInfof.pos); err != nil { + log.Panicf("Failed to truncate AOF %v\n", AOFFileName) } else { - return AofCheckTruncated + return AOFCheckTruncated } } else { - log.Panicf("AOF %v is not valid.Use the --fix potion to try fixing it.\n", aofFilename) + log.Panicf("AOF %v is not valid.Use the --fix potion to try fixing it.\n", AOFFileName) } } - fp.Close() + CheckAOFInfof.fp.Close() - return AofCheckOk + return AOFCheckOk } diff --git a/internal/commands/table.go b/internal/commands/table.go index 3f4feb13..cb5db75c 100644 --- a/internal/commands/table.go +++ b/internal/commands/table.go @@ -34,7 +34,7 @@ var redisCommands = map[string]redisCommand{ 0, 0, 0, - 0, + 0, }, }, }, diff --git a/internal/config/config.go b/internal/config/config.go index 1d304d9b..40450eda 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,7 +3,6 @@ package config import ( "bytes" "fmt" - "io/ioutil" "os" "runtime" @@ -20,11 +19,9 @@ type tomlSource struct { ElastiCachePSync string `toml:"elasticache_psync"` // restore mode - RDBFilePath string `toml:"rdb_file_path"` - AofDirName string `toml:"aof_dirname"` - AofFileName string `toml:"aof_filename"` - AOFFilePath string `toml:"aof_file_path"` // add the aof path - TruncateToTimestamp int64 `toml:"truncate-to-timestamp"` //When reading an AOF file, truncate the file by timestamp. + RDBFilePath string `toml:"rdb_file_path"` + AOFFilePath string `toml:"aof_file_path"` // add the aof path + AOFTruncateToTimestamp int64 `toml:"truncate-to-timestamp"` //When reading an AOF file, truncate the file by timestamp. } type tomlTarget struct { @@ -79,14 +76,12 @@ func init() { Config.Source.ElastiCachePSync = "" // restore Config.Source.RDBFilePath = "" - Config.Source.AOFFilePath = "/home/hwy/appendonlydir" - Config.Source.TruncateToTimestamp = 0 - Config.Source.AofDirName = "/home/hwy/appendonlydir" - Config.Source.AofFileName = "appendonly.aof" + Config.Source.AOFFilePath = "" + Config.Source.AOFTruncateToTimestamp = 0 // target Config.Target.Type = "standalone" Config.Target.Version = 5.0 - Config.Target.Address = "localhos:6379" + Config.Target.Address = "localhost:6379" Config.Target.Username = "" Config.Target.Password = "" Config.Target.IsTLS = false @@ -107,7 +102,7 @@ func init() { func LoadFromFile(filename string) { - buf, err := ioutil.ReadFile(filename) + buf, err := os.ReadFile(filename) if err != nil { panic(err.Error()) } diff --git a/internal/rdb/rdb.go b/internal/rdb/rdb.go index 935707ed..fd1172f4 100644 --- a/internal/rdb/rdb.go +++ b/internal/rdb/rdb.go @@ -106,7 +106,7 @@ func (ld *Loader) parseRDBEntry(rd *bufio.Reader) { defer UpdateRDBSentSize() // read one entry tick := time.Tick(time.Second * 1) - for { + for true { typeByte := structure.ReadByte(rd) switch typeByte { case kFlagIdle: diff --git a/internal/reader/aof_reader.go b/internal/reader/aof_reader.go index a688edbe..1bb86c3e 100644 --- a/internal/reader/aof_reader.go +++ b/internal/reader/aof_reader.go @@ -34,10 +34,13 @@ func (r *aofReader) StartRead() chan *entry.Entry { r.ch = make(chan *entry.Entry, 1024) go func() { - aof.AofLoadManifestFromDisk() - am := aof.AOFINFO.GetAofManifest() + aof.AOFFileInfo = *(aof.NewAOFFileInfo()) + aof.AOFLoadManifestFromDisk() + am := aof.AOFFileInfo.GetAOFManifest() if am == nil { + paths := path.Join(aof.AOFFileInfo.GetAOFDirName(), aof.AOFFileInfo.GetAOFFileName()) + aof.CheckAOFMain(paths) log.Infof("start send AOF。path=[%s]", r.path) fi, err := os.Stat(r.path) if err != nil { @@ -46,12 +49,13 @@ func (r *aofReader) StartRead() chan *entry.Entry { statistics.Metrics.AofFileSize = uint64(fi.Size()) statistics.Metrics.AofReceivedSize = uint64(fi.Size()) aofLoader := aof.NewLoader(r.path, r.ch) - paths := path.Join(aof.AOFINFO.GetAofdirName(), aof.AOFINFO.GetAofFilename()) + _ = aofLoader.LoadSingleAppendOnlyFile(paths, r.ch) log.Infof("Send AOF finished. path=[%s]", r.path) close(r.ch) } else { - + paths := path.Join(aof.AOFFileInfo.GetAOFDirName(), aof.GetAOFManifestFileName()) + aof.CheckAOFMain(paths) log.Infof("start send AOF。path=[%s]", r.path) fi, err := os.Stat(r.path) if err != nil { @@ -60,7 +64,7 @@ func (r *aofReader) StartRead() chan *entry.Entry { statistics.Metrics.AofFileSize = uint64(fi.Size()) statistics.Metrics.AofReceivedSize = uint64(fi.Size()) aofLoader := aof.NewLoader(r.path, r.ch) - _ = aofLoader.LoadAppendOnlyFile(aof.AOFINFO.GetAofManifest(), r.ch) + _ = aofLoader.LoadAppendOnlyFile(aof.AOFFileInfo.GetAOFManifest(), r.ch) log.Infof("Send AOF finished. path=[%s]", r.path) close(r.ch) } diff --git a/restore.toml b/restore.toml index 787e19b1..d43e0d54 100644 --- a/restore.toml +++ b/restore.toml @@ -5,9 +5,7 @@ version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... # Path to the dump.rdb file. Absolute path or relative path. Note # that relative paths are relative to the dir directory. rdb_file_path = "" - aof_dirname="/home/hwy/appendonlydir" - aof_filename="appendonly.aof" - aof_file_path="/home/hwy/appendonlydir" + aof_file_path="appendonlydir"//Please use an absolute path. truncate-to-timestamp=0 [target] type = "standalone" # standalone or cluster From 28bee4249d276c4e4a239cfc4874531828113359 Mon Sep 17 00:00:00 2001 From: hwy1314 <120362432+hwy1314@users.noreply.github.com> Date: Tue, 29 Aug 2023 19:04:57 +0800 Subject: [PATCH 7/8] Fixed a bug where line breaks cross the bufio. Reader cache area causing AOF read errors, and updated the test samples. --- .github/workflows/release.yml | 1 + cmd/redis-shake/main.go | 13 +- cmd/redis-shake/main_test.go | 63 ------ go.mod | 7 +- go.sum | 202 ++++++++++++++++- internal/aof/aof.go | 36 ++- internal/aof/aof_check.go | 26 +-- internal/commands/keys.go | 14 +- internal/commands/table.go | 1 - internal/config/config.go | 15 +- internal/rdb/rdb.go | 9 +- internal/rdb/rdb_check.go | 24 +- internal/rdb/structure/string.go | 1 + internal/reader/aof_reader.go | 6 +- internal/statistics/statistics.go | 3 +- restore.toml | 6 +- test/aof_test/aof_test.go | 207 ++++++++++++++++++ .../appendonlydir}/appendonly.aof.manifest | 0 test/aof_test/test_aof_restore.toml | 55 +++++ 19 files changed, 534 insertions(+), 155 deletions(-) delete mode 100644 cmd/redis-shake/main_test.go create mode 100644 test/aof_test/aof_test.go rename {appendonlydir => test/aof_test/appendonlydir}/appendonly.aof.manifest (100%) create mode 100644 test/aof_test/test_aof_restore.toml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ae03357..e8f933fd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: jobs: build: + permissions: write-all runs-on: ubuntu-latest steps: - name: Checkout diff --git a/cmd/redis-shake/main.go b/cmd/redis-shake/main.go index 07c3aa23..b26cb6f6 100644 --- a/cmd/redis-shake/main.go +++ b/cmd/redis-shake/main.go @@ -2,11 +2,6 @@ package main import ( "fmt" - "net/http" - _ "net/http/pprof" - "os" - "runtime" - "github.com/alibaba/RedisShake/internal/commands" "github.com/alibaba/RedisShake/internal/config" "github.com/alibaba/RedisShake/internal/filter" @@ -14,6 +9,10 @@ import ( "github.com/alibaba/RedisShake/internal/reader" "github.com/alibaba/RedisShake/internal/statistics" "github.com/alibaba/RedisShake/internal/writer" + "net/http" + _ "net/http/pprof" + "os" + "runtime" ) func main() { @@ -83,11 +82,11 @@ func main() { var theReader reader.Reader if config.Config.Type == "sync" { theReader = reader.NewPSyncReader(source.Address, source.Username, source.Password, source.IsTLS, source.ElastiCachePSync) - } else if config.Config.Type == "restore" { // TODO: new aof reader + } else if config.Config.Type == "restore" { if source.RDBFilePath != "" { theReader = reader.NewRDBReader(source.RDBFilePath) } else { - theReader = reader.NewAOFReader(source.AOFFilePath) // 如果是mp-aof 用户传入 manifest文件的地址 ，其他的传递aof的地址 + theReader = reader.NewAOFReader(source.AOFFilePath) } } else if config.Config.Type == "scan" { diff --git a/cmd/redis-shake/main_test.go b/cmd/redis-shake/main_test.go deleted file mode 100644 index b0d38dce..00000000 --- a/cmd/redis-shake/main_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package main - -import ( - "fmt" - "os" - "testing" - - "github.com/go-redis/redis" -) - -func TestMainFunction(t *testing.T) { - - os.Args = []string{"redis-shake", "/home/hwy/kaiyuan/restore.toml"} - main() - - client := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", - Password: "", - DB: 0, - }) - - pong, err := client.Ping().Result() - if err != nil { - t.Fatalf("Failed to connect to Redis: %v", err) - } - fmt.Println("Connected to Redis:", pong) - - expected := map[string]string{ - "kl": "kl", - "key0": "2022-03-29 17:25:54.592593", - "key1": "2022-03-29 17:25:54.876326", - "key2": "2022-03-29 17:25:52.871918", - "key3": "2022-03-29 17:25:53.034060", - "key4": "2022-03-29 17:25:53.196913", - "key5": "2022-03-29 17:25:53.356234", - "key6": "2022-03-29 17:25:53.513544", - "key7": "2022-03-29 17:25:53.671556", - "key8": "2022-03-29 17:25:53.861237", - "key9": "2022-03-29 17:25:54.020518", - "key10": "2022-03-29 17:25:54.177881", - "key11": "2022-03-29 17:25:54.337640", - } - for key, value := range expected { - result, err := client.Get(key).Result() - if err != nil { - t.Fatalf("Failed to read key %s from Redis: %v", key, err) - } - - if result != value { - t.Errorf("Value for key %s is incorrect. Expected: %s, Got: %s", key, value, result) - } - } - - result, err := client.SMembers("superpowers").Result() - if err != nil { - t.Fatalf("Failed to read set from Redis: %v", err) - } - strings := result[0] - if strings != "reflexes" { - t.Errorf("read set wrong") - } - -} diff --git a/go.mod b/go.mod index db88342c..d0c58329 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,16 @@ module github.com/alibaba/RedisShake go 1.17 require ( + github.com/go-redis/redis v6.15.9+incompatible github.com/pelletier/go-toml/v2 v2.0.0-beta.3 github.com/rs/zerolog v1.28.0 github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/tools v0.10.0 + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.27.10 // indirect + golang.org/x/sys v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index a7f1240b..9ea34504 100644 --- a/go.sum +++ b/go.sum @@ -5,11 +5,80 @@ github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= +github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= +github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= +github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pelletier/go-toml/v2 v2.0.0-beta.3 h1:PNCTU4naEJ8mKal97P3A2qDU74QRQGlv4FXiL1XDqi4= github.com/pelletier/go-toml/v2 v2.0.0-beta.3/go.mod h1:aNseLYu/uKskg0zpr/kbr2z8yGuWtotWf/0BpGIAL2Y= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -19,18 +88,143 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942 h1:t0lM6y/M5IiUZyvbBTcngso8SZEZICH7is9B6g/obVU= github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= -golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/aof/aof.go b/internal/aof/aof.go index 0c8215c9..32918afd 100644 --- a/internal/aof/aof.go +++ b/internal/aof/aof.go @@ -12,7 +12,6 @@ import ( "time" "unicode" - "github.com/alibaba/RedisShake/internal/config" "github.com/alibaba/RedisShake/internal/entry" "github.com/alibaba/RedisShake/internal/log" "github.com/alibaba/RedisShake/internal/rdb" @@ -309,13 +308,13 @@ func AOFInfoDup(orig *AOFInfo) *AOFInfo { } func AOFInfoFormat(buf string, ai *AOFInfo) string { - var FileNameRepr string + var AOFManifestcreate string if StringNeedsRepr(ai.FileName) == 1 { - FileNameRepr = Stringcatrepr("", ai.FileName, len(ai.FileName)) + AOFManifestcreate = Stringcatrepr("", ai.FileName, len(ai.FileName)) } var ret string - if FileNameRepr != "" { - ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, FileNameRepr, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) + if AOFManifestcreate != "" { + ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, AOFManifestcreate, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) } else { ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, ai.FileName, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) } @@ -337,12 +336,12 @@ func (a *INFO) GetAOFDirName() string { return a.AOFDirName } -func NewAOFFileInfo() *INFO { +func NewAOFFileInfo(aofFilePath string) *INFO { return &INFO{ - AOFDirName: filepath.Dir(config.Config.Source.AOFFilePath), + AOFDirName: filepath.Dir(aofFilePath), AOFUseRDBPreamble: 0, AOFManifest: nil, - AOFFileName: filepath.Base(config.Config.Source.AOFFilePath), + AOFFileName: filepath.Base(aofFilePath), AOFCurrentSize: 0, AOFRewriteBaseSize: 0, } @@ -707,7 +706,6 @@ func GetLastIncrAOFName(am *AOFManifest) string { ai, ok := lastnode.value.(AOFInfo) if !ok { - fmt.Printf("Failed to convert lastnode.value to AOFInfo") log.Panicf("Failed to convert lastnode.value to AOFInfo") } return ai.FileName @@ -732,6 +730,7 @@ func StartLoading(size int64, RDBflags int, async int) { statistics.Metrics.LoadingTotalBytes = size log.Infof("The AOF File starts loading.\n") } + func StopLoading(ret int) { statistics.Metrics.Loading = false statistics.Metrics.AsyncLoading = false @@ -900,22 +899,19 @@ func (ld *Loader) LoadSingleAppendOnlyFile(FileName string, ch chan *entry.Entry } sizes += int64(len(line)) len, _ := strconv.ParseInt(string(line[1:len(line)-2]), 10, 64) - - argstring := make([]byte, len) - _, err = reader.Read(argstring) - if err != nil { + argstring := make([]byte, len+2) + argstring, err = reader.ReadBytes('\n') + if err != nil || argstring[len+1] != '\n' { log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) ret = AOFFailed return ret } + /*if ConsumeNewline(argstring[len-2:]) == 0 { + return 0 + }*/ + argstring = argstring[:len] argv = append(argv, string(argstring)) - CRLF := make([]byte, 2) - _, err = reader.Read(CRLF) - if err != nil { - log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) - ret = AOFFailed - return ret - } + sizes += len + 2 } for _, value := range argv { diff --git a/internal/aof/aof_check.go b/internal/aof/aof_check.go index 1bd8ea25..3f976e1a 100644 --- a/internal/aof/aof_check.go +++ b/internal/aof/aof_check.go @@ -22,7 +22,6 @@ const ( AOFResp AOFFileType = "AOF_RESP" AOFRdbPreamble AOFFileType = "AOF_RDB_PREAMBLE" AOFMultiPart AOFFileType = "AOF_MULTI_PART" - rdbCheckMode = 1 ManifestMaxLine = 1024 AOFCheckOk = 0 AOFCheckEmpty = 1 @@ -84,6 +83,7 @@ func GetInputAOFFileType(AOFFilePath string) AOFFileType { func FilelsManifest(AOFFilePath string) bool { var is_manifest bool = false + log.Infof("FIleLsMainifest:%v", AOFFilePath) fp, err := os.Open(AOFFilePath) if err != nil { log.Panicf("Cannot open File %v:%v\n", AOFFilePath, err.Error()) @@ -119,27 +119,27 @@ func FilelsManifest(AOFFilePath string) bool { func FileIsRDB(AOFFilePath string) bool { fp, err := os.Open(AOFFilePath) + if err != nil { log.Panicf("Cannot open File %v:%v\n", AOFFilePath, err.Error()) } + + defer fp.Close() sb, err := os.Stat(AOFFilePath) if err != nil { log.Panicf("cannot stat File: %v\n", AOFFilePath) } size := sb.Size() if size == 0 { - fp.Close() return false } if size >= 8 { sig := make([]byte, 5) _, err := fp.Read(sig) if err == nil && string(sig) == "REDIS" { - fp.Close() return true } } - fp.Close() return false } @@ -180,9 +180,8 @@ func ReadString(rd *bufio.Reader, target *string) int { log.Infof("Expected to read string of %d bytes, which is not in the suitable range\n", len) return 0 } - - // Increase length to also consume \r\n len += 2 + // Increase length to also consume \r\n data := make([]byte, len) if ReadBytes(rd, &data, len) == 0 { return 0 @@ -191,20 +190,19 @@ func ReadString(rd *bufio.Reader, target *string) int { if ConsumeNewline(data[len-2:]) == 0 { return 0 } - - *target = string(data[:len-2]) + data = data[:len-2] //\r\n + *target = string(data) return 1 } func ReadBytes(rd *bufio.Reader, target *[]byte, length int64) int { - var real int64 - n, err := rd.Read(*target) - real = int64(n) - if err != nil || real != length { - log.Infof("Expected to read %d bytes, got %d bytes\n", length, real) + var err error + *target, err = rd.ReadBytes('\n') + if err != nil || (*target)[length-1] != '\n' { + log.Infof("Expected to read %d bytes, got %d bytes\n%s", length, 1, *target) return 0 } - CheckAOFInfof.pos += real + CheckAOFInfof.pos += length return 1 } diff --git a/internal/commands/keys.go b/internal/commands/keys.go index f8045493..0899e055 100644 --- a/internal/commands/keys.go +++ b/internal/commands/keys.go @@ -2,12 +2,11 @@ package commands import ( "fmt" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/utils" "math" "strconv" "strings" - - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/utils" ) // CalcKeys https://redis.io/docs/reference/key-specs/ @@ -115,12 +114,3 @@ findHashTag: } return utils.Crc16(key) & 0x3FFF } - -func LookupCommand(cmd string) int { - _, ok := redisCommands[cmd] - if !ok { - log.Warnf("unknown command. argv=%v", cmd) - return 0 - } - return 1 -} diff --git a/internal/commands/table.go b/internal/commands/table.go index cb5db75c..686e9ade 100644 --- a/internal/commands/table.go +++ b/internal/commands/table.go @@ -18,7 +18,6 @@ var containers = map[string]bool{ "SENTINEL": true, "SLOWLOG": true, } - var redisCommands = map[string]redisCommand{ "LLEN": { "LIST", diff --git a/internal/config/config.go b/internal/config/config.go index 40450eda..25c786f5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -3,10 +3,10 @@ package config import ( "bytes" "fmt" + "github.com/pelletier/go-toml/v2" + "io/ioutil" "os" "runtime" - - "github.com/pelletier/go-toml/v2" ) type tomlSource struct { @@ -20,8 +20,8 @@ type tomlSource struct { // restore mode RDBFilePath string `toml:"rdb_file_path"` - AOFFilePath string `toml:"aof_file_path"` // add the aof path - AOFTruncateToTimestamp int64 `toml:"truncate-to-timestamp"` //When reading an AOF file, truncate the file by timestamp. + AOFFilePath string `toml:"aof_file_path"` + AOFTruncateToTimestamp int64 `toml:"aof_truncate_to_timestamp"` } type tomlTarget struct { @@ -65,7 +65,7 @@ type tomlShakeConfig struct { var Config tomlShakeConfig func init() { - Config.Type = "restore" + Config.Type = "sync" // source Config.Source.Version = 5.0 @@ -78,10 +78,11 @@ func init() { Config.Source.RDBFilePath = "" Config.Source.AOFFilePath = "" Config.Source.AOFTruncateToTimestamp = 0 + // target Config.Target.Type = "standalone" Config.Target.Version = 5.0 - Config.Target.Address = "localhost:6379" + Config.Target.Address = "" Config.Target.Username = "" Config.Target.Password = "" Config.Target.IsTLS = false @@ -102,7 +103,7 @@ func init() { func LoadFromFile(filename string) { - buf, err := os.ReadFile(filename) + buf, err := ioutil.ReadFile(filename) if err != nil { panic(err.Error()) } diff --git a/internal/rdb/rdb.go b/internal/rdb/rdb.go index fd1172f4..45bd426b 100644 --- a/internal/rdb/rdb.go +++ b/internal/rdb/rdb.go @@ -4,11 +4,6 @@ import ( "bufio" "bytes" "encoding/binary" - "io" - "os" - "strconv" - "time" - "github.com/alibaba/RedisShake/internal/config" "github.com/alibaba/RedisShake/internal/entry" "github.com/alibaba/RedisShake/internal/log" @@ -16,6 +11,10 @@ import ( "github.com/alibaba/RedisShake/internal/rdb/types" "github.com/alibaba/RedisShake/internal/statistics" "github.com/alibaba/RedisShake/internal/utils" + "io" + "os" + "strconv" + "time" ) const ( diff --git a/internal/rdb/rdb_check.go b/internal/rdb/rdb_check.go index 231ff0c7..4d724d23 100644 --- a/internal/rdb/rdb_check.go +++ b/internal/rdb/rdb_check.go @@ -62,12 +62,12 @@ func (ld *Loader) CheckParseRDB() int64 { log.Infof("RDB version: %d", version) // read entries - rdbpos := ld.CheckparseRDBEntry(rd) + rdbpos := ld.CheckParseRDBEntry(rd) return rdbpos } -func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { +func (ld *Loader) CheckParseRDBEntry(rd *bufio.Reader) int64 { // for stat var RDBPos int64 var rdbsize int64 = 9 @@ -88,8 +88,8 @@ func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { rdbsize += 1 switch typeByte { case kFlagIdle: - tempidle, tempOffset := structure.ReadLengthWithOffset(rd) - ld.idle = int64(tempidle) + tempIdle, tempOffset := structure.ReadLengthWithOffset(rd) + ld.idle = int64(tempIdle) rdbsize += tempOffset case kFlagFreq: ld.freq = int64(structure.ReadByte(rd)) @@ -117,9 +117,9 @@ func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { log.Infof("RDB AUX fields. key=[%s], value=[%s]", key, value) } case kFlagResizeDB: - dbSize, dbsizeoffset := structure.ReadLengthWithOffset(rd) - expireSize, expiresizeoffset := structure.ReadLengthWithOffset(rd) - rdbsize += dbsizeoffset + expiresizeoffset + dbSize, dbSizeOffset := structure.ReadLengthWithOffset(rd) + expireSize, expireSizeOffset := structure.ReadLengthWithOffset(rd) + rdbsize += dbSizeOffset + expireSizeOffset log.Infof("RDB resize db. dbsize=[%d], expiresize=[%d]", dbSize, expireSize) case kFlagExpireMs: ld.expireMs = int64(structure.ReadUint64(rd)) - time.Now().UnixMilli() @@ -134,15 +134,15 @@ func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { ld.expireMs = 1 } case kFlagSelect: - DBId, DBIDoffset := structure.ReadLengthWithOffset(rd) + DBId, DBIDOffset := structure.ReadLengthWithOffset(rd) ld.nowDBId = int(DBId) - rdbsize += DBIDoffset + rdbsize += DBIDOffset case kEOF: UpdateRDBSize() return rdbsize default: - key, tempoffset := structure.ReadStringWithOffset(rd) - rdbsize += tempoffset + key, tempOffset := structure.ReadStringWithOffset(rd) + rdbsize += tempOffset var value bytes.Buffer anotherReader := io.TeeReader(rd, &value) o, tempOffsets := types.ParseObjectWithOffset(anotherReader, typeByte, key) @@ -170,7 +170,7 @@ func (ld *Loader) CheckparseRDBEntry(rd *bufio.Reader) int64 { v := ld.createValueDump(typeByte, value.Bytes()) - //value 口口口 + //value e.Argv = []string{"restore", key, strconv.FormatInt(ld.expireMs, 10), v} if config.Config.Advanced.RDBRestoreCommandBehavior == "rewrite" { if config.Config.Target.Version < 3.0 { diff --git a/internal/rdb/structure/string.go b/internal/rdb/structure/string.go index ebcbd63c..2b0ae7e4 100644 --- a/internal/rdb/structure/string.go +++ b/internal/rdb/structure/string.go @@ -42,6 +42,7 @@ func ReadString(rd io.Reader) string { } return string(ReadBytes(rd, int(length))) } + func ReadStringWithOffset(rd io.Reader) (string, int64) { length, special, offset, err := readEncodedLengthWithOffset(rd) if err != nil { diff --git a/internal/reader/aof_reader.go b/internal/reader/aof_reader.go index 1bb86c3e..b6c6494a 100644 --- a/internal/reader/aof_reader.go +++ b/internal/reader/aof_reader.go @@ -34,14 +34,14 @@ func (r *aofReader) StartRead() chan *entry.Entry { r.ch = make(chan *entry.Entry, 1024) go func() { - aof.AOFFileInfo = *(aof.NewAOFFileInfo()) + aof.AOFFileInfo = *(aof.NewAOFFileInfo(r.path)) aof.AOFLoadManifestFromDisk() am := aof.AOFFileInfo.GetAOFManifest() if am == nil { paths := path.Join(aof.AOFFileInfo.GetAOFDirName(), aof.AOFFileInfo.GetAOFFileName()) - aof.CheckAOFMain(paths) - log.Infof("start send AOF。path=[%s]", r.path) + aof.CheckAOFMain(r.path) + log.Infof("start send AOF path=[%s]", r.path) fi, err := os.Stat(r.path) if err != nil { log.Panicf("NewAOFReader: os.Stat error：%s", err.Error()) diff --git a/internal/statistics/statistics.go b/internal/statistics/statistics.go index e6d6367b..d3d52abb 100644 --- a/internal/statistics/statistics.go +++ b/internal/statistics/statistics.go @@ -27,7 +27,7 @@ type metrics struct { RdbReceivedSize uint64 `json:"rdb_received_size"` RdbSendSize uint64 `json:"rdb_send_size"` - //loading aof + //loading Loading bool `json:"loading"` AsyncLoading bool `json:"async_loading"` LoadingStartTime int64 `json:"loading_start_time"` @@ -39,6 +39,7 @@ type metrics struct { AofAppliedOffset uint64 `json:"aof_applied_offset"` AofFileSize uint64 `json:"aof_file_size"` AofReceivedSize uint64 `json:"aof_received_size"` + // for performance debug InQueueEntriesCount uint64 `json:"in_queue_entries_count"` UnansweredBytesCount uint64 `json:"unanswered_bytes_count"` diff --git a/restore.toml b/restore.toml index d43e0d54..a510e51a 100644 --- a/restore.toml +++ b/restore.toml @@ -4,9 +4,9 @@ type = "restore" version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... # Path to the dump.rdb file. Absolute path or relative path. Note # that relative paths are relative to the dir directory. - rdb_file_path = "" - aof_file_path="appendonlydir"//Please use an absolute path. - truncate-to-timestamp=0 +rdb_file_path = "dump.rdb" +aof_file_path = "" +aof_truncate_to_timestamp=0 [target] type = "standalone" # standalone or cluster # When the target is a cluster, write the address of one of the nodes. diff --git a/test/aof_test/aof_test.go b/test/aof_test/aof_test.go new file mode 100644 index 00000000..dae3d59e --- /dev/null +++ b/test/aof_test/aof_test.go @@ -0,0 +1,207 @@ +package aof_test + +import ( + "fmt" + "net/http" + "os" + "runtime" + "testing" + + "github.com/alibaba/RedisShake/internal/commands" + "github.com/alibaba/RedisShake/internal/config" + "github.com/alibaba/RedisShake/internal/filter" + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/reader" + "github.com/alibaba/RedisShake/internal/statistics" + "github.com/alibaba/RedisShake/internal/writer" + "github.com/go-redis/redis" +) + +const ( + AOFRestorePath = "/test_aof_restore.toml" + AppendOnlyAoFPath = "/appendonlydir/appendonly.aof.manifest" +) + +func AOFMain(configFile string, aofFilePath string) { + + // load aof config file + config.LoadFromFile(configFile) + + log.Init() + log.Infof("GOOS: %s, GOARCH: %s", runtime.GOOS, runtime.GOARCH) + log.Infof("Ncpu: %d, GOMAXPROCS: %d", config.Config.Advanced.Ncpu, runtime.GOMAXPROCS(0)) + log.Infof("pid: %d", os.Getpid()) + log.Infof("pprof_port: %d", config.Config.Advanced.PprofPort) + + // start pprof + if config.Config.Advanced.PprofPort != 0 { + go func() { + err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Config.Advanced.PprofPort), nil) + if err != nil { + log.PanicError(err) + } + }() + } + + // start statistics + if config.Config.Advanced.MetricsPort != 0 { + statistics.Metrics.Address = config.Config.Source.Address + go func() { + log.Infof("metrics url: http://localhost:%d", config.Config.Advanced.MetricsPort) + mux := http.NewServeMux() + mux.HandleFunc("/", statistics.Handler) + err := http.ListenAndServe(fmt.Sprintf("localhost:%d", config.Config.Advanced.MetricsPort), mux) + if err != nil { + log.PanicError(err) + } + }() + } + + // create writer + var theWriter writer.Writer + target := &config.Config.Target + switch config.Config.Target.Type { + case "standalone": + theWriter = writer.NewRedisWriter(target.Address, target.Username, target.Password, target.IsTLS) + case "cluster": + theWriter = writer.NewRedisClusterWriter(target.Address, target.Username, target.Password, target.IsTLS) + default: + log.Panicf("unknown target type: %s", target.Type) + } + + var theReader reader.Reader + + theReader = reader.NewAOFReader(aofFilePath) + + fmt.Printf("the aof path:%v\n", aofFilePath) + ch := theReader.StartRead() + + // start sync + statistics.Init() + id := uint64(0) + for e := range ch { + statistics.UpdateInQueueEntriesCount(uint64(len(ch))) + // calc arguments + e.Id = id + id++ + e.CmdName, e.Group, e.Keys = commands.CalcKeys(e.Argv) + e.Slots = commands.CalcSlots(e.Keys) + + // filter + code := filter.Filter(e) + statistics.UpdateEntryId(e.Id) + if code == filter.Allow { + theWriter.Write(e) + statistics.AddAllowEntriesCount() + } else if code == filter.Disallow { + // do something + statistics.AddDisallowEntriesCount() + } else { + log.Panicf("error when run lua filter. entry: %s", e.ToString()) + } + } + theWriter.Close() + log.Infof("finished.") +} + +// if you use this test you need start redis in port 6379s +func TestMainFunction(t *testing.T) { + + // os.Args = []string{"redis-shake", "/home/hwy/kaiyuan/restore.toml"} + wdPath, err := os.Getwd() + if err != nil { + panic(err) + } + + configPath := wdPath + AOFRestorePath + aofFilePath := wdPath + AppendOnlyAoFPath + fmt.Printf("configPath:%v, aofFilepath:%v\n", configPath, aofFilePath) + + AOFMain(configPath, aofFilePath) //restore aof + + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", + DB: 0, + }) + + pong, err := client.Ping().Result() + if err != nil { + t.Fatalf("Failed to connect to Redis: %v", err) + } + fmt.Println("Connected to Redis:", pong) + /* + for i := 11; i <= 10000; i++ { + value := strconv.Itoa(i) + score := float64(i) + z := redis.Z{Score: score, Member: value} + err := client.ZAdd("myzset", z).Err() + fmt.Println(value) + if err != nil { + fmt.Println("Failed to write data to Redis:", err) + return + } + } + */ + // 读取整个有序集合 + zsetValues, err := client.ZRangeWithScores("myzset", 0, -1).Result() + if err != nil { + fmt.Println("Failed to read data from Redis:", err) + return + } + + // 遍历有序集合中的元素和分数 + for _, z := range zsetValues { + member := z.Member.(string) + score := z.Score + fmt.Printf("Member: %s, Score: %f\n", member, score) + } + /* + expected := map[string]string{ + "kl": "kl", + "key0": "2022-03-29 17:25:54.592593", + "key1": "2022-03-29 17:25:54.876326", + "key2": "2022-03-29 17:25:52.871918", + "key3": "2022-03-29 17:25:53.034060", + "key4": "2022-03-29 17:25:53.196913", + "key5": "2022-03-29 17:25:53.356234", + "key6": "2022-03-29 17:25:53.513544", + "key7": "2022-03-29 17:25:53.671556", + "key8": "2022-03-29 17:25:53.861237", + "key9": "2022-03-29 17:25:54.020518", + "key10": "2022-03-29 17:25:54.177881", + "key11": "2022-03-29 17:25:54.337640", + } + */ + /*for key, value := range expected { + result, err := client.Get(key).Result() + if err != nil { + t.Fatalf("Failed to read key %s from Redis: %v", key, err) + } + + if result != value { + t.Errorf("Value for key %s is incorrect. Expected: %s, Got: %s", key, value, result) + } + } + + for key := 11; key <= 10000; key++ { + result, err := client.Get(strconv.Itoa(key)).Result() + if err != nil { + t.Fatalf("Failed to read key %v from Redis: %v", key, err) + } + + if result != strconv.Itoa(key) { + t.Errorf("Value for key %v is incorrect. Expected: %v, Got: %v", key, key, result) + } + }*/ + + /*result, err := client.SMembers("superpowers").Result() + if err != nil { + t.Fatalf("Failed to read set from Redis: %v", err) + } + strings := result[0] + if strings != "reflexes" { + t.Errorf("read set wrong") + }*/ + +} diff --git a/appendonlydir/appendonly.aof.manifest b/test/aof_test/appendonlydir/appendonly.aof.manifest similarity index 100% rename from appendonlydir/appendonly.aof.manifest rename to test/aof_test/appendonlydir/appendonly.aof.manifest diff --git a/test/aof_test/test_aof_restore.toml b/test/aof_test/test_aof_restore.toml new file mode 100644 index 00000000..e5d2f762 --- /dev/null +++ b/test/aof_test/test_aof_restore.toml @@ -0,0 +1,55 @@ +type = "restore" + +[source] +version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... +# Path to the dump.rdb file. Absolute path or relative path. Note +# that relative paths are relative to the dir directory. +rdb_file_path = "" +aof_file_path="/appendonly" #Please use an absolute path. +aof_truncate_to_timestamp=0 +[target] +type = "standalone" # standalone or cluster +# When the target is a cluster, write the address of one of the nodes. +# redis-shake will obtain other nodes through the `cluster nodes` command. +version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... +address = "127.0.0.1:6379" +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false + +[advanced] +dir = "data" + +# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores +ncpu = 3 + +# pprof port, 0 means disable +pprof_port = 0 + +# metric port, 0 means disable +metrics_port = 0 + +# log +log_file = "redis-shake.log" +log_level = "info" # debug, info or warn +log_interval = 5 # in seconds + +# redis-shake gets key and value from rdb file, and uses RESTORE command to +# create the key in target redis. Redis RESTORE will return a "Target key name +# is busy" error when key already exists. You can use this configuration item +# to change the default behavior of restore: +# panic: redis-shake will stop when meet "Target key name is busy" error. +# rewrite: redis-shake will replace the key with new value. +# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. +rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip + +# pipeline +pipeline_count_limit = 1024 + +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default. This amount is normally 1gb. +target_redis_client_max_querybuf_len = 1024_000_000 + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. +target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file From 8f79194b7e58ea54ac91db27865c623725d77d2f Mon Sep 17 00:00:00 2001 From: hwy1314 <120362432+hwy1314@users.noreply.github.com> Date: Mon, 4 Sep 2023 21:00:04 +0800 Subject: [PATCH 8/8] Fixed a bug regarding aof offset and changed the way timestamps are checked. --- internal/aof/aof.go | 26 ++++++++++++-------- internal/aof/aof_check.go | 46 ++++++++++++++++++++++------------- internal/commands/keys.go | 5 ++-- internal/reader/aof_reader.go | 5 ++-- 4 files changed, 51 insertions(+), 31 deletions(-) diff --git a/internal/aof/aof.go b/internal/aof/aof.go index 32918afd..05f53995 100644 --- a/internal/aof/aof.go +++ b/internal/aof/aof.go @@ -673,10 +673,9 @@ func AOFLoadManifestFromDisk() { } am := AOFLoadManifestFromFile(am_Filepath) - if am != nil { - AOFFileInfo.AOFManifest = am - } - + //if am != nil { + AOFFileInfo.AOFManifest = am + //} } func GetNewIncrAOFName(am *AOFManifest) string { @@ -813,7 +812,7 @@ func GetBaseAndIncrAppendOnlyFilesNum(am *AOFManifest) int { return num } -func (ld *Loader) LoadSingleAppendOnlyFile(FileName string, ch chan *entry.Entry) int { +func (ld *Loader) LoadSingleAppendOnlyFile(FileName string, ch chan *entry.Entry, LastFile bool) int { ret := AOFOK AOFFilepath := MakePath(AOFFileInfo.AOFDirName, FileName) var sizes int64 = 0 @@ -844,13 +843,13 @@ func (ld *Loader) LoadSingleAppendOnlyFile(FileName string, ch chan *entry.Entry return ret } } else { + sizes += 5 log.Infof("Reading RDB Base File on AOF loading...") ldRDB := rdb.NewLoader(AOFFilepath, ch) ldRDB.ParseRDB() return AOFOK //Skipped RDB checksum and has not been processed yet. } - sizes += 5 reader := bufio.NewReader(fp) for { @@ -918,7 +917,9 @@ func (ld *Loader) LoadSingleAppendOnlyFile(FileName string, ch chan *entry.Entry e.Argv = append(e.Argv, value) } ld.ch <- e - + if sizes >= CheckAOFInfof.pos && LastFile { + break + } } } @@ -974,7 +975,7 @@ func (ld *Loader) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Entry) int BaseSize = GetAppendOnlyFileSize(AOFName, nil) lastFile = totalNum start = Ustime() - ret = ld.LoadSingleAppendOnlyFile(AOFName, ch) + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch, false) if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) { log.Infof("DB loaded from Base File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) } @@ -1001,6 +1002,7 @@ func (ld *Loader) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Entry) int return ret } } + totalNum-- } if am.incrAOFList.len > 0 { @@ -1019,7 +1021,11 @@ func (ld *Loader) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Entry) int lastFile = totalNum AOFNum++ start = Ustime() - ret = ld.LoadSingleAppendOnlyFile(AOFName, ch) + if lastFile == 1 { + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch, true) + } else { + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch, false) + } if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) { log.Infof("DB loaded from incr File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) } @@ -1043,7 +1049,7 @@ func (ld *Loader) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Entry) int } ln = ListNext(&li) } - + totalNum-- } AOFFileInfo.AOFCurrentSize = totalSize diff --git a/internal/aof/aof_check.go b/internal/aof/aof_check.go index 3f976e1a..210b741d 100644 --- a/internal/aof/aof_check.go +++ b/internal/aof/aof_check.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "io" + "math" "os" "path" @@ -107,7 +108,7 @@ func FilelsManifest(AOFFilePath string) bool { log.Panicf("cannot read File: %v\n", AOFFilePath) } } - if lines[0] == '#' { + if lines[0] == '#' || len(lines) < 4 { continue } else if lines[:4] == "file" { is_manifest = true @@ -199,7 +200,7 @@ func ReadBytes(rd *bufio.Reader, target *[]byte, length int64) int { var err error *target, err = rd.ReadBytes('\n') if err != nil || (*target)[length-1] != '\n' { - log.Infof("Expected to read %d bytes, got %d bytes\n%s", length, 1, *target) + log.Infof("AOF format error:%s", *target) return 0 } CheckAOFInfof.pos += length @@ -265,13 +266,18 @@ func AOFLoadManifestFromFile(am_Filepath string) *AOFManifest { if err != nil { if err == io.EOF { if linenum == 0 { - log.Panicf("Found an empty AOF manifest") + log.Infof("Found an empty AOF manifest") + am = nil + return am } else { break } } else { - log.Panicf("Read AOF manifest failed") + log.Infof("Read AOF manifest failed") + am = nil + return am + } } @@ -280,17 +286,21 @@ func AOFLoadManifestFromFile(am_Filepath string) *AOFManifest { continue } if !strings.Contains(buf, "\n") { - log.Panicf("The AOF manifest File contains too long line") + log.Infof("The AOF manifest File contains too long line") + return nil } line = strings.Trim(buf, " \t\r\n") if len(line) == 0 { - log.Panicf("Invalid AOF manifest File format") + log.Infof("Invalid AOF manifest File format") + return nil } argc := 0 argv, argc = SplitArgs(line) if argc < 6 || argc%2 != 0 { - log.Panicf("Invalid AOF manifest File format") + log.Infof("Invalid AOF manifest File format") + am = nil + return am } ai = AOFInfoCreate() for i := 0; i < argc; i += 2 { @@ -370,8 +380,6 @@ func ProcessAnnotations(rd *bufio.Reader, Filename string, lastFile bool) int { if err != nil { log.Panicf("Failed to read annotations from AOF %v, aborting...\n", Filename) } - CheckAOFInfof.pos += int64(len(buf)) + 2 - CheckAOFInfof.toTimestamp = config.Config.Source.AOFTruncateToTimestamp if CheckAOFInfof.toTimestamp != 0 && strings.HasPrefix(string(buf), "TS:") { var ts int64 ts, err = strconv.ParseInt(strings.TrimPrefix(string(buf), "TS:"), 10, 64) @@ -380,6 +388,7 @@ func ProcessAnnotations(rd *bufio.Reader, Filename string, lastFile bool) int { } if ts <= CheckAOFInfof.toTimestamp { + CheckAOFInfof.pos += int64(len(buf)) + 2 return 1 } @@ -393,14 +402,14 @@ func ProcessAnnotations(rd *bufio.Reader, Filename string, lastFile bool) int { } // Truncate remaining AOF if exceeding 'toTimestamp' - if err := CheckAOFInfof.fp.Truncate(CheckAOFInfof.pos); err != nil { + /*if err := CheckAOFInfof.fp.Truncate(CheckAOFInfof.pos); err != nil { log.Panicf("Failed to truncate AOF %v to timestamp %d\n", Filename, CheckAOFInfof.toTimestamp) - } else { - - return 0 - } + } else {*/ + //CheckAOFInfof.pos += int64(len(buf)) + 2 + return 0 + //} } - + CheckAOFInfof.pos += int64(len(buf)) + 2 return 1 } @@ -418,7 +427,8 @@ func CheckMultipartAOF(DirPath string, ManifestFilePath string, fix int) { if am.BaseAOFInfo != nil { AOFFileName := am.BaseAOFInfo.FileName AOFFilePath := MakePath(DirPath, AOFFileName) - lastFile := (AOFNum + 1) == totalNum + AOFNum++ + lastFile := AOFNum == totalNum AOFPreable := FileIsRDB(AOFFilePath) if AOFPreable { log.Infof("Start to check Base AOF (RDB format).\n") @@ -437,7 +447,8 @@ func CheckMultipartAOF(DirPath string, ManifestFilePath string, fix int) { ai := ln.value.(*AOFInfo) AOFFileName := ai.FileName AOFFilePath := MakePath(DirPath, AOFFileName) - lastFile := (AOFNum + 1) == totalNum + AOFNum++ + lastFile := AOFNum == totalNum ret = CheckSingleAOF(AOFFileName, AOFFilePath, lastFile, fix, false) OutPutAOFStyle(ret, AOFFileName, "INCR AOF") ln = ln.next @@ -455,6 +466,7 @@ func CheckOldStyleAOF(AOFFilePath string, fix int, preamble bool) { } func CheckSingleAOF(AOFFileName, AOFFilePath string, lastFile bool, fix int, preamble bool) int { var rdbpos int64 = 0 + CheckAOFInfof.toTimestamp = config.Config.Source.AOFTruncateToTimestamp multi := 0 CheckAOFInfof.pos = 0 buf := make([]byte, 1) diff --git a/internal/commands/keys.go b/internal/commands/keys.go index 0899e055..1094821c 100644 --- a/internal/commands/keys.go +++ b/internal/commands/keys.go @@ -2,11 +2,12 @@ package commands import ( "fmt" - "github.com/alibaba/RedisShake/internal/log" - "github.com/alibaba/RedisShake/internal/utils" "math" "strconv" "strings" + + "github.com/alibaba/RedisShake/internal/log" + "github.com/alibaba/RedisShake/internal/utils" ) // CalcKeys https://redis.io/docs/reference/key-specs/ diff --git a/internal/reader/aof_reader.go b/internal/reader/aof_reader.go index b6c6494a..b0e7969f 100644 --- a/internal/reader/aof_reader.go +++ b/internal/reader/aof_reader.go @@ -35,6 +35,7 @@ func (r *aofReader) StartRead() chan *entry.Entry { go func() { aof.AOFFileInfo = *(aof.NewAOFFileInfo(r.path)) + aof.AOFLoadManifestFromDisk() am := aof.AOFFileInfo.GetAOFManifest() @@ -49,8 +50,7 @@ func (r *aofReader) StartRead() chan *entry.Entry { statistics.Metrics.AofFileSize = uint64(fi.Size()) statistics.Metrics.AofReceivedSize = uint64(fi.Size()) aofLoader := aof.NewLoader(r.path, r.ch) - - _ = aofLoader.LoadSingleAppendOnlyFile(paths, r.ch) + _ = aofLoader.LoadSingleAppendOnlyFile(paths, r.ch, true) log.Infof("Send AOF finished. path=[%s]", r.path) close(r.ch) } else { @@ -68,6 +68,7 @@ func (r *aofReader) StartRead() chan *entry.Entry { log.Infof("Send AOF finished. path=[%s]", r.path) close(r.ch) } + }() return r.ch