From 5ee9b6b2a56b9bc6454b53f7cd9a119c203ea3e8 Mon Sep 17 00:00:00 2001 From: code021 Date: Sun, 31 Dec 2023 18:03:16 +0800 Subject: [PATCH 01/13] =?UTF-8?q?=E9=80=9A=E8=BF=87file-rotatelogs?= =?UTF-8?q?=E3=80=81lfshook=E5=AF=B9logrus=E8=BF=9B=E8=A1=8C=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E5=88=87=E5=89=B2=EF=BC=88=E5=8D=95=E5=85=83=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E6=98=AF:logruslyHook=5Ftest.go=5F=20=EF=BC=89?= =?UTF-8?q?=E5=92=8C=E6=9B=B4=E6=96=B0go.mod?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/root.go | 86 ++++++++++++++---- go.mod | 53 +++++++---- go.sum | 55 +++++++++++ test/_cmd/logruslyHook_test.go_ | 156 ++++++++++++++++++++++++++++++++ 4 files changed, 312 insertions(+), 38 deletions(-) create mode 100644 test/_cmd/logruslyHook_test.go_ diff --git a/cmd/root.go b/cmd/root.go index bf414e0..72283f6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,8 +4,9 @@ import ( "bytes" "database/sql" "fmt" + rotatelogs "github.com/lestrrat-go/file-rotatelogs" "github.com/mitchellh/go-homedir" - "io" + "github.com/rifflock/lfshook" "math" "os" "os/signal" @@ -24,7 +25,8 @@ import ( "github.com/spf13/viper" ) -var log = logrus.New() +// var log = logrus.New() +var log *logrus.Logger var cfgFile string var selFromYml bool @@ -48,22 +50,9 @@ func startDataTransfer(connStr *connect.DbConnStr) { exitChan := make(chan os.Signal) signal.Notify(exitChan, os.Interrupt, os.Kill, syscall.SIGTERM) go exitHandle(exitChan) - // 创建运行日志目录 - logDir, _ := filepath.Abs(CreateDateDir("")) - // 输出调用文件以及方法位置 - log.SetReportCaller(true) - f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := f.Close(); err != nil { - log.Fatal(err) - } - }() - // log信息重定向到平面文件 - multiWriter := io.MultiWriter(os.Stdout, f) - log.SetOutput(multiWriter) + + logDir := newLogger() //初始化logrus日志和定义日志文件切割 + start := time.Now() // map结构，表名以及该表用来迁移查询源库的语句 var tableMap map[string][]string @@ -200,6 +189,67 @@ func startDataTransfer(connStr *connect.DbConnStr) { log.Info(fmt.Sprintf("All complete totalTime %s The Report Dir %s", cost, logDir)) } +func newLogger() string { + // 创建运行日志目录 + logDir, _ := filepath.Abs(CreateDateDir("")) + if log != nil { + return logDir + } + + infoLogPath := filepath.Join(logDir, "run.log") + // 创建rotatelogs的Logger实例 + infoRotator, err := rotatelogs.New( + infoLogPath+".%Y%m%d%H%M", + rotatelogs.WithLinkName(infoLogPath), // 生成软链接指向最新日志文件 + rotatelogs.WithMaxAge(24*time.Hour), // 日志文件最大保留时间 + rotatelogs.WithRotationTime(time.Hour), // 日志切割时间间隔 + rotatelogs.WithRotationSize(50*1024*1024), // 日志文件最大大小（例如50MB） + ) + if err != nil { + fmt.Printf("Failed to create rotatelogs logger: %v", err) + return logDir + } + + // 为 Error 级别日志设置 rotatelogs + errorLogPath := filepath.Join(logDir, "run_error.log") + errorRotator, err := rotatelogs.New( + errorLogPath+".%Y%m%d%H%M", + rotatelogs.WithLinkName(errorLogPath), + rotatelogs.WithMaxAge(7*24*time.Hour), + rotatelogs.WithRotationTime(24*time.Hour), + //rotatelogs.WithRotationSize(10*1024*1024), + ) + if err != nil { + fmt.Printf("failed to create rotatelogs for error: %v", err) + } + + // lfshook 决定哪些日志级别可用日志分割 + writeMap := lfshook.WriterMap{ + //logrus.PanicLevel: rl, + //logrus.FatalLevel: rl, + logrus.ErrorLevel: errorRotator, + //logrus.WarnLevel: rl, + logrus.InfoLevel: infoRotator, + //logrus.DebugLevel: rl, + } + + // 配置 lfshook + hook := lfshook.NewHook(writeMap, &logrus.TextFormatter{ + // 设置日期格式 + TimestampFormat: "2006.01.02 - 15:04:05", + }) + + log = logrus.New() + // 输出调用文件以及方法位置 + log.SetReportCaller(true) + // 设置将日志输出到标准输出（默认的输出为stderr,标准错误） + log.SetOutput(os.Stdout) + log.AddHook(hook) + + fmt.Println(infoLogPath) + return logDir +} + // 自动对表分析，然后生成每个表用来迁移查询源库SQL的集合(全表查询或者分页查询) // 自动分析是否有排除的表名 // 最后返回map结构即 表:[查询SQL] diff --git a/go.mod b/go.mod index 0327760..e3f1c57 100644 --- a/go.mod +++ b/go.mod @@ -3,34 +3,47 @@ module OracleSync2MySQL go 1.20 require ( - github.com/fatih/color v1.15.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/go-sql-driver/mysql v1.7.1 // indirect - github.com/godror/godror v0.37.0 // indirect - github.com/godror/knownpb v0.1.0 // indirect + github.com/fatih/color v1.16.0 + github.com/go-sql-driver/mysql v1.7.1 + github.com/godror/godror v0.41.0 + github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092 + github.com/mitchellh/go-homedir v1.1.0 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/cobra v1.8.0 + github.com/spf13/viper v1.18.2 +) + +require ( + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/godror/knownpb v0.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092 // indirect + github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sijms/go-ora/v2 v2.7.9 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/afero v1.9.5 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/cobra v1.7.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.16.0 // indirect - github.com/subosito/gotenv v1.4.2 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 86b831a..9442e3b 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,7 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -57,21 +58,33 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/godror/godror v0.37.0 h1:3wR3/1msywDE49PzuXh9UUiwWOBNri0RVQQcu3HU4UY= github.com/godror/godror v0.37.0/go.mod h1:jW1+pN+z/V0h28p9XZXVNtEvfZP/2EBfaSjKJLp3E4g= +github.com/godror/godror v0.41.0 h1:LVwpbfYmGrxIy7nBmv9w7VJxKlmRei6he4lyHgKCEF0= +github.com/godror/godror v0.41.0/go.mod h1:i8YtVTHUJKfFT3wTat4A9UoqScUtZXiYB9Rf3SVARgc= github.com/godror/knownpb v0.1.0 h1:dJPK8s/I3PQzGGaGcUStL2zIaaICNzKKAK8BzP1uLio= github.com/godror/knownpb v0.1.0/go.mod h1:4nRFbQo1dDuwKnblRXDxrfCFYeT4hjg3GjMqef58eRE= +github.com/godror/knownpb v0.1.1 h1:A4J7jdx7jWBhJm18NntafzSC//iZDHkDi1+juwQ5pTI= +github.com/godror/knownpb v0.1.1/go.mod h1:4nRFbQo1dDuwKnblRXDxrfCFYeT4hjg3GjMqef58eRE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -143,6 +156,11 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092 h1:u9I3sJ+uTakxnRrvuYJGsEi4SvEMN+yB47WWGDHHxIk= github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092/go.mod h1:CxUy8nDvutaC1pOfaG9TRoYwdHHqoNstSPPKhomC9k8= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -152,46 +170,71 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= +github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sijms/go-ora/v2 v2.7.9 h1:FvPwsyNtAOywDKlgjrgCpGkL0s49ZA/ShTBgEAfYKE0= github.com/sijms/go-ora/v2 v2.7.9/go.mod h1:EHxlY6x7y9HAsdfumurRfTd+v8NrEOTR3Xl4FWlH6xk= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -202,6 +245,10 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -219,6 +266,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -335,6 +384,8 @@ golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -346,6 +397,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -491,6 +544,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/test/_cmd/logruslyHook_test.go_ b/test/_cmd/logruslyHook_test.go_ new file mode 100644 index 0000000..4b16228 --- /dev/null +++ b/test/_cmd/logruslyHook_test.go_ @@ -0,0 +1,156 @@ +package _cmd + +import ( + "fmt" + rotatelogs "github.com/lestrrat-go/file-rotatelogs" + "github.com/rifflock/lfshook" + "github.com/sirupsen/logrus" + "math/rand" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +var log *logrus.Logger + +// 测试日志文件是否按大小进行切割 +func TestLogRotationBySize(t *testing.T) { + logDir := newLogger() + fmt.Println(logDir) + log.Info("running SourceDB check connect") + + // 生成足够的日志以触发文件切割 + for i := 0; i < 100000; i++ { + g := generateRandomString(33) + fmt.Printf("Test os.Stdout log entry %d,%s", i, g) + log.Infof("Test info log entry %d,%s", i, g) + log.Errorf("Test error log entry %d,%s", i, g) + } + + // 检查是否创建了多个日志文件 + //files, err := os.ReadDir(logDir) + //if err != nil { + // t.Fatalf("Failed to list log directory: %v", err) + //} + + // 至少应该有两个日志文件 + //if len(files) < 2 { + // t.Errorf("Expected multiple log files due to rotation, but found %d", len(files)) + //} + + //清理测试日志文件 + //cleanupTestLogs(logDir) +} + +// 清理测试生成的日志文件 +func cleanupTestLogs(dir string) { + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if strings.Contains(path, "run") { + os.Remove(path) + } + return nil + }) +} + +// CreateDateDir 根据当前日期来创建文件夹 +func CreateDateDir(basePath string) string { + folderName := "log/" + time.Now().Format("2006_01_02_15_04_05") + folderPath := filepath.Join(basePath, folderName) + if _, err := os.Stat(folderPath); os.IsNotExist(err) { + // 必须分成两步 + // 先创建文件夹 + err := os.MkdirAll(folderPath, 0777) //级联创建目录 + if err != nil { + fmt.Println("create directory log failed ", err) + } + // 再修改权限 + err = os.Chmod(folderPath, 0777) + if err != nil { + fmt.Println("chmod directory log failed ", err) + } + } + return folderPath +} + +func newLogger() string { + // 创建运行日志目录 + logDir, _ := filepath.Abs(CreateDateDir("")) + if log != nil { + return logDir + } + + logFilePath := logDir + "/" + "run.log" + // 创建rotatelogs的Logger实例 + rl, err := rotatelogs.New( + logFilePath+".%Y%m%d%H%M", + rotatelogs.WithLinkName(logFilePath), // 生成软链接指向最新日志文件 + rotatelogs.WithMaxAge(24*time.Hour), // 日志文件最大保留时间 + rotatelogs.WithRotationTime(time.Hour), // 日志切割时间间隔 + rotatelogs.WithRotationSize(1024*1024), // 日志文件最大大小（例如50MB） + ) + if err != nil { + fmt.Println("Failed to create rotatelogs logger:", err) + return logDir + } + + // 为 Error 级别日志设置 rotatelogs + errorLogPath := filepath.Join(logDir, "run_error.log") + errorRotator, err := rotatelogs.New( + errorLogPath+".%Y%m%d%H%M", + rotatelogs.WithLinkName(errorLogPath), + rotatelogs.WithMaxAge(7*24*time.Hour), + rotatelogs.WithRotationTime(24*time.Hour), + //rotatelogs.WithRotationSize(10*1024*1024), + ) + if err != nil { + fmt.Printf("failed to create rotatelogs for error: %v", err) + } + + // lfshook 决定哪些日志级别可用日志分割 + writeMap := lfshook.WriterMap{ + //logrus.PanicLevel: rl, + //logrus.FatalLevel: rl, + logrus.ErrorLevel: errorRotator, + //logrus.WarnLevel: rl, + logrus.InfoLevel: rl, + //logrus.DebugLevel: rl, + } + + //logrus.AddHook(lfshook.NewHook( + // lfshook.WriterMap{ + // logrus.InfoLevel: rl, + // logrus.ErrorLevel: rl, + // }, + // &logrus.TextFormatter{}, + //)) + + // 配置 lfshook + hook := lfshook.NewHook(writeMap, &logrus.TextFormatter{ + // 设置日期格式 + TimestampFormat: "2006.01.02 - 15:04:05", + }) + + log = logrus.New() + log.SetReportCaller(true) + //multiWriter := io.MultiWriter(os.Stdout, rl) + log.SetOutput(os.Stdout) + fmt.Println(logFilePath) + log.AddHook(hook) + + return logDir +} + +// generateRandomString 生成长度为 n 的随机字符串 +func generateRandomString(n int) string { + charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + var seededRand *rand.Rand = rand.New( + rand.NewSource(time.Now().UnixNano())) + + b := make([]byte, n) + for i := range b { + b[i] = charset[seededRand.Intn(len(charset))] + } + return string(b) +} From bda9dad99dac2ceb9b1afeb85ce976368f32b182 Mon Sep 17 00:00:00 2001 From: xls-web Date: Tue, 2 Jan 2024 10:49:13 +0800 Subject: [PATCH 02/13] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E4=BD=BF=E7=94=A8newlo?= =?UTF-8?q?gger=E6=9D=A5=E5=88=9D=E5=A7=8B=E5=8C=96=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/compare.go | 30 ++++++++++++-------------- cmd/create.go | 58 +++++++++++++++++++++++++------------------------- 2 files changed, 43 insertions(+), 45 deletions(-) diff --git a/cmd/compare.go b/cmd/compare.go index bdbe8fc..3677973 100644 --- a/cmd/compare.go +++ b/cmd/compare.go @@ -5,9 +5,6 @@ import ( "github.com/liushuochen/gotable" "github.com/spf13/cobra" "github.com/spf13/viper" - "io" - "os" - "path/filepath" "strconv" "time" ) @@ -40,20 +37,21 @@ var compareDbCmd = &cobra.Command{ } else { // 不指定-s选项，查询源库所有表名 tableMap = fetchTableMap(pageSize, excludeTab) } + newLogger() //初始化logrus日志和定义日志文件切割 // 创建运行日志目录 - logDir, _ := filepath.Abs(CreateDateDir("")) - f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := f.Close(); err != nil { - log.Fatal(err) // 或设置到函数返回值中 - } - }() - // log信息重定向到平面文件 - multiWriter := io.MultiWriter(os.Stdout, f) - log.SetOutput(multiWriter) + //logDir, _ := filepath.Abs(CreateDateDir("")) + //f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) + //if err != nil { + // log.Fatal(err) + //} + //defer func() { + // if err := f.Close(); err != nil { + // log.Fatal(err) // 或设置到函数返回值中 + // } + //}() + //// log信息重定向到平面文件 + //multiWriter := io.MultiWriter(os.Stdout, f) + //log.SetOutput(multiWriter) // 以下开始调用比对表行数的方法 start := time.Now() // 用于控制协程goroutine运行时候的并发数,例如3个一批，3个一批的goroutine并发运行 diff --git a/cmd/create.go b/cmd/create.go index fc7a9d5..3ebda74 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -4,8 +4,6 @@ import ( "fmt" "github.com/liushuochen/gotable" "github.com/spf13/viper" - "io" - "os" "path/filepath" "strconv" "strings" @@ -45,20 +43,22 @@ var createTableCmd = &cobra.Command{ } else { // 不指定-s选项，查询源库所有表名 tableMap = fetchTableMap(pageSize, excludeTab) } + logDir := newLogger() //初始化logrus日志和定义日志文件切割 + //logDir := newLogger() //初始化logrus日志和定义日志文件切割 // 创建运行日志目录 - logDir, _ := filepath.Abs(CreateDateDir("")) - f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := f.Close(); err != nil { - log.Fatal(err) // 或设置到函数返回值中 - } - }() - // log信息重定向到平面文件 - multiWriter := io.MultiWriter(os.Stdout, f) - log.SetOutput(multiWriter) + //logDir, _ := filepath.Abs(CreateDateDir("")) + //f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) + //if err != nil { + // log.Fatal(err) + //} + //defer func() { + // if err := f.Close(); err != nil { + // log.Fatal(err) // 或设置到函数返回值中 + // } + //}() + //// log信息重定向到平面文件 + //multiWriter := io.MultiWriter(os.Stdout, f) + //log.SetOutput(multiWriter) // 实例初始化，调用接口中创建目标表的方法 var db Database start := time.Now() @@ -91,20 +91,20 @@ var onlyDataCmd = &cobra.Command{ connStr := getConn() // 创建运行日志目录 logDir, _ := filepath.Abs(CreateDateDir("")) - // 输出调用文件以及方法位置 - log.SetReportCaller(true) - f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := f.Close(); err != nil { - log.Fatal(err) - } - }() - // log信息重定向到平面文件 - multiWriter := io.MultiWriter(os.Stdout, f) - log.SetOutput(multiWriter) + //// 输出调用文件以及方法位置 + //log.SetReportCaller(true) + //f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) + //if err != nil { + // log.Fatal(err) + //} + //defer func() { + // if err := f.Close(); err != nil { + // log.Fatal(err) + // } + //}() + //// log信息重定向到平面文件 + //multiWriter := io.MultiWriter(os.Stdout, f) + //log.SetOutput(multiWriter) start := time.Now() // map结构，表名以及该表用来迁移查询源库的语句 var tableMap map[string][]string From 08f1d97ed53e7c2af0bcf0e919dd9a33d39299f4 Mon Sep 17 00:00:00 2001 From: xls-web Date: Tue, 2 Jan 2024 11:10:19 +0800 Subject: [PATCH 03/13] =?UTF-8?q?defer=20close=E8=B0=83=E4=BC=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/app.go | 6 +++++- cmd/create.go | 13 +++++++++++-- cmd/root.go | 31 +++++++++++++++++++++++-------- cmd/tablemeta.go | 18 +++++++++++++++--- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/cmd/app.go b/cmd/app.go index 8c097cc..517e429 100644 --- a/cmd/app.go +++ b/cmd/app.go @@ -194,7 +194,11 @@ func cleanDBconn() { if err != nil { log.Error(err) } - defer rows.Close() + defer func(rows *sql.Rows) { + if err := rows.Close(); err != nil { + log.Fatal(err) + } + }(rows) for rows.Next() { var id string err = rows.Scan(&id) diff --git a/cmd/create.go b/cmd/create.go index 3ebda74..e7e15fe 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -1,6 +1,7 @@ package cmd import ( + "database/sql" "fmt" "github.com/liushuochen/gotable" "github.com/spf13/viper" @@ -113,13 +114,21 @@ var onlyDataCmd = &cobra.Command{ log.Info("running SourceDB check connect") // 生成源库数据库连接 PrepareSrc(connStr) - defer srcDb.Close() + defer func(srcDb *sql.DB) { + if err := srcDb.Close(); err != nil { + log.Fatal(err) + } + }(srcDb) // 每页的分页记录数,仅全库迁移时有效 pageSize := viper.GetInt("pageSize") log.Info("running TargetDB check connect") // 生成目标库的数据库连接 PrepareDest(connStr) - defer destDb.Close() + defer func(destDb *sql.DB) { + if err := destDb.Close(); err != nil { + log.Fatal(err) + } + }(destDb) // 以下是迁移数据前的准备工作，获取要迁移的表名以及该表查询源库的sql语句(如果有主键生成该表的分页查询切片集合，没有主键的统一是全表查询sql) if selFromYml { // 如果用了-s选项，从配置文件中获取表名以及sql语句 tableMap = viper.GetStringMapStringSlice("tables") diff --git a/cmd/root.go b/cmd/root.go index 72283f6..df85051 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -61,13 +61,21 @@ func startDataTransfer(connStr *connect.DbConnStr) { log.Info("running SourceDB check connect") // 生成源库数据库连接 PrepareSrc(connStr) - defer srcDb.Close() + defer func(srcDb *sql.DB) { + if err := srcDb.Close(); err != nil { + log.Fatal(err) + } + }(srcDb) // 每页的分页记录数,仅全库迁移时有效 pageSize := viper.GetInt("pageSize") log.Info("running TargetDB check connect") // 生成目标库的数据库连接 PrepareDest(connStr) - defer destDb.Close() + defer func(destDb *sql.DB) { + if err := destDb.Close(); err != nil { + log.Fatal(err) + } + }(destDb) // 以下是迁移数据前的准备工作，获取要迁移的表名以及该表查询源库的sql语句(如果有主键生成该表的分页查询切片集合，没有主键的统一是全表查询sql) if selFromYml { // 如果用了-s选项，从配置文件中获取表名以及sql语句 tableMap = viper.GetStringMapStringSlice("tables") @@ -169,7 +177,10 @@ func startDataTransfer(connStr *connect.DbConnStr) { return } ymlConfig := []string{connStr.SrcHost + "-" + connStr.SrcUserName, connStr.DestHost + "-" + connStr.DestDatabase, strconv.Itoa(maxParallel), strconv.Itoa(pageSize), strconv.Itoa(len(excludeTab))} - tblConfig.AddRow(ymlConfig) + err = tblConfig.AddRow(ymlConfig) + if err != nil { + fmt.Println("tblConfig Add row failed: ", err.Error()) + } fmt.Println(tblConfig) // 输出迁移摘要 table, err := gotable.Create("Object", "BeginTime", "EndTime", "FailedTotal", "ElapsedTime") @@ -225,11 +236,11 @@ func newLogger() string { // lfshook 决定哪些日志级别可用日志分割 writeMap := lfshook.WriterMap{ - //logrus.PanicLevel: rl, - //logrus.FatalLevel: rl, + logrus.PanicLevel: errorRotator, + logrus.FatalLevel: errorRotator, logrus.ErrorLevel: errorRotator, - //logrus.WarnLevel: rl, - logrus.InfoLevel: infoRotator, + logrus.WarnLevel: infoRotator, + logrus.InfoLevel: infoRotator, //logrus.DebugLevel: rl, } @@ -274,7 +285,11 @@ func fetchTableMap(pageSize int, excludeTable []string) (tableMap map[string][]s } // 查询下源库总共的表，获取到表名 rows, err := srcDb.Query(sqlStr) - defer rows.Close() + defer func(rows *sql.Rows) { + if err := rows.Close(); err != nil { + log.Fatal(err) + } + }(rows) if err != nil { log.Error(fmt.Sprintf("Query "+sqlStr+" failed,\nerr:%v\n", err)) return diff --git a/cmd/tablemeta.go b/cmd/tablemeta.go index d394af2..39f4fd7 100644 --- a/cmd/tablemeta.go +++ b/cmd/tablemeta.go @@ -186,7 +186,11 @@ func (tb *Table) IdxCreate(logDir string, tableName string, ch chan struct{}, id if err != nil { log.Error(err) } - defer rows.Close() + defer func(rows *sql.Rows) { + if err := rows.Close(); err != nil { + log.Fatal(err) + } + }(rows) // 从sql结果集遍历，获取到创建语句 for rows.Next() { if err := rows.Scan(&destIdxSql); err != nil { @@ -214,7 +218,11 @@ func (tb *Table) SeqCreate(logDir string) (ret []string) { if err != nil { log.Error(err) } - defer rows.Close() + defer func(rows *sql.Rows) { + if err := rows.Close(); err != nil { + log.Fatal(err) + } + }(rows) idx := 0 for rows.Next() { idx += 1 @@ -268,7 +276,11 @@ func (tb *Table) FkCreate(logDir string) (ret []string) { if err != nil { log.Error(err) } - defer rows.Close() + defer func(rows *sql.Rows) { + if err := rows.Close(); err != nil { + log.Fatal(err) + } + }(rows) idx := 0 for rows.Next() { idx += 1 From caa52fa948f489a1eeee4979361e2112fd2322cd Mon Sep 17 00:00:00 2001 From: code021 Date: Fri, 31 May 2024 20:35:58 +0800 Subject: [PATCH 04/13] =?UTF-8?q?=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/app.go | 8 +-- cmd/compare.go | 30 ++++++----- cmd/create.go | 73 ++++++++++++-------------- cmd/root.go | 111 +++++++++------------------------------- cmd/tablemeta.go | 29 ++++------- cmd/version.go | 2 +- connect/connect.go | 1 + example.yml | 2 +- go.mod | 27 +++++----- go.sum | 47 ++++++++--------- macos_compile_remote.sh | 77 ++++++++++++++-------------- readme.md | 5 ++ readme_cn.md | 5 ++ 13 files changed, 172 insertions(+), 245 deletions(-) diff --git a/cmd/app.go b/cmd/app.go index 517e429..5b221f1 100644 --- a/cmd/app.go +++ b/cmd/app.go @@ -189,16 +189,12 @@ func StrVal(value interface{}) string { } func cleanDBconn() { - // 遍历正在执行的客户端，使用kill query 命令kill所有查询id + // 遍历正在执行的客户端，使用kill query 命令kill所有查询id,避免目标数据库仍在执行额外sql rows, err := destDb.Query("select id from information_schema.PROCESSLIST where info like '/* goapp%';") if err != nil { log.Error(err) } - defer func(rows *sql.Rows) { - if err := rows.Close(); err != nil { - log.Fatal(err) - } - }(rows) + defer rows.Close() for rows.Next() { var id string err = rows.Scan(&id) diff --git a/cmd/compare.go b/cmd/compare.go index 3677973..bdbe8fc 100644 --- a/cmd/compare.go +++ b/cmd/compare.go @@ -5,6 +5,9 @@ import ( "github.com/liushuochen/gotable" "github.com/spf13/cobra" "github.com/spf13/viper" + "io" + "os" + "path/filepath" "strconv" "time" ) @@ -37,21 +40,20 @@ var compareDbCmd = &cobra.Command{ } else { // 不指定-s选项，查询源库所有表名 tableMap = fetchTableMap(pageSize, excludeTab) } - newLogger() //初始化logrus日志和定义日志文件切割 // 创建运行日志目录 - //logDir, _ := filepath.Abs(CreateDateDir("")) - //f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - //if err != nil { - // log.Fatal(err) - //} - //defer func() { - // if err := f.Close(); err != nil { - // log.Fatal(err) // 或设置到函数返回值中 - // } - //}() - //// log信息重定向到平面文件 - //multiWriter := io.MultiWriter(os.Stdout, f) - //log.SetOutput(multiWriter) + logDir, _ := filepath.Abs(CreateDateDir("")) + f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) + if err != nil { + log.Fatal(err) + } + defer func() { + if err := f.Close(); err != nil { + log.Fatal(err) // 或设置到函数返回值中 + } + }() + // log信息重定向到平面文件 + multiWriter := io.MultiWriter(os.Stdout, f) + log.SetOutput(multiWriter) // 以下开始调用比对表行数的方法 start := time.Now() // 用于控制协程goroutine运行时候的并发数,例如3个一批，3个一批的goroutine并发运行 diff --git a/cmd/create.go b/cmd/create.go index e7e15fe..d3b73ac 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -1,10 +1,11 @@ package cmd import ( - "database/sql" "fmt" "github.com/liushuochen/gotable" "github.com/spf13/viper" + "io" + "os" "path/filepath" "strconv" "strings" @@ -44,22 +45,20 @@ var createTableCmd = &cobra.Command{ } else { // 不指定-s选项，查询源库所有表名 tableMap = fetchTableMap(pageSize, excludeTab) } - logDir := newLogger() //初始化logrus日志和定义日志文件切割 - //logDir := newLogger() //初始化logrus日志和定义日志文件切割 // 创建运行日志目录 - //logDir, _ := filepath.Abs(CreateDateDir("")) - //f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - //if err != nil { - // log.Fatal(err) - //} - //defer func() { - // if err := f.Close(); err != nil { - // log.Fatal(err) // 或设置到函数返回值中 - // } - //}() - //// log信息重定向到平面文件 - //multiWriter := io.MultiWriter(os.Stdout, f) - //log.SetOutput(multiWriter) + logDir, _ := filepath.Abs(CreateDateDir("")) + f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) + if err != nil { + log.Fatal(err) + } + defer func() { + if err := f.Close(); err != nil { + log.Fatal(err) // 或设置到函数返回值中 + } + }() + // log信息重定向到平面文件 + multiWriter := io.MultiWriter(os.Stdout, f) + log.SetOutput(multiWriter) // 实例初始化，调用接口中创建目标表的方法 var db Database start := time.Now() @@ -78,7 +77,7 @@ var createTableCmd = &cobra.Command{ // 这里等待上面所有迁移数据的goroutine协程任务完成才会接着运行下面的主程序，如果这里不wait，上面还在迁移行数据的goroutine会被强制中断 wg2.Wait() cost := time.Since(start) - log.Info("Table structure synced from MySQL to PolarDB ,Source Table Total ", tableCount, " Failed Total ", strconv.Itoa(failedCount)) + log.Info("Table structure synced finish,Source Table Total ", tableCount, " Failed Total ", strconv.Itoa(failedCount)) fmt.Println("Table Create finish elapsed time ", cost) }, } @@ -92,20 +91,20 @@ var onlyDataCmd = &cobra.Command{ connStr := getConn() // 创建运行日志目录 logDir, _ := filepath.Abs(CreateDateDir("")) - //// 输出调用文件以及方法位置 - //log.SetReportCaller(true) - //f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - //if err != nil { - // log.Fatal(err) - //} - //defer func() { - // if err := f.Close(); err != nil { - // log.Fatal(err) - // } - //}() - //// log信息重定向到平面文件 - //multiWriter := io.MultiWriter(os.Stdout, f) - //log.SetOutput(multiWriter) + // 输出调用文件以及方法位置 + log.SetReportCaller(true) + f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) + if err != nil { + log.Fatal(err) + } + defer func() { + if err := f.Close(); err != nil { + log.Fatal(err) + } + }() + // log信息重定向到平面文件 + multiWriter := io.MultiWriter(os.Stdout, f) + log.SetOutput(multiWriter) start := time.Now() // map结构，表名以及该表用来迁移查询源库的语句 var tableMap map[string][]string @@ -114,21 +113,13 @@ var onlyDataCmd = &cobra.Command{ log.Info("running SourceDB check connect") // 生成源库数据库连接 PrepareSrc(connStr) - defer func(srcDb *sql.DB) { - if err := srcDb.Close(); err != nil { - log.Fatal(err) - } - }(srcDb) + defer srcDb.Close() // 每页的分页记录数,仅全库迁移时有效 pageSize := viper.GetInt("pageSize") log.Info("running TargetDB check connect") // 生成目标库的数据库连接 PrepareDest(connStr) - defer func(destDb *sql.DB) { - if err := destDb.Close(); err != nil { - log.Fatal(err) - } - }(destDb) + defer destDb.Close() // 以下是迁移数据前的准备工作，获取要迁移的表名以及该表查询源库的sql语句(如果有主键生成该表的分页查询切片集合，没有主键的统一是全表查询sql) if selFromYml { // 如果用了-s选项，从配置文件中获取表名以及sql语句 tableMap = viper.GetStringMapStringSlice("tables") diff --git a/cmd/root.go b/cmd/root.go index df85051..5a6f308 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,9 +4,8 @@ import ( "bytes" "database/sql" "fmt" - rotatelogs "github.com/lestrrat-go/file-rotatelogs" "github.com/mitchellh/go-homedir" - "github.com/rifflock/lfshook" + "io" "math" "os" "os/signal" @@ -25,8 +24,7 @@ import ( "github.com/spf13/viper" ) -// var log = logrus.New() -var log *logrus.Logger +var log = logrus.New() var cfgFile string var selFromYml bool @@ -50,9 +48,22 @@ func startDataTransfer(connStr *connect.DbConnStr) { exitChan := make(chan os.Signal) signal.Notify(exitChan, os.Interrupt, os.Kill, syscall.SIGTERM) go exitHandle(exitChan) - - logDir := newLogger() //初始化logrus日志和定义日志文件切割 - + // 创建运行日志目录 + logDir, _ := filepath.Abs(CreateDateDir("")) + // 输出调用文件以及方法位置 + log.SetReportCaller(true) + f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) + if err != nil { + log.Fatal(err) + } + defer func() { + if err := f.Close(); err != nil { + log.Fatal(err) + } + }() + // log信息重定向到平面文件 + multiWriter := io.MultiWriter(os.Stdout, f) + log.SetOutput(multiWriter) start := time.Now() // map结构，表名以及该表用来迁移查询源库的语句 var tableMap map[string][]string @@ -61,21 +72,13 @@ func startDataTransfer(connStr *connect.DbConnStr) { log.Info("running SourceDB check connect") // 生成源库数据库连接 PrepareSrc(connStr) - defer func(srcDb *sql.DB) { - if err := srcDb.Close(); err != nil { - log.Fatal(err) - } - }(srcDb) + defer srcDb.Close() // 每页的分页记录数,仅全库迁移时有效 pageSize := viper.GetInt("pageSize") log.Info("running TargetDB check connect") // 生成目标库的数据库连接 PrepareDest(connStr) - defer func(destDb *sql.DB) { - if err := destDb.Close(); err != nil { - log.Fatal(err) - } - }(destDb) + defer destDb.Close() // 以下是迁移数据前的准备工作，获取要迁移的表名以及该表查询源库的sql语句(如果有主键生成该表的分页查询切片集合，没有主键的统一是全表查询sql) if selFromYml { // 如果用了-s选项，从配置文件中获取表名以及sql语句 tableMap = viper.GetStringMapStringSlice("tables") @@ -177,10 +180,7 @@ func startDataTransfer(connStr *connect.DbConnStr) { return } ymlConfig := []string{connStr.SrcHost + "-" + connStr.SrcUserName, connStr.DestHost + "-" + connStr.DestDatabase, strconv.Itoa(maxParallel), strconv.Itoa(pageSize), strconv.Itoa(len(excludeTab))} - err = tblConfig.AddRow(ymlConfig) - if err != nil { - fmt.Println("tblConfig Add row failed: ", err.Error()) - } + tblConfig.AddRow(ymlConfig) fmt.Println(tblConfig) // 输出迁移摘要 table, err := gotable.Create("Object", "BeginTime", "EndTime", "FailedTotal", "ElapsedTime") @@ -200,67 +200,6 @@ func startDataTransfer(connStr *connect.DbConnStr) { log.Info(fmt.Sprintf("All complete totalTime %s The Report Dir %s", cost, logDir)) } -func newLogger() string { - // 创建运行日志目录 - logDir, _ := filepath.Abs(CreateDateDir("")) - if log != nil { - return logDir - } - - infoLogPath := filepath.Join(logDir, "run.log") - // 创建rotatelogs的Logger实例 - infoRotator, err := rotatelogs.New( - infoLogPath+".%Y%m%d%H%M", - rotatelogs.WithLinkName(infoLogPath), // 生成软链接指向最新日志文件 - rotatelogs.WithMaxAge(24*time.Hour), // 日志文件最大保留时间 - rotatelogs.WithRotationTime(time.Hour), // 日志切割时间间隔 - rotatelogs.WithRotationSize(50*1024*1024), // 日志文件最大大小（例如50MB） - ) - if err != nil { - fmt.Printf("Failed to create rotatelogs logger: %v", err) - return logDir - } - - // 为 Error 级别日志设置 rotatelogs - errorLogPath := filepath.Join(logDir, "run_error.log") - errorRotator, err := rotatelogs.New( - errorLogPath+".%Y%m%d%H%M", - rotatelogs.WithLinkName(errorLogPath), - rotatelogs.WithMaxAge(7*24*time.Hour), - rotatelogs.WithRotationTime(24*time.Hour), - //rotatelogs.WithRotationSize(10*1024*1024), - ) - if err != nil { - fmt.Printf("failed to create rotatelogs for error: %v", err) - } - - // lfshook 决定哪些日志级别可用日志分割 - writeMap := lfshook.WriterMap{ - logrus.PanicLevel: errorRotator, - logrus.FatalLevel: errorRotator, - logrus.ErrorLevel: errorRotator, - logrus.WarnLevel: infoRotator, - logrus.InfoLevel: infoRotator, - //logrus.DebugLevel: rl, - } - - // 配置 lfshook - hook := lfshook.NewHook(writeMap, &logrus.TextFormatter{ - // 设置日期格式 - TimestampFormat: "2006.01.02 - 15:04:05", - }) - - log = logrus.New() - // 输出调用文件以及方法位置 - log.SetReportCaller(true) - // 设置将日志输出到标准输出（默认的输出为stderr,标准错误） - log.SetOutput(os.Stdout) - log.AddHook(hook) - - fmt.Println(infoLogPath) - return logDir -} - // 自动对表分析，然后生成每个表用来迁移查询源库SQL的集合(全表查询或者分页查询) // 自动分析是否有排除的表名 // 最后返回map结构即 表:[查询SQL] @@ -285,11 +224,7 @@ func fetchTableMap(pageSize int, excludeTable []string) (tableMap map[string][]s } // 查询下源库总共的表，获取到表名 rows, err := srcDb.Query(sqlStr) - defer func(rows *sql.Rows) { - if err := rows.Close(); err != nil { - log.Fatal(err) - } - }(rows) + defer rows.Close() if err != nil { log.Error(fmt.Sprintf("Query "+sqlStr+" failed,\nerr:%v\n", err)) return @@ -395,7 +330,7 @@ func prepareSqlStr(tableName string, pageSize int) (sqlList []string) { startNum = ((curStartPage - 1) * pageSize) + 1 } endNum := startNum + pageSize - 1 - sqlStr = fmt.Sprintf("SELECT %s FROM (SELECT A.*, ROWNUM RN FROM (SELECT * FROM \"%s\") A WHERE ROWNUM <= %s) WHERE RN >=%s", colNameFull, tableName, strconv.Itoa(endNum), strconv.Itoa(startNum)) + sqlStr = fmt.Sprintf("SELECT %s FROM (SELECT A.*, ROWNUM RNcolumn FROM (SELECT * FROM \"%s\") A WHERE ROWNUM <= %s) WHERE RNcolumn >=%s", colNameFull, tableName, strconv.Itoa(endNum), strconv.Itoa(startNum)) sqlList = append(sqlList, sqlStr) } return sqlList diff --git a/cmd/tablemeta.go b/cmd/tablemeta.go index 39f4fd7..a771c1d 100644 --- a/cmd/tablemeta.go +++ b/cmd/tablemeta.go @@ -82,8 +82,8 @@ func (tb *Table) TableCreate(logDir string, tblName string, ch chan struct{}) { if err := rows.Scan(&newTable.columnName, &newTable.dataType, &newTable.characterMaximumLength, &newTable.isNullable, &colDefaultValue, &newTable.numericPrecision, &newTable.numericScale, &newTable.columnComment, &newTable.avgColLen, &newTable.ordinalPosition); err != nil { log.Error(err) } - // 判断下默认值是否是null，go语言中不能直接把null值转成字符串 - if !colDefaultValue.Valid { + // 判断colDefaultValue的长度，如果len大于0说明就是有非null的默认值，如果len为0说明在源库的默认值就是null + if len([]rune(colDefaultValue.String)) > 0 { newTable.columnDefault = colDefaultValue.String } else { newTable.columnDefault = "null" @@ -154,8 +154,13 @@ func (tb *Table) TableCreate(logDir string, tblName string, ch chan struct{}) { default: newTable.destType = newTable.dataType } + // 列注释,每个列字段的注释使用comment 注释的文字进行拼接 + colComment := "" + if newTable.columnComment != "null" { + colComment = fmt.Sprintf(" comment '%s'", newTable.columnComment) + } // 在目标库创建的语句 - createTblSql += fmt.Sprintf("`%s` %s %s %s,", newTable.columnName, newTable.destType, newTable.destNullable, newTable.destDefault) + createTblSql += fmt.Sprintf("`%s` %s %s %s %s,", newTable.columnName, newTable.destType, newTable.destNullable, newTable.destDefault, colComment) if newTable.ordinalPosition == colTotal { createTblSql = createTblSql[:len(createTblSql)-1] + ")" // 最后一个列字段结尾去掉逗号,并且加上语句的右括号 } @@ -186,11 +191,7 @@ func (tb *Table) IdxCreate(logDir string, tableName string, ch chan struct{}, id if err != nil { log.Error(err) } - defer func(rows *sql.Rows) { - if err := rows.Close(); err != nil { - log.Fatal(err) - } - }(rows) + defer rows.Close() // 从sql结果集遍历，获取到创建语句 for rows.Next() { if err := rows.Scan(&destIdxSql); err != nil { @@ -218,11 +219,7 @@ func (tb *Table) SeqCreate(logDir string) (ret []string) { if err != nil { log.Error(err) } - defer func(rows *sql.Rows) { - if err := rows.Close(); err != nil { - log.Fatal(err) - } - }(rows) + defer rows.Close() idx := 0 for rows.Next() { idx += 1 @@ -276,11 +273,7 @@ func (tb *Table) FkCreate(logDir string) (ret []string) { if err != nil { log.Error(err) } - defer func(rows *sql.Rows) { - if err := rows.Close(); err != nil { - log.Fatal(err) - } - }(rows) + defer rows.Close() idx := 0 for rows.Next() { idx += 1 diff --git a/cmd/version.go b/cmd/version.go index f36b3ba..ba4eeee 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -var ver = "0.0.9" +var ver = "0.1.2" func init() { rootCmd.AddCommand(versionCmd) diff --git a/connect/connect.go b/connect/connect.go index 81f9d79..53e3edb 100644 --- a/connect/connect.go +++ b/connect/connect.go @@ -1,5 +1,6 @@ package connect +// DbConnStr related with config.yml type DbConnStr struct { SrcHost string SrcUserName string diff --git a/example.yml b/example.yml index 948ad51..3df2b58 100644 --- a/example.yml +++ b/example.yml @@ -7,7 +7,7 @@ src: dest: host: 192.168.1.37 port: 3306 - database: test_polar + database: test username: root password: 11111 pageSize: 100000 diff --git a/go.mod b/go.mod index e3f1c57..51cb558 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,13 @@ module OracleSync2MySQL -go 1.20 +go 1.21 + +toolchain go1.22.3 require ( - github.com/fatih/color v1.16.0 - github.com/go-sql-driver/mysql v1.7.1 - github.com/godror/godror v0.41.0 + github.com/fatih/color v1.17.0 + github.com/go-sql-driver/mysql v1.8.1 + github.com/godror/godror v0.44.0 github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092 github.com/mitchellh/go-homedir v1.1.0 github.com/sirupsen/logrus v1.9.3 @@ -14,21 +16,18 @@ require ( ) require ( + filippo.io/edwards25519 v1.1.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/godror/knownpb v0.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect - github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/pelletier/go-toml/v2 v2.1.1 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sijms/go-ora/v2 v2.7.9 // indirect @@ -40,10 +39,10 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/protobuf v1.32.0 // indirect + golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9442e3b..4265c61 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -58,8 +60,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -73,14 +75,16 @@ github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/godror/godror v0.37.0 h1:3wR3/1msywDE49PzuXh9UUiwWOBNri0RVQQcu3HU4UY= github.com/godror/godror v0.37.0/go.mod h1:jW1+pN+z/V0h28p9XZXVNtEvfZP/2EBfaSjKJLp3E4g= -github.com/godror/godror v0.41.0 h1:LVwpbfYmGrxIy7nBmv9w7VJxKlmRei6he4lyHgKCEF0= -github.com/godror/godror v0.41.0/go.mod h1:i8YtVTHUJKfFT3wTat4A9UoqScUtZXiYB9Rf3SVARgc= +github.com/godror/godror v0.44.0 h1:tW0oDotJDoC5GTl5saOAwxmlozDy15ImjsbC7UVKEVA= +github.com/godror/godror v0.44.0/go.mod h1:oRlxogABC1Tr5u/zYF3EjHE1fYvAeNBS9MJ8bq1hVkU= github.com/godror/knownpb v0.1.0 h1:dJPK8s/I3PQzGGaGcUStL2zIaaICNzKKAK8BzP1uLio= github.com/godror/knownpb v0.1.0/go.mod h1:4nRFbQo1dDuwKnblRXDxrfCFYeT4hjg3GjMqef58eRE= github.com/godror/knownpb v0.1.1 h1:A4J7jdx7jWBhJm18NntafzSC//iZDHkDi1+juwQ5pTI= @@ -156,11 +160,6 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= -github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= -github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= -github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092 h1:u9I3sJ+uTakxnRrvuYJGsEi4SvEMN+yB47WWGDHHxIk= github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092/go.mod h1:CxUy8nDvutaC1pOfaG9TRoYwdHHqoNstSPPKhomC9k8= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -178,15 +177,12 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= -github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= -github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -222,8 +218,8 @@ github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMV github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -231,6 +227,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= @@ -266,8 +263,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= -golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 h1:Mi0bCswbz+9cXmwFAdxoo5GPFMKONUpua6iUdtQS7lk= +golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -384,8 +381,8 @@ golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -397,8 +394,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -544,8 +541,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= diff --git a/macos_compile_remote.sh b/macos_compile_remote.sh index 48f8fec..b4aa3f2 100644 --- a/macos_compile_remote.sh +++ b/macos_compile_remote.sh @@ -4,52 +4,55 @@ export LANG=en_US.UTF-8 version=$1 # clean old binary zip file -cd /Users/kay/go/src/OracleSync2MySQL/binary +cd /Users/kay/Documents/GoProj/OracleSync2MySQL/binary rm -rf OracleSync2MySQL.exe rm -rf example.yml rm -rf OracleSync2MySQL-MacOS-x64-v$version.zip rm -rf OracleSync2MySQL-win-x64-v$version.zip rm -rf OracleSync2MySQL-linux-x64-v$version.zip -# macos compile -cd /Users/kay/go/src/OracleSync2MySQL -/Users/kay/go/go1.20.6/bin/go clean -/Users/kay/go/go1.20.6/bin/go build -o OracleSync2MySQL OracleSync2MySQL +# MacOS macos compile +cd /Users/kay/Documents/GoProj/OracleSync2MySQL +/Users/kay/sdk/go1.22.0/bin/go clean +/Users/kay/sdk/go1.22.0/bin/go build -o OracleSync2MySQL OracleSync2MySQL zip -r OracleSync2MySQL-MacOS-x64-v$version.zip OracleSync2MySQL example.yml instantclient mv OracleSync2MySQL-MacOS-x64-v$version.zip binary -# ssh remote linux server and compile -ssh root@192.168.125.129 "rm -rf /root/go/src/OracleSync2MySQL/cmd" -ssh root@192.168.125.129 "rm -rf /root/go/src/OracleSync2MySQL/connect" -ssh root@192.168.125.129 "rm -rf /root/go/src/OracleSync2MySQL/go.mod" -ssh root@192.168.125.129 "rm -rf /root/go/src/OracleSync2MySQL/go.sum" -ssh root@192.168.125.129 "rm -rf /root/go/src/OracleSync2MySQL/main.go" -ssh root@192.168.125.129 "rm -rf /root/go/src/OracleSync2MySQL/*.yml" -ssh root@192.168.125.129 "rm -rf /root/go/src/OracleSync2MySQL/*.zip" -scp -r /Users/kay/go/src/OracleSync2MySQL/cmd/ root@192.168.125.129:/root/go/src/OracleSync2MySQL -scp -r /Users/kay/go/src/OracleSync2MySQL/connect/ root@192.168.125.129:/root/go/src/OracleSync2MySQL -scp /Users/kay/go/src/OracleSync2MySQL/go.mod root@192.168.125.129:/root/go/src/OracleSync2MySQL -scp /Users/kay/go/src/OracleSync2MySQL/go.sum root@192.168.125.129:/root/go/src/OracleSync2MySQL -scp /Users/kay/go/src/OracleSync2MySQL/main.go root@192.168.125.129:/root/go/src/OracleSync2MySQL -scp /Users/kay/go/src/OracleSync2MySQL/*.yml root@192.168.125.129:/root/go/src/OracleSync2MySQL -ssh root@192.168.125.129 "rm -rf /root/go/src/OracleSync2MySQL/*.zip" -ssh root@192.168.125.129 "cd /root/go/src/OracleSync2MySQL && /usr/local/go/bin/go clean" -ssh root@192.168.125.129 "cd /root/go/src/OracleSync2MySQL && /usr/local/go/bin/go build -o OracleSync2MySQL OracleSync2MySQL" -ssh root@192.168.125.129 "cd /root/go/src/OracleSync2MySQL && zip -r OracleSync2MySQL-linux-x64-v$version.zip OracleSync2MySQL example.yml instantclient" -scp root@192.168.125.129:/root/go/src/OracleSync2MySQL/OracleSync2MySQL-linux-x64-v$version.zip /Users/kay/go/src/OracleSync2MySQL/binary -# ssh remote Windows server and compile + + +# Linux ssh remote linux server and compile +ssh root@192.168.74.129 "rm -rf /root/go/src/OracleSync2MySQL/cmd" +ssh root@192.168.74.129 "rm -rf /root/go/src/OracleSync2MySQL/connect" +ssh root@192.168.74.129 "rm -rf /root/go/src/OracleSync2MySQL/go.mod" +ssh root@192.168.74.129 "rm -rf /root/go/src/OracleSync2MySQL/go.sum" +ssh root@192.168.74.129 "rm -rf /root/go/src/OracleSync2MySQL/main.go" +ssh root@192.168.74.129 "rm -rf /root/go/src/OracleSync2MySQL/*.yml" +ssh root@192.168.74.129 "rm -rf /root/go/src/OracleSync2MySQL/*.zip" +scp -r /Users/kay/Documents/GoProj/OracleSync2MySQL/cmd/ root@192.168.74.129:/root/go/src/OracleSync2MySQL +scp -r /Users/kay/Documents/GoProj/OracleSync2MySQL/connect/ root@192.168.74.129:/root/go/src/OracleSync2MySQL +scp /Users/kay/Documents/GoProj/OracleSync2MySQL/go.mod root@192.168.74.129:/root/go/src/OracleSync2MySQL +scp /Users/kay/Documents/GoProj/OracleSync2MySQL/go.sum root@192.168.74.129:/root/go/src/OracleSync2MySQL +scp /Users/kay/Documents/GoProj/OracleSync2MySQL/main.go root@192.168.74.129:/root/go/src/OracleSync2MySQL +scp /Users/kay/Documents/GoProj/OracleSync2MySQL/*.yml root@192.168.74.129:/root/go/src/OracleSync2MySQL +ssh root@192.168.74.129 "rm -rf /root/go/src/OracleSync2MySQL/*.zip" +ssh root@192.168.74.129 "cd /root/go/src/OracleSync2MySQL && /usr/local/go/bin/go clean" +ssh root@192.168.74.129 "cd /root/go/src/OracleSync2MySQL && /usr/local/go/bin/go build -o OracleSync2MySQL OracleSync2MySQL" +ssh root@192.168.74.129 "cd /root/go/src/OracleSync2MySQL && zip -r OracleSync2MySQL-linux-x64-v$version.zip OracleSync2MySQL example.yml instantclient" +scp root@192.168.74.129:/root/go/src/OracleSync2MySQL/OracleSync2MySQL-linux-x64-v$version.zip /Users/kay/Documents/GoProj/OracleSync2MySQL/binary + + +# Windows ssh remote Windows server and compile ssh administrator@192.168.149.80 "cd C:\go\src\OracleSync2MySQL && C:\Users\Administrator\sdk\go1.20.6\bin\go clean" ssh administrator@192.168.149.80 "cd C:\go\src\OracleSync2MySQL && del /f /s /q *.zip *.yml go* main.go cmd connect" -scp -r /Users/kay/go/src/OracleSync2MySQL/cmd/ administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" -scp -r /Users/kay/go/src/OracleSync2MySQL/connect/ administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" -scp -r /Users/kay/go/src/OracleSync2MySQL/test/ administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" -scp /Users/kay/go/src/OracleSync2MySQL/go.mod administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" -scp /Users/kay/go/src/OracleSync2MySQL/go.sum administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" -scp /Users/kay/go/src/OracleSync2MySQL/main.go administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" -scp /Users/kay/go/src/OracleSync2MySQL/*.yml administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" +scp -r /Users/kay/Documents/GoProj/OracleSync2MySQL/cmd/ administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" +scp -r /Users/kay/Documents/GoProj/OracleSync2MySQL/connect/ administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" +scp -r /Users/kay/Documents/GoProj/OracleSync2MySQL/test/ administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" +scp /Users/kay/Documents/GoProj/OracleSync2MySQL/go.mod administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" +scp /Users/kay/Documents/GoProj/OracleSync2MySQL/go.sum administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" +scp /Users/kay/Documents/GoProj/OracleSync2MySQL/main.go administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" +scp /Users/kay/Documents/GoProj/OracleSync2MySQL/*.yml administrator@192.168.149.80:"C:\go\src\OracleSync2MySQL" ssh administrator@192.168.149.80 "cd C:\go\src\OracleSync2MySQL && C:\Users\Administrator\sdk\go1.20.6\bin\go build -o OracleSync2MySQL.exe OracleSync2MySQL" - -# pack zip file windows -cd /Users/kay/go/src/OracleSync2MySQL/binary -scp administrator@192.168.149.80:"C:/go/src/OracleSync2MySQL/OracleSync2MySQL.exe" /Users/kay/go/src/OracleSync2MySQL/binary -cp /Users/kay/go/src/OracleSync2MySQL/example.yml . +# pack zip file from windows to local mac +cd /Users/kay/Documents/GoProj/OracleSync2MySQL/binary +scp administrator@192.168.149.80:"C:/go/src/OracleSync2MySQL/OracleSync2MySQL.exe" /Users/kay/Documents/GoProj/OracleSync2MySQL/binary +cp /Users/kay/Documents/GoProj/OracleSync2MySQL/example.yml . zip -r OracleSync2MySQL-win-x64-v$version.zip OracleSync2MySQL.exe example.yml instantclient \ No newline at end of file diff --git a/readme.md b/readme.md index 57cb2ea..8ba98e5 100644 --- a/readme.md +++ b/readme.md @@ -253,6 +253,11 @@ OracleSync2MySQL.exe --config example.yml onlyData -s ``` ## change history +### v0.1.0 +2024-03-29 +fix some error + + ### v0.0.9 2023-11-27 fix -s mode with upper table name diff --git a/readme_cn.md b/readme_cn.md index 068e362..2c1856f 100644 --- a/readme_cn.md +++ b/readme_cn.md @@ -241,6 +241,11 @@ OracleSync2MySQL.exe --config example.yml onlyData -s ``` ## change history +### v0.1.0 +2024-03-29 +修复分页查询遇到别名冲突问题 + + ### v0.0.9 2023-11-27 -s自定义迁移的时候，读取配置文件中的表名并统一转为大小 From 21672cc024b809f581512dfcff201913cef4949b Mon Sep 17 00:00:00 2001 From: code021 Date: Sat, 1 Jun 2024 11:47:26 +0800 Subject: [PATCH 05/13] =?UTF-8?q?=E6=9B=B4=E6=96=B0go.mod?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.bat | 1 + cmd/tablemeta.go | 15 ++++++++++++++- go.mod | 2 +- go.sum | 2 ++ 4 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 build.bat diff --git a/build.bat b/build.bat new file mode 100644 index 0000000..4ac8687 --- /dev/null +++ b/build.bat @@ -0,0 +1 @@ +go build -o OracleSync2MySQL -ldflags "-X main.Version=0.1.2" \ No newline at end of file diff --git a/cmd/tablemeta.go b/cmd/tablemeta.go index a771c1d..40e3451 100644 --- a/cmd/tablemeta.go +++ b/cmd/tablemeta.go @@ -15,6 +15,7 @@ var failedCount int type Database interface { // TableCreate (logDir string, tableMap map[string][]string) (result []string) 单线程 + TableCheck(logDir string, tblName string) string //检查目标表是否存在 TableCreate(logDir string, tblName string, ch chan struct{}) IdxCreate(logDir string, tableName string, ch chan struct{}, id int) SeqCreate(logDir string) (ret []string) @@ -49,6 +50,16 @@ type Table struct { viewSql string } +func (tb *Table) TableCheck(logDir string, tblName string) string { + checkSql := fmt.Sprintf("SHOW TABLES LIKE '%s'", tblName) + var result string + err := destDb.QueryRow(checkSql).Scan(&result) + if err != nil { + log.Error(err) + } + return result +} + func (tb *Table) TableCreate(logDir string, tblName string, ch chan struct{}) { defer wg2.Done() var newTable Table @@ -148,7 +159,9 @@ func (tb *Table) TableCreate(logDir string, tblName string, ch chan struct{}) { newTable.destType = "datetime" case "CLOB", "NCLOB", "LONG": newTable.destType = "longtext" - case "BLOB", "RAW", "LONG RAW": + case "BLOB": + newTable.destType = "longtext" + case "RAW", "LONG RAW": newTable.destType = "longblob" // 其余类型，源库使用什么类型，目标库就使用什么类型 default: diff --git a/go.mod b/go.mod index 51cb558..698c128 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 // indirect + golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect google.golang.org/protobuf v1.34.1 // indirect diff --git a/go.sum b/go.sum index 4265c61..52927bc 100644 --- a/go.sum +++ b/go.sum @@ -265,6 +265,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 h1:Mi0bCswbz+9cXmwFAdxoo5GPFMKONUpua6iUdtQS7lk= golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From 4fa1b6637d1590eb9ce8175067be8736e3c91220 Mon Sep 17 00:00:00 2001 From: xls-web Date: Mon, 3 Jun 2024 19:43:02 +0800 Subject: [PATCH 06/13] =?UTF-8?q?=E4=BC=98=E5=8C=96logerror=E8=BE=93?= =?UTF-8?q?=E5=87=BAinsertsql=E5=92=8CreplaceALL=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/root.go | 4 ++-- cmd/tablemeta.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 5a6f308..c49aad7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -434,7 +434,7 @@ func runMigration(logDir string, startPage int, tableName string, sqlStr string, stmt, err := txn.Prepare(insertSql) //prepare里的方法CopyIn只是把copy语句拼接好并返回，并非直接执行copy if err != nil { log.Error("txn.Prepare(insertSql) failed table[", tableName, "] ", err) - LogError(logDir, "errorTableData ", tableName, err) + LogError(logDir, "errorTableData ", insertSql, err) //responseChannel <- fmt.Sprintf("data error %s", tableName) <-ch // 通道向外发送数据 return @@ -481,7 +481,7 @@ func runMigration(logDir string, startPage int, tableName string, sqlStr string, stmt, err := txn.Prepare(insertSql) //prepare里的方法CopyIn只是把copy语句拼接好并返回，并非直接执行copy if err != nil { log.Error("txn.Prepare(insertSql) failed table[", tableName, "] ", err) - LogError(logDir, "errorTableData ", tableName, err) + LogError(logDir, "errorTableData ", insertSql, err) //responseChannel <- fmt.Sprintf("data error %s", tableName) <-ch // 通道向外发送数据 return diff --git a/cmd/tablemeta.go b/cmd/tablemeta.go index 40e3451..18a4dc3 100644 --- a/cmd/tablemeta.go +++ b/cmd/tablemeta.go @@ -405,9 +405,9 @@ func (tb *Table) ViewCreate(logDir string) (ret []string) { dbRet = strings.ReplaceAll(dbRet, "--", "-- -- ") dbRet = strings.ReplaceAll(dbRet, "\"", "`") dbRet = strings.ReplaceAll(dbRet, "NVL(", "IFNULL(") - dbRet = strings.ReplaceAll(dbRet, "unistr('\0030')", "0") - dbRet = strings.ReplaceAll(dbRet, "unistr('\0031')", "1") - dbRet = strings.ReplaceAll(dbRet, "unistr('\0033')", "3") + dbRet = strings.ReplaceAll(dbRet, "UNISTR('\0030')", "0") + dbRet = strings.ReplaceAll(dbRet, "UNISTR('\0031')", "1") + dbRet = strings.ReplaceAll(dbRet, "UNISTR('\0033')", "3") if len(viewName) > 0 { sqlStr := "create or replace view " + viewName + " as " + dbRet log.Info("[", idx, "] create view ", viewName) From a2efb40ad2bc6d4f903e11cced1ede16f3eb11d4 Mon Sep 17 00:00:00 2001 From: xls-web Date: Wed, 5 Jun 2024 10:19:58 +0800 Subject: [PATCH 07/13] =?UTF-8?q?=E4=B8=BA=E4=B8=8D=E5=90=8C=E7=9A=84sql?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E5=AE=8C=E5=96=84goapp=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/root.go | 8 ++++---- cmd/tablemeta.go | 25 ++++++------------------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 91e324f..5883bee 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -317,7 +317,7 @@ func prepareSqlStr(tableName string, pageSize int) (sqlList []string) { return } // 根据当前表总数以及每页的页记录大小pageSize，自动计算需要多少页记录数，即总共循环多少次，如果表没有数据，后面判断下切片长度再做处理 - sql2 := "/* goapp */" + "select ceil(count(*)/" + strconv.Itoa(pageSize) + ") as total_page_num from " + "\"" + tableName + "\"" + sql2 := "/* goapp count */" + "select ceil(count(*)/" + strconv.Itoa(pageSize) + ") as total_page_num from " + "\"" + tableName + "\"" //以下是直接使用QueryRow err = srcDb.QueryRow(sql2).Scan(&totalPageNum) if err != nil { @@ -345,7 +345,7 @@ func runMigration(logDir string, startPage int, tableName string, sqlStr string, log.Info(fmt.Sprintf("%v Taskid[%d] Processing TableData %v ", time.Now().Format("2006-01-02 15:04:05.000000"), startPage, tableName)) start := time.Now() // 直接查询,即查询全表或者分页查询(SELECT t.* FROM (SELECT id FROM test ORDER BY id LIMIT ?, ?) temp LEFT JOIN test t ON temp.id = t.id;) - sqlStr = "/* goapp */" + sqlStr + sqlStr = "/* goapp query */" + sqlStr // 查询源库的sql rows, err := srcDb.Query(sqlStr) //传入参数之后执行 defer rows.Close() @@ -437,7 +437,7 @@ func runMigration(logDir string, startPage int, tableName string, sqlStr string, stmt, err := txn.Prepare(insertSql) //prepare里的方法CopyIn只是把copy语句拼接好并返回，并非直接执行copy if err != nil { log.Error("txn.Prepare(insertSql) failed table[", tableName, "] ", err) - LogError(logDir, "errorTableData ", insertSql, err) + LogError(logDir, "errorTableData ", tableName, err) //responseChannel <- fmt.Sprintf("data error %s", tableName) <-ch // 通道向外发送数据 return @@ -484,7 +484,7 @@ func runMigration(logDir string, startPage int, tableName string, sqlStr string, stmt, err := txn.Prepare(insertSql) //prepare里的方法CopyIn只是把copy语句拼接好并返回，并非直接执行copy if err != nil { log.Error("txn.Prepare(insertSql) failed table[", tableName, "] ", err) - LogError(logDir, "errorTableData ", insertSql, err) + LogError(logDir, "errorTableData ", tableName, err) //responseChannel <- fmt.Sprintf("data error %s", tableName) <-ch // 通道向外发送数据 return diff --git a/cmd/tablemeta.go b/cmd/tablemeta.go index 761ff9d..dfe4d6e 100644 --- a/cmd/tablemeta.go +++ b/cmd/tablemeta.go @@ -15,7 +15,6 @@ var failedCount int type Database interface { // TableCreate (logDir string, tableMap map[string][]string) (result []string) 单线程 - TableCheck(logDir string, tblName string) string //检查目标表是否存在 TableCreate(logDir string, tblName string, ch chan struct{}) IdxCreate(logDir string, tableName string, ch chan struct{}, id int) SeqCreate(logDir string) (ret []string) @@ -50,16 +49,6 @@ type Table struct { viewSql string } -func (tb *Table) TableCheck(logDir string, tblName string) string { - checkSql := fmt.Sprintf("SHOW TABLES LIKE '%s'", tblName) - var result string - err := destDb.QueryRow(checkSql).Scan(&result) - if err != nil { - log.Error(err) - } - return result -} - func (tb *Table) TableCreate(logDir string, tblName string, ch chan struct{}) { defer wg2.Done() var newTable Table @@ -161,9 +150,7 @@ func (tb *Table) TableCreate(logDir string, tblName string, ch chan struct{}) { newTable.destType = "datetime" case "CLOB", "NCLOB", "LONG": newTable.destType = "longtext" - case "BLOB": - newTable.destType = "longtext" - case "RAW", "LONG RAW": + case "BLOB", "RAW", "LONG RAW": newTable.destType = "longblob" // 其余类型，源库使用什么类型，目标库就使用什么类型 default: @@ -217,7 +204,7 @@ func (tb *Table) IdxCreate(logDir string, tableName string, ch chan struct{}, id log.Error(err) } LogOutput(logDir, "createSql", destIdxSql) - destIdxSql = "/* goapp */" + destIdxSql + destIdxSql = "/* goapp idx */" + destIdxSql // 创建目标索引，主键、其余约束 if !metaData { if _, err = destDb.Exec(destIdxSql); err != nil { @@ -264,7 +251,7 @@ func (tb *Table) SeqCreate(logDir string) (ret []string) { if len(match) == 2 { autoColName := match[1] // 创建目标数据库该表表的自增列索引 - sqlAutoColIdx := "/* goapp */" + "create index ids_" + tableName + "_" + autoColName + "_" + strconv.Itoa(idx) + " on " + tableName + "(" + autoColName + ")" + sqlAutoColIdx := "/* goapp seq idx */" + "create index ids_" + tableName + "_" + autoColName + "_" + strconv.Itoa(idx) + " on " + tableName + "(" + autoColName + ")" log.Info("[", idx, "] create auto_increment for table ", tableName) LogOutput(logDir, "createSql", sqlAutoColIdx+";") if !metaData { @@ -276,7 +263,7 @@ func (tb *Table) SeqCreate(logDir string) (ret []string) { } // 更改目标数据库该表的列属性为自增列 - sqlModifyAuto := "/* goapp */" + "alter table " + tableName + " modify " + autoColName + " bigint auto_increment" + sqlModifyAuto := "/* goapp seq auto */" + "alter table " + tableName + " modify " + autoColName + " bigint auto_increment" LogOutput(logDir, "createSql", sqlModifyAuto+";") if !metaData { if _, err = destDb.Exec(sqlModifyAuto); err != nil { @@ -312,7 +299,7 @@ func (tb *Table) FkCreate(logDir string) (ret []string) { log.Error(err) } log.Info("[", idx, "] create foreign key for table ", tableName) - sqlStr = "/* goapp */" + sqlStr + sqlStr = "/* goapp fk */" + sqlStr LogOutput(logDir, "createSql", sqlStr) if !metaData { if _, err = destDb.Exec(sqlStr); err != nil { @@ -356,7 +343,7 @@ func (tb *Table) NormalIdx(logDir string) (ret []string) { } log.Info("[", idx, "] create normal index for table ", tableName) LogOutput(logDir, "createSql", createSql+";") - createSql = "/* goapp */" + createSql + createSql = "/* goapp normal idx */" + createSql if !metaData { if _, err = destDb.Exec(createSql); err != nil { log.Error(createSql, " create normal index failed ", err) From 261a049ea0d6ddabac1720ec5acca8835f33a8b3 Mon Sep 17 00:00:00 2001 From: code021 Date: Wed, 5 Jun 2024 21:40:44 +0800 Subject: [PATCH 08/13] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=88=86=E9=A1=B5sql?= =?UTF-8?q?=E8=AE=A1=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/root.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/root.go b/cmd/root.go index 5883bee..4f2639e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -324,6 +324,10 @@ func prepareSqlStr(tableName string, pageSize int) (sqlList []string) { log.Fatal(sql2, " exec failed ", err) return } + if totalPageNum == 1 { + sqlList = append(sqlList, fmt.Sprintf("SELECT %s FROM \"%s\"", colNameFull, tableName)) + return sqlList + } // 以下生成分页查询语句 for i := 0; i < totalPageNum; i++ { // 使用小于而不是小于等于，否则会多生成一条分页查询边界外的sql，即此sql查询源表没有数据，也会导致后面迁移数据有多个无用的goroutine curStartPage := i + 1 From 2d52f287c2faf455e749f784368f24e00b5777df Mon Sep 17 00:00:00 2001 From: xls-web Date: Thu, 6 Jun 2024 16:00:09 +0800 Subject: [PATCH 09/13] =?UTF-8?q?=E4=BD=BF=E7=94=A8zap=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=88=87=E5=89=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/compare.go | 17 -- cmd/create.go | 33 ---- cmd/root.go | 23 +-- go.mod | 7 +- go.sum | 11 ++ pkg/flagx/conf.go | 83 ++++++++++ pkg/flagx/options.go | 23 +++ pkg/logger/logger.go | 283 ++++++++++++++++++++++++++++++++++ pkg/logger/unit.go | 50 ++++++ pkg/sirupsen/logrus/logrus.go | 42 +++++ 10 files changed, 501 insertions(+), 71 deletions(-) create mode 100644 pkg/flagx/conf.go create mode 100644 pkg/flagx/options.go create mode 100644 pkg/logger/logger.go create mode 100644 pkg/logger/unit.go create mode 100644 pkg/sirupsen/logrus/logrus.go diff --git a/cmd/compare.go b/cmd/compare.go index bdbe8fc..0e00cf4 100644 --- a/cmd/compare.go +++ b/cmd/compare.go @@ -5,9 +5,6 @@ import ( "github.com/liushuochen/gotable" "github.com/spf13/cobra" "github.com/spf13/viper" - "io" - "os" - "path/filepath" "strconv" "time" ) @@ -40,20 +37,6 @@ var compareDbCmd = &cobra.Command{ } else { // 不指定-s选项，查询源库所有表名 tableMap = fetchTableMap(pageSize, excludeTab) } - // 创建运行日志目录 - logDir, _ := filepath.Abs(CreateDateDir("")) - f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := f.Close(); err != nil { - log.Fatal(err) // 或设置到函数返回值中 - } - }() - // log信息重定向到平面文件 - multiWriter := io.MultiWriter(os.Stdout, f) - log.SetOutput(multiWriter) // 以下开始调用比对表行数的方法 start := time.Now() // 用于控制协程goroutine运行时候的并发数,例如3个一批，3个一批的goroutine并发运行 diff --git a/cmd/create.go b/cmd/create.go index d3b73ac..9402fb6 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -4,9 +4,6 @@ import ( "fmt" "github.com/liushuochen/gotable" "github.com/spf13/viper" - "io" - "os" - "path/filepath" "strconv" "strings" "time" @@ -45,20 +42,6 @@ var createTableCmd = &cobra.Command{ } else { // 不指定-s选项，查询源库所有表名 tableMap = fetchTableMap(pageSize, excludeTab) } - // 创建运行日志目录 - logDir, _ := filepath.Abs(CreateDateDir("")) - f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := f.Close(); err != nil { - log.Fatal(err) // 或设置到函数返回值中 - } - }() - // log信息重定向到平面文件 - multiWriter := io.MultiWriter(os.Stdout, f) - log.SetOutput(multiWriter) // 实例初始化，调用接口中创建目标表的方法 var db Database start := time.Now() @@ -89,22 +72,6 @@ var onlyDataCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { // 获取配置文件中的数据库连接字符串 connStr := getConn() - // 创建运行日志目录 - logDir, _ := filepath.Abs(CreateDateDir("")) - // 输出调用文件以及方法位置 - log.SetReportCaller(true) - f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := f.Close(); err != nil { - log.Fatal(err) - } - }() - // log信息重定向到平面文件 - multiWriter := io.MultiWriter(os.Stdout, f) - log.SetOutput(multiWriter) start := time.Now() // map结构，表名以及该表用来迁移查询源库的语句 var tableMap map[string][]string diff --git a/cmd/root.go b/cmd/root.go index 4f2639e..873a873 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,22 +1,21 @@ package cmd import ( + "OracleSync2MySQL/pkg/sirupsen/logrus" "bytes" "database/sql" "fmt" "github.com/mitchellh/go-homedir" - "io" "math" "os" "os/signal" - "path/filepath" "strconv" "strings" "sync" "syscall" "time" - "github.com/sirupsen/logrus" + //"github.com/sirupsen/logrus" "github.com/spf13/cobra" "OracleSync2MySQL/connect" @@ -24,7 +23,7 @@ import ( "github.com/spf13/viper" ) -var log = logrus.New() +var log, logDir = logrus.New() var cfgFile string var selFromYml bool var metaData bool @@ -49,22 +48,6 @@ func startDataTransfer(connStr *connect.DbConnStr) { exitChan := make(chan os.Signal) signal.Notify(exitChan, os.Interrupt, os.Kill, syscall.SIGTERM) go exitHandle(exitChan) - // 创建运行日志目录 - logDir, _ := filepath.Abs(CreateDateDir("")) - // 输出调用文件以及方法位置 - log.SetReportCaller(true) - f, err := os.OpenFile(logDir+"/"+"run.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, os.ModePerm) - if err != nil { - log.Fatal(err) - } - defer func() { - if err := f.Close(); err != nil { - log.Fatal(err) - } - }() - // log信息重定向到平面文件 - multiWriter := io.MultiWriter(os.Stdout, f) - log.SetOutput(multiWriter) start := time.Now() // map结构，表名以及该表用来迁移查询源库的语句 var tableMap map[string][]string diff --git a/go.mod b/go.mod index 698c128..10e9a7d 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/godror/godror v0.44.0 github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092 github.com/mitchellh/go-homedir v1.1.0 - github.com/sirupsen/logrus v1.9.3 + //github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 ) @@ -23,11 +23,14 @@ require ( github.com/godror/knownpb v0.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect + github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sijms/go-ora/v2 v2.7.9 // indirect @@ -39,10 +42,12 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 52927bc..9720240 100644 --- a/go.sum +++ b/go.sum @@ -160,6 +160,11 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible h1:Y6sqxHMyB1D2YSzWkLibYKgg+SwmyFU9dF2hn6MdTj4= +github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible/go.mod h1:ZQnN8lSECaebrkQytbHj4xNgtg8CR7RYXnPok8e0EHA= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092 h1:u9I3sJ+uTakxnRrvuYJGsEi4SvEMN+yB47WWGDHHxIk= github.com/liushuochen/gotable v0.0.0-20221119160816-1113793e7092/go.mod h1:CxUy8nDvutaC1pOfaG9TRoYwdHHqoNstSPPKhomC9k8= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -179,6 +184,7 @@ github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZ github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -220,6 +226,7 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -246,6 +253,8 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -550,6 +559,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/flagx/conf.go b/pkg/flagx/conf.go new file mode 100644 index 0000000..c9215a5 --- /dev/null +++ b/pkg/flagx/conf.go @@ -0,0 +1,83 @@ +package flagx + +import ( + "fmt" + + "OracleSync2MySQL/pkg/gozero/core/mapping" + "io/ioutil" + "log" + "os" + "path" + "strings" +) + +/** + * @Description + * @Author panwutian + * @Date 5/7/2022 2:42 PM + **/ + +var loaders = map[string]func([]byte, interface{}) error{ + ".json": LoadFromJsonBytes, + ".yaml": LoadFromYamlBytes, + ".yml": LoadFromYamlBytes, +} + +// Load loads config into v from file, .json, .yaml and .yml are acceptable. +func Load(file string, v interface{}, opts ...Option) error { + content, err := ioutil.ReadFile(file) + if err != nil { + return err + } + + loader, ok := loaders[strings.ToLower(path.Ext(file))] + if !ok { + return fmt.Errorf("unrecognized file type: %s", file) + } + + var opt options + for _, o := range opts { + o(&opt) + } + + if opt.env { + return loader([]byte(os.ExpandEnv(string(content))), v) + } + + return loader(content, v) +} + +// LoadConfig loads config into v from file, .json, .yaml and .yml are acceptable. +// Deprecated: use Load instead. +func LoadConfig(file string, v interface{}, opts ...Option) error { + return Load(file, v, opts...) +} + +// LoadFromJsonBytes loads config into v from content json bytes. +func LoadFromJsonBytes(content []byte, v interface{}) error { + return mapping.UnmarshalJsonBytes(content, v) +} + +// LoadConfigFromJsonBytes loads config into v from content json bytes. +// Deprecated: use LoadFromJsonBytes instead. +func LoadConfigFromJsonBytes(content []byte, v interface{}) error { + return LoadFromJsonBytes(content, v) +} + +// LoadFromYamlBytes loads config into v from content yaml bytes. +func LoadFromYamlBytes(content []byte, v interface{}) error { + return mapping.UnmarshalYamlBytes(content, v) +} + +// LoadConfigFromYamlBytes loads config into v from content yaml bytes. +// Deprecated: use LoadFromYamlBytes instead. +func LoadConfigFromYamlBytes(content []byte, v interface{}) error { + return LoadFromYamlBytes(content, v) +} + +// MustLoad loads config into v from path, exits on error. +func MustLoad(path string, v interface{}, opts ...Option) { + if err := Load(path, v, opts...); err != nil { + log.Fatalf("error: config file %s, %s", path, err.Error()) + } +} diff --git a/pkg/flagx/options.go b/pkg/flagx/options.go new file mode 100644 index 0000000..307df31 --- /dev/null +++ b/pkg/flagx/options.go @@ -0,0 +1,23 @@ +package flagx + +/** + * @Description + * @Author panwutian + * @Date 5/7/2022 2:43 PM + **/ + +type ( + // Option defines the method to customize the config options. + Option func(opt *options) + + options struct { + env bool + } +) + +// UseEnv customizes the config to use environment variables. +func UseEnv() Option { + return func(opt *options) { + opt.env = true + } +} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go new file mode 100644 index 0000000..1ed7552 --- /dev/null +++ b/pkg/logger/logger.go @@ -0,0 +1,283 @@ +package logger + +import ( + "encoding/json" + "fmt" + rotatelogs "github.com/lestrrat-go/file-rotatelogs" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" + "gopkg.in/yaml.v3" + "io" + "os" + "time" +) + +const ( + TimeDivision = "time" + SizeDivision = "size" + + _defaultEncoding = "console" + _defaultDivision = "size" + _defaultUnit = Hour +) + +var ( + _encoderNameToConstructor = map[string]func(zapcore.EncoderConfig) zapcore.Encoder{ + "console": func(encoderConfig zapcore.EncoderConfig) zapcore.Encoder { + return zapcore.NewConsoleEncoder(encoderConfig) + }, + "json": func(encoderConfig zapcore.EncoderConfig) zapcore.Encoder { + return zapcore.NewJSONEncoder(encoderConfig) + }, + } +) + +type ZapLogger struct { + zl *zap.Logger +} + +type LogOptions struct { + // Encoding sets the logger's encoding. Valid values are "json" and + // "console", as well as any third-party encodings registered via + // RegisterEncoder. + Encoding string `json:"encoding" yaml:"encoding" toml:"encoding"` + InfoFilename string `json:"info_filename" yaml:"info_filename" toml:"info_filename"` + ErrorFilename string `json:"error_filename" yaml:"error_filename" toml:"error_filename"` + MaxSize int `json:"max_size" yaml:"max_size" toml:"max_size"` + MaxBackups int `json:"max_backups" yaml:"max_backups" toml:"max_backups"` + MaxAge int `json:"max_age" yaml:"max_age" toml:"max_age"` + Compress bool `json:"compress" yaml:"compress" toml:"compress"` + Division string `json:"division" yaml:"division" toml:"division"` + LevelSeparate bool `json:"level_separate" yaml:"level_separate" toml:"level_separate"` + TimeUnit TimeUnit `json:"time_unit" yaml:"time_unit" toml:"time_unit"` + Stacktrace bool `json:"stacktrace" yaml:"stacktrace" toml:"stacktrace"` + EncodeTime string `json:"encode_time" yaml:"encode_time" toml:"encode_time"` + closeDisplay int + caller bool +} + +func infoLevel() zap.LevelEnablerFunc { + return zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl < zapcore.WarnLevel + }) +} + +func warnLevel() zap.LevelEnablerFunc { + return zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl >= zapcore.WarnLevel + }) +} + +func New() *LogOptions { + return &LogOptions{ + Division: _defaultDivision, + LevelSeparate: false, + TimeUnit: _defaultUnit, + Encoding: _defaultEncoding, + caller: false, + } +} + +//func NewFromToml(confPath string) *LogOptions { +// var c *LogOptions +// if _, err := toml.DecodeFile(confPath, &c); err != nil { +// panic(err) +// } +// return c +//} + +func NewFromYaml(confPath string) *LogOptions { + var c *LogOptions + file, err := os.ReadFile(confPath) + if err != nil { + fmt.Printf("yamlFile.Get err #%v ", err) + } + err = yaml.Unmarshal(file, &c) + if err != nil { + fmt.Printf("error: %v", err) + } + return c +} + +func NewFromJson(confPath string) *LogOptions { + var c *LogOptions + file, err := os.ReadFile(confPath) + if err != nil { + fmt.Printf("yamlFile.Get err #%v ", err) + } + err = json.Unmarshal(file, &c) + if err != nil { + fmt.Printf("error: %v", err) + } + return c +} + +func (c *LogOptions) SetDivision(division string) { + c.Division = division +} + +func (c *LogOptions) SetEncodeTime(format string) { + c.EncodeTime = format +} + +func (c *LogOptions) CloseConsoleDisplay() { + c.closeDisplay = 1 +} + +func (c *LogOptions) SetCaller(b bool) { + c.caller = b +} + +func (c *LogOptions) SetTimeUnit(t TimeUnit) { + c.TimeUnit = t +} + +func (c *LogOptions) SetErrorFile(path string) { + c.LevelSeparate = true + c.ErrorFilename = path +} + +func (c *LogOptions) SetInfoFile(path string) { + c.InfoFilename = path +} + +func (c *LogOptions) SetEncoding(encoding string) { + c.Encoding = encoding +} + +// isOutput whether set output file +func (c *LogOptions) isOutput() bool { + return c.InfoFilename != "" +} + +func (c *LogOptions) InitLogger() *ZapLogger { + var ( + logger *zap.Logger + infoHook, warnHook io.Writer + wsInfo []zapcore.WriteSyncer + wsWarn []zapcore.WriteSyncer + ) + + if c.Encoding == "" { + c.Encoding = _defaultEncoding + } + if c.EncodeTime == "" { + c.EncodeTime = RFC3339 + } + encoder := _encoderNameToConstructor[c.Encoding] + + encoderConfig := zapcore.EncoderConfig{ + TimeKey: "time", + LevelKey: "level", + NameKey: "logger", + CallerKey: "file", + MessageKey: "msg", + StacktraceKey: "stacktrace", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.LowercaseLevelEncoder, + EncodeTime: zapcore.TimeEncoderOfLayout(c.EncodeTime), + EncodeDuration: zapcore.SecondsDurationEncoder, + EncodeCaller: zapcore.FullCallerEncoder, + } + + if c.closeDisplay == 0 { + wsInfo = append(wsInfo, zapcore.AddSync(os.Stdout)) + wsWarn = append(wsWarn, zapcore.AddSync(os.Stdout)) + } + + // zapcore WriteSyncer setting + if c.isOutput() { + switch c.Division { + case TimeDivision: + infoHook = c.timeDivisionWriter(c.InfoFilename) + if c.LevelSeparate { + warnHook = c.timeDivisionWriter(c.ErrorFilename) + } + case SizeDivision: + infoHook = c.sizeDivisionWriter(c.InfoFilename) + if c.LevelSeparate { + warnHook = c.sizeDivisionWriter(c.ErrorFilename) + } + } + wsInfo = append(wsInfo, zapcore.AddSync(infoHook)) + } + + if c.ErrorFilename != "" { + wsWarn = append(wsWarn, zapcore.AddSync(warnHook)) + } + + opts := make([]zap.Option, 0) + cos := make([]zapcore.Core, 0) + + if c.LevelSeparate { + cos = append( + cos, + zapcore.NewCore(encoder(encoderConfig), zapcore.NewMultiWriteSyncer(wsInfo...), infoLevel()), + zapcore.NewCore(encoder(encoderConfig), zapcore.NewMultiWriteSyncer(wsWarn...), warnLevel()), + ) + } else { + cos = append( + cos, + zapcore.NewCore(encoder(encoderConfig), zapcore.NewMultiWriteSyncer(wsInfo...), zap.InfoLevel), + ) + } + + opts = append(opts, zap.Development()) + + if c.Stacktrace { + opts = append(opts, zap.AddStacktrace(zapcore.WarnLevel)) + } + + if c.caller { + opts = append(opts, zap.AddCaller()) + } + + logger = zap.New(zapcore.NewTee(cos...), opts...) + zlog := &ZapLogger{zl: logger} + return zlog +} + +func (c *LogOptions) sizeDivisionWriter(filename string) io.Writer { + hook := &lumberjack.Logger{ + Filename: filename, + MaxSize: c.MaxSize, + MaxBackups: c.MaxBackups, + MaxAge: c.MaxSize, + Compress: c.Compress, + } + return hook +} + +func (c *LogOptions) timeDivisionWriter(filename string) io.Writer { + hook, err := rotatelogs.New( + filename+c.TimeUnit.Format(), + rotatelogs.WithMaxAge(time.Duration(int64(24*time.Hour)*int64(c.MaxAge))), + rotatelogs.WithRotationTime(c.TimeUnit.RotationGap()), + ) + + if err != nil { + panic(err) + } + return hook +} + +func (l *ZapLogger) Error(args ...interface{}) { + l.zl.Error(fmt.Sprint(args...)) +} + +func (l *ZapLogger) Fatal(args ...interface{}) { + l.zl.Fatal(fmt.Sprint(args...)) +} + +func (l *ZapLogger) Info(args ...interface{}) { + l.zl.Info(fmt.Sprint(args...)) +} + +func (l *ZapLogger) Debug(args ...interface{}) { + l.zl.Debug(fmt.Sprint(args...)) +} + +func (l *ZapLogger) Warn(args ...interface{}) { + l.zl.Warn(fmt.Sprint(args...)) +} diff --git a/pkg/logger/unit.go b/pkg/logger/unit.go new file mode 100644 index 0000000..ffa91d3 --- /dev/null +++ b/pkg/logger/unit.go @@ -0,0 +1,50 @@ +package logger + +import ( + "time" +) + +type TimeUnit string + +const ( + Minute = "minute" + Hour = "hour" + Day = "day" + Month = "month" + Year = "year" + RFC3339 = "2006-01-02T15:04:05Z07:00" +) + +func (t TimeUnit) Format() string { + switch t { + case Minute: + return ".%Y%m%d%H%M" + case Hour: + return ".%Y%m%d%H" + case Day: + return ".%Y%m%d" + case Month: + return ".%Y%m" + case Year: + return ".%Y" + default: + return ".%Y%m%d" + } +} + +func (t TimeUnit) RotationGap() time.Duration { + switch t { + case Minute: + return time.Minute + case Hour: + return time.Hour + case Day: + return time.Hour * 24 + case Month: + return time.Hour * 24 * 30 + case Year: + return time.Hour * 24 * 365 + default: + return time.Hour * 24 + } +} diff --git a/pkg/sirupsen/logrus/logrus.go b/pkg/sirupsen/logrus/logrus.go new file mode 100644 index 0000000..992aaaf --- /dev/null +++ b/pkg/sirupsen/logrus/logrus.go @@ -0,0 +1,42 @@ +package logrus + +import ( + "OracleSync2MySQL/pkg/logger" + "fmt" + "os" + "path/filepath" + "time" +) + +func New() (*logger.ZapLogger, string) { + c := logger.New() + logDir, _ := filepath.Abs(CreateDateDir("")) + c.SetDivision("time") // 设置归档方式，"time"时间归档 "size" 文件大小归档，文件大小等可以在配置文件配置 + c.SetTimeUnit(logger.Day) // 时间归档 可以设置切割单位 + c.SetEncoding("console") // 输出格式 "json" 或者 "console" + c.Stacktrace = true // 添加 Stacktrace, 默认false + c.SetInfoFile(logDir + "/" + "run.log") // 设置info级别日志 + c.SetErrorFile(logDir + "/" + "run_err.log") // 设置warn级别日志 + c.SetEncodeTime("2006-01-02 15:04:05") // 设置时间格式 + return c.InitLogger(), logDir // 初始化 +} + +// CreateDateDir 根据当前日期来创建文件夹 +func CreateDateDir(basePath string) string { + folderName := "log/" + time.Now().Format("2006_01_02_15_04_05") + folderPath := filepath.Join(basePath, folderName) + if _, err := os.Stat(folderPath); os.IsNotExist(err) { + // 必须分成两步 + // 先创建文件夹 + err := os.MkdirAll(folderPath, 0777) //级联创建目录 + if err != nil { + fmt.Println("create directory log failed ", err) + } + // 再修改权限 + err = os.Chmod(folderPath, 0777) + if err != nil { + fmt.Println("chmod directory log failed ", err) + } + } + return folderPath +} From 8c28931bb6ff9a97eae35974097e182f520b3c36 Mon Sep 17 00:00:00 2001 From: code021 Date: Fri, 7 Jun 2024 22:35:11 +0800 Subject: [PATCH 10/13] =?UTF-8?q?=E6=8E=92=E9=99=A4exclude=E5=92=8C?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=8E=B7=E5=8F=96seq=E5=92=8C=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E8=87=AA=E5=A2=9E=E7=9A=84=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/compare.go | 2 +- cmd/create.go | 4 ++-- cmd/root.go | 3 ++- cmd/tablemeta.go | 33 +++++++++++++++++++++++++++++++-- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/cmd/compare.go b/cmd/compare.go index 0e00cf4..32f5eda 100644 --- a/cmd/compare.go +++ b/cmd/compare.go @@ -27,7 +27,7 @@ var compareDbCmd = &cobra.Command{ // 每页的分页记录数,仅全库迁移时有效 pageSize := viper.GetInt("pageSize") // 从配置文件中获取需要排除的表 - excludeTab := viper.GetStringSlice("exclude") + excludeTab = viper.GetStringSlice("exclude") PrepareSrc(connStr) PrepareDest(connStr) var tableMap map[string][]string diff --git a/cmd/create.go b/cmd/create.go index 9402fb6..e3c4db6 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -32,7 +32,7 @@ var createTableCmd = &cobra.Command{ // 每页的分页记录数,仅全库迁移时有效 pageSize := viper.GetInt("pageSize") // 从配置文件中获取需要排除的表 - excludeTab := viper.GetStringSlice("exclude") + excludeTab = viper.GetStringSlice("exclude") PrepareSrc(connStr) PrepareDest(connStr) var tableMap map[string][]string @@ -76,7 +76,7 @@ var onlyDataCmd = &cobra.Command{ // map结构，表名以及该表用来迁移查询源库的语句 var tableMap map[string][]string // 从配置文件中获取需要排除的表 - excludeTab := viper.GetStringSlice("exclude") + excludeTab = viper.GetStringSlice("exclude") log.Info("running SourceDB check connect") // 生成源库数据库连接 PrepareSrc(connStr) diff --git a/cmd/root.go b/cmd/root.go index 873a873..f6f9d62 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -27,6 +27,7 @@ var log, logDir = logrus.New() var cfgFile string var selFromYml bool var metaData bool +var excludeTab []string var wg sync.WaitGroup var wg2 sync.WaitGroup @@ -52,7 +53,7 @@ func startDataTransfer(connStr *connect.DbConnStr) { // map结构，表名以及该表用来迁移查询源库的语句 var tableMap map[string][]string // 从配置文件中获取需要排除的表 - excludeTab := viper.GetStringSlice("exclude") + excludeTab = viper.GetStringSlice("exclude") log.Info("running SourceDB check connect") // 生成源库数据库连接 PrepareSrc(connStr) diff --git a/cmd/tablemeta.go b/cmd/tablemeta.go index dfe4d6e..bde806f 100644 --- a/cmd/tablemeta.go +++ b/cmd/tablemeta.go @@ -1,6 +1,7 @@ package cmd import ( + "bytes" "database/sql" "fmt" "regexp" @@ -224,7 +225,20 @@ func (tb *Table) SeqCreate(logDir string) (ret []string) { startTime := time.Now() failedCount = 0 var dbRet, tableName string - rows, err := srcDb.Query("select table_name,trigger_body from user_triggers where upper(trigger_type) ='BEFORE EACH ROW'") + srcTableSql := "select table_name,trigger_body from user_triggers where upper(trigger_type) ='BEFORE EACH ROW' %s union select table_name,trigger_body from dba_triggers where upper(trigger_type) ='BEFORE EACH ROW' %s" + buffer := bytes.NewBufferString(" ") + if len(excludeTab) > 0 { + buffer.WriteString(" and table_name not in ( ") + for index, tabName := range excludeTab { + if index < len(excludeTab)-1 { + buffer.WriteString("'" + tabName + "'" + ",") + } else { + buffer.WriteString("'" + tabName + "'" + ")") + } + } + } + srcTableSql = fmt.Sprintf(srcTableSql, buffer.String(), buffer.String()) + rows, err := srcDb.Query(srcTableSql) if err != nil { log.Error(err) } @@ -286,7 +300,22 @@ func (tb *Table) FkCreate(logDir string) (ret []string) { startTime := time.Now() failedCount = 0 var tableName, sqlStr string - rows, err := srcDb.Query("SELECT B.TABLE_NAME,'ALTER TABLE ' || B.TABLE_NAME || ' ADD CONSTRAINT ' ||\n B.CONSTRAINT_NAME || ' FOREIGN KEY (' ||\n (SELECT listagg(A.COLUMN_NAME,',') within group(order by a.position)\n FROM USER_CONS_COLUMNS A\n WHERE A.CONSTRAINT_NAME = B.CONSTRAINT_NAME) || ') REFERENCES ' ||\n (SELECT B1.table_name FROM USER_CONSTRAINTS B1\n WHERE B1.CONSTRAINT_NAME = B.R_CONSTRAINT_NAME) || '(' ||\n (SELECT listagg(A.COLUMN_NAME,',') within group(order by a.position)\n FROM USER_CONS_COLUMNS A\n WHERE A.CONSTRAINT_NAME = B.R_CONSTRAINT_NAME) || ');'\nFROM USER_CONSTRAINTS B\nWHERE B.CONSTRAINT_TYPE = 'R' ") + srcTableSql := "SELECT B.TABLE_NAME,'ALTER TABLE ' || B.TABLE_NAME || ' ADD CONSTRAINT ' ||\n B.CONSTRAINT_NAME || ' FOREIGN KEY (' ||\n (SELECT listagg(A.COLUMN_NAME,',') within group(order by a.position)\n FROM USER_CONS_COLUMNS A\n WHERE A.CONSTRAINT_NAME = B.CONSTRAINT_NAME) || ') REFERENCES ' ||\n (SELECT B1.table_name FROM USER_CONSTRAINTS B1\n WHERE B1.CONSTRAINT_NAME = B.R_CONSTRAINT_NAME) || '(' ||\n (SELECT listagg(A.COLUMN_NAME,',') within group(order by a.position)\n FROM USER_CONS_COLUMNS A\n WHERE A.CONSTRAINT_NAME = B.R_CONSTRAINT_NAME) || ');'\nFROM USER_CONSTRAINTS B\nWHERE B.CONSTRAINT_TYPE = 'R' %s" + srcTableSql = srcTableSql + "union SELECT B.TABLE_NAME,'ALTER TABLE ' || B.TABLE_NAME || ' ADD CONSTRAINT ' ||\n B.CONSTRAINT_NAME || ' FOREIGN KEY (' ||\n (SELECT listagg(A.COLUMN_NAME,',') within group(order by a.position)\n FROM DBA_CONS_COLUMNS A\n WHERE A.CONSTRAINT_NAME = B.CONSTRAINT_NAME) || ') REFERENCES ' ||\n (SELECT B1.table_name FROM DBA_CONSTRAINTS B1\n WHERE B1.CONSTRAINT_NAME = B.R_CONSTRAINT_NAME) || '(' ||\n (SELECT listagg(A.COLUMN_NAME,',') within group(order by a.position)\n FROM DBA_CONS_COLUMNS A\n WHERE A.CONSTRAINT_NAME = B.R_CONSTRAINT_NAME) || ');'\nFROM DBA_CONSTRAINTS B\nWHERE B.CONSTRAINT_TYPE = 'R' %s" + buffer := bytes.NewBufferString(" ") + if len(excludeTab) > 0 { + buffer.WriteString(" and B.TABLE_NAME not in ( ") + for index, tabName := range excludeTab { + if index < len(excludeTab)-1 { + buffer.WriteString("'" + tabName + "'" + ",") + } else { + buffer.WriteString("'" + tabName + "'" + ")") + } + } + } + srcTableSql = fmt.Sprintf(srcTableSql, buffer.String(), buffer.String()) + rows, err := srcDb.Query(srcTableSql) + if err != nil { log.Error(err) } From e5bf57f503e255a94f2b4b23b9fd45d039797525 Mon Sep 17 00:00:00 2001 From: code021 Date: Sun, 9 Jun 2024 11:29:18 +0800 Subject: [PATCH 11/13] =?UTF-8?q?=E7=B2=BE=E7=AE=80=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/flagx/conf.go | 83 -------------------------------------------- pkg/flagx/options.go | 23 ------------ 2 files changed, 106 deletions(-) delete mode 100644 pkg/flagx/conf.go delete mode 100644 pkg/flagx/options.go diff --git a/pkg/flagx/conf.go b/pkg/flagx/conf.go deleted file mode 100644 index c9215a5..0000000 --- a/pkg/flagx/conf.go +++ /dev/null @@ -1,83 +0,0 @@ -package flagx - -import ( - "fmt" - - "OracleSync2MySQL/pkg/gozero/core/mapping" - "io/ioutil" - "log" - "os" - "path" - "strings" -) - -/** - * @Description - * @Author panwutian - * @Date 5/7/2022 2:42 PM - **/ - -var loaders = map[string]func([]byte, interface{}) error{ - ".json": LoadFromJsonBytes, - ".yaml": LoadFromYamlBytes, - ".yml": LoadFromYamlBytes, -} - -// Load loads config into v from file, .json, .yaml and .yml are acceptable. -func Load(file string, v interface{}, opts ...Option) error { - content, err := ioutil.ReadFile(file) - if err != nil { - return err - } - - loader, ok := loaders[strings.ToLower(path.Ext(file))] - if !ok { - return fmt.Errorf("unrecognized file type: %s", file) - } - - var opt options - for _, o := range opts { - o(&opt) - } - - if opt.env { - return loader([]byte(os.ExpandEnv(string(content))), v) - } - - return loader(content, v) -} - -// LoadConfig loads config into v from file, .json, .yaml and .yml are acceptable. -// Deprecated: use Load instead. -func LoadConfig(file string, v interface{}, opts ...Option) error { - return Load(file, v, opts...) -} - -// LoadFromJsonBytes loads config into v from content json bytes. -func LoadFromJsonBytes(content []byte, v interface{}) error { - return mapping.UnmarshalJsonBytes(content, v) -} - -// LoadConfigFromJsonBytes loads config into v from content json bytes. -// Deprecated: use LoadFromJsonBytes instead. -func LoadConfigFromJsonBytes(content []byte, v interface{}) error { - return LoadFromJsonBytes(content, v) -} - -// LoadFromYamlBytes loads config into v from content yaml bytes. -func LoadFromYamlBytes(content []byte, v interface{}) error { - return mapping.UnmarshalYamlBytes(content, v) -} - -// LoadConfigFromYamlBytes loads config into v from content yaml bytes. -// Deprecated: use LoadFromYamlBytes instead. -func LoadConfigFromYamlBytes(content []byte, v interface{}) error { - return LoadFromYamlBytes(content, v) -} - -// MustLoad loads config into v from path, exits on error. -func MustLoad(path string, v interface{}, opts ...Option) { - if err := Load(path, v, opts...); err != nil { - log.Fatalf("error: config file %s, %s", path, err.Error()) - } -} diff --git a/pkg/flagx/options.go b/pkg/flagx/options.go deleted file mode 100644 index 307df31..0000000 --- a/pkg/flagx/options.go +++ /dev/null @@ -1,23 +0,0 @@ -package flagx - -/** - * @Description - * @Author panwutian - * @Date 5/7/2022 2:43 PM - **/ - -type ( - // Option defines the method to customize the config options. - Option func(opt *options) - - options struct { - env bool - } -) - -// UseEnv customizes the config to use environment variables. -func UseEnv() Option { - return func(opt *options) { - opt.env = true - } -} From 85e463e519b0289810ec1dbf80f98b204ff37ef4 Mon Sep 17 00:00:00 2001 From: xls-web Date: Thu, 13 Jun 2024 15:56:16 +0800 Subject: [PATCH 12/13] srcSchema --- cmd/app.go | 3 +++ connect/connect.go | 1 + example.yml | 1 + 3 files changed, 5 insertions(+) diff --git a/cmd/app.go b/cmd/app.go index 138768c..116e711 100644 --- a/cmd/app.go +++ b/cmd/app.go @@ -22,6 +22,7 @@ import ( var srcDb *sql.DB var destDb *sql.DB var oracleConnStr godror.ConnectionParams +var srcSchema string func getConn() (connStr *connect.DbConnStr) { connStr = new(connect.DbConnStr) @@ -30,6 +31,7 @@ func getConn() (connStr *connect.DbConnStr) { connStr.SrcPassword = viper.GetString("src.password") connStr.SrcDatabase = viper.GetString("src.database") connStr.SrcPort = viper.GetInt("src.port") + connStr.SrcSchema = viper.GetString("src.schema") connStr.DestHost = viper.GetString("dest.host") connStr.DestPort = viper.GetInt("dest.port") connStr.DestUserName = viper.GetString("dest.username") @@ -46,6 +48,7 @@ func PrepareSrc(connStr *connect.DbConnStr) { srcPassword := connStr.SrcPassword srcDatabase := connStr.SrcDatabase srcPort := connStr.SrcPort + srcSchema = connStr.SrcSchema //srcConn := fmt.Sprintf("oracle://%s:%s@%s:%d/%s?LOB FETCH=POST", srcUserName, srcPassword, srcHost, srcPort, srcDatabase) //fmt.Println(srcConn) var err error diff --git a/connect/connect.go b/connect/connect.go index 53e3edb..282d8dc 100644 --- a/connect/connect.go +++ b/connect/connect.go @@ -7,6 +7,7 @@ type DbConnStr struct { SrcPassword string SrcDatabase string SrcPort int + SrcSchema string DestHost string DestPort int DestUserName string diff --git a/example.yml b/example.yml index 3df2b58..0208e08 100644 --- a/example.yml +++ b/example.yml @@ -4,6 +4,7 @@ src: database: orcl username: admin password: oracle +# schema: test dest: host: 192.168.1.37 port: 3306 From 2f75d1170b8063f487fc1247cd3850d21d096f7c Mon Sep 17 00:00:00 2001 From: xls-web Date: Thu, 13 Jun 2024 15:58:15 +0800 Subject: [PATCH 13/13] =?UTF-8?q?=E6=9B=B4=E6=96=B0mod?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 23 +++++++++++++---------- go.sum | 12 ++++++++++++ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 10e9a7d..390f059 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,14 @@ require ( github.com/mitchellh/go-homedir v1.1.0 //github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 - github.com/spf13/viper v1.18.2 + github.com/spf13/viper v1.19.0 +) + +require ( + github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible + go.uber.org/zap v1.27.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -23,7 +30,6 @@ require ( github.com/godror/knownpb v0.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible // indirect github.com/lestrrat-go/strftime v1.0.6 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -31,7 +37,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sijms/go-ora/v2 v2.7.9 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -42,12 +48,9 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9720240..51109e1 100644 --- a/go.sum +++ b/go.sum @@ -193,6 +193,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= +github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sijms/go-ora/v2 v2.7.9 h1:FvPwsyNtAOywDKlgjrgCpGkL0s49ZA/ShTBgEAfYKE0= @@ -221,6 +223,8 @@ github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -276,6 +280,8 @@ golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0 h1:Mi0bCswbz+9cXmwFAdxoo5GPF golang.org/x/exp v0.0.0-20240530194437-404ba88c7ed0/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 h1:LoYXNGAShUG3m/ehNk4iFctuhGX/+R1ZpfJ4/ia80JM= +golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -394,6 +400,8 @@ golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -407,6 +415,8 @@ golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -554,6 +564,8 @@ google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cn google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=