diff --git a/DEPS.bzl b/DEPS.bzl index 2996df116e67d..eecd5c9f479d5 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -121,45 +121,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/aclements/go-moremath/com_github_aclements_go_moremath-v0.0.0-20210112150236-f10218a38794.zip", ], ) - go_repository( - name = "com_github_ajstarks_deck", - build_file_proto_mode = "disable_global", - importpath = "github.com/ajstarks/deck", - sha256 = "68bad2e38bf5b01e6bbd7b9bbdba35da94dac72bc4ba41f8ea5fe92aa836a3c3", - strip_prefix = "github.com/ajstarks/deck@v0.0.0-20200831202436-30c9fc6549a9", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/ajstarks/deck/com_github_ajstarks_deck-v0.0.0-20200831202436-30c9fc6549a9.zip", - "http://ats.apps.svc/gomod/github.com/ajstarks/deck/com_github_ajstarks_deck-v0.0.0-20200831202436-30c9fc6549a9.zip", - "https://cache.hawkingrei.com/gomod/github.com/ajstarks/deck/com_github_ajstarks_deck-v0.0.0-20200831202436-30c9fc6549a9.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ajstarks/deck/com_github_ajstarks_deck-v0.0.0-20200831202436-30c9fc6549a9.zip", - ], - ) - go_repository( - name = "com_github_ajstarks_deck_generate", - build_file_proto_mode = "disable_global", - importpath = "github.com/ajstarks/deck/generate", - sha256 = "dce1cbc4cb42ac26512dd0bccf997baeea99fb4595cd419a28e8566d2d7c7ba8", - strip_prefix = "github.com/ajstarks/deck/generate@v0.0.0-20210309230005-c3f852c02e19", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/ajstarks/deck/generate/com_github_ajstarks_deck_generate-v0.0.0-20210309230005-c3f852c02e19.zip", - "http://ats.apps.svc/gomod/github.com/ajstarks/deck/generate/com_github_ajstarks_deck_generate-v0.0.0-20210309230005-c3f852c02e19.zip", - "https://cache.hawkingrei.com/gomod/github.com/ajstarks/deck/generate/com_github_ajstarks_deck_generate-v0.0.0-20210309230005-c3f852c02e19.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ajstarks/deck/generate/com_github_ajstarks_deck_generate-v0.0.0-20210309230005-c3f852c02e19.zip", - ], - ) - go_repository( - name = "com_github_ajstarks_svgo", - build_file_proto_mode = "disable_global", - importpath = "github.com/ajstarks/svgo", - sha256 = "e25b5dbb6cc86d2a0b5db08aad757c534681c2cecb30d84746e09c661cbd7c6f", - strip_prefix = "github.com/ajstarks/svgo@v0.0.0-20211024235047-1546f124cd8b", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20211024235047-1546f124cd8b.zip", - "http://ats.apps.svc/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20211024235047-1546f124cd8b.zip", - "https://cache.hawkingrei.com/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20211024235047-1546f124cd8b.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ajstarks/svgo/com_github_ajstarks_svgo-v0.0.0-20211024235047-1546f124cd8b.zip", - ], - ) go_repository( name = "com_github_alecthomas_chroma_v2", build_file_proto_mode = "disable_global", @@ -307,13 +268,13 @@ def go_deps(): name = "com_github_andybalholm_brotli", build_file_proto_mode = "disable_global", importpath = "github.com/andybalholm/brotli", - sha256 = "f5ae9b2f3260a22ff3f3445fff081d3ef12ee1aa3c0b87eadc59b5a8fb2cdef0", - strip_prefix = "github.com/andybalholm/brotli@v1.0.5", + sha256 = "d183c2a1277d2784861bfa89e3903ecba31ef08237c3e92fbdfd4ab0b551154e", + strip_prefix = "github.com/andybalholm/brotli@v1.1.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/andybalholm/brotli/com_github_andybalholm_brotli-v1.0.5.zip", - "http://ats.apps.svc/gomod/github.com/andybalholm/brotli/com_github_andybalholm_brotli-v1.0.5.zip", - "https://cache.hawkingrei.com/gomod/github.com/andybalholm/brotli/com_github_andybalholm_brotli-v1.0.5.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/andybalholm/brotli/com_github_andybalholm_brotli-v1.0.5.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/andybalholm/brotli/com_github_andybalholm_brotli-v1.1.1.zip", + "http://ats.apps.svc/gomod/github.com/andybalholm/brotli/com_github_andybalholm_brotli-v1.1.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/andybalholm/brotli/com_github_andybalholm_brotli-v1.1.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/andybalholm/brotli/com_github_andybalholm_brotli-v1.1.1.zip", ], ) go_repository( @@ -329,6 +290,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/antihax/optional/com_github_antihax_optional-v1.0.0.zip", ], ) + go_repository( + name = "com_github_antlr4_go_antlr_v4", + build_file_proto_mode = "disable_global", + importpath = "github.com/antlr4-go/antlr/v4", + sha256 = "2fb455ab1f46e53a3cfa678306a6e405f291b34858222db9240335c018ae9554", + strip_prefix = "github.com/antlr4-go/antlr/v4@v4.13.1", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/antlr4-go/antlr/v4/com_github_antlr4_go_antlr_v4-v4.13.1.zip", + "http://ats.apps.svc/gomod/github.com/antlr4-go/antlr/v4/com_github_antlr4_go_antlr_v4-v4.13.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/antlr4-go/antlr/v4/com_github_antlr4_go_antlr_v4-v4.13.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/antlr4-go/antlr/v4/com_github_antlr4_go_antlr_v4-v4.13.1.zip", + ], + ) go_repository( name = "com_github_antonboom_errname", build_file_proto_mode = "disable_global", @@ -369,16 +343,16 @@ def go_deps(): ], ) go_repository( - name = "com_github_apache_arrow_go_v12", + name = "com_github_apache_arrow_go_v18", build_file_proto_mode = "disable_global", - importpath = "github.com/apache/arrow/go/v12", - sha256 = "5eb05ed9c2c5e164503b00912b7b2456400578de29e7e8a8956a41acd861ab5b", - strip_prefix = "github.com/apache/arrow/go/v12@v12.0.1", + importpath = "github.com/apache/arrow-go/v18", + sha256 = "98a849b1afa6b1fb1dfea8421f14576f25eaa698bad20f9533658c240332cc2c", + strip_prefix = "github.com/joechenrh/arrow-go/v18@v18.0.0-20251104113423-8cacfcd24f0d", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/apache/arrow/go/v12/com_github_apache_arrow_go_v12-v12.0.1.zip", - "http://ats.apps.svc/gomod/github.com/apache/arrow/go/v12/com_github_apache_arrow_go_v12-v12.0.1.zip", - "https://cache.hawkingrei.com/gomod/github.com/apache/arrow/go/v12/com_github_apache_arrow_go_v12-v12.0.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/apache/arrow/go/v12/com_github_apache_arrow_go_v12-v12.0.1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/joechenrh/arrow-go/v18/com_github_joechenrh_arrow_go_v18-v18.0.0-20251104113423-8cacfcd24f0d.zip", + "http://ats.apps.svc/gomod/github.com/joechenrh/arrow-go/v18/com_github_joechenrh_arrow_go_v18-v18.0.0-20251104113423-8cacfcd24f0d.zip", + "https://cache.hawkingrei.com/gomod/github.com/joechenrh/arrow-go/v18/com_github_joechenrh_arrow_go_v18-v18.0.0-20251104113423-8cacfcd24f0d.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/joechenrh/arrow-go/v18/com_github_joechenrh_arrow_go_v18-v18.0.0-20251104113423-8cacfcd24f0d.zip", ], ) go_repository( @@ -398,13 +372,13 @@ def go_deps(): name = "com_github_apache_thrift", build_file_proto_mode = "disable_global", importpath = "github.com/apache/thrift", - sha256 = "50d5c610df30fa2a6039394d5142382b7d9938870dfb12ef46bddfa3da250893", - strip_prefix = "github.com/apache/thrift@v0.16.0", + sha256 = "c96231bd10eb2488974c977c1f1bafdce90ec56710d193f54f5771a379835a99", + strip_prefix = "github.com/apache/thrift@v0.21.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/apache/thrift/com_github_apache_thrift-v0.16.0.zip", - "http://ats.apps.svc/gomod/github.com/apache/thrift/com_github_apache_thrift-v0.16.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/apache/thrift/com_github_apache_thrift-v0.16.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/apache/thrift/com_github_apache_thrift-v0.16.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/apache/thrift/com_github_apache_thrift-v0.21.0.zip", + "http://ats.apps.svc/gomod/github.com/apache/thrift/com_github_apache_thrift-v0.21.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/apache/thrift/com_github_apache_thrift-v0.21.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/apache/thrift/com_github_apache_thrift-v0.21.0.zip", ], ) go_repository( @@ -498,6 +472,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ashanbrown/makezero/v2/com_github_ashanbrown_makezero_v2-v2.0.1.zip", ], ) + go_repository( + name = "com_github_atomicgo_cursor", + build_file_proto_mode = "disable_global", + importpath = "github.com/atomicgo/cursor", + sha256 = "b77665289a1a6dc750cf3752537e5b5e67abdeb3e98547ce59d2eef210bf36cf", + strip_prefix = "github.com/atomicgo/cursor@v0.0.1", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/atomicgo/cursor/com_github_atomicgo_cursor-v0.0.1.zip", + "http://ats.apps.svc/gomod/github.com/atomicgo/cursor/com_github_atomicgo_cursor-v0.0.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/atomicgo/cursor/com_github_atomicgo_cursor-v0.0.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/atomicgo/cursor/com_github_atomicgo_cursor-v0.0.1.zip", + ], + ) go_repository( name = "com_github_aws_aws_sdk_go", build_file_proto_mode = "disable_global", @@ -797,19 +784,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/bombsimon/wsl/v5/com_github_bombsimon_wsl_v5-v5.1.0.zip", ], ) - go_repository( - name = "com_github_boombuler_barcode", - build_file_proto_mode = "disable_global", - importpath = "github.com/boombuler/barcode", - sha256 = "812c5beeaa87864227f9d92a9ae71792dc0e6302a33737a91aabe1e511cde42b", - strip_prefix = "github.com/boombuler/barcode@v1.0.1", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/boombuler/barcode/com_github_boombuler_barcode-v1.0.1.zip", - "http://ats.apps.svc/gomod/github.com/boombuler/barcode/com_github_boombuler_barcode-v1.0.1.zip", - "https://cache.hawkingrei.com/gomod/github.com/boombuler/barcode/com_github_boombuler_barcode-v1.0.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/boombuler/barcode/com_github_boombuler_barcode-v1.0.1.zip", - ], - ) go_repository( name = "com_github_breml_bidichk", build_file_proto_mode = "disable_global", @@ -1204,13 +1178,13 @@ def go_deps(): name = "com_github_cncf_udpa_go", build_file_proto_mode = "disable_global", importpath = "github.com/cncf/udpa/go", - sha256 = "a449fa94e58117a79c17577e39f72f695c4876f74cbd9142d512278192ca90aa", - strip_prefix = "github.com/cncf/udpa/go@v0.0.0-20210930031921-04548b0d99d4", + sha256 = "f2a2fee0b2024946ddd3b7ec5cd06a6d318cdb8421a8d5afff4c2fd69f1e74a7", + strip_prefix = "github.com/cncf/udpa/go@v0.0.0-20191209042840-269d4d468f6f", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20210930031921-04548b0d99d4.zip", - "http://ats.apps.svc/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20210930031921-04548b0d99d4.zip", - "https://cache.hawkingrei.com/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20210930031921-04548b0d99d4.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20210930031921-04548b0d99d4.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20191209042840-269d4d468f6f.zip", + "http://ats.apps.svc/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20191209042840-269d4d468f6f.zip", + "https://cache.hawkingrei.com/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20191209042840-269d4d468f6f.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cncf/udpa/go/com_github_cncf_udpa_go-v0.0.0-20191209042840-269d4d468f6f.zip", ], ) go_repository( @@ -1226,6 +1200,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20231128003011-0fa0005c9caa.zip", ], ) + go_repository( + name = "com_github_cockroachdb_apd_v3", + build_file_proto_mode = "disable_global", + importpath = "github.com/cockroachdb/apd/v3", + sha256 = "6ad54bb71a36fba8ca6725a00d916e51815a4c68de54096313ca6fffda6c87c2", + strip_prefix = "github.com/cockroachdb/apd/v3@v3.2.1", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/cockroachdb/apd/v3/com_github_cockroachdb_apd_v3-v3.2.1.zip", + "http://ats.apps.svc/gomod/github.com/cockroachdb/apd/v3/com_github_cockroachdb_apd_v3-v3.2.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/cockroachdb/apd/v3/com_github_cockroachdb_apd_v3-v3.2.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cockroachdb/apd/v3/com_github_cockroachdb_apd_v3-v3.2.1.zip", + ], + ) go_repository( name = "com_github_cockroachdb_datadriven", build_file_proto_mode = "disable_global", @@ -1343,19 +1330,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/codegangsta/inject/com_github_codegangsta_inject-v0.0.0-20150114235600-33e0aa1cb7c0.zip", ], ) - go_repository( - name = "com_github_colinmarc_hdfs_v2", - build_file_proto_mode = "disable_global", - importpath = "github.com/colinmarc/hdfs/v2", - sha256 = "6a40084f999e3ddbd9a8566b1333646424201fc2ad28aa1a40ddf51aaf8fbc51", - strip_prefix = "github.com/colinmarc/hdfs/v2@v2.1.1", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/colinmarc/hdfs/v2/com_github_colinmarc_hdfs_v2-v2.1.1.zip", - "http://ats.apps.svc/gomod/github.com/colinmarc/hdfs/v2/com_github_colinmarc_hdfs_v2-v2.1.1.zip", - "https://cache.hawkingrei.com/gomod/github.com/colinmarc/hdfs/v2/com_github_colinmarc_hdfs_v2-v2.1.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/colinmarc/hdfs/v2/com_github_colinmarc_hdfs_v2-v2.1.1.zip", - ], - ) go_repository( name = "com_github_containerd_cgroups_v3", build_file_proto_mode = "disable_global", @@ -1512,6 +1486,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/creack/pty/com_github_creack_pty-v1.1.11.zip", ], ) + go_repository( + name = "com_github_creasty_defaults", + build_file_proto_mode = "disable_global", + importpath = "github.com/creasty/defaults", + sha256 = "d9984bcd4b7326a6066f58bc94b46fe8657e50e1ba0a3ef6eb592b0ff96e6712", + strip_prefix = "github.com/creasty/defaults@v1.8.0", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/creasty/defaults/com_github_creasty_defaults-v1.8.0.zip", + "http://ats.apps.svc/gomod/github.com/creasty/defaults/com_github_creasty_defaults-v1.8.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/creasty/defaults/com_github_creasty_defaults-v1.8.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/creasty/defaults/com_github_creasty_defaults-v1.8.0.zip", + ], + ) go_repository( name = "com_github_curioswitch_go_reassign", build_file_proto_mode = "disable_global", @@ -2110,19 +2097,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/flosch/pongo2/v4/com_github_flosch_pongo2_v4-v4.0.2.zip", ], ) - go_repository( - name = "com_github_fogleman_gg", - build_file_proto_mode = "disable_global", - importpath = "github.com/fogleman/gg", - sha256 = "792f7a3ea9eea31b7947dabaf9d5a307389245069078e4bf435d76cb0505439c", - strip_prefix = "github.com/fogleman/gg@v1.3.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.3.0.zip", - "http://ats.apps.svc/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.3.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.3.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/fogleman/gg/com_github_fogleman_gg-v1.3.0.zip", - ], - ) go_repository( name = "com_github_frankban_quicktest", build_file_proto_mode = "disable_global", @@ -2305,58 +2279,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-errors/errors/com_github_go_errors_errors-v1.4.2.zip", ], ) - go_repository( - name = "com_github_go_fonts_dejavu", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-fonts/dejavu", - sha256 = "c2094ce49cfc24b7b7a041e54d924e311322b73a8e56db28ff179fcd403b4111", - strip_prefix = "github.com/go-fonts/dejavu@v0.1.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-fonts/dejavu/com_github_go_fonts_dejavu-v0.1.0.zip", - "http://ats.apps.svc/gomod/github.com/go-fonts/dejavu/com_github_go_fonts_dejavu-v0.1.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/go-fonts/dejavu/com_github_go_fonts_dejavu-v0.1.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-fonts/dejavu/com_github_go_fonts_dejavu-v0.1.0.zip", - ], - ) - go_repository( - name = "com_github_go_fonts_latin_modern", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-fonts/latin-modern", - sha256 = "037085a80ad108287e772d064d64bb72deb62514de84ef610506bc079f330ec0", - strip_prefix = "github.com/go-fonts/latin-modern@v0.2.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-fonts/latin-modern/com_github_go_fonts_latin_modern-v0.2.0.zip", - "http://ats.apps.svc/gomod/github.com/go-fonts/latin-modern/com_github_go_fonts_latin_modern-v0.2.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/go-fonts/latin-modern/com_github_go_fonts_latin_modern-v0.2.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-fonts/latin-modern/com_github_go_fonts_latin_modern-v0.2.0.zip", - ], - ) - go_repository( - name = "com_github_go_fonts_liberation", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-fonts/liberation", - sha256 = "bd7561251c221fe0fd8cd4c361b062a5796f6f3a1096968b8fecdd61eb82d8fe", - strip_prefix = "github.com/go-fonts/liberation@v0.2.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-fonts/liberation/com_github_go_fonts_liberation-v0.2.0.zip", - "http://ats.apps.svc/gomod/github.com/go-fonts/liberation/com_github_go_fonts_liberation-v0.2.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/go-fonts/liberation/com_github_go_fonts_liberation-v0.2.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-fonts/liberation/com_github_go_fonts_liberation-v0.2.0.zip", - ], - ) - go_repository( - name = "com_github_go_fonts_stix", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-fonts/stix", - sha256 = "51ea5a38b9fda7854af60f280dbd8b40a3e5b5a48eb00d3f8d4e43de3f514ecf", - strip_prefix = "github.com/go-fonts/stix@v0.1.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-fonts/stix/com_github_go_fonts_stix-v0.1.0.zip", - "http://ats.apps.svc/gomod/github.com/go-fonts/stix/com_github_go_fonts_stix-v0.1.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/go-fonts/stix/com_github_go_fonts_stix-v0.1.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-fonts/stix/com_github_go_fonts_stix-v0.1.0.zip", - ], - ) go_repository( name = "com_github_go_gl_glfw", build_file_proto_mode = "disable_global", @@ -2409,19 +2331,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-kit/log/com_github_go_kit_log-v0.2.1.zip", ], ) - go_repository( - name = "com_github_go_latex_latex", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-latex/latex", - sha256 = "c58be686b31679ad0a51a5d70e60df92fb4bb50a16727caa58b4a67b33f16509", - strip_prefix = "github.com/go-latex/latex@v0.0.0-20210823091927-c0d11ff05a81", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-latex/latex/com_github_go_latex_latex-v0.0.0-20210823091927-c0d11ff05a81.zip", - "http://ats.apps.svc/gomod/github.com/go-latex/latex/com_github_go_latex_latex-v0.0.0-20210823091927-c0d11ff05a81.zip", - "https://cache.hawkingrei.com/gomod/github.com/go-latex/latex/com_github_go_latex_latex-v0.0.0-20210823091927-c0d11ff05a81.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-latex/latex/com_github_go_latex_latex-v0.0.0-20210823091927-c0d11ff05a81.zip", - ], - ) go_repository( name = "com_github_go_ldap_ldap_v3", build_file_proto_mode = "disable_global", @@ -2617,19 +2526,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-openapi/validate/com_github_go_openapi_validate-v0.22.1.zip", ], ) - go_repository( - name = "com_github_go_pdf_fpdf", - build_file_proto_mode = "disable_global", - importpath = "github.com/go-pdf/fpdf", - sha256 = "03a6909fc346ac972b008b77585ac3954d76b416c33b4b64dc22c5f35f0e1edb", - strip_prefix = "github.com/go-pdf/fpdf@v0.6.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/go-pdf/fpdf/com_github_go_pdf_fpdf-v0.6.0.zip", - "http://ats.apps.svc/gomod/github.com/go-pdf/fpdf/com_github_go_pdf_fpdf-v0.6.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/go-pdf/fpdf/com_github_go_pdf_fpdf-v0.6.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/go-pdf/fpdf/com_github_go_pdf_fpdf-v0.6.0.zip", - ], - ) go_repository( name = "com_github_go_playground_locales", build_file_proto_mode = "disable_global", @@ -2868,26 +2764,26 @@ def go_deps(): name = "com_github_goccy_go_json", build_file_proto_mode = "disable_global", importpath = "github.com/goccy/go-json", - sha256 = "ed9043ee01cc46557c74bcecc625db37ffe3a5c7af219f390a287f44a40c2520", - strip_prefix = "github.com/goccy/go-json@v0.10.2", + sha256 = "d89fa5215f5c2e0077b67dbc6e043db1b6427e93bdf5faacb7c536afca95e454", + strip_prefix = "github.com/goccy/go-json@v0.10.4", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/goccy/go-json/com_github_goccy_go_json-v0.10.2.zip", - "http://ats.apps.svc/gomod/github.com/goccy/go-json/com_github_goccy_go_json-v0.10.2.zip", - "https://cache.hawkingrei.com/gomod/github.com/goccy/go-json/com_github_goccy_go_json-v0.10.2.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/goccy/go-json/com_github_goccy_go_json-v0.10.2.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/goccy/go-json/com_github_goccy_go_json-v0.10.4.zip", + "http://ats.apps.svc/gomod/github.com/goccy/go-json/com_github_goccy_go_json-v0.10.4.zip", + "https://cache.hawkingrei.com/gomod/github.com/goccy/go-json/com_github_goccy_go_json-v0.10.4.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/goccy/go-json/com_github_goccy_go_json-v0.10.4.zip", ], ) go_repository( - name = "com_github_goccy_go_reflect", + name = "com_github_goccy_go_yaml", build_file_proto_mode = "disable_global", - importpath = "github.com/goccy/go-reflect", - sha256 = "d5d5b55be60c40d1ecfbd13a7e89c3fb5363e8b7cd07e2827f7e987944c41458", - strip_prefix = "github.com/goccy/go-reflect@v1.2.0", + importpath = "github.com/goccy/go-yaml", + sha256 = "d696eddbab891896b3a70334d04cfb0208cdc0e18544bebdf95926e1f64df310", + strip_prefix = "github.com/goccy/go-yaml@v1.17.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/goccy/go-reflect/com_github_goccy_go_reflect-v1.2.0.zip", - "http://ats.apps.svc/gomod/github.com/goccy/go-reflect/com_github_goccy_go_reflect-v1.2.0.zip", - "https://cache.hawkingrei.com/gomod/github.com/goccy/go-reflect/com_github_goccy_go_reflect-v1.2.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/goccy/go-reflect/com_github_goccy_go_reflect-v1.2.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/goccy/go-yaml/com_github_goccy_go_yaml-v1.17.1.zip", + "http://ats.apps.svc/gomod/github.com/goccy/go-yaml/com_github_goccy_go_yaml-v1.17.1.zip", + "https://cache.hawkingrei.com/gomod/github.com/goccy/go-yaml/com_github_goccy_go_yaml-v1.17.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/goccy/go-yaml/com_github_goccy_go_yaml-v1.17.1.zip", ], ) go_repository( @@ -2968,19 +2864,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/goji/httpauth/com_github_goji_httpauth-v0.0.0-20160601135302-2da839ab0f4d.zip", ], ) - go_repository( - name = "com_github_golang_freetype", - build_file_proto_mode = "disable_global", - importpath = "github.com/golang/freetype", - sha256 = "cdcb9e6a14933dcbf167b44dcd5083fc6a2e52c4fae8fb79747c691efeb7d84e", - strip_prefix = "github.com/golang/freetype@v0.0.0-20170609003504-e2365dfdc4a0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/golang/freetype/com_github_golang_freetype-v0.0.0-20170609003504-e2365dfdc4a0.zip", - "http://ats.apps.svc/gomod/github.com/golang/freetype/com_github_golang_freetype-v0.0.0-20170609003504-e2365dfdc4a0.zip", - "https://cache.hawkingrei.com/gomod/github.com/golang/freetype/com_github_golang_freetype-v0.0.0-20170609003504-e2365dfdc4a0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/golang/freetype/com_github_golang_freetype-v0.0.0-20170609003504-e2365dfdc4a0.zip", - ], - ) go_repository( name = "com_github_golang_glog", build_file_proto_mode = "disable_global", @@ -3271,13 +3154,13 @@ def go_deps(): name = "com_github_google_flatbuffers", build_file_proto_mode = "disable_global", importpath = "github.com/google/flatbuffers", - sha256 = "0c0a4aab1c6029141d655bc7fdc07e22dd06f3f64ebbf7a2250b870ef7aac7ee", - strip_prefix = "github.com/google/flatbuffers@v2.0.8+incompatible", + sha256 = "d067355c553528de4ca6e75a64013ee7336a48c2e98d9eb2d4c7803cc6051dac", + strip_prefix = "github.com/google/flatbuffers@v24.3.25+incompatible", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v2.0.8+incompatible.zip", - "http://ats.apps.svc/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v2.0.8+incompatible.zip", - "https://cache.hawkingrei.com/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v2.0.8+incompatible.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v2.0.8+incompatible.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v24.3.25+incompatible.zip", + "http://ats.apps.svc/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v24.3.25+incompatible.zip", + "https://cache.hawkingrei.com/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v24.3.25+incompatible.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/google/flatbuffers/com_github_google_flatbuffers-v24.3.25+incompatible.zip", ], ) go_repository( @@ -3501,6 +3384,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/googleapis/gax-go/v2/com_github_googleapis_gax_go_v2-v2.12.3.zip", ], ) + go_repository( + name = "com_github_gookit_color", + build_file_proto_mode = "disable_global", + importpath = "github.com/gookit/color", + sha256 = "c5295c810538f77bebb2cf9e34cdfa92adfdec75486234358956ebe877e685f9", + strip_prefix = "github.com/gookit/color@v1.5.4", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/gookit/color/com_github_gookit_color-v1.5.4.zip", + "http://ats.apps.svc/gomod/github.com/gookit/color/com_github_gookit_color-v1.5.4.zip", + "https://cache.hawkingrei.com/gomod/github.com/gookit/color/com_github_gookit_color-v1.5.4.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/gookit/color/com_github_gookit_color-v1.5.4.zip", + ], + ) go_repository( name = "com_github_gophercloud_gophercloud", build_file_proto_mode = "disable_global", @@ -3753,6 +3649,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/guptarohit/asciigraph/com_github_guptarohit_asciigraph-v0.5.5.zip", ], ) + go_repository( + name = "com_github_hamba_avro_v2", + build_file_proto_mode = "disable_global", + importpath = "github.com/hamba/avro/v2", + sha256 = "43e5bb66c63e312431112307c7e40dd4ae772819169a48dfacb725fd737d46d2", + strip_prefix = "github.com/hamba/avro/v2@v2.27.0", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/hamba/avro/v2/com_github_hamba_avro_v2-v2.27.0.zip", + "http://ats.apps.svc/gomod/github.com/hamba/avro/v2/com_github_hamba_avro_v2-v2.27.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/hamba/avro/v2/com_github_hamba_avro_v2-v2.27.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/hamba/avro/v2/com_github_hamba_avro_v2-v2.27.0.zip", + ], + ) go_repository( name = "com_github_hashicorp_consul_api", build_file_proto_mode = "disable_global", @@ -3883,19 +3792,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/hashicorp/go-rootcerts/com_github_hashicorp_go_rootcerts-v1.0.2.zip", ], ) - go_repository( - name = "com_github_hashicorp_go_uuid", - build_file_proto_mode = "disable_global", - importpath = "github.com/hashicorp/go-uuid", - sha256 = "4b8a152aba5b6db8093f240f11f6999c4401c01e13458228ee023c0682cc5c1d", - strip_prefix = "github.com/hashicorp/go-uuid@v0.0.0-20180228145832-27454136f036", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/hashicorp/go-uuid/com_github_hashicorp_go_uuid-v0.0.0-20180228145832-27454136f036.zip", - "http://ats.apps.svc/gomod/github.com/hashicorp/go-uuid/com_github_hashicorp_go_uuid-v0.0.0-20180228145832-27454136f036.zip", - "https://cache.hawkingrei.com/gomod/github.com/hashicorp/go-uuid/com_github_hashicorp_go_uuid-v0.0.0-20180228145832-27454136f036.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/hashicorp/go-uuid/com_github_hashicorp_go_uuid-v0.0.0-20180228145832-27454136f036.zip", - ], - ) go_repository( name = "com_github_hashicorp_go_version", build_file_proto_mode = "disable_global", @@ -4117,19 +4013,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/iris-contrib/schema/com_github_iris_contrib_schema-v0.0.6.zip", ], ) - go_repository( - name = "com_github_jcmturner_gofork", - build_file_proto_mode = "disable_global", - importpath = "github.com/jcmturner/gofork", - sha256 = "dd93724b2bb93705db6f5c8a86f48802cbb4a831883cb82c4ce5b50f5d7fc854", - strip_prefix = "github.com/jcmturner/gofork@v0.0.0-20180107083740-2aebee971930", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/jcmturner/gofork/com_github_jcmturner_gofork-v0.0.0-20180107083740-2aebee971930.zip", - "http://ats.apps.svc/gomod/github.com/jcmturner/gofork/com_github_jcmturner_gofork-v0.0.0-20180107083740-2aebee971930.zip", - "https://cache.hawkingrei.com/gomod/github.com/jcmturner/gofork/com_github_jcmturner_gofork-v0.0.0-20180107083740-2aebee971930.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/jcmturner/gofork/com_github_jcmturner_gofork-v0.0.0-20180107083740-2aebee971930.zip", - ], - ) go_repository( name = "com_github_jedib0t_go_pretty_v6", build_file_proto_mode = "disable_global", @@ -4312,19 +4195,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/johannesboyne/gofakes3/com_github_johannesboyne_gofakes3-v0.0.0-20230506070712-04da935ef877.zip", ], ) - go_repository( - name = "com_github_johncgriffin_overflow", - build_file_proto_mode = "disable_global", - importpath = "github.com/JohnCGriffin/overflow", - sha256 = "8ad4da840214861386d243127290666cc54eb914d1f4a8856523481876af2a09", - strip_prefix = "github.com/JohnCGriffin/overflow@v0.0.0-20211019200055-46fa312c352c", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/JohnCGriffin/overflow/com_github_johncgriffin_overflow-v0.0.0-20211019200055-46fa312c352c.zip", - "http://ats.apps.svc/gomod/github.com/JohnCGriffin/overflow/com_github_johncgriffin_overflow-v0.0.0-20211019200055-46fa312c352c.zip", - "https://cache.hawkingrei.com/gomod/github.com/JohnCGriffin/overflow/com_github_johncgriffin_overflow-v0.0.0-20211019200055-46fa312c352c.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/JohnCGriffin/overflow/com_github_johncgriffin_overflow-v0.0.0-20211019200055-46fa312c352c.zip", - ], - ) go_repository( name = "com_github_joho_sqltocsv", build_file_proto_mode = "disable_global", @@ -4442,19 +4312,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/julz/importas/com_github_julz_importas-v0.2.0.zip", ], ) - go_repository( - name = "com_github_jung_kurt_gofpdf", - build_file_proto_mode = "disable_global", - importpath = "github.com/jung-kurt/gofpdf", - sha256 = "f0fa70ade137185bbff2f016831a2a456eaadc8d14bc7bf24f0229211820c078", - strip_prefix = "github.com/jung-kurt/gofpdf@v1.0.3-0.20190309125859-24315acbbda5", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/jung-kurt/gofpdf/com_github_jung_kurt_gofpdf-v1.0.3-0.20190309125859-24315acbbda5.zip", - "http://ats.apps.svc/gomod/github.com/jung-kurt/gofpdf/com_github_jung_kurt_gofpdf-v1.0.3-0.20190309125859-24315acbbda5.zip", - "https://cache.hawkingrei.com/gomod/github.com/jung-kurt/gofpdf/com_github_jung_kurt_gofpdf-v1.0.3-0.20190309125859-24315acbbda5.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/jung-kurt/gofpdf/com_github_jung_kurt_gofpdf-v1.0.3-0.20190309125859-24315acbbda5.zip", - ], - ) go_repository( name = "com_github_karamaru_alpha_copyloopvar", build_file_proto_mode = "disable_global", @@ -4546,19 +4403,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/kataras/tunnel/com_github_kataras_tunnel-v0.0.4.zip", ], ) - go_repository( - name = "com_github_kballard_go_shellquote", - build_file_proto_mode = "disable_global", - importpath = "github.com/kballard/go-shellquote", - sha256 = "ae4cb7b097dc4eb0c248dff00ed3bbf0f36984c4162ad1d615266084e58bd6cc", - strip_prefix = "github.com/kballard/go-shellquote@v0.0.0-20180428030007-95032a82bc51", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/kballard/go-shellquote/com_github_kballard_go_shellquote-v0.0.0-20180428030007-95032a82bc51.zip", - "http://ats.apps.svc/gomod/github.com/kballard/go-shellquote/com_github_kballard_go_shellquote-v0.0.0-20180428030007-95032a82bc51.zip", - "https://cache.hawkingrei.com/gomod/github.com/kballard/go-shellquote/com_github_kballard_go_shellquote-v0.0.0-20180428030007-95032a82bc51.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/kballard/go-shellquote/com_github_kballard_go_shellquote-v0.0.0-20180428030007-95032a82bc51.zip", - ], - ) go_repository( name = "com_github_kimmachinegun_automemlimit", build_file_proto_mode = "disable_global", @@ -4658,13 +4502,13 @@ def go_deps(): name = "com_github_klauspost_cpuid_v2", build_file_proto_mode = "disable_global", importpath = "github.com/klauspost/cpuid/v2", - sha256 = "415f9c5ed79ee9b123cf13923a9ab86a34d1f7d417c02b2e00db0d8e34fda037", - strip_prefix = "github.com/klauspost/cpuid/v2@v2.2.7", + sha256 = "59cd12e9094495dda4ce5f344bcbaa4238dfa7bb0eba66f514ec0bacd5ce99e5", + strip_prefix = "github.com/klauspost/cpuid/v2@v2.2.9", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.2.7.zip", - "http://ats.apps.svc/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.2.7.zip", - "https://cache.hawkingrei.com/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.2.7.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.2.7.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.2.9.zip", + "http://ats.apps.svc/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.2.9.zip", + "https://cache.hawkingrei.com/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.2.9.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/klauspost/cpuid/v2/com_github_klauspost_cpuid_v2-v2.2.9.zip", ], ) go_repository( @@ -5135,6 +4979,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/maratori/testpackage/com_github_maratori_testpackage-v1.1.1.zip", ], ) + go_repository( + name = "com_github_marvinjwendt_testza", + build_file_proto_mode = "disable_global", + importpath = "github.com/MarvinJWendt/testza", + sha256 = "fd9b3bca02ff1677bf933a2dafa50b4a9eb5118fe793de704af6e1b4b57162e1", + strip_prefix = "github.com/MarvinJWendt/testza@v0.4.2", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/MarvinJWendt/testza/com_github_marvinjwendt_testza-v0.4.2.zip", + "http://ats.apps.svc/gomod/github.com/MarvinJWendt/testza/com_github_marvinjwendt_testza-v0.4.2.zip", + "https://cache.hawkingrei.com/gomod/github.com/MarvinJWendt/testza/com_github_marvinjwendt_testza-v0.4.2.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/MarvinJWendt/testza/com_github_marvinjwendt_testza-v0.4.2.zip", + ], + ) go_repository( name = "com_github_masterminds_goutils", build_file_proto_mode = "disable_global", @@ -5239,19 +5096,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/mattn/go-runewidth/com_github_mattn_go_runewidth-v0.0.16.zip", ], ) - go_repository( - name = "com_github_mattn_go_sqlite3", - build_file_proto_mode = "disable_global", - importpath = "github.com/mattn/go-sqlite3", - sha256 = "0114d2df439ddeb03eef49a4bf2cc8fb69665c0d76494463cafa7d189a16e0f9", - strip_prefix = "github.com/mattn/go-sqlite3@v1.14.15", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/mattn/go-sqlite3/com_github_mattn_go_sqlite3-v1.14.15.zip", - "http://ats.apps.svc/gomod/github.com/mattn/go-sqlite3/com_github_mattn_go_sqlite3-v1.14.15.zip", - "https://cache.hawkingrei.com/gomod/github.com/mattn/go-sqlite3/com_github_mattn_go_sqlite3-v1.14.15.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/mattn/go-sqlite3/com_github_mattn_go_sqlite3-v1.14.15.zip", - ], - ) go_repository( name = "com_github_matttproud_golang_protobuf_extensions", build_file_proto_mode = "disable_global", @@ -5577,6 +5421,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/nbutton23/zxcvbn-go/com_github_nbutton23_zxcvbn_go-v0.0.0-20210217022336-fa2cb2858354.zip", ], ) + go_repository( + name = "com_github_ncruces_go_strftime", + build_file_proto_mode = "disable_global", + importpath = "github.com/ncruces/go-strftime", + sha256 = "3c46ee9c9db8fde8ce93c768a8701fa01f630bab0cfff338481cde768fe561ac", + strip_prefix = "github.com/ncruces/go-strftime@v0.1.9", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/ncruces/go-strftime/com_github_ncruces_go_strftime-v0.1.9.zip", + "http://ats.apps.svc/gomod/github.com/ncruces/go-strftime/com_github_ncruces_go_strftime-v0.1.9.zip", + "https://cache.hawkingrei.com/gomod/github.com/ncruces/go-strftime/com_github_ncruces_go_strftime-v0.1.9.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ncruces/go-strftime/com_github_ncruces_go_strftime-v0.1.9.zip", + ], + ) go_repository( name = "com_github_ncw_directio", build_file_proto_mode = "disable_global", @@ -5902,19 +5759,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pbnjay/memory/com_github_pbnjay_memory-v0.0.0-20210728143218-7b4eea64cf58.zip", ], ) - go_repository( - name = "com_github_pborman_getopt", - build_file_proto_mode = "disable_global", - importpath = "github.com/pborman/getopt", - sha256 = "2c7e5c93709a3b3302d63f8239679d5b0c33f1dc0e1a18ce8167fb97df09f90a", - strip_prefix = "github.com/pborman/getopt@v0.0.0-20180729010549-6fdd0a2c7117", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pborman/getopt/com_github_pborman_getopt-v0.0.0-20180729010549-6fdd0a2c7117.zip", - "http://ats.apps.svc/gomod/github.com/pborman/getopt/com_github_pborman_getopt-v0.0.0-20180729010549-6fdd0a2c7117.zip", - "https://cache.hawkingrei.com/gomod/github.com/pborman/getopt/com_github_pborman_getopt-v0.0.0-20180729010549-6fdd0a2c7117.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pborman/getopt/com_github_pborman_getopt-v0.0.0-20180729010549-6fdd0a2c7117.zip", - ], - ) go_repository( name = "com_github_pelletier_go_toml", build_file_proto_mode = "disable_global", @@ -5967,43 +5811,17 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/phayes/freeport/com_github_phayes_freeport-v0.0.0-20180830031419-95f893ade6f2.zip", ], ) - go_repository( - name = "com_github_phpdave11_gofpdf", - build_file_proto_mode = "disable_global", - importpath = "github.com/phpdave11/gofpdf", - sha256 = "4db05258f281b40d8a17392fd71648779ea758a9aa506a8d1346ded737ede43f", - strip_prefix = "github.com/phpdave11/gofpdf@v1.4.2", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/phpdave11/gofpdf/com_github_phpdave11_gofpdf-v1.4.2.zip", - "http://ats.apps.svc/gomod/github.com/phpdave11/gofpdf/com_github_phpdave11_gofpdf-v1.4.2.zip", - "https://cache.hawkingrei.com/gomod/github.com/phpdave11/gofpdf/com_github_phpdave11_gofpdf-v1.4.2.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/phpdave11/gofpdf/com_github_phpdave11_gofpdf-v1.4.2.zip", - ], - ) - go_repository( - name = "com_github_phpdave11_gofpdi", - build_file_proto_mode = "disable_global", - importpath = "github.com/phpdave11/gofpdi", - sha256 = "09b728136cf290f4ee87aa47b60f2f9df2b3f4f64119ff10f12319bc3438b58d", - strip_prefix = "github.com/phpdave11/gofpdi@v1.0.13", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/phpdave11/gofpdi/com_github_phpdave11_gofpdi-v1.0.13.zip", - "http://ats.apps.svc/gomod/github.com/phpdave11/gofpdi/com_github_phpdave11_gofpdi-v1.0.13.zip", - "https://cache.hawkingrei.com/gomod/github.com/phpdave11/gofpdi/com_github_phpdave11_gofpdi-v1.0.13.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/phpdave11/gofpdi/com_github_phpdave11_gofpdi-v1.0.13.zip", - ], - ) go_repository( name = "com_github_pierrec_lz4_v4", build_file_proto_mode = "disable_global", importpath = "github.com/pierrec/lz4/v4", - sha256 = "5dadfc447d593c4a8a75520b9f048142d725e4d966d48883ece2380c16081900", - strip_prefix = "github.com/pierrec/lz4/v4@v4.1.15", + sha256 = "bd2e8ef13800ca42205b0d4085a927a6d012b82cfa831769be4830036e953bec", + strip_prefix = "github.com/pierrec/lz4/v4@v4.1.21", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.15.zip", - "http://ats.apps.svc/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.15.zip", - "https://cache.hawkingrei.com/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.15.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.15.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.21.zip", + "http://ats.apps.svc/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.21.zip", + "https://cache.hawkingrei.com/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.21.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pierrec/lz4/v4/com_github_pierrec_lz4_v4-v4.1.21.zip", ], ) go_repository( @@ -6357,6 +6175,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/prometheus/prometheus/com_github_prometheus_prometheus-v0.50.1.zip", ], ) + go_repository( + name = "com_github_pterm_pterm", + build_file_proto_mode = "disable_global", + importpath = "github.com/pterm/pterm", + sha256 = "846d01170c0383aceaf576d9ed864b6c27998a0e2ba5e09d046e56b921bb735d", + strip_prefix = "github.com/pterm/pterm@v0.12.40", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pterm/pterm/com_github_pterm_pterm-v0.12.40.zip", + "http://ats.apps.svc/gomod/github.com/pterm/pterm/com_github_pterm_pterm-v0.12.40.zip", + "https://cache.hawkingrei.com/gomod/github.com/pterm/pterm/com_github_pterm_pterm-v0.12.40.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pterm/pterm/com_github_pterm_pterm-v0.12.40.zip", + ], + ) go_repository( name = "com_github_qri_io_jsonpointer", build_file_proto_mode = "disable_global", @@ -6552,19 +6383,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/russross/blackfriday/v2/com_github_russross_blackfriday_v2-v2.1.0.zip", ], ) - go_repository( - name = "com_github_ruudk_golang_pdf417", - build_file_proto_mode = "disable_global", - importpath = "github.com/ruudk/golang-pdf417", - sha256 = "f0006c0f60789da76c1b3fef73bb63f5581744fbe3ab5973ec718b40c6822f69", - strip_prefix = "github.com/ruudk/golang-pdf417@v0.0.0-20201230142125-a7e3863a1245", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/ruudk/golang-pdf417/com_github_ruudk_golang_pdf417-v0.0.0-20201230142125-a7e3863a1245.zip", - "http://ats.apps.svc/gomod/github.com/ruudk/golang-pdf417/com_github_ruudk_golang_pdf417-v0.0.0-20201230142125-a7e3863a1245.zip", - "https://cache.hawkingrei.com/gomod/github.com/ruudk/golang-pdf417/com_github_ruudk_golang_pdf417-v0.0.0-20201230142125-a7e3863a1245.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/ruudk/golang-pdf417/com_github_ruudk_golang_pdf417-v0.0.0-20201230142125-a7e3863a1245.zip", - ], - ) go_repository( name = "com_github_ryancurrah_gomodguard", build_file_proto_mode = "disable_global", @@ -7072,6 +6890,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/stbenjam/no-sprintf-host-port/com_github_stbenjam_no_sprintf_host_port-v0.2.0.zip", ], ) + go_repository( + name = "com_github_stoewer_go_strcase", + build_file_proto_mode = "disable_global", + importpath = "github.com/stoewer/go-strcase", + sha256 = "42448455706d43dc6b51dc007ceb9726f2b928cb3137c82dc4cfce4e4c565d83", + strip_prefix = "github.com/stoewer/go-strcase@v1.3.0", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/stoewer/go-strcase/com_github_stoewer_go_strcase-v1.3.0.zip", + "http://ats.apps.svc/gomod/github.com/stoewer/go-strcase/com_github_stoewer_go_strcase-v1.3.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/stoewer/go-strcase/com_github_stoewer_go_strcase-v1.3.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/stoewer/go-strcase/com_github_stoewer_go_strcase-v1.3.0.zip", + ], + ) go_repository( name = "com_github_stretchr_objx", build_file_proto_mode = "disable_global", @@ -7111,6 +6942,32 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/subosito/gotenv/com_github_subosito_gotenv-v1.4.1.zip", ], ) + go_repository( + name = "com_github_substrait_io_substrait", + build_file_proto_mode = "disable_global", + importpath = "github.com/substrait-io/substrait", + sha256 = "fbc11abf7516a4650fb3fdc4f1fd2314dfa92260885022f96d7a3f69cf708b13", + strip_prefix = "github.com/substrait-io/substrait@v0.69.0", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/substrait-io/substrait/com_github_substrait_io_substrait-v0.69.0.zip", + "http://ats.apps.svc/gomod/github.com/substrait-io/substrait/com_github_substrait_io_substrait-v0.69.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/substrait-io/substrait/com_github_substrait_io_substrait-v0.69.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/substrait-io/substrait/com_github_substrait_io_substrait-v0.69.0.zip", + ], + ) + go_repository( + name = "com_github_substrait_io_substrait_go_v3", + build_file_proto_mode = "disable_global", + importpath = "github.com/substrait-io/substrait-go/v3", + sha256 = "e24bdbfea7ee3147f7f21d9cca6ce0fca0faeebec320e3c1d58055a58a953b01", + strip_prefix = "github.com/substrait-io/substrait-go/v3@v3.3.0", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/substrait-io/substrait-go/v3/com_github_substrait_io_substrait_go_v3-v3.3.0.zip", + "http://ats.apps.svc/gomod/github.com/substrait-io/substrait-go/v3/com_github_substrait_io_substrait_go_v3-v3.3.0.zip", + "https://cache.hawkingrei.com/gomod/github.com/substrait-io/substrait-go/v3/com_github_substrait_io_substrait_go_v3-v3.3.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/substrait-io/substrait-go/v3/com_github_substrait_io_substrait_go_v3-v3.3.0.zip", + ], + ) go_repository( name = "com_github_tdakkota_asciicheck", build_file_proto_mode = "disable_global", @@ -7672,32 +7529,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/xiang90/probing/com_github_xiang90_probing-v0.0.0-20221125231312-a49e3df8f510.zip", ], ) - go_repository( - name = "com_github_xitongsys_parquet_go", - build_file_proto_mode = "disable_global", - importpath = "github.com/xitongsys/parquet-go", - sha256 = "5b9473cce95cf094d398348fd394002b656ae1363bb5c33c1338fcdcd57e1b33", - strip_prefix = "github.com/xitongsys/parquet-go@v1.6.3-0.20240520233950-75e935fc3e17", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.6.3-0.20240520233950-75e935fc3e17.zip", - "http://ats.apps.svc/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.6.3-0.20240520233950-75e935fc3e17.zip", - "https://cache.hawkingrei.com/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.6.3-0.20240520233950-75e935fc3e17.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/xitongsys/parquet-go/com_github_xitongsys_parquet_go-v1.6.3-0.20240520233950-75e935fc3e17.zip", - ], - ) - go_repository( - name = "com_github_xitongsys_parquet_go_source", - build_file_proto_mode = "disable_global", - importpath = "github.com/xitongsys/parquet-go-source", - sha256 = "9fa786105465c7da0b4d0a3f334b5d284cce486229a0631e5bd962e4dc67cd50", - strip_prefix = "github.com/xitongsys/parquet-go-source@v0.0.0-20200817004010-026bad9b25d0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/xitongsys/parquet-go-source/com_github_xitongsys_parquet_go_source-v0.0.0-20200817004010-026bad9b25d0.zip", - "http://ats.apps.svc/gomod/github.com/xitongsys/parquet-go-source/com_github_xitongsys_parquet_go_source-v0.0.0-20200817004010-026bad9b25d0.zip", - "https://cache.hawkingrei.com/gomod/github.com/xitongsys/parquet-go-source/com_github_xitongsys_parquet_go_source-v0.0.0-20200817004010-026bad9b25d0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/xitongsys/parquet-go-source/com_github_xitongsys_parquet_go_source-v0.0.0-20200817004010-026bad9b25d0.zip", - ], - ) go_repository( name = "com_github_xo_terminfo", build_file_proto_mode = "disable_global", @@ -7724,6 +7555,19 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/github.com/xordataexchange/crypt/com_github_xordataexchange_crypt-v0.0.3-0.20170626215501-b2862e3d0a77.zip", ], ) + go_repository( + name = "com_github_xyproto_randomstring", + build_file_proto_mode = "disable_global", + importpath = "github.com/xyproto/randomstring", + sha256 = "58ea0c70496fc698c4597625395b3f35321df09be013ee880a6ce10e94969261", + strip_prefix = "github.com/xyproto/randomstring@v1.0.5", + urls = [ + "http://bazel-cache.pingcap.net:8080/gomod/github.com/xyproto/randomstring/com_github_xyproto_randomstring-v1.0.5.zip", + "http://ats.apps.svc/gomod/github.com/xyproto/randomstring/com_github_xyproto_randomstring-v1.0.5.zip", + "https://cache.hawkingrei.com/gomod/github.com/xyproto/randomstring/com_github_xyproto_randomstring-v1.0.5.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/xyproto/randomstring/com_github_xyproto_randomstring-v1.0.5.zip", + ], + ) go_repository( name = "com_github_yagipy_maintidx", build_file_proto_mode = "disable_global", @@ -9401,19 +9245,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/cloud.google.com/go/workflows/com_google_cloud_go_workflows-v1.12.5.zip", ], ) - go_repository( - name = "com_lukechampine_uint128", - build_file_proto_mode = "disable_global", - importpath = "lukechampine.com/uint128", - sha256 = "9ff6e9ad553a69fdb961ab2d92f92cda183ef616a6709c15972c2d4bedf33de5", - strip_prefix = "lukechampine.com/uint128@v1.2.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/lukechampine.com/uint128/com_lukechampine_uint128-v1.2.0.zip", - "http://ats.apps.svc/gomod/lukechampine.com/uint128/com_lukechampine_uint128-v1.2.0.zip", - "https://cache.hawkingrei.com/gomod/lukechampine.com/uint128/com_lukechampine_uint128-v1.2.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/lukechampine.com/uint128/com_lukechampine_uint128-v1.2.0.zip", - ], - ) go_repository( name = "com_shuralyov_dmitri_gpu_mtl", build_file_proto_mode = "disable_global", @@ -9466,19 +9297,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/stathat.com/c/consistent/com_stathat_c_consistent-v1.0.0.zip", ], ) - go_repository( - name = "ht_sr_git_~sbinet_gg", - build_file_proto_mode = "disable_global", - importpath = "git.sr.ht/~sbinet/gg", - sha256 = "435103529c4f24aecf7e4550bc816db2482dda4ee0123d337daba99971a8c498", - strip_prefix = "git.sr.ht/~sbinet/gg@v0.3.1", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/git.sr.ht/~sbinet/gg/ht_sr_git_~sbinet_gg-v0.3.1.zip", - "http://ats.apps.svc/gomod/git.sr.ht/~sbinet/gg/ht_sr_git_~sbinet_gg-v0.3.1.zip", - "https://cache.hawkingrei.com/gomod/git.sr.ht/~sbinet/gg/ht_sr_git_~sbinet_gg-v0.3.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/git.sr.ht/~sbinet/gg/ht_sr_git_~sbinet_gg-v0.3.1.zip", - ], - ) go_repository( name = "in_gopkg_check_v1", build_file_proto_mode = "disable_global", @@ -9531,71 +9349,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/gopkg.in/ini.v1/in_gopkg_ini_v1-v1.67.0.zip", ], ) - go_repository( - name = "in_gopkg_jcmturner_aescts_v1", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/jcmturner/aescts.v1", - sha256 = "8bfd83c7204032fb16946202d5d643bd9a7e618005bd39578f29030a7d51dcf9", - strip_prefix = "gopkg.in/jcmturner/aescts.v1@v1.0.1", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/gopkg.in/jcmturner/aescts.v1/in_gopkg_jcmturner_aescts_v1-v1.0.1.zip", - "http://ats.apps.svc/gomod/gopkg.in/jcmturner/aescts.v1/in_gopkg_jcmturner_aescts_v1-v1.0.1.zip", - "https://cache.hawkingrei.com/gomod/gopkg.in/jcmturner/aescts.v1/in_gopkg_jcmturner_aescts_v1-v1.0.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/gopkg.in/jcmturner/aescts.v1/in_gopkg_jcmturner_aescts_v1-v1.0.1.zip", - ], - ) - go_repository( - name = "in_gopkg_jcmturner_dnsutils_v1", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/jcmturner/dnsutils.v1", - sha256 = "4fb8b6a5471cb6dda1d0aabd1e01e4d54cb5ee83c395849916392b19153f5203", - strip_prefix = "gopkg.in/jcmturner/dnsutils.v1@v1.0.1", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/gopkg.in/jcmturner/dnsutils.v1/in_gopkg_jcmturner_dnsutils_v1-v1.0.1.zip", - "http://ats.apps.svc/gomod/gopkg.in/jcmturner/dnsutils.v1/in_gopkg_jcmturner_dnsutils_v1-v1.0.1.zip", - "https://cache.hawkingrei.com/gomod/gopkg.in/jcmturner/dnsutils.v1/in_gopkg_jcmturner_dnsutils_v1-v1.0.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/gopkg.in/jcmturner/dnsutils.v1/in_gopkg_jcmturner_dnsutils_v1-v1.0.1.zip", - ], - ) - go_repository( - name = "in_gopkg_jcmturner_goidentity_v3", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/jcmturner/goidentity.v3", - sha256 = "1be44bee93d9080ce89f40827c57e8a396b7c801e2d19a1f5446a4325afa755e", - strip_prefix = "gopkg.in/jcmturner/goidentity.v3@v3.0.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/gopkg.in/jcmturner/goidentity.v3/in_gopkg_jcmturner_goidentity_v3-v3.0.0.zip", - "http://ats.apps.svc/gomod/gopkg.in/jcmturner/goidentity.v3/in_gopkg_jcmturner_goidentity_v3-v3.0.0.zip", - "https://cache.hawkingrei.com/gomod/gopkg.in/jcmturner/goidentity.v3/in_gopkg_jcmturner_goidentity_v3-v3.0.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/gopkg.in/jcmturner/goidentity.v3/in_gopkg_jcmturner_goidentity_v3-v3.0.0.zip", - ], - ) - go_repository( - name = "in_gopkg_jcmturner_gokrb5_v7", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/jcmturner/gokrb5.v7", - sha256 = "f7e772eaadb923044924cb86b7a6ed34a3386df831705bb62b6a47dc0819a94b", - strip_prefix = "gopkg.in/jcmturner/gokrb5.v7@v7.3.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/gopkg.in/jcmturner/gokrb5.v7/in_gopkg_jcmturner_gokrb5_v7-v7.3.0.zip", - "http://ats.apps.svc/gomod/gopkg.in/jcmturner/gokrb5.v7/in_gopkg_jcmturner_gokrb5_v7-v7.3.0.zip", - "https://cache.hawkingrei.com/gomod/gopkg.in/jcmturner/gokrb5.v7/in_gopkg_jcmturner_gokrb5_v7-v7.3.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/gopkg.in/jcmturner/gokrb5.v7/in_gopkg_jcmturner_gokrb5_v7-v7.3.0.zip", - ], - ) - go_repository( - name = "in_gopkg_jcmturner_rpc_v1", - build_file_proto_mode = "disable_global", - importpath = "gopkg.in/jcmturner/rpc.v1", - sha256 = "83d897b60ecb5a66d25232b775ed04c182ca8e02431f351b3768d4d2876d07ae", - strip_prefix = "gopkg.in/jcmturner/rpc.v1@v1.1.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/gopkg.in/jcmturner/rpc.v1/in_gopkg_jcmturner_rpc_v1-v1.1.0.zip", - "http://ats.apps.svc/gomod/gopkg.in/jcmturner/rpc.v1/in_gopkg_jcmturner_rpc_v1-v1.1.0.zip", - "https://cache.hawkingrei.com/gomod/gopkg.in/jcmturner/rpc.v1/in_gopkg_jcmturner_rpc_v1-v1.1.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/gopkg.in/jcmturner/rpc.v1/in_gopkg_jcmturner_rpc_v1-v1.1.0.zip", - ], - ) go_repository( name = "in_gopkg_mgo_v2", build_file_proto_mode = "disable_global", @@ -10194,19 +9947,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/rsc.io/binaryregexp/io_rsc_binaryregexp-v0.2.0.zip", ], ) - go_repository( - name = "io_rsc_pdf", - build_file_proto_mode = "disable_global", - importpath = "rsc.io/pdf", - sha256 = "79bf310e399cf0e2d8aa61536750d2a6999c5ca884e7a27faf88d3701cd5ba8f", - strip_prefix = "rsc.io/pdf@v0.1.1", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/rsc.io/pdf/io_rsc_pdf-v0.1.1.zip", - "http://ats.apps.svc/gomod/rsc.io/pdf/io_rsc_pdf-v0.1.1.zip", - "https://cache.hawkingrei.com/gomod/rsc.io/pdf/io_rsc_pdf-v0.1.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/rsc.io/pdf/io_rsc_pdf-v0.1.1.zip", - ], - ) go_repository( name = "io_rsc_quote_v3", build_file_proto_mode = "disable_global", @@ -10259,19 +9999,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/codeberg.org/chavacava/garif/org_codeberg_chavacava_garif-v0.2.0.zip", ], ) - go_repository( - name = "org_gioui", - build_file_proto_mode = "disable_global", - importpath = "gioui.org", - sha256 = "fcbab2a0ea09ff775c1ff4fa99299d95b94aad496b1ac329e3c7389119168fc0", - strip_prefix = "gioui.org@v0.0.0-20210308172011-57750fc8a0a6", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/gioui.org/org_gioui-v0.0.0-20210308172011-57750fc8a0a6.zip", - "http://ats.apps.svc/gomod/gioui.org/org_gioui-v0.0.0-20210308172011-57750fc8a0a6.zip", - "https://cache.hawkingrei.com/gomod/gioui.org/org_gioui-v0.0.0-20210308172011-57750fc8a0a6.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/gioui.org/org_gioui-v0.0.0-20210308172011-57750fc8a0a6.zip", - ], - ) go_repository( name = "org_go_simpler_musttag", build_file_proto_mode = "disable_global", @@ -10458,13 +10185,13 @@ def go_deps(): name = "org_golang_x_image", build_file_proto_mode = "disable_global", importpath = "golang.org/x/image", - sha256 = "56176a4d4d47910d61df9a77aa66a8469ae79fa18b7f5821c43bef1ef212116d", - strip_prefix = "golang.org/x/image@v0.0.0-20220302094943-723b81ca9867", + sha256 = "4a44b498934a95e8f84e8374530de0cab38d81fcd558898d4880c3c5ce1efe47", + strip_prefix = "golang.org/x/image@v0.0.0-20190802002840-cff245a6509b", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20220302094943-723b81ca9867.zip", - "http://ats.apps.svc/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20220302094943-723b81ca9867.zip", - "https://cache.hawkingrei.com/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20220302094943-723b81ca9867.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20220302094943-723b81ca9867.zip", + "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20190802002840-cff245a6509b.zip", + "http://ats.apps.svc/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20190802002840-cff245a6509b.zip", + "https://cache.hawkingrei.com/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20190802002840-cff245a6509b.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/image/org_golang_x_image-v0.0.0-20190802002840-cff245a6509b.zip", ], ) go_repository( @@ -10666,91 +10393,52 @@ def go_deps(): name = "org_golang_x_xerrors", build_file_proto_mode = "disable_global", importpath = "golang.org/x/xerrors", - sha256 = "df5dd109153c94d2f5c9601d28f558871094e37c42f8e3875f36db858d8be9f9", - strip_prefix = "golang.org/x/xerrors@v0.0.0-20231012003039-104605ab7028", + sha256 = "07ee9f680118861ee732ce0df4553b834383b87e0519fb9a0990c51d7abd6885", + strip_prefix = "golang.org/x/xerrors@v0.0.0-20240903120638-7835f813f4da", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/xerrors/org_golang_x_xerrors-v0.0.0-20231012003039-104605ab7028.zip", - "http://ats.apps.svc/gomod/golang.org/x/xerrors/org_golang_x_xerrors-v0.0.0-20231012003039-104605ab7028.zip", - "https://cache.hawkingrei.com/gomod/golang.org/x/xerrors/org_golang_x_xerrors-v0.0.0-20231012003039-104605ab7028.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/xerrors/org_golang_x_xerrors-v0.0.0-20231012003039-104605ab7028.zip", + "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/xerrors/org_golang_x_xerrors-v0.0.0-20240903120638-7835f813f4da.zip", + "http://ats.apps.svc/gomod/golang.org/x/xerrors/org_golang_x_xerrors-v0.0.0-20240903120638-7835f813f4da.zip", + "https://cache.hawkingrei.com/gomod/golang.org/x/xerrors/org_golang_x_xerrors-v0.0.0-20240903120638-7835f813f4da.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/xerrors/org_golang_x_xerrors-v0.0.0-20240903120638-7835f813f4da.zip", ], ) go_repository( name = "org_gonum_v1_gonum", build_file_proto_mode = "disable_global", importpath = "gonum.org/v1/gonum", - sha256 = "abdfee15ce7c9d2cd96b66468d3ae28d6054add4efbfc1b15fadfe3613f3d362", - strip_prefix = "gonum.org/v1/gonum@v0.11.0", + sha256 = "7a1b124a144b2c97a29829464d4b7258e04235c1fb14bbcea902086618414a43", + strip_prefix = "gonum.org/v1/gonum@v0.15.1", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.11.0.zip", - "http://ats.apps.svc/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.11.0.zip", - "https://cache.hawkingrei.com/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.11.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.11.0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.15.1.zip", + "http://ats.apps.svc/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.15.1.zip", + "https://cache.hawkingrei.com/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.15.1.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/gonum.org/v1/gonum/org_gonum_v1_gonum-v0.15.1.zip", ], ) go_repository( name = "org_gonum_v1_netlib", build_file_proto_mode = "disable_global", importpath = "gonum.org/v1/netlib", - sha256 = "eeaeb60f410b86f59d97f15c5ef89096dc72aeb42bae55141738bf9866893938", - strip_prefix = "gonum.org/v1/netlib@v0.0.0-20190313105609-8cb42192e0e0", + sha256 = "35405098d5c2e9ac83607b51a59d9f4e2947166cc7d42aa8fc4bb6dc7be3fb1c", + strip_prefix = "gonum.org/v1/netlib@v0.0.0-20181029234149-ec6d1f5cefe6", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/gonum.org/v1/netlib/org_gonum_v1_netlib-v0.0.0-20190313105609-8cb42192e0e0.zip", - "http://ats.apps.svc/gomod/gonum.org/v1/netlib/org_gonum_v1_netlib-v0.0.0-20190313105609-8cb42192e0e0.zip", - "https://cache.hawkingrei.com/gomod/gonum.org/v1/netlib/org_gonum_v1_netlib-v0.0.0-20190313105609-8cb42192e0e0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/gonum.org/v1/netlib/org_gonum_v1_netlib-v0.0.0-20190313105609-8cb42192e0e0.zip", + "http://bazel-cache.pingcap.net:8080/gomod/gonum.org/v1/netlib/org_gonum_v1_netlib-v0.0.0-20181029234149-ec6d1f5cefe6.zip", + "http://ats.apps.svc/gomod/gonum.org/v1/netlib/org_gonum_v1_netlib-v0.0.0-20181029234149-ec6d1f5cefe6.zip", + "https://cache.hawkingrei.com/gomod/gonum.org/v1/netlib/org_gonum_v1_netlib-v0.0.0-20181029234149-ec6d1f5cefe6.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/gonum.org/v1/netlib/org_gonum_v1_netlib-v0.0.0-20181029234149-ec6d1f5cefe6.zip", ], ) go_repository( - name = "org_gonum_v1_plot", + name = "org_modernc_gc_v3", build_file_proto_mode = "disable_global", - importpath = "gonum.org/v1/plot", - sha256 = "eaa47ad966b3b67325c1f3ae704d566332c573b7cca79016cb4ffe82155aab39", - strip_prefix = "gonum.org/v1/plot@v0.10.1", + importpath = "modernc.org/gc/v3", + sha256 = "d533e65369fe0dfcd19c725226a30e47b184092603ec7ee38dd5d056a2ad1474", + strip_prefix = "modernc.org/gc/v3@v3.0.0-20240107210532-573471604cb6", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.10.1.zip", - "http://ats.apps.svc/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.10.1.zip", - "https://cache.hawkingrei.com/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.10.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/gonum.org/v1/plot/org_gonum_v1_plot-v0.10.1.zip", - ], - ) - go_repository( - name = "org_modernc_cc_v3", - build_file_proto_mode = "disable_global", - importpath = "modernc.org/cc/v3", - sha256 = "fe3aeb761e55ce77a95b297321a122b4273aeffe1c08f48fc99310e065211f74", - strip_prefix = "modernc.org/cc/v3@v3.40.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/cc/v3/org_modernc_cc_v3-v3.40.0.zip", - "http://ats.apps.svc/gomod/modernc.org/cc/v3/org_modernc_cc_v3-v3.40.0.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/cc/v3/org_modernc_cc_v3-v3.40.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/cc/v3/org_modernc_cc_v3-v3.40.0.zip", - ], - ) - go_repository( - name = "org_modernc_ccgo_v3", - build_file_proto_mode = "disable_global", - importpath = "modernc.org/ccgo/v3", - sha256 = "bfc293300cd1ce656ba0ce0cee1f508afec2518bc4214a6b10ccfad6e8e6046e", - strip_prefix = "modernc.org/ccgo/v3@v3.16.13", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/ccgo/v3/org_modernc_ccgo_v3-v3.16.13.zip", - "http://ats.apps.svc/gomod/modernc.org/ccgo/v3/org_modernc_ccgo_v3-v3.16.13.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/ccgo/v3/org_modernc_ccgo_v3-v3.16.13.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/ccgo/v3/org_modernc_ccgo_v3-v3.16.13.zip", - ], - ) - go_repository( - name = "org_modernc_ccorpus", - build_file_proto_mode = "disable_global", - importpath = "modernc.org/ccorpus", - sha256 = "3831b62a73a379b81ac927e17e3e9ffe2d44ad07c934505e1ae24eea8a26a6d3", - strip_prefix = "modernc.org/ccorpus@v1.11.6", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/ccorpus/org_modernc_ccorpus-v1.11.6.zip", - "http://ats.apps.svc/gomod/modernc.org/ccorpus/org_modernc_ccorpus-v1.11.6.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/ccorpus/org_modernc_ccorpus-v1.11.6.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/ccorpus/org_modernc_ccorpus-v1.11.6.zip", + "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/gc/v3/org_modernc_gc_v3-v3.0.0-20240107210532-573471604cb6.zip", + "http://ats.apps.svc/gomod/modernc.org/gc/v3/org_modernc_gc_v3-v3.0.0-20240107210532-573471604cb6.zip", + "https://cache.hawkingrei.com/gomod/modernc.org/gc/v3/org_modernc_gc_v3-v3.0.0-20240107210532-573471604cb6.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/gc/v3/org_modernc_gc_v3-v3.0.0-20240107210532-573471604cb6.zip", ], ) go_repository( @@ -10766,30 +10454,17 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/golex/org_modernc_golex-v1.1.0.zip", ], ) - go_repository( - name = "org_modernc_httpfs", - build_file_proto_mode = "disable_global", - importpath = "modernc.org/httpfs", - sha256 = "0b5314649c1327a199397eb6fd52b3ce41c9d3bc6dd2a4dea565b5fb87c13f41", - strip_prefix = "modernc.org/httpfs@v1.0.6", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/httpfs/org_modernc_httpfs-v1.0.6.zip", - "http://ats.apps.svc/gomod/modernc.org/httpfs/org_modernc_httpfs-v1.0.6.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/httpfs/org_modernc_httpfs-v1.0.6.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/httpfs/org_modernc_httpfs-v1.0.6.zip", - ], - ) go_repository( name = "org_modernc_libc", build_file_proto_mode = "disable_global", importpath = "modernc.org/libc", - sha256 = "5f98bedf9f0663b3b87555904ee41b82fe9d8e9ac5c47c9fac9a42a7fe232313", - strip_prefix = "modernc.org/libc@v1.22.2", + sha256 = "e7857dd7ac722f43be7f13a43db632f53b67067010416f330b8b80664af9b619", + strip_prefix = "modernc.org/libc@v1.41.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/libc/org_modernc_libc-v1.22.2.zip", - "http://ats.apps.svc/gomod/modernc.org/libc/org_modernc_libc-v1.22.2.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/libc/org_modernc_libc-v1.22.2.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/libc/org_modernc_libc-v1.22.2.zip", + "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/libc/org_modernc_libc-v1.41.0.zip", + "http://ats.apps.svc/gomod/modernc.org/libc/org_modernc_libc-v1.41.0.zip", + "https://cache.hawkingrei.com/gomod/modernc.org/libc/org_modernc_libc-v1.41.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/libc/org_modernc_libc-v1.41.0.zip", ], ) go_repository( @@ -10809,26 +10484,13 @@ def go_deps(): name = "org_modernc_memory", build_file_proto_mode = "disable_global", importpath = "modernc.org/memory", - sha256 = "f79e8ada14c36d08817ee2bf6b2103f65c1a61a91b042b59016465869624043c", - strip_prefix = "modernc.org/memory@v1.5.0", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/memory/org_modernc_memory-v1.5.0.zip", - "http://ats.apps.svc/gomod/modernc.org/memory/org_modernc_memory-v1.5.0.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/memory/org_modernc_memory-v1.5.0.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/memory/org_modernc_memory-v1.5.0.zip", - ], - ) - go_repository( - name = "org_modernc_opt", - build_file_proto_mode = "disable_global", - importpath = "modernc.org/opt", - sha256 = "294b1b80137cb86292c8893481d545eee90b17b84b6ad1dcb2e6c9bb523a2d9e", - strip_prefix = "modernc.org/opt@v0.1.3", + sha256 = "fb640d04a514b88f7d988e8b79b41f46e7958da5ea3e69505199c4059138d189", + strip_prefix = "modernc.org/memory@v1.7.2", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/opt/org_modernc_opt-v0.1.3.zip", - "http://ats.apps.svc/gomod/modernc.org/opt/org_modernc_opt-v0.1.3.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/opt/org_modernc_opt-v0.1.3.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/opt/org_modernc_opt-v0.1.3.zip", + "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/memory/org_modernc_memory-v1.7.2.zip", + "http://ats.apps.svc/gomod/modernc.org/memory/org_modernc_memory-v1.7.2.zip", + "https://cache.hawkingrei.com/gomod/modernc.org/memory/org_modernc_memory-v1.7.2.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/memory/org_modernc_memory-v1.7.2.zip", ], ) go_repository( @@ -10861,13 +10523,13 @@ def go_deps(): name = "org_modernc_sqlite", build_file_proto_mode = "disable_global", importpath = "modernc.org/sqlite", - sha256 = "be0501f87458962a00c8fb07d1f131af010a534cd6ffb654c570be35b9b608d5", - strip_prefix = "modernc.org/sqlite@v1.18.2", + sha256 = "3add9bde5a932e2a52cd2b50a1be276332d91f196767a5be740afa3ee037afad", + strip_prefix = "modernc.org/sqlite@v1.29.6", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.18.2.zip", - "http://ats.apps.svc/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.18.2.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.18.2.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.18.2.zip", + "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.29.6.zip", + "http://ats.apps.svc/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.29.6.zip", + "https://cache.hawkingrei.com/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.29.6.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/sqlite/org_modernc_sqlite-v1.29.6.zip", ], ) go_repository( @@ -10883,19 +10545,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/strutil/org_modernc_strutil-v1.2.0.zip", ], ) - go_repository( - name = "org_modernc_tcl", - build_file_proto_mode = "disable_global", - importpath = "modernc.org/tcl", - sha256 = "f966db0dd1ccbc7f8d5ac2e752b64c3be343aa3f92215ed98b6f2a51b7abbb64", - strip_prefix = "modernc.org/tcl@v1.13.2", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/tcl/org_modernc_tcl-v1.13.2.zip", - "http://ats.apps.svc/gomod/modernc.org/tcl/org_modernc_tcl-v1.13.2.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/tcl/org_modernc_tcl-v1.13.2.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/tcl/org_modernc_tcl-v1.13.2.zip", - ], - ) go_repository( name = "org_modernc_token", build_file_proto_mode = "disable_global", @@ -10922,19 +10571,6 @@ def go_deps(): "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/y/org_modernc_y-v1.1.0.zip", ], ) - go_repository( - name = "org_modernc_z", - build_file_proto_mode = "disable_global", - importpath = "modernc.org/z", - sha256 = "5be23ef96669963e52d25b787d71028fff4fe1c468dec20aac59c9512caa2eb7", - strip_prefix = "modernc.org/z@v1.5.1", - urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/modernc.org/z/org_modernc_z-v1.5.1.zip", - "http://ats.apps.svc/gomod/modernc.org/z/org_modernc_z-v1.5.1.zip", - "https://cache.hawkingrei.com/gomod/modernc.org/z/org_modernc_z-v1.5.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/modernc.org/z/org_modernc_z-v1.5.1.zip", - ], - ) go_repository( name = "org_mongodb_go_mongo_driver", build_file_proto_mode = "disable_global", diff --git a/br/pkg/storage/gcs.go b/br/pkg/storage/gcs.go index 22b65a4f77e2d..c8d4e53dbb5d4 100644 --- a/br/pkg/storage/gcs.go +++ b/br/pkg/storage/gcs.go @@ -594,8 +594,6 @@ type gcsObjectReader struct { prefetchSize int // reader context used for implement `io.Seek` - // currently, lightning depends on package `xitongsys/parquet-go` to read parquet file and it needs `io.Seeker` - // See: https://github.com/xitongsys/parquet-go/blob/207a3cee75900b2b95213627409b7bac0f190bb3/source/source.go#L9-L10 ctx context.Context } diff --git a/br/pkg/storage/s3.go b/br/pkg/storage/s3.go index 42b31b611e22b..6f7b78b5e6b8d 100644 --- a/br/pkg/storage/s3.go +++ b/br/pkg/storage/s3.go @@ -1018,8 +1018,6 @@ type s3ObjectReader struct { pos int64 rangeInfo RangeInfo // reader context used for implement `io.Seek` - // currently, lightning depends on package `xitongsys/parquet-go` to read parquet file and it needs `io.Seeker` - // See: https://github.com/xitongsys/parquet-go/blob/207a3cee75900b2b95213627409b7bac0f190bb3/source/source.go#L9-L10 ctx context.Context prefetchSize int } diff --git a/go.mod b/go.mod index dd3b2d9f18492..6ae3d61311c7e 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/YangKeao/go-mysql-driver v0.0.0-20240627104025-dd5589458cfa github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 + github.com/apache/arrow-go/v18 v18.0.0 github.com/apache/skywalking-eyes v0.4.0 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/ashanbrown/forbidigo/v2 v2.1.0 @@ -118,8 +119,6 @@ require ( github.com/uber/jaeger-client-go v2.22.1+incompatible github.com/vbauerster/mpb/v7 v7.5.3 github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f - github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 - github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 github.com/zyedidia/generic v1.2.1 go.etcd.io/etcd/api/v3 v3.5.15 go.etcd.io/etcd/client/pkg/v3 v3.5.15 @@ -156,24 +155,22 @@ require ( require ( codeberg.org/chavacava/garif v0.2.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect - github.com/apache/arrow/go/v12 v12.0.1 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect - github.com/goccy/go-reflect v1.2.0 // indirect - github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/ldez/grignotin v0.9.0 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/qri-io/jsonpointer v0.1.1 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/tidwall/gjson v1.14.4 // indirect @@ -197,7 +194,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.2 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/apache/thrift v0.16.0 // indirect + github.com/apache/thrift v0.21.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmatcuk/doublestar/v2 v2.0.4 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect @@ -220,7 +217,7 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/goccy/go-json v0.10.4 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/glog v1.2.4 // indirect @@ -312,7 +309,7 @@ require ( golang.org/x/crypto v0.41.0 // indirect golang.org/x/exp/typeparams v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/mod v0.27.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect @@ -330,6 +327,7 @@ require ( ) replace ( + github.com/apache/arrow-go/v18 => github.com/joechenrh/arrow-go/v18 v18.0.0-20251104113423-8cacfcd24f0d github.com/go-ldap/ldap/v3 => github.com/YangKeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 github.com/pingcap/tidb/pkg/parser => ./pkg/parser diff --git a/go.sum b/go.sum index ba116af3e2c03..d5efb5179685c 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,6 @@ codeberg.org/chavacava/garif v0.2.0/go.mod h1:P2BPbVbT4QcvLZrORc2T29szK3xEOlnl0G dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= -git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= @@ -71,8 +69,6 @@ github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= -github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= @@ -91,27 +87,19 @@ github.com/YangKeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 h1:+OqGGFc2YHFd github.com/YangKeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= -github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 h1:Q/yk4z/cHUVZfgTqtD09qeYBxHwshQAjVRX73qs8UH0= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg= -github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= github.com/apache/skywalking-eyes v0.4.0 h1:O13kdRU6FCEZevfD01mdhTgCZLLfPZIQ0GXZrLl7FpQ= github.com/apache/skywalking-eyes v0.4.0/go.mod h1:WblDbBgOLsLN0FJEBa9xj6PhuUA/J6spKYVTG4/F8Ls= -github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= +github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= @@ -119,7 +107,6 @@ github.com/ashanbrown/forbidigo/v2 v2.1.0 h1:NAxZrWqNUQiDz19FKScQ/xvwzmij6BiOw3S github.com/ashanbrown/forbidigo/v2 v2.1.0/go.mod h1:0zZfdNAuZIL7rSComLGthgc/9/n2FqspBOH90xlCHdA= github.com/ashanbrown/makezero v1.2.0 h1:/2Lp1bypdmK9wDIq7uWBlDF1iMUpIIS4A+pF6C9IEUU= github.com/ashanbrown/makezero v1.2.0/go.mod h1:dxlPhHbDMC6N6xICzFBSK+4njQDdK8euNO0qjQMtGY4= -github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= @@ -140,8 +127,6 @@ github.com/blacktear23/go-proxyprotocol v1.0.6 h1:eTt6UMpEnq59NjON49b3Cay8Dm0sCs github.com/blacktear23/go-proxyprotocol v1.0.6/go.mod h1:FSCbgnRZrQXazBLL5snfBbrcFSMtcmUDhSRb9OfFA1o= github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI= github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw= -github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/butuzov/mirror v1.3.0 h1:HdWCXzmwlQHdVhwvsfBb2Au0r3HyINry3bDWLYXiKoc= github.com/butuzov/mirror v1.3.0/go.mod h1:AEij0Z8YMALaq4yQj9CPPVYOyJQyiexpQEQgihajRfI= github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= @@ -170,11 +155,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudfoundry/gosigar v1.3.6 h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc= github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= @@ -189,7 +169,6 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ= github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= github.com/coocood/freecache v1.2.1 h1:/v1CqMq45NFH9mp/Pt142reundeBM0dVUD3osQBeu/U= @@ -229,7 +208,6 @@ github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa5 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ= github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4= github.com/dolthub/swiss v0.2.1 h1:gs2osYs5SJkAaH5/ggVJqXQxRXtWshF6uE0lgR/Y3Gw= @@ -242,8 +220,6 @@ github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FM github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= @@ -255,8 +231,6 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -272,19 +246,12 @@ github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= -github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= -github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= -github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= -github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= -github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= @@ -296,22 +263,16 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= -github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-reflect v1.2.0 h1:O0T8rZCuNmGXewnATuKYnkL0xm6o8UNOJZd/gOkb9ms= -github.com/goccy/go-reflect v1.2.0/go.mod h1:n0oYZn8VcV2CkWTxi8B9QjkCoq6GTtCEdfmR66YhFtE= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -322,7 +283,6 @@ github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXe github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= @@ -338,9 +298,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -359,7 +317,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -377,8 +334,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= -github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -391,8 +348,6 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -428,7 +383,6 @@ github.com/google/skylark v0.0.0-20181101142754-a5f7082aabed h1:rZdD1GeRTHD1aG+V github.com/google/skylark v0.0.0-20181101142754-a5f7082aabed/go.mod h1:CKSX6SxHW1vp20ZNaeGe3TFFBIwCG6vaYrpAiOzX+NA= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= @@ -470,7 +424,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= -github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -491,7 +444,6 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY= github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= -github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jedib0t/go-pretty/v6 v6.2.2 h1:o3McN0rQ4X+IU+HduppSp9TwRdGLRW2rhJXy9CJaCRw= github.com/jedib0t/go-pretty/v6 v6.2.2/go.mod h1:+nE9fyyHGil+PuISTCrp7avEdo6bqoMwqZnuiK2r2a0= github.com/jellydator/ttlcache/v3 v3.0.1 h1:cHgCSMS7TdQcoprXnWUptJZzyFsqs18Lt8VVhRuZYVU= @@ -511,11 +463,12 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joechenrh/arrow-go/v18 v18.0.0-20251104113423-8cacfcd24f0d h1:4YsAGqzXtxoKwnxrr4DTk7QFEoTa4Tmh46uGYK3w3uw= +github.com/joechenrh/arrow-go/v18 v18.0.0-20251104113423-8cacfcd24f0d/go.mod h1:xBWjSZfpqLkg1mHuVUSF8lMTUfE7SdEUl/VVVd79MIw= github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877 h1:O7syWuYGzre3s73s+NkgB8e0ZvsIVhT/zxNU7V1gHK8= github.com/johannesboyne/gofakes3 v0.0.0-20230506070712-04da935ef877/go.mod h1:AxgWC4DDX54O2WDoQO1Ceabtn6IbktjU/7bigor+66g= github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df h1:Zrb0IbuLOGHL7nrO2WrcuNWgDTlzFv3zY69QMx4ggQE= @@ -529,11 +482,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karamaru-alpha/copyloopvar v1.2.1 h1:wmZaZYIjnJ0b5UoKDjUHrikcV0zuPyyxI4SVplLd2CI= github.com/karamaru-alpha/copyloopvar v1.2.1/go.mod h1:nFmMlFNlClC2BPvNaHMdkirmTJxVCY0lhxBtlfOypMM= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3M= @@ -542,21 +492,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -591,8 +535,6 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -600,7 +542,6 @@ github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRC github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgechev/revive v1.11.0 h1:b/gLLpBE427o+Xmd8G58gSA+KtBwxWinH/A565Awh0w= github.com/mgechev/revive v1.11.0/go.mod h1:tI0oLF/2uj+InHCBLrrqfTKfjtFTBCFFfG05auyzgdw= @@ -658,18 +599,14 @@ github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= -github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe h1:vHpqOnPlnkba8iSxU4j/CvDSS9J4+F4473esQsYLGoE= github.com/petermattis/goid v0.0.0-20250813065127-a731cc31b4fe/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= -github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= -github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= -github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d h1:eHcokyHxm7HVM+7+Qy1zZwC7NhX9wVNX8oQDcSZw1qI= github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d/go.mod h1:KiO2zumBCWx7yoVYoFRpb+DNrwEPk1pR1LF7NvOACMQ= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -732,7 +669,6 @@ github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJ github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64= github.com/qri-io/jsonschema v0.2.1 h1:NNFoKms+kut6ABPf6xiKNM5214jzxAhDBrPHCJ97Wg0= github.com/qri-io/jsonschema v0.2.1/go.mod h1:g7DPkiOsK1xv6T/Ao5scXRkd+yTFygcANPBaaqW+VrI= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -743,14 +679,11 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= -github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU= @@ -791,7 +724,6 @@ github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:8ZnTA2 github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:tNZjgbYncKL5HxvDULAr/mWDmFz4B7H8yrXEDlnoIiw= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -876,13 +808,9 @@ github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f h1:9DDCDwOyEy github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f/go.mod h1:8sdOQnirw1PrcnTJYkmW1iOHtUmblMmGdUOHyWYycLI= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= -github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 h1:mr+7gGPUasLmH3/5Iv1zwQwiY0WgGO21Ym7Q4FVw+xs= -github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17/go.mod h1:u9udtIEWeBkphB2isZ8V8xVIMWgcUobH+7FRMO/Ld6c= -github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= -github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= -github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -943,7 +871,6 @@ go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucg go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.starlark.net v0.0.0-20210223155950-e043a3d3c984/go.mod h1:t3mmBBPzAVvK0L0n1drDmrQsJ8FoIx4INCqVMTr/Zo0= @@ -978,7 +905,6 @@ go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -994,37 +920,22 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/exp/typeparams v0.0.0-20250620022241-b7579e27df2b h1:KdrhdYPDUvJTvrDK9gdjfFd6JTk8vA1WJoldYSi0kHo= golang.org/x/exp/typeparams v0.0.0-20250620022241-b7579e27df2b/go.mod h1:LKZHyeOpPuZcMgxeHjJp4p5yvxrCX1xDvH10zYHhjjQ= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= -golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1045,7 +956,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -1095,7 +1005,6 @@ golang.org/x/net v0.0.0-20220517181318-183a9ca12b87/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -1162,9 +1071,7 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1181,7 +1088,6 @@ golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1210,7 +1116,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -1230,7 +1135,6 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1244,7 +1148,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1275,14 +1178,11 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= @@ -1301,20 +1201,12 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= -gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= -gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0= +gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= -gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1393,8 +1285,6 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs= @@ -1411,9 +1301,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1425,11 +1313,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -1461,7 +1344,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= honnef.co/go/tools v0.6.1 h1:R094WgE8K4JirYjBaOpz/AvTyUu/3wbmAoskKN/pxTI= honnef.co/go/tools v0.6.1/go.mod h1:3puzxxljPCe8RGJX7BIy1plGbxEOZni5mR2aXe3/uk4= k8s.io/api v0.29.11 h1:6FwDo33f1WX5Yu0RQTX9YAd3wth8Ik0B4SXQKsoQfbk= @@ -1474,54 +1356,7 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= -modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= -modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= -modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= -modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= -modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= -modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= -modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= -modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= -modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= -modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= -modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= -modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= -modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= -modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= -modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= -modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= -modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= -modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= -modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= -modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= -modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= -modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= -modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= -modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= -modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= -modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= -modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= -modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/lightning/pkg/importer/BUILD.bazel b/lightning/pkg/importer/BUILD.bazel index f78ef53b0ea33..73be6f4921f39 100644 --- a/lightning/pkg/importer/BUILD.bazel +++ b/lightning/pkg/importer/BUILD.bazel @@ -182,8 +182,6 @@ go_test( "@com_github_tikv_pd_client//:client", "@com_github_tikv_pd_client//http", "@com_github_tikv_pd_client//pkg/caller", - "@com_github_xitongsys_parquet_go//writer", - "@com_github_xitongsys_parquet_go_source//buffer", "@io_etcd_go_etcd_client_v3//:client", "@io_etcd_go_etcd_tests_v3//integration", "@org_uber_go_mock//gomock", diff --git a/lightning/pkg/importer/chunk_process.go b/lightning/pkg/importer/chunk_process.go index 4980bf35080ed..512e4cdc2ba20 100644 --- a/lightning/pkg/importer/chunk_process.go +++ b/lightning/pkg/importer/chunk_process.go @@ -107,7 +107,7 @@ func openParser( case mydump.SourceTypeSQL: parser = mydump.NewChunkParser(ctx, cfg.TiDB.SQLMode, reader, blockBufSize, ioWorkers) case mydump.SourceTypeParquet: - parser, err = mydump.NewParquetParser(ctx, store, reader, chunk.FileMeta.Path) + parser, err = mydump.NewParquetParser(ctx, store, reader, chunk.FileMeta.Path, chunk.FileMeta.ParquetMeta) if err != nil { return nil, err } diff --git a/lightning/pkg/importer/get_pre_info.go b/lightning/pkg/importer/get_pre_info.go index 8ca2115f9739c..0f8b529512c51 100644 --- a/lightning/pkg/importer/get_pre_info.go +++ b/lightning/pkg/importer/get_pre_info.go @@ -491,7 +491,10 @@ func (p *PreImportInfoGetterImpl) ReadFirstNRowsByFileMeta(ctx context.Context, case mydump.SourceTypeSQL: parser = mydump.NewChunkParser(ctx, p.cfg.TiDB.SQLMode, reader, blockBufSize, p.ioWorkers) case mydump.SourceTypeParquet: - parser, err = mydump.NewParquetParser(ctx, p.srcStorage, reader, dataFileMeta.Path) + parser, err = mydump.NewParquetParser( + ctx, p.srcStorage, reader, + dataFileMeta.Path, mydump.GetDefaultParquetMeta(), + ) if err != nil { return nil, nil, errors.Trace(err) } @@ -661,7 +664,10 @@ func (p *PreImportInfoGetterImpl) sampleDataFromTable( case mydump.SourceTypeSQL: parser = mydump.NewChunkParser(ctx, p.cfg.TiDB.SQLMode, reader, blockBufSize, p.ioWorkers) case mydump.SourceTypeParquet: - parser, err = mydump.NewParquetParser(ctx, p.srcStorage, reader, sampleFile.Path) + parser, err = mydump.NewParquetParser( + ctx, p.srcStorage, reader, + sampleFile.Path, mydump.GetDefaultParquetMeta(), + ) if err != nil { return 0.0, false, errors.Trace(err) } diff --git a/lightning/pkg/importer/get_pre_info_test.go b/lightning/pkg/importer/get_pre_info_test.go index aa7d8157ea87b..7b1af9422da8f 100644 --- a/lightning/pkg/importer/get_pre_info_test.go +++ b/lightning/pkg/importer/get_pre_info_test.go @@ -20,13 +20,13 @@ import ( "context" "database/sql" "fmt" - "slices" "strings" "testing" "github.com/DATA-DOG/go-sqlmock" mysql_sql_driver "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" + "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/lightning/pkg/importer/mock" ropts "github.com/pingcap/tidb/lightning/pkg/importer/opts" "github.com/pingcap/tidb/pkg/errno" @@ -36,8 +36,6 @@ import ( "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/types" "github.com/stretchr/testify/require" - pqt_buf_src "github.com/xitongsys/parquet-go-source/buffer" - pqtwriter "github.com/xitongsys/parquet-go/writer" ) type colDef struct { @@ -253,26 +251,24 @@ func TestGetPreInfoGetAllTableStructures(t *testing.T) { } } -func generateParquetData(t *testing.T) []byte { - type parquetStruct struct { - ID int64 `parquet:"name=id, type=INT64"` - Name string `parquet:"name=name, type=BYTE_ARRAY"` - } - pf, err := pqt_buf_src.NewBufferFile(make([]byte, 0)) +func readParquetData(t *testing.T) []byte { + s, err := storage.ParseBackend("./testdata", nil) require.NoError(t, err) - pw, err := pqtwriter.NewParquetWriter(pf, new(parquetStruct), 4) + + store, err := storage.NewWithDefaultOpt(context.Background(), s) require.NoError(t, err) - for i := range 10 { - require.NoError(t, pw.Write(parquetStruct{ - ID: int64(i + 1), - Name: fmt.Sprintf("name_%d", i+1), - })) - } - require.NoError(t, pw.WriteStop()) - require.NoError(t, pf.Close()) - bf, ok := pf.(pqt_buf_src.BufferFile) - require.True(t, ok) - return slices.Clone(bf.Bytes()) + defer store.Close() + + reader, err := store.Open(context.Background(), "test.parquet", nil) + require.NoError(t, err) + defer reader.Close() + + bs := make([]byte, 1024) + l, err := reader.Read(bs) + bs = bs[:l] + require.NoError(t, err) + + return bs } func TestGetPreInfoReadFirstRow(t *testing.T) { @@ -282,7 +278,6 @@ func TestGetPreInfoReadFirstRow(t *testing.T) { 111,"aaa" 222,"bbb" `) - pqtData := generateParquetData(t) const testSQLData01 string = `INSERT INTO db01.tbl01 (ival, sval) VALUES (333, 'ccc'); INSERT INTO db01.tbl01 (ival, sval) VALUES (444, 'ddd');` testDataInfos := []struct { @@ -349,7 +344,7 @@ INSERT INTO db01.tbl01 (ival, sval) VALUES (444, 'ddd');` }, { FileName: "/db01/tbl01/data.005.parquet", - Data: pqtData, + Data: readParquetData(t), FirstN: 3, ExpectFirstRowDatums: [][]types.Datum{ { diff --git a/lightning/pkg/importer/table_import.go b/lightning/pkg/importer/table_import.go index 620aed35060b9..c6f4449668eb0 100644 --- a/lightning/pkg/importer/table_import.go +++ b/lightning/pkg/importer/table_import.go @@ -29,7 +29,6 @@ import ( dmysql "github.com/go-sql-driver/mysql" "github.com/pingcap/errors" "github.com/pingcap/failpoint" - "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/br/pkg/version" "github.com/pingcap/tidb/lightning/pkg/web" "github.com/pingcap/tidb/pkg/errno" @@ -788,13 +787,6 @@ ChunkLoop: break } - if chunk.FileMeta.Type == mydump.SourceTypeParquet { - // TODO: use the compressed size of the chunk to conduct memory control - if _, err = getChunkCompressedSizeForParquet(ctx, chunk, rc.store); err != nil { - return nil, errors.Trace(err) - } - } - restoreWorker := rc.regionWorkers.Apply() wg.Add(1) go func(w *worker.Worker, cr *chunkProcessor) { @@ -1201,42 +1193,6 @@ func (tr *TableImporter) postProcess( return true, nil } -func getChunkCompressedSizeForParquet( - ctx context.Context, - chunk *checkpoints.ChunkCheckpoint, - store storage.ExternalStorage, -) (int64, error) { - reader, err := mydump.OpenReader(ctx, &chunk.FileMeta, store, storage.DecompressConfig{ - ZStdDecodeConcurrency: 1, - }) - if err != nil { - return 0, errors.Trace(err) - } - parser, err := mydump.NewParquetParser(ctx, store, reader, chunk.FileMeta.Path) - if err != nil { - _ = reader.Close() - return 0, errors.Trace(err) - } - //nolint: errcheck - defer parser.Close() - err = parser.Reader.ReadFooter() - if err != nil { - return 0, errors.Trace(err) - } - rowGroups := parser.Reader.Footer.GetRowGroups() - var maxRowGroupSize int64 - for _, rowGroup := range rowGroups { - var rowGroupSize int64 - columnChunks := rowGroup.GetColumns() - for _, columnChunk := range columnChunks { - columnChunkSize := columnChunk.MetaData.GetTotalCompressedSize() - rowGroupSize += columnChunkSize - } - maxRowGroupSize = max(maxRowGroupSize, rowGroupSize) - } - return maxRowGroupSize, nil -} - func updateStatsMeta(ctx context.Context, db *sql.DB, tableID int64, count int) { s := common.SQLWithRetry{ DB: db, diff --git a/lightning/pkg/importer/table_import_test.go b/lightning/pkg/importer/table_import_test.go index f36d03d2ea652..4ea8f199bb961 100644 --- a/lightning/pkg/importer/table_import_test.go +++ b/lightning/pkg/importer/table_import_test.go @@ -2398,39 +2398,3 @@ func TestGetDDLStatus(t *testing.T) { require.Equal(t, model.JobStateRunning, status.state) require.Equal(t, int64(123)+int64(456), status.rowCount) } - -func TestGetChunkCompressedSizeForParquet(t *testing.T) { - dir := "./testdata/" - fileName := "000000_0.parquet" - store, err := storage.NewLocalStorage(dir) - require.NoError(t, err) - - dataFiles := make([]mydump.FileInfo, 0) - dataFiles = append(dataFiles, mydump.FileInfo{ - TableName: filter.Table{Schema: "db", Name: "table"}, - FileMeta: mydump.SourceFileMeta{ - Path: fileName, - Type: mydump.SourceTypeParquet, - Compression: mydump.CompressionNone, - SortKey: "99", - FileSize: 192, - }, - }) - - chunk := checkpoints.ChunkCheckpoint{ - Key: checkpoints.ChunkCheckpointKey{Path: dataFiles[0].FileMeta.Path, Offset: 0}, - FileMeta: dataFiles[0].FileMeta, - Chunk: mydump.Chunk{ - Offset: 0, - EndOffset: 192, - PrevRowIDMax: 0, - RowIDMax: 100, - }, - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - compressedSize, err := getChunkCompressedSizeForParquet(ctx, &chunk, store) - require.NoError(t, err) - require.Equal(t, compressedSize, int64(192)) -} diff --git a/lightning/pkg/importer/testdata/test.parquet b/lightning/pkg/importer/testdata/test.parquet new file mode 100644 index 0000000000000..5c46f14ded682 Binary files /dev/null and b/lightning/pkg/importer/testdata/test.parquet differ diff --git a/pkg/disttask/importinto/encode_and_sort_operator.go b/pkg/disttask/importinto/encode_and_sort_operator.go index fa979c29967b3..7c52c45fce315 100644 --- a/pkg/disttask/importinto/encode_and_sort_operator.go +++ b/pkg/disttask/importinto/encode_and_sort_operator.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/tidb/pkg/disttask/operator" "github.com/pingcap/tidb/pkg/executor/importer" "github.com/pingcap/tidb/pkg/lightning/backend/external" + "github.com/pingcap/tidb/pkg/lightning/mydump" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/resourcemanager/pool/workerpool" "github.com/pingcap/tidb/pkg/resourcemanager/util" @@ -52,6 +53,7 @@ type encodeAndSortOperator struct { ctx context.Context cancel context.CancelFunc collector execute.Collector + pool *mydump.Pool taskID, subtaskID int64 tableImporter *importer.TableImporter @@ -71,12 +73,14 @@ func newEncodeAndSortOperator( collector execute.Collector, subtaskID int64, concurrency int, + memPool *mydump.Pool, ) *encodeAndSortOperator { subCtx, cancel := context.WithCancel(ctx) op := &encodeAndSortOperator{ ctx: subCtx, cancel: cancel, collector: collector, + pool: memPool, taskID: executor.taskID, subtaskID: subtaskID, tableImporter: executor.tableImporter, @@ -197,7 +201,7 @@ func (w *chunkWorker) HandleTask(task *importStepMinimalTask, _ func(workerpool. // we don't use the input send function, it makes workflow more complex // we send result to errCh and handle it here. executor := newImportMinimalTaskExecutor(task) - if err := executor.Run(w.ctx, w.dataWriter, w.indexWriter, w.op.collector); err != nil { + if err := executor.Run(w.ctx, w.dataWriter, w.indexWriter, w.op.collector, w.op.pool); err != nil { w.op.onError(err) } } @@ -228,12 +232,17 @@ func subtaskPrefix(taskID, subtaskID int64) string { return path.Join(strconv.Itoa(int(taskID)), strconv.Itoa(int(subtaskID))) } -func getWriterMemorySizeLimit(resource *proto.StepResource, plan *importer.Plan) ( +func getWriterMemorySizeLimit(resource *proto.StepResource, plan *importer.Plan, encodeStep bool) ( dataKVMemSizePerCon, perIndexKVMemSizePerCon uint64) { indexKVGroupCnt := getNumOfIndexGenKV(plan.DesiredTableInfo) - memPerCon := resource.Mem.Capacity() / int64(plan.ThreadCnt) - // we use half of the total available memory for data writer, and the other half - // for encoding and other stuffs, it's an experience value, might not optimal. + + // We use a portion of the total available memory for data writer, which is depended + // on the data format, and the other half for encoding and other stuffs, it's an + // experience value, might not optimal. + memForWriter := mydump.GetMemoryForWriter( + encodeStep, plan.ParquetFileMemoryUsage, + plan.ThreadCnt, int(resource.Mem.Capacity())) + // Then we divide those memory into indexKVGroupCnt + 3 shares, data KV writer // takes 3 shares, and each index KV writer takes 1 share. // suppose we have memPerCon = 2G @@ -243,7 +252,7 @@ func getWriterMemorySizeLimit(resource *proto.StepResource, plan *importer.Plan) // | 1 | 768/256 MiB | // | 5 | 384/128 MiB | // | 13 | 192/64 MiB | - memPerShare := float64(memPerCon) / 2 / float64(indexKVGroupCnt+3) + memPerShare := float64(memForWriter) / float64(indexKVGroupCnt+3) return uint64(memPerShare * 3), uint64(memPerShare) } diff --git a/pkg/disttask/importinto/encode_and_sort_operator_test.go b/pkg/disttask/importinto/encode_and_sort_operator_test.go index 3c24800d1d306..75e14f36d9fb2 100644 --- a/pkg/disttask/importinto/encode_and_sort_operator_test.go +++ b/pkg/disttask/importinto/encode_and_sort_operator_test.go @@ -233,7 +233,7 @@ func TestGetWriterMemorySizeLimit(t *testing.T) { }, &importer.Plan{ DesiredTableInfo: info, ThreadCnt: 1, - }) + }, false) require.Equal(t, c.dataKVMemSizePerCon, dataKVMemSizePerCon, c.createSQL) if c.numOfIndexGenKV > 0 { require.Equal(t, c.perIndexKVMemSizePerCon, perIndexKVMemSizePerCon, c.createSQL) diff --git a/pkg/disttask/importinto/subtask_executor.go b/pkg/disttask/importinto/subtask_executor.go index 8770c6eb30099..fa51b0b148084 100644 --- a/pkg/disttask/importinto/subtask_executor.go +++ b/pkg/disttask/importinto/subtask_executor.go @@ -30,6 +30,7 @@ import ( "github.com/pingcap/tidb/pkg/lightning/backend/local" "github.com/pingcap/tidb/pkg/lightning/checkpoints" "github.com/pingcap/tidb/pkg/lightning/log" + "github.com/pingcap/tidb/pkg/lightning/mydump" verify "github.com/pingcap/tidb/pkg/lightning/verification" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/resourcegroup" @@ -42,7 +43,7 @@ import ( // MiniTaskExecutor is the interface for a minimal task executor. // exported for testing. type MiniTaskExecutor interface { - Run(ctx context.Context, dataWriter, indexWriter backend.EngineWriter, collector execute.Collector) error + Run(ctx context.Context, dataWriter, indexWriter backend.EngineWriter, collector execute.Collector, pool *mydump.Pool) error } // importMinimalTaskExecutor is a minimal task executor for IMPORT INTO. @@ -62,6 +63,7 @@ func (e *importMinimalTaskExecutor) Run( ctx context.Context, dataWriter, indexWriter backend.EngineWriter, collector execute.Collector, + pool *mydump.Pool, ) error { logger := logutil.BgLogger().With(zap.Stringer("type", proto.ImportInto), zap.Int64("table-id", e.mTtask.Plan.TableInfo.ID)) logger.Info("execute chunk") @@ -70,8 +72,15 @@ func (e *importMinimalTaskExecutor) Run( failpoint.Return(errors.New("occur an error when sort chunk")) }) failpoint.InjectCall("syncBeforeSortChunk") - chunkCheckpoint := toChunkCheckpoint(e.mTtask.Chunk) sharedVars := e.mTtask.SharedVars + + chunkCheckpoint := toChunkCheckpoint(e.mTtask.Chunk) + chunkCheckpoint.FileMeta.ParquetMeta = mydump.ParquetFileMeta{ + MemoryPool: pool, + MemoryUsage: sharedVars.TableImporter.ParquetFileMemoryUsage, + Loc: sharedVars.TableImporter.Location, + } + checksum := verify.NewKVGroupChecksumWithKeyspace(sharedVars.TableImporter.GetKeySpace()) if sharedVars.TableImporter.IsLocalSort() { if err := importer.ProcessChunk( diff --git a/pkg/disttask/importinto/task_executor.go b/pkg/disttask/importinto/task_executor.go index b774fe62df575..ee0360d563452 100644 --- a/pkg/disttask/importinto/task_executor.go +++ b/pkg/disttask/importinto/task_executor.go @@ -45,6 +45,7 @@ import ( "github.com/pingcap/tidb/pkg/lightning/config" "github.com/pingcap/tidb/pkg/lightning/log" "github.com/pingcap/tidb/pkg/lightning/metric" + "github.com/pingcap/tidb/pkg/lightning/mydump" "github.com/pingcap/tidb/pkg/lightning/verification" "github.com/pingcap/tidb/pkg/meta/autoid" "github.com/pingcap/tidb/pkg/sessionctx" @@ -77,6 +78,7 @@ type importStepExecutor struct { wg sync.WaitGroup summary execute.SubtaskSummary + memPool *mydump.Pool } func getTableImporter( @@ -154,7 +156,8 @@ func (s *importStepExecutor) Init(ctx context.Context) (err error) { s.tableImporter.CheckDiskQuota(s.importCtx) }() } - s.dataKVMemSizePerCon, s.perIndexKVMemSizePerCon = getWriterMemorySizeLimit(s.GetResource(), s.tableImporter.Plan) + s.dataKVMemSizePerCon, s.perIndexKVMemSizePerCon = getWriterMemorySizeLimit( + s.GetResource(), s.tableImporter.Plan, true) s.dataBlockSize = external.GetAdjustedBlockSize(s.dataKVMemSizePerCon, tidbconfig.MaxTxnEntrySizeLimit) s.indexBlockSize = external.GetAdjustedBlockSize(s.perIndexKVMemSizePerCon, external.DefaultBlockSize) s.logger.Info("KV writer memory buf info", @@ -222,8 +225,12 @@ func (s *importStepExecutor) RunSubtask(ctx context.Context, subtask *proto.Subt } s.sharedVars.Store(subtaskMeta.ID, sharedVars) + if s.memPool == nil { + s.memPool = mydump.GetPool(int(s.GetResource().Mem.Capacity())) + } + source := operator.NewSimpleDataChannel(make(chan *importStepMinimalTask)) - op := newEncodeAndSortOperator(ctx, s, sharedVars, s, subtask.ID, int(s.GetResource().CPU.Capacity())) + op := newEncodeAndSortOperator(ctx, s, sharedVars, s, subtask.ID, int(s.GetResource().CPU.Capacity()), s.memPool) op.SetSource(source) pipeline := operator.NewAsyncPipeline(op) if err = pipeline.Execute(); err != nil { @@ -365,7 +372,7 @@ func (m *mergeSortStepExecutor) Init(ctx context.Context) error { return err } m.sortStore = store - dataKVMemSizePerCon, perIndexKVMemSizePerCon := getWriterMemorySizeLimit(m.GetResource(), &m.taskMeta.Plan) + dataKVMemSizePerCon, perIndexKVMemSizePerCon := getWriterMemorySizeLimit(m.GetResource(), &m.taskMeta.Plan, false) m.dataKVPartSize = max(external.MinUploadPartSize, int64(dataKVMemSizePerCon*uint64(external.MaxMergingFilesPerThread)/10000)) m.indexKVPartSize = max(external.MinUploadPartSize, int64(perIndexKVMemSizePerCon*uint64(external.MaxMergingFilesPerThread)/10000)) diff --git a/pkg/executor/importer/chunk_process.go b/pkg/executor/importer/chunk_process.go index e1ae52b7b6067..9cca73cf3fa3d 100644 --- a/pkg/executor/importer/chunk_process.go +++ b/pkg/executor/importer/chunk_process.go @@ -69,7 +69,7 @@ func parserEncodeReader(parser mydump.Parser, endOffset int64, filename string) return } - err = parser.ReadRow() + err = parser.ReadRowUnsafe() // todo: we can implement a ScannedPos which don't return error, will change it later. currOffset, _ := parser.ScannedPos() switch errors.Cause(err) { diff --git a/pkg/executor/importer/import.go b/pkg/executor/importer/import.go index db539176ec979..fa72ea2f21528 100644 --- a/pkg/executor/importer/import.go +++ b/pkg/executor/importer/import.go @@ -26,6 +26,7 @@ import ( "slices" "strings" "sync" + "time" "unicode/utf8" "github.com/docker/go-units" @@ -249,6 +250,13 @@ type Plan struct { // ref https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-column-assignments Restrictive bool + // Location is used to convert time type for parquet, as we assume that time stored + // in parquet is always adjusted to UTC, see + // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp + Location *time.Location + // ParquetFileMemoryUsage is the estimated memory usage of each parquet parser. + ParquetFileMemoryUsage int + SQLMode mysql.SQLMode // Charset is the charset of the data file when file is CSV or TSV. // it might be nil when using LOAD DATA and no charset is specified. @@ -478,6 +486,7 @@ func NewImportPlan(ctx context.Context, userSctx sessionctx.Context, plan *plann FieldNullDef: defaultFieldNullDef, LineFieldsInfo: lineFieldsInfo, + Location: userSctx.GetSessionVars().Location(), SQLMode: userSctx.GetSessionVars().SQLMode, ImportantSysVars: getImportantSysVars(userSctx), @@ -1306,6 +1315,21 @@ func (e *LoadDataController) InitDataFiles(ctx context.Context) error { } } + failpoint.Inject("skipReadFiles", func() { + failpoint.Goto("afterReadFiles") + }) + + // Fill memory usage info + if sourceType == mydump.SourceTypeParquet && len(dataFiles) > 0 { + _, _, memoryUsage, err := mydump.SampleStatisticsFromParquet(ctx, *dataFiles[0], e.dataStore) + e.Plan.ParquetFileMemoryUsage = memoryUsage + if err != nil { + return errors.Trace(err) + } + } + + failpoint.Label("afterReadFiles") + e.dataFiles = dataFiles e.TotalFileSize = totalSize @@ -1438,6 +1462,7 @@ func (e *LoadDataController) GetParser( e.dataStore, reader, dataFileInfo.Remote.Path, + dataFileInfo.Remote.ParquetMeta, ) } if err != nil { diff --git a/pkg/executor/importer/import_test.go b/pkg/executor/importer/import_test.go index 66ed933eb71ca..b4f20d0b37afd 100644 --- a/pkg/executor/importer/import_test.go +++ b/pkg/executor/importer/import_test.go @@ -38,6 +38,7 @@ import ( "github.com/pingcap/tidb/pkg/planner/core/operator/physicalop" plannerutil "github.com/pingcap/tidb/pkg/planner/util" "github.com/pingcap/tidb/pkg/sessionctx/vardef" + "github.com/pingcap/tidb/pkg/testkit/testfailpoint" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/dbterror/exeerrors" "github.com/pingcap/tidb/pkg/util/logutil" @@ -433,6 +434,7 @@ func TestSupportedSuffixForServerDisk(t *testing.T) { fileNames: []string{"file3.PARQUET", "file3.parquet.gz", "file3.PARQUET.GZIP", "file3.parquet.zstd", "file3.parquet.zst", "file3.parquet.snappy", "file3.parquet.snappy"}, }, } + testfailpoint.Enable(t, "github.com/pingcap/tidb/pkg/executor/importer/skipReadFiles", "return()") for _, testcase := range testcases { for _, fileName := range testcase.fileNames { c.Format = DataFormatAuto diff --git a/pkg/executor/importer/table_import.go b/pkg/executor/importer/table_import.go index 86b52fe1411dd..04d24b0c97af1 100644 --- a/pkg/executor/importer/table_import.go +++ b/pkg/executor/importer/table_import.go @@ -100,6 +100,7 @@ type Chunk struct { Type mydump.SourceType Compression mydump.Compression Timestamp int64 + ParquetMeta mydump.ParquetFileMeta } // prepareSortDir creates a new directory for import, remove previous sort directory if exists. @@ -304,6 +305,7 @@ func (ti *TableImporter) getParser(ctx context.Context, chunk *checkpoints.Chunk }, Remote: &chunk.FileMeta, } + parser, err := ti.LoadDataController.GetParser(ctx, info) if err != nil { return nil, err @@ -457,6 +459,7 @@ func (e *LoadDataController) PopulateChunks(ctx context.Context) (chunksMap map[ Type: region.FileMeta.Type, Compression: region.FileMeta.Compression, Timestamp: timestamp, + ParquetMeta: region.FileMeta.ParquetMeta, }) } @@ -469,12 +472,7 @@ func (e *LoadDataController) PopulateChunks(ctx context.Context) (chunksMap map[ func (ti *TableImporter) getTotalRawFileSize(indexCnt int64) int64 { var totalSize int64 for _, file := range ti.dataFiles { - size := file.RealSize - if file.Type == mydump.SourceTypeParquet { - // parquet file is compressed, thus estimates with a factor of 2 - size *= 2 - } - totalSize += size + totalSize += file.RealSize } return totalSize * indexCnt } diff --git a/pkg/lightning/mydump/BUILD.bazel b/pkg/lightning/mydump/BUILD.bazel index a71422f556eb5..edfa5c75df55e 100644 --- a/pkg/lightning/mydump/BUILD.bazel +++ b/pkg/lightning/mydump/BUILD.bazel @@ -3,11 +3,15 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "mydump", srcs = [ + "allocator.go", + "append_only_allocator.go", "bytes.go", "charset_convertor.go", "csv_parser.go", "loader.go", "parquet_parser.go", + "parquet_type_converter.go", + "parquet_writer.go", "parser.go", "parser_generated.go", "reader.go", @@ -24,6 +28,7 @@ go_library( "//pkg/lightning/common", "//pkg/lightning/config", "//pkg/lightning/log", + "//pkg/lightning/membuf", "//pkg/lightning/metric", "//pkg/lightning/worker", "//pkg/parser", @@ -39,14 +44,17 @@ go_library( "//pkg/util/set", "//pkg/util/sqlescape", "//pkg/util/table-filter", + "//pkg/util/timeutil", "//pkg/util/zeropool", + "@com_github_apache_arrow_go_v18//arrow/memory", + "@com_github_apache_arrow_go_v18//parquet", + "@com_github_apache_arrow_go_v18//parquet/compress", + "@com_github_apache_arrow_go_v18//parquet/file", + "@com_github_apache_arrow_go_v18//parquet/schema", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_spkg_bom//:bom", - "@com_github_xitongsys_parquet_go//parquet", - "@com_github_xitongsys_parquet_go//reader", - "@com_github_xitongsys_parquet_go//source", "@org_golang_x_sync//errgroup", "@org_golang_x_text//encoding", "@org_golang_x_text//encoding/charmap", @@ -60,6 +68,7 @@ go_test( name = "mydump_test", timeout = "short", srcs = [ + "allocator_test.go", "charset_convertor_test.go", "csv_parser_test.go", "loader_test.go", @@ -95,15 +104,14 @@ go_test( "//pkg/util/logutil", "//pkg/util/table-filter", "//pkg/util/table-router", + "@com_github_apache_arrow_go_v18//parquet", + "@com_github_apache_arrow_go_v18//parquet/schema", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", - "@com_github_xitongsys_parquet_go//parquet", - "@com_github_xitongsys_parquet_go//writer", - "@com_github_xitongsys_parquet_go_source//local", "@org_uber_go_goleak//:goleak", "@org_uber_go_mock//gomock", "@org_uber_go_zap//:zap", diff --git a/pkg/lightning/mydump/allocator.go b/pkg/lightning/mydump/allocator.go new file mode 100644 index 0000000000000..9fc218ea6c9fe --- /dev/null +++ b/pkg/lightning/mydump/allocator.go @@ -0,0 +1,130 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mydump + +import ( + "unsafe" + + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/pingcap/tidb/pkg/lightning/log" + "github.com/pingcap/tidb/pkg/lightning/membuf" +) + +var ( + // arenaSize is the size of each arena + arenaSize = 64 << 20 + + // maxParquetMemoryPercent defines the maximum percentage of memory used for parquet parser + // Because less memory for writer can make more small files, which may affect performance + // merge and ingest steps, so we set a limit here. + maxParquetMemoryPercent = 40 +) + +// GetMemoryForWriter gets the memory for writer +func GetMemoryForWriter(encodeStep bool, parquetMemUsage, threadCnt, totalMem int) int64 { + memPerCon := totalMem / threadCnt + + writerPercent := 50 + if encodeStep { + upperLimit := totalMem * maxParquetMemoryPercent / 100 + if parquetMemUsage >= upperLimit { + log.L().Warn("parquet parser memory usage is too high, may cause OOM") + } + actualUsage := min(parquetMemUsage*threadCnt, upperLimit) + parserPercent := (actualUsage*100/totalMem + 9) / 10 * 10 + writerPercent = (100 - parserPercent) / 2 + if parserPercent <= 20 { + writerPercent = 50 + } + } + + // Use half of the remaining memory for writer + return int64(memPerCon * writerPercent / 100) +} + +// Pool manages a pool of reusable byte buffers to reduce memory allocation overhead. +// It uses a buffered channel to store and reuse buffers efficiently. +type Pool struct { + blockSize int + blockCache chan []byte + limit int + // As we may not be able to open all files concurrently due to memory usage, + // we use a memory limiter to limit the memory usage of parquet parser. + limiter *membuf.Limiter +} + +// GetPool gets a pool with the given capacity. +func GetPool(capacity int) *Pool { + limit := capacity * maxParquetMemoryPercent / 100 + return &Pool{ + blockSize: arenaSize, + blockCache: make(chan []byte, (limit+arenaSize-1)/arenaSize), + limit: limit, + limiter: membuf.NewLimiter(limit), + } +} + +// Acquire acquires memory from the pool. +func (p *Pool) Acquire(quota int) { + p.limiter.Acquire(quota) +} + +// Release releases memory to the pool. +func (p *Pool) Release(quota int) { + p.limiter.Release(quota) +} + +// Get retrieves a buffer from the pool or allocates a new one if the pool is empty. +func (p *Pool) Get() []byte { + select { + case buf := <-p.blockCache: + return buf + default: + return make([]byte, p.blockSize) + } +} + +func (p *Pool) Put(buf []byte) { + if buf == nil { + return + } + + select { + case p.blockCache <- buf: + default: + // Pool is full, discard the buffer + } +} + +// addressOf returns the address of a buffer, return 0 if the buffer is nil or empty. +// This is used to create unique identifiers for tracking buffer allocations. +func addressOf(buf []byte) uintptr { + if buf == nil || cap(buf) == 0 { + return 0 + } + buf = buf[:1] + return uintptr(unsafe.Pointer(&buf[0])) +} + +type arena interface { + allocate(int) []byte + free([]byte) + reset() +} + +type AllocatorWithClose interface { + memory.Allocator + Close() +} diff --git a/pkg/lightning/mydump/allocator_test.go b/pkg/lightning/mydump/allocator_test.go new file mode 100644 index 0000000000000..370f97832feb0 --- /dev/null +++ b/pkg/lightning/mydump/allocator_test.go @@ -0,0 +1,72 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mydump + +import ( + "context" + "math/rand" + "runtime" + "sync" + "testing" + "time" +) + +func TestSimpleAllocator(t *testing.T) { + arenaSize = 16 << 20 + + pool := GetPool(16 << 23) + a := NewAppendOnlyAllocator(pool, 0) + + var ( + lk sync.Mutex + wg sync.WaitGroup + ) + + allocSize := []int{16 << 10, 32 << 10, 64 << 10, 128 << 10, 256 << 10, 512 << 10} + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + allocFunc := func(ctx context.Context) { + defer wg.Done() + for { + select { + case <-ctx.Done(): + return + default: + lk.Lock() + bufSize := allocSize[rand.Intn(len(allocSize))] + buf := a.Allocate(bufSize) + lk.Unlock() + + // hold for sometimes + time.Sleep(time.Millisecond) + + lk.Lock() + a.Free(buf) + lk.Unlock() + } + } + } + + numCPU := runtime.NumCPU() + for i := 0; i < numCPU*2; i++ { + wg.Add(1) + go allocFunc(ctx) + } + wg.Wait() + + a.check() +} diff --git a/pkg/lightning/mydump/append_only_allocator.go b/pkg/lightning/mydump/append_only_allocator.go new file mode 100644 index 0000000000000..84ec294f465c8 --- /dev/null +++ b/pkg/lightning/mydump/append_only_allocator.go @@ -0,0 +1,174 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mydump + +import ( + "sync" + "sync/atomic" +) + +type appendOnlySlice struct { + buf []byte + offset int + counter int + mu sync.Mutex +} + +func (s *appendOnlySlice) allocate(size int) []byte { + s.mu.Lock() + defer s.mu.Unlock() + + if s.offset+size <= len(s.buf) { + start := s.offset + s.offset += size + s.counter++ + return s.buf[start : start+size] + } + return nil +} + +func (s *appendOnlySlice) free() { + s.mu.Lock() + defer s.mu.Unlock() + + if s.counter > 0 { + s.counter-- + if s.counter == 0 { + s.offset = 0 + } + } +} + +type appendOnlyAllocator struct { + pool *Pool + slices []*appendOnlySlice + slicesMutex sync.RWMutex + mapper sync.Map + + memUsage int + + nextAllocIdx atomic.Int32 + + externalMemoryCurrent atomic.Int64 + externalMemoryMax atomic.Int64 +} + +// NewAppendOnlyAllocator creates a new appendOnlyAllocator with the given memory pool +func NewAppendOnlyAllocator(pool *Pool, memUsage int) *appendOnlyAllocator { + memUsage = min(memUsage, pool.limit) + pool.Acquire(memUsage) + alloc := &appendOnlyAllocator{ + pool: pool, + memUsage: memUsage, + } + for range 1 { + alloc.slices = append(alloc.slices, &appendOnlySlice{buf: pool.Get()}) + } + return alloc +} + +func (a *appendOnlyAllocator) Allocate(size int) []byte { + if size >= arenaSize { + buf := make([]byte, size) + a.mapper.Store(addressOf(buf), -1) + + // Many not accurate but enough for estimation + current := a.externalMemoryCurrent.Add(int64(size)) + if current > a.externalMemoryMax.Load() { + a.externalMemoryMax.Store(current) + } + return buf + } + + if buf := a.allocate(size); buf != nil { + return buf + } + return a.getAndAllocate(size) +} + +func (a *appendOnlyAllocator) allocate(size int) []byte { + a.slicesMutex.RLock() + defer a.slicesMutex.RUnlock() + + sliceLen := len(a.slices) + idx := int(a.nextAllocIdx.Add(1)) % sliceLen + for i := range sliceLen { + sel := (idx + i) % sliceLen + if buf := a.slices[sel].allocate(size); buf != nil { + a.mapper.Store(addressOf(buf), sel) + return buf + } + } + + return nil +} + +func (a *appendOnlyAllocator) getAndAllocate(size int) []byte { + a.slicesMutex.Lock() + defer a.slicesMutex.Unlock() + + newSlice := &appendOnlySlice{buf: a.pool.Get()} + buf := newSlice.allocate(size) + a.slices = append(a.slices, newSlice) + a.mapper.Store(addressOf(buf), len(a.slices)-1) + return buf +} + +func (a *appendOnlyAllocator) Free(buf []byte) { + addr := addressOf(buf) + v, ok := a.mapper.Load(addr) + if !ok { + return + } + + a.slicesMutex.RLock() + defer a.slicesMutex.RUnlock() + + id, _ := v.(int) + if id == -1 { + a.externalMemoryCurrent.Add(-int64(len(buf))) + } else { + a.slices[id].free() + } + + a.mapper.Delete(addr) +} + +func (a *appendOnlyAllocator) Reallocate(size int, buf []byte) []byte { + a.Free(buf) + return a.Allocate(size) +} + +func (a *appendOnlyAllocator) Allocated() int { + return arenaSize*len(a.slices) + int(a.externalMemoryMax.Load()) +} + +func (a *appendOnlyAllocator) Close() { + for _, s := range a.slices { + a.pool.Put(s.buf) + } + a.pool.Release(a.memUsage) +} + +func (a *appendOnlyAllocator) check() { + a.mapper.Range(func(key, value any) bool { + panic("memory leak detected in appendOnlyAllocator") + }) + for _, s := range a.slices { + if s.counter != 0 { + panic("memory leak detected in appendOnlyAllocator") + } + } +} diff --git a/pkg/lightning/mydump/csv_parser.go b/pkg/lightning/mydump/csv_parser.go index 37106ff47f181..c2ed29ececd59 100644 --- a/pkg/lightning/mydump/csv_parser.go +++ b/pkg/lightning/mydump/csv_parser.go @@ -628,6 +628,12 @@ func (parser *CSVParser) replaceEOF(err error, replaced error) error { return replaced } +// ReadRowUnsafe implements the Parser interface. +// TODO(joechenrh): implement unsafe read for CSV parser +func (parser *CSVParser) ReadRowUnsafe() error { + return parser.ReadRow() +} + // ReadRow reads a row from the datafile. func (parser *CSVParser) ReadRow() error { row := &parser.lastRow diff --git a/pkg/lightning/mydump/loader.go b/pkg/lightning/mydump/loader.go index d150b97011a48..bc0c75bd32781 100644 --- a/pkg/lightning/mydump/loader.go +++ b/pkg/lightning/mydump/loader.go @@ -21,6 +21,7 @@ import ( "sort" "strings" "sync" + "time" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -89,6 +90,13 @@ type MDTableMeta struct { IsRowOrdered bool } +// ParquetFileMeta contains some analyzed metadata for a parquet file by MyDumper Loader. +type ParquetFileMeta struct { + MemoryUsage int // memory usage for reader + MemoryPool *Pool + Loc *time.Location +} + // SourceFileMeta contains some analyzed metadata for a source file by MyDumper Loader. type SourceFileMeta struct { Path string @@ -104,7 +112,10 @@ type SourceFileMeta struct { // If the file is parquet, RealSize is the estimated data size after convert // to row oriented storage. RealSize int64 - Rows int64 // only for parquet + Rows int64 + + // ParquetMeta store meta only used for parquet + ParquetMeta ParquetFileMeta } // NewMDTableMeta creates an Mydumper table meta with specified character set. @@ -260,6 +271,7 @@ type RawFile struct { type parquetInfo struct { rowSize float64 // the average row size of the parquet file compressionRatio float64 // the estimated compression ratio of the parquet file + memoryUsage int // sampled memory usage for streaming parquet read } type mdLoaderSetup struct { @@ -476,6 +488,7 @@ func (s *mdLoaderSetup) setup(ctx context.Context) error { info.FileMeta.RealSize = int64(float64(info.FileMeta.FileSize) * pinfo.compressionRatio) // Postpone reading the row count to `MakeTableRegion` if necessary. info.FileMeta.Rows = int64(float64(info.FileMeta.RealSize) / pinfo.rowSize) + info.FileMeta.ParquetMeta.MemoryUsage = pinfo.memoryUsage if m, ok := metric.FromContext(ctx); ok { m.RowsCounter.WithLabelValues(metric.StateTotalRestore, info.TableName.String()).Add(float64(info.FileMeta.Rows)) @@ -618,7 +631,8 @@ func (s *mdLoaderSetup) constructFileInfo(ctx context.Context, f RawFile) (*File // Only sample once for each table _, loaded := s.sampledParquetInfos.LoadOrStore(tableName, parquetInfo{}) if !loaded { - rows, rowSize, err := SampleParquetRowSize(ctx, info.FileMeta, s.loader.GetStore()) + rows, rowSize, memoryUsage, err := + SampleStatisticsFromParquet(ctx, info.FileMeta, s.loader.GetStore()) if err != nil { logger.Error("fail to sample parquet row size", zap.String("category", "loader"), zap.String("schema", res.Schema), zap.String("table", res.Name), @@ -626,7 +640,7 @@ func (s *mdLoaderSetup) constructFileInfo(ctx context.Context, f RawFile) (*File return nil, errors.Trace(err) } compressionRatio := float64(info.FileMeta.FileSize) / (rowSize * float64(rows)) - s.sampledParquetInfos.Store(tableName, parquetInfo{rowSize, compressionRatio}) + s.sampledParquetInfos.Store(tableName, parquetInfo{rowSize, compressionRatio, memoryUsage}) } } @@ -945,46 +959,3 @@ func SampleFileCompressRatio(ctx context.Context, fileMeta SourceFileMeta, store } return float64(tot) / float64(pos), nil } - -// SampleParquetRowSize samples row size of the parquet file. -func SampleParquetRowSize(ctx context.Context, fileMeta SourceFileMeta, store storage.ExternalStorage) (int64, float64, error) { - totalRowCount, err := ReadParquetFileRowCountByFile(ctx, store, fileMeta) - if totalRowCount == 0 || err != nil { - return 0, 0, err - } - - reader, err := store.Open(ctx, fileMeta.Path, nil) - if err != nil { - return 0, 0, err - } - parser, err := NewParquetParser(ctx, store, reader, fileMeta.Path) - if err != nil { - //nolint: errcheck - reader.Close() - return 0, 0, err - } - //nolint: errcheck - defer parser.Close() - - var ( - rowSize int64 - rowCount int64 - ) - for { - err = parser.ReadRow() - if err != nil { - if errors.Cause(err) == io.EOF { - break - } - return 0, 0, err - } - lastRow := parser.LastRow() - rowCount++ - rowSize += int64(lastRow.Length) - parser.RecycleRow(lastRow) - if rowSize > maxSampleParquetDataSize || rowCount > maxSampleParquetRowCount { - break - } - } - return parser.Reader.Footer.NumRows, float64(rowSize) / float64(rowCount), nil -} diff --git a/pkg/lightning/mydump/loader_test.go b/pkg/lightning/mydump/loader_test.go index 6f8fec0258149..73006fdf18f23 100644 --- a/pkg/lightning/mydump/loader_test.go +++ b/pkg/lightning/mydump/loader_test.go @@ -22,10 +22,13 @@ import ( "math/rand" "os" "path/filepath" + "strconv" "strings" "testing" "time" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/schema" "github.com/pingcap/failpoint" "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/pkg/lightning/common" @@ -37,8 +40,6 @@ import ( router "github.com/pingcap/tidb/pkg/util/table-router" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/xitongsys/parquet-go/parquet" - "github.com/xitongsys/parquet-go/writer" ) type testMydumpLoaderSuite struct { @@ -1133,53 +1134,67 @@ func testSampleParquetDataSize(t *testing.T, count int) { store, err := storage.NewLocalStorage(s.sourceDir) require.NoError(t, err) - type row struct { - ID int64 `parquet:"name=id, type=INT64"` - Key string `parquet:"name=key, type=BYTE_ARRAY, encoding=PLAIN_DICTIONARY"` - Value string `parquet:"name=value, type=BYTE_ARRAY, encoding=PLAIN_DICTIONARY"` - } - ctx, cancel := context.WithCancel(context.Background()) defer cancel() - byteArray := make([]byte, 0, 40*1024) - bf := bytes.NewBuffer(byteArray) - pwriter, err := writer.NewParquetWriterFromWriter(bf, new(row), 4) - require.NoError(t, err) - pwriter.RowGroupSize = 128 * 1024 * 1024 //128M - pwriter.PageSize = 8 * 1024 //8K - pwriter.CompressionType = parquet.CompressionCodec_SNAPPY + fileName := "test_1.t1.parquet" + + totalRowSize := int64(0) seed := time.Now().Unix() t.Logf("seed: %d. To reproduce the random behaviour, manually set `rand.New(rand.NewSource(seed))`", seed) rnd := rand.New(rand.NewSource(seed)) - totalRowSize := 0 - for i := range count { - kl := rnd.Intn(20) + 1 - key := make([]byte, kl) - kl, err = rnd.Read(key) - require.NoError(t, err) - vl := rnd.Intn(20) + 1 - value := make([]byte, vl) - vl, err = rnd.Read(value) - require.NoError(t, err) - - totalRowSize += kl + vl + 8 - row := row{ - ID: int64(i), - Key: string(key[:kl]), - Value: string(value[:vl]), - } - err = pwriter.Write(row) - require.NoError(t, err) + pc := []md.ParquetColumn{ + { + Name: "id", + Type: parquet.Types.Int64, + Converted: schema.ConvertedTypes.None, + Gen: func(numRows int) (any, []int16) { + data := make([]int64, numRows) + defLevels := make([]int16, numRows) + for i := range numRows { + data[i] = int64(i) + defLevels[i] = 1 + } + totalRowSize += int64(8 * numRows) + return data, defLevels + }, + }, + { + Name: "key", + Type: parquet.Types.ByteArray, + Converted: schema.ConvertedTypes.UTF8, + Gen: func(numRows int) (any, []int16) { + data := make([]parquet.ByteArray, numRows) + defLevels := make([]int16, numRows) + for i := range numRows { + s := strconv.Itoa(rnd.Intn(100) + 1) + data[i] = parquet.ByteArray(s) + defLevels[i] = 1 + totalRowSize += int64(len(s)) + } + return data, defLevels + }, + }, + { + Name: "value", + Type: parquet.Types.ByteArray, + Converted: schema.ConvertedTypes.UTF8, + Gen: func(numRows int) (any, []int16) { + data := make([]parquet.ByteArray, numRows) + defLevels := make([]int16, numRows) + for i := range numRows { + s := strconv.Itoa(rnd.Intn(100) + 1) + data[i] = parquet.ByteArray(s) + defLevels[i] = 1 + totalRowSize += int64(len(s)) + } + return data, defLevels + }, + }, } - err = pwriter.WriteStop() - require.NoError(t, err) - - fileName := "test_1.t1.parquet" - err = store.WriteFile(ctx, fileName, bf.Bytes()) - require.NoError(t, err) + md.WriteParquetFile(s.sourceDir, fileName, pc, count) - rowCount, rowSize, err := md.SampleParquetRowSize(ctx, md.SourceFileMeta{ + rowCount, rowSize, _, err := md.SampleStatisticsFromParquet(ctx, md.SourceFileMeta{ Path: fileName, }, store) require.NoError(t, err) diff --git a/pkg/lightning/mydump/parquet_parser.go b/pkg/lightning/mydump/parquet_parser.go index 189ecc78a82e0..a5105cd3c8e35 100644 --- a/pkg/lightning/mydump/parquet_parser.go +++ b/pkg/lightning/mydump/parquet_parser.go @@ -15,570 +15,430 @@ package mydump import ( - "bytes" "context" - "encoding/binary" "fmt" "io" - "math/big" - "reflect" "strings" "time" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/schema" "github.com/pingcap/errors" "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/pkg/lightning/log" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/logutil" - "github.com/xitongsys/parquet-go/parquet" - preader "github.com/xitongsys/parquet-go/reader" - "github.com/xitongsys/parquet-go/source" + "github.com/pingcap/tidb/pkg/util/timeutil" + "github.com/pingcap/tidb/pkg/util/zeropool" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) const ( - batchReadRowSize = 32 + // defaultBufSize specifies the default size of skip buffer. + // Skip buffer is used when reading data from the cloud. If there is a gap between the current + // read position and the last read position, these data is stored in this buffer to avoid + // potentially reopening the underlying file when the gap size is less than the buffer size. + defaultBufSize = 64 * 1024 +) - // if a parquet if small than this threshold, parquet will load the whole file in a byte slice to - // optimize the read performance - smallParquetFileThreshold = 256 * 1024 * 1024 - // jan011970 is the date of unix epoch in julian day, - jan011970 = 2440588 - secPerDay = 24 * 60 * 60 +func estimateRowSize(row []types.Datum) int { + length := 0 + for _, v := range row { + if v.IsNull() { + continue + } + if v.Kind() == types.KindString { + length += len(v.GetBytes()) + } else { + length += 8 + } + } + return length +} - utcTimeLayout = "2006-01-02 15:04:05.999999Z" - timeLayout = "2006-01-02 15:04:05.999999" -) +type innerReader[T parquet.ColumnTypes] interface { + Type() parquet.Type + Descriptor() *schema.Column -// ParquetParser parses a parquet file for import -// It implements the Parser interface. -type ParquetParser struct { - Reader *preader.ParquetReader - columns []string - columnMetas []*parquet.SchemaElement - rows []any - readRows int64 - curStart int64 - curIndex int - lastRow Row - logger log.Logger - - readSeekCloser ReadSeekCloser -} - -// readerWrapper is a used for implement `source.ParquetFile` -type readerWrapper struct { - ReadSeekCloser - store storage.ExternalStorage - ctx context.Context - // current file path - path string + ReadBatchInPage(batchSize int64, values []T, defLvls, repLvls []int16) (int64, int, error) + HasNext() bool + + Close() error } -func (*readerWrapper) Write(_ []byte) (n int, err error) { - return 0, errors.New("unsupported operation") +type columnDumper interface { + Type() parquet.Type + SetReader(colReader file.ColumnChunkReader) + + Next(*types.Datum) bool + ReadNextBatch() int + + Close() error } -func (r *readerWrapper) Open(name string) (source.ParquetFile, error) { - if len(name) == 0 { - name = r.path - } - reader, err := r.store.Open(r.ctx, name, nil) - if err != nil { - return nil, errors.Trace(err) +type generalColumnDumper[T parquet.ColumnTypes, R innerReader[T]] struct { + reader R + batchSize int64 + valueOffset int + valuesBuffered int + + levelOffset int64 + levelsBuffered int64 + defLevels []int16 + repLevels []int16 + values []T + + closed bool + + setter setter[T] +} + +// newGeneralColumnDumper creates a new generic column dumper +func newGeneralColumnDumper[T parquet.ColumnTypes, R innerReader[T]]( + batchSize int, getter setter[T], +) *generalColumnDumper[T, R] { + return &generalColumnDumper[T, R]{ + batchSize: int64(batchSize), + defLevels: make([]int16, batchSize), + repLevels: make([]int16, batchSize), + values: make([]T, batchSize), + setter: getter, } - return &readerWrapper{ - ReadSeekCloser: reader, - store: r.store, - ctx: r.ctx, - path: name, - }, nil } -func (*readerWrapper) Create(_ string) (source.ParquetFile, error) { - return nil, errors.New("unsupported operation") +func (dump *generalColumnDumper[T, R]) SetReader(colReader file.ColumnChunkReader) { + dump.reader, _ = colReader.(R) } -// bytesReaderWrapper is a wrapper of bytes.Reader used for implement `source.ParquetFile` -type bytesReaderWrapper struct { - *bytes.Reader - rawBytes []byte - // current file path - path string +func (dump *generalColumnDumper[T, R]) Type() parquet.Type { + return dump.reader.Type() } -func (*bytesReaderWrapper) Close() error { - return nil +func (dump *generalColumnDumper[T, R]) Close() error { + if dump.closed { + return nil + } + + err := dump.reader.Close() + dump.closed = true + return err } -func (*bytesReaderWrapper) Create(_ string) (source.ParquetFile, error) { - return nil, errors.New("unsupported operation") +func (dump *generalColumnDumper[T, R]) ReadNextBatch() int { + // ReadBatchInPage reads a batch of values from the current page. + // And the values returned may be shallow copies from the internal page buffer. + //nolint: errcheck + dump.levelsBuffered, dump.valuesBuffered, _ = dump.reader.ReadBatchInPage( + dump.batchSize, + dump.values, + dump.defLevels, + dump.repLevels, + ) + + dump.valueOffset = 0 + dump.levelOffset = 0 + return int(dump.levelsBuffered) +} + +// Next reads the next value with proper level handling +func (dump *generalColumnDumper[T, R]) Next(d *types.Datum) bool { + if dump.levelOffset == dump.levelsBuffered { + if !dump.reader.HasNext() { + return false + } + dump.ReadNextBatch() + if dump.levelsBuffered == 0 { + return false + } + } + + // Check definition level for NULL handling + defLevel := dump.defLevels[dump.levelOffset] + dump.levelOffset++ + + if defLevel < dump.reader.Descriptor().MaxDefinitionLevel() { + d.SetNull() + return true + } + + value := dump.values[dump.valueOffset] + dump.valueOffset++ + dump.setter(value, d) + return true +} + +func createColumnDumper(tp parquet.Type, converted *convertedType, loc *time.Location, batchSize int) columnDumper { + switch tp { + case parquet.Types.Boolean: + return newGeneralColumnDumper[bool, *file.BooleanColumnChunkReader](batchSize, getBoolData) + case parquet.Types.Int32: + return newGeneralColumnDumper[int32, *file.Int32ColumnChunkReader](batchSize, getInt32Getter(converted, loc)) + case parquet.Types.Int64: + return newGeneralColumnDumper[int64, *file.Int64ColumnChunkReader](batchSize, getInt64Getter(converted, loc)) + case parquet.Types.Float: + return newGeneralColumnDumper[float32, *file.Float32ColumnChunkReader](batchSize, getFloat32Data) + case parquet.Types.Double: + return newGeneralColumnDumper[float64, *file.Float64ColumnChunkReader](batchSize, getFloat64Data) + case parquet.Types.Int96: + return newGeneralColumnDumper[parquet.Int96, *file.Int96ColumnChunkReader](batchSize, getInt96Getter(converted, loc)) + case parquet.Types.ByteArray: + return newGeneralColumnDumper[parquet.ByteArray, *file.ByteArrayColumnChunkReader](batchSize, getByteArrayGetter(converted)) + case parquet.Types.FixedLenByteArray: + return newGeneralColumnDumper[parquet.FixedLenByteArray, *file.FixedLenByteArrayColumnChunkReader](batchSize, getFixedLenByteArrayGetter(converted)) + default: + return nil + } } -func (*bytesReaderWrapper) Write(_ []byte) (n int, err error) { - return 0, errors.New("unsupported operation") +// convertedType is older representation of the logical type in parquet +// ref: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md +type convertedType struct { + converted schema.ConvertedType + decimalMeta schema.DecimalMetadata } -func (r *bytesReaderWrapper) Open(name string) (source.ParquetFile, error) { - if len(name) > 0 && name != r.path { - panic(fmt.Sprintf("Open with a different name is not supported! current: '%s', new: '%s'", r.path, name)) +// parquetFileWrapper is a wrapper for storage.ReadSeekCloser +// It implements io.ReaderAt interface to read parquet file using arrow-go. +type parquetFileWrapper struct { + ctx context.Context + + storage.ReadSeekCloser + lastOff int64 + skipBuf []byte + + // current file path and store, used to open file + store storage.ExternalStorage + path string +} + +func (pf *parquetFileWrapper) readNBytes(p []byte) (int, error) { + n, err := io.ReadFull(pf, p) + if err != nil && err != io.EOF { + return 0, errors.Trace(err) + } + if n != len(p) { + return n, errors.Errorf("Error reading %d bytes, only read %d bytes", len(p), n) } - return &bytesReaderWrapper{ - Reader: bytes.NewReader(r.rawBytes), - rawBytes: r.rawBytes, - path: r.path, - }, nil + return n, nil } -// OpenParquetReader opens a parquet file and returns a handle that can at least read the file. -func OpenParquetReader( - ctx context.Context, - store storage.ExternalStorage, - path string, - size int64, -) (source.ParquetFile, error) { - if size <= smallParquetFileThreshold { - fileBytes, err := store.ReadFile(ctx, path) - if err != nil { - return nil, err +// ReadAt implemement ReaderAt interface +func (pf *parquetFileWrapper) ReadAt(p []byte, off int64) (int, error) { + // We want to minimize the number of Seek call as much as possible, + // since the underlying reader may require reopening the file. + gap := int(off - pf.lastOff) + if gap < 0 || gap > cap(pf.skipBuf) { + if _, err := pf.Seek(off, io.SeekStart); err != nil { + return 0, err + } + } else { + pf.skipBuf = pf.skipBuf[:gap] + if read, err := pf.readNBytes(pf.skipBuf); err != nil { + return read, err } - return &bytesReaderWrapper{ - Reader: bytes.NewReader(fileBytes), - rawBytes: fileBytes, - path: path, - }, nil } - r, err := store.Open(ctx, path, nil) + read, err := pf.readNBytes(p) if err != nil { - return nil, err + return read, err } - return &readerWrapper{ - ReadSeekCloser: r, - store: store, - ctx: ctx, - path: path, - }, nil + pf.lastOff = off + int64(read) + + return len(p), nil } -// ReadParquetFileRowCountByFile reads the parquet file row count. -// It is a special func to fetch parquet file row count fast. -func ReadParquetFileRowCountByFile( - ctx context.Context, - store storage.ExternalStorage, - fileMeta SourceFileMeta, -) (int64, error) { - r, err := store.Open(ctx, fileMeta.Path, nil) +// Seek implemement Seeker interface +func (pf *parquetFileWrapper) Seek(offset int64, whence int) (int64, error) { + newOffset, err := pf.ReadSeekCloser.Seek(offset, whence) + pf.lastOff = newOffset + return newOffset, err +} + +func (*parquetFileWrapper) Write(_ []byte) (n int, err error) { + return 0, errors.New("unsupported operation") +} + +func (pf *parquetFileWrapper) Open() (parquet.ReaderAtSeekerOpener, error) { + reader, err := pf.store.Open(pf.ctx, pf.path, nil) if err != nil { - return 0, errors.Trace(err) + return nil, errors.Trace(err) } - wrapper := &readerWrapper{ - ReadSeekCloser: r, - store: store, - ctx: ctx, - path: fileMeta.Path, + newPf := &parquetFileWrapper{ + ReadSeekCloser: reader, + store: pf.store, + ctx: pf.ctx, + path: pf.path, + skipBuf: make([]byte, defaultBufSize), } + return newPf, nil +} - //nolint: errcheck - defer wrapper.Close() +// ParquetParser parses a parquet file for import +// It implements the Parser interface. +type ParquetParser struct { + reader *file.Reader + colMetas []convertedType + columnNames []string - res := new(preader.ParquetReader) - res.NP = 1 - res.PFile = wrapper - if err = res.ReadFooter(); err != nil { - return 0, err - } + alloc memory.Allocator + + dumpers []columnDumper + + rowPool *zeropool.Pool[[]types.Datum] + + curRowGroup int + totalRowGroup int + + curRowInGroup int // number of rows read in current group + totalRowsInGroup int // total rows in current group + totalRows int // total rows in this file + totalRowsRead int64 // total rows read + totalBytesRead int // total bytes read, estimated by all the read datum. - return res.Footer.NumRows, nil + lastRow Row + logger log.Logger } -// NewParquetParser generates a parquet parser. -func NewParquetParser( - ctx context.Context, - store storage.ExternalStorage, - r storage.ReadSeekCloser, - path string, -) (*ParquetParser, error) { - // check to avoid wrapping twice - wrapper, ok := r.(source.ParquetFile) - if !ok { - wrapper = &readerWrapper{ - ReadSeekCloser: r, - store: store, - ctx: ctx, - path: path, - } - } +// Init initializes the Parquet parser and allocate necessary buffers +func (pp *ParquetParser) Init(loc *time.Location) error { + meta := pp.reader.MetaData() - // FIXME: need to bench what the best value for the concurrent reader number - reader, err := preader.NewParquetReader(wrapper, nil, 2) - if err != nil { - return nil, errors.Trace(err) + pp.curRowGroup, pp.totalRowGroup, pp.totalRows = -1, pp.reader.NumRowGroups(), int(meta.NumRows) + + numCols := meta.Schema.NumColumns() + pp.dumpers = make([]columnDumper, numCols) + + if loc == nil { + loc = timeutil.SystemLocation() + } + for i := range numCols { + pp.dumpers[i] = createColumnDumper(meta.Schema.Column(i).PhysicalType(), &pp.colMetas[i], loc, 128) } - columns := make([]string, 0, len(reader.Footer.Schema)-1) - columnMetas := make([]*parquet.SchemaElement, 0, len(reader.Footer.Schema)-1) - for i, c := range reader.SchemaHandler.SchemaElements { - if c.GetNumChildren() == 0 { - // we need to use the raw name, SchemaElement.Name might be prefixed with PARGO_PERFIX_ - columns = append(columns, strings.ToLower(reader.SchemaHandler.GetExName(i))) - // transfer old ConvertedType to LogicalType - columnMeta := c - if c.ConvertedType != nil && c.LogicalType == nil { - newMeta := *c - columnMeta = &newMeta - if err := convertToLogicType(columnMeta); err != nil { - return nil, err - } - } - columnMetas = append(columnMetas, columnMeta) - } + return nil +} + +// resetReader is used to reclaim the memory used by the column reader. +func (pp *ParquetParser) resetReader() { + for _, d := range pp.dumpers { + //nolint: errcheck + d.Close() } +} - return &ParquetParser{ - Reader: reader, - columns: columns, - columnMetas: columnMetas, - logger: log.Wrap(logutil.Logger(ctx)), - readSeekCloser: wrapper, - }, nil -} - -func convertToLogicType(se *parquet.SchemaElement) error { - logicalType := &parquet.LogicalType{} - switch *se.ConvertedType { - case parquet.ConvertedType_UTF8: - logicalType.STRING = &parquet.StringType{} - case parquet.ConvertedType_ENUM: - logicalType.ENUM = &parquet.EnumType{} - case parquet.ConvertedType_DECIMAL: - logicalType.DECIMAL = &parquet.DecimalType{ - Scale: *se.Scale, - Precision: *se.Precision, - } - case parquet.ConvertedType_DATE: - logicalType.DATE = &parquet.DateType{} - case parquet.ConvertedType_TIME_MILLIS: - logicalType.TIME = &parquet.TimeType{ - IsAdjustedToUTC: true, - Unit: &parquet.TimeUnit{ - MILLIS: parquet.NewMilliSeconds(), - }, - } - case parquet.ConvertedType_TIME_MICROS: - logicalType.TIME = &parquet.TimeType{ - IsAdjustedToUTC: true, - Unit: &parquet.TimeUnit{ - MICROS: parquet.NewMicroSeconds(), - }, - } - case parquet.ConvertedType_TIMESTAMP_MILLIS: - logicalType.TIMESTAMP = &parquet.TimestampType{ - IsAdjustedToUTC: true, - Unit: &parquet.TimeUnit{ - MILLIS: parquet.NewMilliSeconds(), - }, - } - case parquet.ConvertedType_TIMESTAMP_MICROS: - logicalType.TIMESTAMP = &parquet.TimestampType{ - IsAdjustedToUTC: true, - Unit: &parquet.TimeUnit{ - MICROS: parquet.NewMicroSeconds(), - }, - } - case parquet.ConvertedType_UINT_8: - logicalType.INTEGER = &parquet.IntType{ - BitWidth: 8, - IsSigned: false, - } - case parquet.ConvertedType_UINT_16: - logicalType.INTEGER = &parquet.IntType{ - BitWidth: 16, - IsSigned: false, +// ReadRows read several rows internally and store them in the row buffer. +func (pp *ParquetParser) readSingleRows(row []types.Datum) error { + // Move to next row group + if pp.curRowInGroup == pp.totalRowsInGroup { + if pp.curRowGroup >= 0 { + pp.resetReader() } - case parquet.ConvertedType_UINT_32: - logicalType.INTEGER = &parquet.IntType{ - BitWidth: 32, - IsSigned: false, - } - case parquet.ConvertedType_UINT_64: - logicalType.INTEGER = &parquet.IntType{ - BitWidth: 64, - IsSigned: false, - } - case parquet.ConvertedType_INT_8: - logicalType.INTEGER = &parquet.IntType{ - BitWidth: 8, - IsSigned: true, - } - case parquet.ConvertedType_INT_16: - logicalType.INTEGER = &parquet.IntType{ - BitWidth: 16, - IsSigned: true, + pp.curRowGroup++ + if pp.curRowGroup >= pp.totalRowGroup { + return io.EOF } - case parquet.ConvertedType_INT_32: - logicalType.INTEGER = &parquet.IntType{ - BitWidth: 32, - IsSigned: true, + + rowGroup := pp.reader.RowGroup(pp.curRowGroup) + for c := range len(pp.dumpers) { + colReader, err := rowGroup.Column(c) + if err != nil { + return errors.Trace(err) + } + pp.dumpers[c].SetReader(colReader) } - case parquet.ConvertedType_INT_64: - logicalType.INTEGER = &parquet.IntType{ - BitWidth: 64, - IsSigned: true, + pp.curRowInGroup, pp.totalRowsInGroup = 0, int(pp.reader.MetaData().RowGroups[pp.curRowGroup].NumRows) + } + + // Read in this group + for col, dumper := range pp.dumpers { + if ok := dumper.Next(&row[col]); !ok { + return errors.New("error get data") } - case parquet.ConvertedType_JSON: - logicalType.JSON = &parquet.JsonType{} - case parquet.ConvertedType_BSON: - logicalType.BSON = &parquet.BsonType{} - // case parquet.ConvertedType_INTERVAL, parquet.ConvertedType_MAP, parquet.ConvertedType_MAP_KEY_VALUE, parquet.ConvertedType_LIST: - default: - return errors.Errorf("unsupported type: '%s'", *se.ConvertedType) } - se.LogicalType = logicalType + + pp.totalBytesRead += estimateRowSize(row) + pp.totalRowsRead++ + pp.curRowInGroup++ return nil } // Pos returns the currently row number of the parquet file func (pp *ParquetParser) Pos() (pos int64, rowID int64) { - return pp.curStart + int64(pp.curIndex), pp.lastRow.RowID + return int64(pp.totalRowsRead), pp.lastRow.RowID } -// SetPos sets the position in a parquet file. -// It implements the Parser interface. +// SetPos implements the Parser interface. +// For parquet file, this interface will read and discard the first `pos` rows, +// and set the current row ID to `rowID` func (pp *ParquetParser) SetPos(pos int64, rowID int64) error { - if pos < pp.curStart { - panic("don't support seek back yet") - } pp.lastRow.RowID = rowID - if pos < pp.curStart+int64(len(pp.rows)) { - pp.curIndex = int(pos - pp.curStart) - pp.readRows = pos - return nil - } + row := pp.rowPool.Get() + defer pp.rowPool.Put(row) - if pos > pp.curStart+int64(len(pp.rows)) { - if err := pp.Reader.SkipRows(pos - pp.curStart - int64(len(pp.rows))); err != nil { - return errors.Trace(err) + // TODO(joechenrh): skip rows use underlying SkipRow interface + // For now it's ok, since only UTs use this interface + for range pos { + if err := pp.readSingleRows(row); err != nil { + return err } } - pp.curStart = pos - pp.readRows = pos - pp.curIndex = 0 - if len(pp.rows) > 0 { - pp.rows = pp.rows[:0] - } return nil } // ScannedPos implements the Parser interface. -// For parquet it's parquet file's reader current position. +// For parquet we use the size of all read datum to estimate the scanned position. func (pp *ParquetParser) ScannedPos() (int64, error) { - return pp.readSeekCloser.Seek(0, io.SeekCurrent) + return int64(pp.totalBytesRead), nil } // Close closes the parquet file of the parser. // It implements the Parser interface. func (pp *ParquetParser) Close() error { - pp.Reader.ReadStop() - return pp.Reader.PFile.Close() -} - -// ReadRow reads a row in the parquet file by the parser. -// It implements the Parser interface. -func (pp *ParquetParser) ReadRow() error { - pp.lastRow.RowID++ - pp.lastRow.Length = 0 - if pp.curIndex >= len(pp.rows) { - if pp.readRows >= pp.Reader.GetNumRows() { - return io.EOF - } - count := batchReadRowSize - if pp.Reader.GetNumRows()-pp.readRows < int64(count) { - count = int(pp.Reader.GetNumRows() - pp.readRows) - } - - var err error - pp.rows, err = pp.Reader.ReadByNumber(count) - if err != nil { - return errors.Trace(err) + defer func() { + if a, ok := pp.alloc.(interface{ Close() }); ok { + a.Close() } - pp.curStart = pp.readRows - pp.readRows += int64(len(pp.rows)) - pp.curIndex = 0 - } - - row := pp.rows[pp.curIndex] - pp.curIndex++ + }() - v := reflect.ValueOf(row) - length := v.NumField() - if cap(pp.lastRow.Row) < length { - pp.lastRow.Row = make([]types.Datum, length) - } else { - pp.lastRow.Row = pp.lastRow.Row[:length] - } - for i := range length { - pp.lastRow.Length += getDatumLen(v.Field(i)) - if err := setDatumValue(&pp.lastRow.Row[i], v.Field(i), pp.columnMetas[i], pp.logger); err != nil { - return err - } - } - return nil + pp.logger.Info("[parquet parser test] Close parquet parser") + pp.resetReader() + return pp.reader.Close() } -func getDatumLen(v reflect.Value) int { - if v.Kind() == reflect.Ptr { - if v.IsNil() { - return 0 - } - return getDatumLen(v.Elem()) - } - if v.Kind() == reflect.String { - return len(v.String()) +func (pp *ParquetParser) ReadRow() error { + if err := pp.ReadRowUnsafe(); err != nil { + return err } - return 8 -} - -// convert a parquet value to Datum -// -// See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md -func setDatumValue(d *types.Datum, v reflect.Value, meta *parquet.SchemaElement, logger log.Logger) error { - switch v.Kind() { - case reflect.Bool: - if v.Bool() { - d.SetUint64(1) - } else { - d.SetUint64(0) - } - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - d.SetUint64(v.Uint()) - case reflect.Int8, reflect.Int16: - d.SetInt64(v.Int()) - case reflect.Int32, reflect.Int64: - return setDatumByInt(d, v.Int(), meta) - case reflect.String: - setDatumByString(d, v.String(), meta) - case reflect.Float32, reflect.Float64: - d.SetFloat64(v.Float()) - case reflect.Ptr: - if !v.IsNil() { - return setDatumValue(d, v.Elem(), meta, logger) - } - d.SetNull() - default: - logger.Error("unknown value", zap.Stringer("kind", v.Kind()), - zap.String("type", v.Type().Name()), zap.Reflect("value", v.Interface())) - return errors.Errorf("unknown value: %v", v) + for i, d := range pp.lastRow.Row { + pp.lastRow.Row[i] = *d.Clone() } return nil } -func setDatumByString(d *types.Datum, v string, meta *parquet.SchemaElement) { - if meta.LogicalType != nil && meta.LogicalType.DECIMAL != nil { - v = binaryToDecimalStr([]byte(v), int(meta.LogicalType.DECIMAL.Scale)) - } - if meta.Type != nil && *meta.Type == parquet.Type_INT96 && len(v) == 96/8 { - ts := int96ToTime([]byte(v)) - ts = ts.UTC() - v = ts.Format(utcTimeLayout) - } - d.SetString(v, "utf8mb4_bin") -} - -func binaryToDecimalStr(rawBytes []byte, scale int) string { - negative := rawBytes[0] > 127 - if negative { - for i := range rawBytes { - rawBytes[i] = ^rawBytes[i] - } - for i := len(rawBytes) - 1; i >= 0; i-- { - rawBytes[i]++ - if rawBytes[i] != 0 { - break - } - } - } +// ReadRowUnsafe reads a row in the parquet file by the parser. +// It implements the Parser interface. +// Return io.EOF if reaching the end of the file. +func (pp *ParquetParser) ReadRowUnsafe() error { + pp.lastRow.RowID++ + pp.lastRow.Length = 0 - intValue := big.NewInt(0) - intValue = intValue.SetBytes(rawBytes) - val := fmt.Sprintf("%0*d", scale, intValue) - dotIndex := len(val) - scale - var res strings.Builder - if negative { - res.WriteByte('-') - } - if dotIndex == 0 { - res.WriteByte('0') - } else { - res.WriteString(val[:dotIndex]) + row := pp.rowPool.Get() + if err := pp.readSingleRows(row); err != nil { + pp.rowPool.Put(row) + return err } - if scale > 0 { - res.WriteByte('.') - res.WriteString(val[dotIndex:]) - } - return res.String() -} -// when the value type is int32/int64, convert to value to target logical type in tidb -func setDatumByInt(d *types.Datum, v int64, meta *parquet.SchemaElement) error { - if meta.ConvertedType == nil && meta.LogicalType == nil { - d.SetInt64(v) - return nil - } - - logicalType := meta.LogicalType - switch { - case logicalType.DECIMAL != nil: - if logicalType.DECIMAL.Scale == 0 { - d.SetInt64(v) - return nil - } - minLen := logicalType.DECIMAL.Scale + 1 - if v < 0 { - minLen++ - } - val := fmt.Sprintf("%0*d", minLen, v) - dotIndex := len(val) - int(*meta.Scale) - d.SetString(val[:dotIndex]+"."+val[dotIndex:], "utf8mb4_bin") - case logicalType.DATE != nil: - dateStr := time.Unix(v*86400, 0).Format(time.DateOnly) - d.SetString(dateStr, "utf8mb4_bin") - case logicalType.TIMESTAMP != nil: - // convert all timestamp types (datetime/timestamp) to string - timeStr := formatTime(v, logicalType.TIMESTAMP.Unit, timeLayout, - utcTimeLayout, logicalType.TIMESTAMP.IsAdjustedToUTC) - d.SetString(timeStr, "utf8mb4_bin") - case logicalType.TIME != nil: - // convert all timestamp types (datetime/timestamp) to string - timeStr := formatTime(v, logicalType.TIME.Unit, "15:04:05.999999", "15:04:05.999999Z", - logicalType.TIME.IsAdjustedToUTC) - d.SetString(timeStr, "utf8mb4_bin") - default: - d.SetInt64(v) - } + pp.lastRow.Row = row + pp.lastRow.Length = estimateRowSize(row) return nil } -func formatTime(v int64, units *parquet.TimeUnit, format, utcFormat string, utc bool) string { - var t time.Time - if units.MICROS != nil { - t = time.UnixMicro(v) - } else if units.MILLIS != nil { - t = time.UnixMilli(v) - } else { - // nano - t = time.Unix(0, v) - } - t = t.UTC() - if utc { - return t.Format(utcFormat) - } - return t.Format(format) -} - // LastRow gets the last row parsed by the parser. // It implements the Parser interface. func (pp *ParquetParser) LastRow() Row { @@ -586,13 +446,14 @@ func (pp *ParquetParser) LastRow() Row { } // RecycleRow implements the Parser interface. -func (*ParquetParser) RecycleRow(_ Row) { +func (pp *ParquetParser) RecycleRow(row Row) { + pp.rowPool.Put(row.Row) } // Columns returns the _lower-case_ column names corresponding to values in // the LastRow. func (pp *ParquetParser) Columns() []string { - return pp.columns + return pp.columnNames } // SetColumns set restored column names to parser @@ -612,26 +473,196 @@ func (pp *ParquetParser) SetRowID(rowID int64) { pp.lastRow.RowID = rowID } -func jdToTime(jd int32, nsec int64) time.Time { - sec := int64(jd-jan011970) * secPerDay - // it's fine not to check the value of nsec - // because it's legall even though it exceeds the maximum. - // See TestNsecOutSideRange. - return time.Unix(sec, nsec) -} - -// FYI: https://github.com/apache/spark/blob/d66a4e82eceb89a274edeb22c2fb4384bed5078b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala#L171-L178 -// INT96 timestamp layout -// -------------------------- -// | 64 bit | 32 bit | -// --------------------------- -// | nano sec | julian day | -// --------------------------- -// NOTE: parquet date can be less than 1970-01-01 that is not supported by TiDB, -// where dt is a negative number but still legal in the context of Go. -// But it will cause errors or potential data inconsistency when importing. -func int96ToTime(parquetDate []byte) time.Time { - nano := binary.LittleEndian.Uint64(parquetDate[:8]) - dt := binary.LittleEndian.Uint32(parquetDate[8:]) - return jdToTime(int32(dt), int64(nano)) +// OpenParquetReader opens a parquet file and returns a handle that can at least read the file. +func OpenParquetReader( + ctx context.Context, + store storage.ExternalStorage, + path string, +) (storage.ReadSeekCloser, error) { + r, err := store.Open(ctx, path, nil) + if err != nil { + return nil, err + } + + pf := &parquetFileWrapper{ + ReadSeekCloser: r, + store: store, + ctx: ctx, + path: path, + skipBuf: make([]byte, defaultBufSize), + } + return pf, nil +} + +// ReadParquetFileRowCountByFile reads the parquet file row count through fileMeta. +func ReadParquetFileRowCountByFile( + ctx context.Context, + store storage.ExternalStorage, + fileMeta SourceFileMeta, +) (int64, error) { + r, err := store.Open(ctx, fileMeta.Path, nil) + if err != nil { + return 0, errors.Trace(err) + } + + reader, err := file.NewParquetReader(&parquetFileWrapper{ReadSeekCloser: r}) + if err != nil { + return 0, errors.Trace(err) + } + + return reader.MetaData().NumRows, nil +} + +// GetDefaultParquetMeta return a default file meta +func GetDefaultParquetMeta() ParquetFileMeta { + return ParquetFileMeta{ + MemoryUsage: 0, + } +} + +// NewParquetParser generates a parquet parser. +func NewParquetParser( + ctx context.Context, + store storage.ExternalStorage, + r storage.ReadSeekCloser, + path string, + meta ParquetFileMeta, +) (*ParquetParser, error) { + logger := log.Wrap(logutil.Logger(ctx)) + logger.Info("Get memory usage of parquet reader", + zap.String("file", path), + zap.String("memory usage", fmt.Sprintf("%d MB", meta.MemoryUsage>>20)), + ) + + workerPool := &errgroup.Group{} + workerPool.SetLimit(8) + + wrapper, ok := r.(*parquetFileWrapper) + if !ok { + wrapper = &parquetFileWrapper{ + ReadSeekCloser: r, + store: store, + ctx: ctx, + path: path, + skipBuf: make([]byte, defaultBufSize), + } + } + + var allocator memory.Allocator + allocator = memory.NewGoAllocator() + if meta.MemoryPool != nil { + allocator = NewAppendOnlyAllocator(meta.MemoryPool, meta.MemoryUsage) + } + + // allocator := memory.NewGoAllocator() + prop := parquet.NewReaderProperties(allocator) + prop.BufferedStreamEnabled = true + prop.BufferSize = 1024 + + reader, err := file.NewParquetReader(wrapper, file.WithReadProps(prop), file.WithPrefetch(8)) + if err != nil { + return nil, errors.Trace(err) + } + + fileSchema := reader.MetaData().Schema + columnMetas := make([]convertedType, fileSchema.NumColumns()) + columnNames := make([]string, 0, fileSchema.NumColumns()) + + for i := range columnMetas { + desc := reader.MetaData().Schema.Column(i) + columnNames = append(columnNames, strings.ToLower(desc.Name())) + + logicalType := desc.LogicalType() + if logicalType.IsValid() { + columnMetas[i].converted, columnMetas[i].decimalMeta = logicalType.ToConvertedType() + } else { + columnMetas[i].converted = desc.ConvertedType() + pnode, _ := desc.SchemaNode().(*schema.PrimitiveNode) + columnMetas[i].decimalMeta = pnode.DecimalMetadata() + } + } + + numColumns := len(columnMetas) + pool := zeropool.New(func() []types.Datum { + return make([]types.Datum, numColumns) + }) + + parser := &ParquetParser{ + reader: reader, + colMetas: columnMetas, + columnNames: columnNames, + alloc: allocator, + logger: logger, + rowPool: &pool, + } + if err := parser.Init(meta.Loc); err != nil { + return nil, errors.Trace(err) + } + + return parser, nil +} + +// SampleStatisticsFromParquet samples row size and memory usage of the parquet file. +func SampleStatisticsFromParquet( + ctx context.Context, + fileMeta SourceFileMeta, + store storage.ExternalStorage, +) ( + rowCount int64, + avgRowSize float64, + memoryUsage int, + err error, +) { + r, err := store.Open(ctx, fileMeta.Path, nil) + if err != nil { + return 0, 0, 0, err + } + + meta := GetDefaultParquetMeta() + meta.MemoryPool = GetPool(10 << 30) // use up to 4GiB memory for sampling + + parser, err := NewParquetParser(ctx, store, r, fileMeta.Path, meta) + if err != nil { + return 0, 0, 0, err + } + + //nolint: errcheck + defer parser.Close() + + var rowSize int64 + + reader := parser.reader + if reader.NumRowGroups() == 0 || reader.MetaData().RowGroups[0].NumRows == 0 { + return 0, 0, 0, nil + } + + totalReadRows := reader.MetaData().RowGroups[0].NumRows + for range totalReadRows { + err = parser.ReadRowUnsafe() + if err != nil { + if errors.Cause(err) == io.EOF { + break + } + return 0, 0, 0, err + } + lastRow := parser.LastRow() + rowSize += int64(lastRow.Length) + parser.RecycleRow(lastRow) + rowCount++ + } + + avgRowSize = float64(rowSize) / float64(rowCount) + + alloc := parser.alloc + a, ok := alloc.(*appendOnlyAllocator) + + if ok { + memoryUsage = a.Allocated() + parser.logger.Info("Get memory usage of parquet reader", + zap.String("memory usage", fmt.Sprintf("%d MB", memoryUsage>>20)), + ) + return rowCount, avgRowSize, memoryUsage, err + } + + return rowCount, avgRowSize, 128 << 20, err } diff --git a/pkg/lightning/mydump/parquet_parser_test.go b/pkg/lightning/mydump/parquet_parser_test.go index 66acfa657c591..212b54643b96a 100644 --- a/pkg/lightning/mydump/parquet_parser_test.go +++ b/pkg/lightning/mydump/parquet_parser_test.go @@ -17,49 +17,60 @@ package mydump import ( "context" "io" - "path/filepath" "strconv" "testing" "time" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/schema" "github.com/pingcap/tidb/br/pkg/storage" "github.com/pingcap/tidb/pkg/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/xitongsys/parquet-go-source/local" - writer2 "github.com/xitongsys/parquet-go/writer" ) func TestParquetParser(t *testing.T) { - type Test struct { - S string `parquet:"name=sS, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` - A int32 `parquet:"name=a_A, type=INT32"` + pc := []ParquetColumn{ + { + Name: "sS", + Type: parquet.Types.ByteArray, + Converted: schema.ConvertedTypes.UTF8, + Gen: func(numRows int) (any, []int16) { + defLevel := make([]int16, numRows) + data := make([]parquet.ByteArray, numRows) + for i := range numRows { + s := strconv.Itoa(i) + data[i] = parquet.ByteArray(s) + defLevel[i] = 1 + } + return data, defLevel + }, + }, + { + Name: "a_A", + Type: parquet.Types.Int64, + Converted: schema.ConvertedTypes.Int64, + Gen: func(numRows int) (any, []int16) { + defLevel := make([]int16, numRows) + data := make([]int64, numRows) + for i := range numRows { + data[i] = int64(i) + defLevel[i] = 1 + } + return data, defLevel + }, + }, } dir := t.TempDir() - // prepare data name := "test123.parquet" - testPath := filepath.Join(dir, name) - pf, err := local.NewLocalFileWriter(testPath) - require.NoError(t, err) - test := &Test{} - writer, err := writer2.NewParquetWriter(pf, test, 2) - require.NoError(t, err) - - for i := range 100 { - test.A = int32(i) - test.S = strconv.Itoa(i) - require.NoError(t, writer.Write(test)) - } - - require.NoError(t, writer.WriteStop()) - require.NoError(t, pf.Close()) + WriteParquetFile(dir, name, pc, 100) store, err := storage.NewLocalStorage(dir) require.NoError(t, err) - r, err := store.Open(context.TODO(), name, nil) + r, err := store.Open(context.Background(), name, nil) require.NoError(t, err) - reader, err := NewParquetParser(context.TODO(), store, r, name) + reader, err := NewParquetParser(context.Background(), store, r, name, GetDefaultParquetMeta()) require.NoError(t, err) defer reader.Close() @@ -94,53 +105,103 @@ func TestParquetParser(t *testing.T) { } func TestParquetVariousTypes(t *testing.T) { - type Test struct { - Date int32 `parquet:"name=date, type=INT32, convertedtype=DATE"` - TimeMillis int32 `parquet:"name=timemillis, type=INT32, isadjustedtoutc=true, convertedtype=TIME_MILLIS"` - TimeMicros int64 `parquet:"name=timemicros, type=INT64, isadjustedtoutc=true, convertedtype=TIME_MICROS"` - TimestampMillis int64 `parquet:"name=timestampmillis, type=INT64, isadjustedtoutc=true, convertedtype=TIMESTAMP_MILLIS"` - TimestampMicros int64 `parquet:"name=timestampmicros, type=INT64, isadjustedtoutc=true, convertedtype=TIMESTAMP_MICROS"` - - Decimal1 int32 `parquet:"name=decimal1, type=INT32, convertedtype=DECIMAL, scale=2, precision=9"` - Decimal2 int32 `parquet:"name=decimal2, type=INT32, convertedtype=DECIMAL, scale=4, precision=4"` - Decimal3 int64 `parquet:"name=decimal3, type=INT64, convertedtype=DECIMAL, scale=2, precision=18"` - Decimal6 int32 `parquet:"name=decimal6, type=INT32, convertedtype=DECIMAL, scale=4, precision=4"` + pc := []ParquetColumn{ + { + Name: "date", + Type: parquet.Types.Int32, + Converted: schema.ConvertedTypes.Date, + Gen: func(_ int) (any, []int16) { + return []int32{18564}, []int16{1} // 2020-10-29 + }, + }, + { + Name: "timemillis", + Type: parquet.Types.Int32, + Converted: schema.ConvertedTypes.TimeMillis, + Gen: func(_ int) (any, []int16) { + return []int32{62775123}, []int16{1} // 17:26:15.123 + }, + }, + { + Name: "timemicros", + Type: parquet.Types.Int64, + Converted: schema.ConvertedTypes.TimeMicros, + Gen: func(_ int) (any, []int16) { + return []int64{62775123456}, []int16{1} // 17:26:15.123456 + }, + }, + { + Name: "timestampmillis", + Type: parquet.Types.Int64, + Converted: schema.ConvertedTypes.TimestampMillis, + Gen: func(_ int) (any, []int16) { + return []int64{1603963672356}, []int16{1} // 2020-10-29T09:27:52.356Z + }, + }, + { + Name: "timestampmicros", + Type: parquet.Types.Int64, + Converted: schema.ConvertedTypes.TimestampMicros, + Gen: func(_ int) (any, []int16) { + return []int64{1603963672356956}, []int16{1} // 2020-10-29T09:27:52.356956Z + }, + }, + { + Name: "decimal1", + Type: parquet.Types.Int32, + Converted: schema.ConvertedTypes.Decimal, + Precision: 9, + Scale: 2, + Gen: func(_ int) (any, []int16) { + return []int32{-12345678}, []int16{1} // -123456.78, + }, + }, + { + Name: "decimal2", + Type: parquet.Types.Int32, + Converted: schema.ConvertedTypes.Decimal, + Precision: 4, + Scale: 4, + Gen: func(_ int) (any, []int16) { + return []int32{456}, []int16{1} // 0.0456 + }, + }, + { + Name: "decimal3", + Type: parquet.Types.Int64, + Converted: schema.ConvertedTypes.Decimal, + Precision: 18, + Scale: 2, + Gen: func(_ int) (any, []int16) { + return []int64{123456789012345678}, []int16{1} // 1234567890123456.78, + }, + }, + { + Name: "decimal6", + Type: parquet.Types.Int32, + Converted: schema.ConvertedTypes.Decimal, + Precision: 4, + Scale: 4, + Gen: func(_ int) (any, []int16) { + return []int32{-1}, []int16{1} // -0.0001 + }, + }, } dir := t.TempDir() // prepare data name := "test123.parquet" - testPath := filepath.Join(dir, name) - pf, err := local.NewLocalFileWriter(testPath) - require.NoError(t, err) - test := &Test{} - writer, err := writer2.NewParquetWriter(pf, test, 2) - require.NoError(t, err) - - v := &Test{ - Date: 18564, // 2020-10-29 - TimeMillis: 62775123, // 17:26:15.123 - TimeMicros: 62775123456, // 17:26:15.123 - TimestampMillis: 1603963672356, // 2020-10-29T09:27:52.356Z - TimestampMicros: 1603963672356956, // 2020-10-29T09:27:52.356956Z - Decimal1: -12345678, // -123456.78 - Decimal2: 456, // 0.0456 - Decimal3: 123456789012345678, // 1234567890123456.78 - Decimal6: -1, // -0.0001 - } - require.NoError(t, writer.Write(v)) - require.NoError(t, writer.WriteStop()) - require.NoError(t, pf.Close()) + WriteParquetFile(dir, name, pc, 1) store, err := storage.NewLocalStorage(dir) require.NoError(t, err) r, err := store.Open(context.TODO(), name, nil) require.NoError(t, err) - reader, err := NewParquetParser(context.TODO(), store, r, name) + reader, err := NewParquetParser(context.TODO(), store, r, name, GetDefaultParquetMeta()) require.NoError(t, err) defer reader.Close() - require.Len(t, reader.columns, 9) + require.Len(t, reader.columnNames, 9) require.NoError(t, reader.ReadRow()) rowValue := []string{ @@ -154,9 +215,27 @@ func TestParquetVariousTypes(t *testing.T) { assert.Equal(t, row[i].GetString(), rowValue[i]) } - type TestDecimal struct { - Decimal1 int32 `parquet:"name=decimal1, type=INT32, convertedtype=DECIMAL, scale=3, precision=5"` - DecimalRef *int32 `parquet:"name=decimal2, type=INT32, convertedtype=DECIMAL, scale=3, precision=5"` + pc = []ParquetColumn{ + { + Name: "decimal1", + Type: parquet.Types.Int32, + Converted: schema.ConvertedTypes.Decimal, + Precision: 5, + Scale: 3, + Gen: func(_ int) (any, []int16) { + return []int32{0, 1000, int32(-1000), 999, int32(-999), 1, int32(-1)}, []int16{1, 1, 1, 1, 1, 1, 1} + }, + }, + { + Name: "decimal2", + Type: parquet.Types.Int32, + Converted: schema.ConvertedTypes.Decimal, + Precision: 5, + Scale: 3, + Gen: func(_ int) (any, []int16) { + return []int32{0, int32(-1000), int32(-999), int32(-1), 0, 0, 0}, []int16{1, 0, 1, 0, 1, 0, 1} + }, + }, } cases := [][]any{ @@ -170,29 +249,11 @@ func TestParquetVariousTypes(t *testing.T) { } fileName := "test.02.parquet" - testPath = filepath.Join(dir, fileName) - pf, err = local.NewLocalFileWriter(testPath) - td := &TestDecimal{} - require.NoError(t, err) - writer, err = writer2.NewParquetWriter(pf, td, 2) - require.NoError(t, err) - for i, testCase := range cases { - val, ok := testCase[0].(int32) - require.True(t, ok) - td.Decimal1 = val - if i%2 == 0 { - td.DecimalRef = &val - } else { - td.DecimalRef = nil - } - assert.NoError(t, writer.Write(td)) - } - require.NoError(t, writer.WriteStop()) - require.NoError(t, pf.Close()) + WriteParquetFile(dir, fileName, pc, 7) r, err = store.Open(context.TODO(), fileName, nil) require.NoError(t, err) - reader, err = NewParquetParser(context.TODO(), store, r, fileName) + reader, err = NewParquetParser(context.TODO(), store, r, fileName, GetDefaultParquetMeta()) require.NoError(t, err) defer reader.Close() @@ -215,24 +276,23 @@ func TestParquetVariousTypes(t *testing.T) { } } - type TestBool struct { - BoolVal bool `parquet:"name=bool_val, type=BOOLEAN"` + pc = []ParquetColumn{ + { + Name: "bool_val", + Type: parquet.Types.Boolean, + Converted: schema.ConvertedTypes.None, + Gen: func(_ int) (any, []int16) { + return []bool{false, true}, []int16{1, 1} + }, + }, } fileName = "test.bool.parquet" - testPath = filepath.Join(dir, fileName) - pf, err = local.NewLocalFileWriter(testPath) - require.NoError(t, err) - writer, err = writer2.NewParquetWriter(pf, new(TestBool), 2) - require.NoError(t, err) - require.NoError(t, writer.Write(&TestBool{false})) - require.NoError(t, writer.Write(&TestBool{true})) - require.NoError(t, writer.WriteStop()) - require.NoError(t, pf.Close()) + WriteParquetFile(dir, fileName, pc, 2) r, err = store.Open(context.TODO(), fileName, nil) require.NoError(t, err) - reader, err = NewParquetParser(context.TODO(), store, r, fileName) + reader, err = NewParquetParser(context.TODO(), store, r, fileName, GetDefaultParquetMeta()) require.NoError(t, err) defer reader.Close() @@ -253,7 +313,7 @@ func TestParquetAurora(t *testing.T) { fileName := "test.parquet" r, err := store.Open(context.TODO(), fileName, nil) require.NoError(t, err) - parser, err := NewParquetParser(context.TODO(), store, r, fileName) + parser, err := NewParquetParser(context.TODO(), store, r, fileName, GetDefaultParquetMeta()) require.NoError(t, err) require.Equal(t, []string{"id", "val1", "val2", "d1", "d2", "d3", "d4", "d5", "d6"}, parser.Columns()) @@ -310,7 +370,7 @@ func TestHiveParquetParser(t *testing.T) { require.NoError(t, err) r, err := store.Open(context.TODO(), name, nil) require.NoError(t, err) - reader, err := NewParquetParser(context.TODO(), store, r, name) + reader, err := NewParquetParser(context.TODO(), store, r, name, GetDefaultParquetMeta()) require.NoError(t, err) defer reader.Close() // UTC+0:00 @@ -327,8 +387,8 @@ func TestHiveParquetParser(t *testing.T) { require.NoError(t, err) lastRow := reader.LastRow() require.Equal(t, 2, len(lastRow.Row)) - require.Equal(t, types.KindString, lastRow.Row[1].Kind()) - ts, err := time.Parse(utcTimeLayout, lastRow.Row[1].GetString()) + require.Equal(t, types.KindMysqlTime, lastRow.Row[1].Kind()) + ts, err := lastRow.Row[1].GetMysqlTime().GoTime(time.UTC) require.NoError(t, err) require.Equal(t, results[i], ts) } diff --git a/pkg/lightning/mydump/parquet_type_converter.go b/pkg/lightning/mydump/parquet_type_converter.go new file mode 100644 index 0000000000000..cdc116921ca9a --- /dev/null +++ b/pkg/lightning/mydump/parquet_type_converter.go @@ -0,0 +1,227 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mydump + +import ( + "fmt" + "math/big" + "strings" + "time" + + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/schema" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" +) + +type setter[T parquet.ColumnTypes] func(T, *types.Datum) + +func binaryToDecimalStr(rawBytes []byte, scale int) string { + negative := rawBytes[0] > 127 + if negative { + for i := range rawBytes { + rawBytes[i] = ^rawBytes[i] + } + for i := len(rawBytes) - 1; i >= 0; i-- { + rawBytes[i]++ + if rawBytes[i] != 0 { + break + } + } + } + + intValue := big.NewInt(0) + intValue = intValue.SetBytes(rawBytes) + val := fmt.Sprintf("%0*d", scale, intValue) + dotIndex := len(val) - scale + var res strings.Builder + if negative { + res.WriteByte('-') + } + if dotIndex == 0 { + res.WriteByte('0') + } else { + res.WriteString(val[:dotIndex]) + } + if scale > 0 { + res.WriteByte('.') + res.WriteString(val[dotIndex:]) + } + return res.String() +} + +func getBoolData(val bool, d *types.Datum) { + if val { + d.SetUint64(1) + } else { + d.SetUint64(0) + } +} + +func getDecimalFromIntImpl(val int64, d *types.Datum, converted *convertedType) { + decimal := converted.decimalMeta + if !decimal.IsSet || decimal.Scale == 0 { + d.SetInt64(val) + } + + minLen := decimal.Scale + 1 + if val < 0 { + minLen++ + } + v := fmt.Sprintf("%0*d", minLen, val) + dotIndex := len(v) - int(decimal.Scale) + d.SetString(v[:dotIndex]+"."+v[dotIndex:], "utf8mb4_bin") +} + +func getInt32Getter(converted *convertedType, loc *time.Location) setter[int32] { + switch converted.converted { + case schema.ConvertedTypes.Decimal: + return func(val int32, d *types.Datum) { + getDecimalFromIntImpl(int64(val), d, converted) + } + case schema.ConvertedTypes.Date: + return func(val int32, d *types.Datum) { + // Convert days since Unix epoch to time.Time + t := time.Unix(int64(val)*86400, 0).In(loc) + mysqlTime := types.NewTime(types.FromGoTime(t), mysql.TypeDate, 0) + d.SetMysqlTime(mysqlTime) + } + case schema.ConvertedTypes.TimeMillis: + return func(val int32, d *types.Datum) { + // Convert milliseconds to time.Time + t := time.UnixMilli(int64(val)).In(loc) + mysqlTime := types.NewTime(types.FromGoTime(t), mysql.TypeTimestamp, 0) + d.SetMysqlTime(mysqlTime) + } + case schema.ConvertedTypes.Int32, schema.ConvertedTypes.Uint32, + schema.ConvertedTypes.Int16, schema.ConvertedTypes.Uint16, + schema.ConvertedTypes.Int8, schema.ConvertedTypes.Uint8, + schema.ConvertedTypes.None: + return func(val int32, d *types.Datum) { + d.SetInt64(int64(val)) + } + } + + return nil +} + +func getInt64Getter(converted *convertedType, loc *time.Location) setter[int64] { + switch converted.converted { + case schema.ConvertedTypes.Uint64, + schema.ConvertedTypes.Uint32, schema.ConvertedTypes.Int32, + schema.ConvertedTypes.Uint16, schema.ConvertedTypes.Int16, + schema.ConvertedTypes.Uint8, schema.ConvertedTypes.Int8: + return func(val int64, d *types.Datum) { + d.SetUint64(uint64(val)) + } + case schema.ConvertedTypes.None, schema.ConvertedTypes.Int64: + return func(val int64, d *types.Datum) { + d.SetInt64(int64(val)) + } + case schema.ConvertedTypes.TimeMicros: + return func(val int64, d *types.Datum) { + // Convert microseconds to time.Time + t := time.UnixMicro(val).In(loc) + mysqlTime := types.NewTime(types.FromGoTime(t), mysql.TypeTimestamp, 0) + d.SetMysqlTime(mysqlTime) + } + case schema.ConvertedTypes.TimestampMillis: + return func(val int64, d *types.Datum) { + // Convert milliseconds to time.Time + t := time.UnixMilli(val).In(loc) + mysqlTime := types.NewTime(types.FromGoTime(t), mysql.TypeTimestamp, 0) + d.SetMysqlTime(mysqlTime) + } + case schema.ConvertedTypes.TimestampMicros: + return func(val int64, d *types.Datum) { + // Convert microseconds to time.Time + t := time.UnixMicro(val).In(loc) + mysqlTime := types.NewTime(types.FromGoTime(t), mysql.TypeTimestamp, 0) + d.SetMysqlTime(mysqlTime) + } + case schema.ConvertedTypes.Decimal: + return func(val int64, d *types.Datum) { + getDecimalFromIntImpl(val, d, converted) + } + } + + return nil +} + +func getInt96Data(val parquet.Int96, d *types.Datum, loc *time.Location) { + // FYI: https://github.com/apache/spark/blob/d66a4e82eceb89a274edeb22c2fb4384bed5078b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetWriteSupport.scala#L171-L178 + // INT96 timestamp layout + // -------------------------- + // | 64 bit | 32 bit | + // --------------------------- + // | nano sec | julian day | + // --------------------------- + // NOTE: + // INT96 is a deprecated type in parquet format to store timestamp, which consists of + // two parts: the first 8 bytes is the nanoseconds within the day, and the last 4 bytes + // is the Julian Day (days since noon on January 1, 4713 BC). And it will be converted it to UTC by + // julian day - 2440588 (Julian Day of the Unix epoch 1970-01-01 00:00:00) + // As julian day is decoded as uint32, so if user store a date before 1970-01-01, the converted time will be wrong + // and possibly to be truncated. + t := val.ToTime().In(loc) + mysqlTime := types.NewTime(types.FromGoTime(t), mysql.TypeTimestamp, 0) + d.SetMysqlTime(mysqlTime) +} + +func getInt96Getter(_ *convertedType, loc *time.Location) setter[parquet.Int96] { + return func(val parquet.Int96, d *types.Datum) { + getInt96Data(val, d, loc) + } +} + +func getFloat32Data(val float32, d *types.Datum) { + d.SetFloat32(val) +} + +func getFloat64Data(val float64, d *types.Datum) { + d.SetFloat64(val) +} + +func getByteArrayGetter(converted *convertedType) setter[parquet.ByteArray] { + switch converted.converted { + case schema.ConvertedTypes.None, schema.ConvertedTypes.BSON, schema.ConvertedTypes.JSON, schema.ConvertedTypes.UTF8, schema.ConvertedTypes.Enum: + return func(val parquet.ByteArray, d *types.Datum) { + d.SetBytesAsString(val, "utf8mb4_bin", uint32(len(val))) + } + case schema.ConvertedTypes.Decimal: + return func(val parquet.ByteArray, d *types.Datum) { + str := binaryToDecimalStr(val, int(converted.decimalMeta.Scale)) + d.SetString(str, "utf8mb4_bin") + } + } + + return nil +} + +func getFixedLenByteArrayGetter(converted *convertedType) setter[parquet.FixedLenByteArray] { + switch converted.converted { + case schema.ConvertedTypes.None, schema.ConvertedTypes.BSON, schema.ConvertedTypes.JSON, schema.ConvertedTypes.UTF8, schema.ConvertedTypes.Enum: + return func(val parquet.FixedLenByteArray, d *types.Datum) { + d.SetBytesAsString(val, "utf8mb4_bin", uint32(len(val))) + } + case schema.ConvertedTypes.Decimal: + return func(val parquet.FixedLenByteArray, d *types.Datum) { + str := binaryToDecimalStr(val, int(converted.decimalMeta.Scale)) + d.SetString(str, "utf8mb4_bin") + } + } + + return nil +} diff --git a/pkg/lightning/mydump/parquet_writer.go b/pkg/lightning/mydump/parquet_writer.go new file mode 100644 index 0000000000000..b28cb71ec6a69 --- /dev/null +++ b/pkg/lightning/mydump/parquet_writer.go @@ -0,0 +1,153 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mydump + +import ( + "context" + "fmt" + + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/compress" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/schema" + "github.com/pingcap/tidb/br/pkg/storage" +) + +// ParquetColumn defines the properties of a column in a Parquet file. +// Used to generate parquet files in tests. +type ParquetColumn struct { + Name string + Type parquet.Type + Converted schema.ConvertedType + TypeLen int + Precision int + Scale int + Gen func(numRows int) (any, []int16) +} + +type writeWrapper struct { + Writer storage.ExternalFileWriter +} + +func (*writeWrapper) Seek(_ int64, _ int) (int64, error) { + return 0, nil +} + +func (*writeWrapper) Read(_ []byte) (int, error) { + return 0, nil +} + +func (w *writeWrapper) Write(b []byte) (int, error) { + return w.Writer.Write(context.Background(), b) +} + +func (w *writeWrapper) Close() error { + return w.Writer.Close(context.Background()) +} + +func getStore(path string) (storage.ExternalStorage, error) { + s, err := storage.ParseBackend(path, nil) + if err != nil { + return nil, err + } + + store, err := storage.NewWithDefaultOpt(context.Background(), s) + if err != nil { + return nil, err + } + return store, nil +} + +// WriteParquetFile writes a simple Parquet file with the specified columns and number of rows. +// It's used for test and DON'T use this function to generate large Parquet files. +func WriteParquetFile(path, fileName string, pcolumns []ParquetColumn, rows int) error { + s, err := getStore(path) + if err != nil { + return err + } + writer, err := s.Create(context.Background(), fileName, nil) + if err != nil { + return err + } + wrapper := &writeWrapper{Writer: writer} + + fields := make([]schema.Node, len(pcolumns)) + opts := make([]parquet.WriterProperty, 0, len(pcolumns)*2) + for i, pc := range pcolumns { + typeLen := -1 + if pc.TypeLen > 0 { + typeLen = pc.TypeLen + } + if fields[i], err = schema.NewPrimitiveNodeConverted( + pc.Name, + parquet.Repetitions.Optional, + pc.Type, pc.Converted, + typeLen, pc.Precision, pc.Scale, + -1, + ); err != nil { + return err + } + opts = append(opts, parquet.WithDictionaryFor(pc.Name, true)) + opts = append(opts, parquet.WithCompressionFor(pc.Name, compress.Codecs.Snappy)) + } + + node, _ := schema.NewGroupNode("schema", parquet.Repetitions.Required, fields, -1) + props := parquet.NewWriterProperties(opts...) + pw := file.NewParquetWriter(wrapper, node, file.WithWriterProps(props)) + //nolint: errcheck + defer pw.Close() + + // Only one row group for simplicity + rgw := pw.AppendRowGroup() + //nolint: errcheck + defer rgw.Close() + + for _, pc := range pcolumns { + cw, err := rgw.NextColumn() + if err != nil { + return err + } + vals, defLevel := pc.Gen(rows) + + switch w := cw.(type) { + case *file.Int64ColumnChunkWriter: + buf, _ := vals.([]int64) + _, err = w.WriteBatch(buf, defLevel, nil) + case *file.Float64ColumnChunkWriter: + buf, _ := vals.([]float64) + _, err = w.WriteBatch(buf, defLevel, nil) + case *file.ByteArrayColumnChunkWriter: + buf, _ := vals.([]parquet.ByteArray) + _, err = w.WriteBatch(buf, defLevel, nil) + case *file.Int32ColumnChunkWriter: + buf, _ := vals.([]int32) + _, err = w.WriteBatch(buf, defLevel, nil) + case *file.BooleanColumnChunkWriter: + buf, _ := vals.([]bool) + _, err = w.WriteBatch(buf, defLevel, nil) + default: + return fmt.Errorf("unsupported column type %T", cw) + } + + if err != nil { + return err + } + if err := cw.Close(); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/lightning/mydump/parser.go b/pkg/lightning/mydump/parser.go index f6d2c2cd30c4c..ddfac8cd6eabb 100644 --- a/pkg/lightning/mydump/parser.go +++ b/pkg/lightning/mydump/parser.go @@ -160,7 +160,15 @@ type Parser interface { // ScannedPos always returns the current file reader pointer's location ScannedPos() (int64, error) Close() error + + // ReadRow reads a row from the datafile. ReadRow() error + + // ReadRowUnsafe reads a row from the datafile, + // and the returned Row is only valid until the + // next call to ReadRow or ReadRowUnsafe. + ReadRowUnsafe() error + LastRow() Row RecycleRow(row Row) @@ -393,6 +401,10 @@ func (parser *ChunkParser) unescapeString(input string) string { return input } +func (parser *ChunkParser) ReadRowUnsafe() error { + return parser.ReadRow() +} + // ReadRow reads a row from the datafile. func (parser *ChunkParser) ReadRow() error { // This parser will recognize contents like: @@ -680,7 +692,7 @@ func OpenReader( ) (reader storage.ReadSeekCloser, err error) { switch { case fileMeta.Type == SourceTypeParquet: - reader, err = OpenParquetReader(ctx, store, fileMeta.Path, fileMeta.FileSize) + reader, err = OpenParquetReader(ctx, store, fileMeta.Path) case fileMeta.Compression != CompressionNone: compressType, err2 := ToStorageCompressType(fileMeta.Compression) if err2 != nil { diff --git a/tools/gen-parquet/BUILD.bazel b/tools/gen-parquet/BUILD.bazel index 08eba16858c31..358a08dc19eea 100644 --- a/tools/gen-parquet/BUILD.bazel +++ b/tools/gen-parquet/BUILD.bazel @@ -6,8 +6,10 @@ go_library( importpath = "github.com/pingcap/tidb/tools/gen-parquet", visibility = ["//visibility:private"], deps = [ - "@com_github_xitongsys_parquet_go//writer", - "@com_github_xitongsys_parquet_go_source//local", + "@com_github_apache_arrow_go_v18//parquet", + "@com_github_apache_arrow_go_v18//parquet/compress", + "@com_github_apache_arrow_go_v18//parquet/file", + "@com_github_apache_arrow_go_v18//parquet/schema", ], ) diff --git a/tools/gen-parquet/main.go b/tools/gen-parquet/main.go index 5f0a98a6c35c8..f7ae34aecb8d2 100644 --- a/tools/gen-parquet/main.go +++ b/tools/gen-parquet/main.go @@ -17,61 +17,142 @@ package main import ( "flag" "fmt" + "io" "log" - "path/filepath" + "os" "strconv" - "github.com/xitongsys/parquet-go-source/local" - "github.com/xitongsys/parquet-go/writer" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/compress" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/schema" ) -var ( - schema = flag.String("schema", "test", "Test schema name") - table = flag.String("table", "parquet", "Test table name") - chunks = flag.Int("chunk", 10, "Chunk files count") - rowNumbers = flag.Int("rows", 1000, "Row number for each test file") - sourceDir = flag.String("dir", "", "test directory path") -) +type writeWrapper struct { + Writer *os.File +} -func genParquetFile(dir, name string, count int) error { - type Test struct { - I int32 `parquet:"name=iVal, type=INT32"` - S string `parquet:"name=s, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"` +func (*writeWrapper) Seek(_ int64, _ int) (int64, error) { + return 0, nil +} + +func (*writeWrapper) Read(_ []byte) (int, error) { + return 0, nil +} + +func (w *writeWrapper) Write(b []byte) (int, error) { + return w.Writer.Write(b) +} + +func (w *writeWrapper) Close() error { + return nil +} + +func getParquetWriter(w io.Writer, rowNames []string, rowTypes []parquet.Type) *file.Writer { + fields := make([]schema.Node, len(rowNames)) + for i, name := range rowNames { + fields[i], _ = schema.NewPrimitiveNode( + name, + parquet.Repetitions.Optional, + rowTypes[i], + -1, 8, + ) } - w, err := local.NewLocalFileWriter(filepath.Join(dir, name)) - if err != nil { - return err + node, _ := schema.NewGroupNode("schema", parquet.Repetitions.Required, fields, -1) + schema := schema.NewSchema(node) + + opts := []parquet.WriterProperty{} + for i := range rowNames { + opts = append(opts, parquet.WithDictionaryFor(schema.Column(i).Name(), true)) + opts = append(opts, parquet.WithCompressionFor(schema.Column(i).Name(), compress.Codecs.Snappy)) } + props := parquet.NewWriterProperties(opts...) - test := &Test{} - dataWriter, err := writer.NewParquetWriter(w, test, 2) + return file.NewParquetWriter(w, schema.Root(), file.WithWriterProps(props)) +} + +func writeColumn(rgw file.SerialRowGroupWriter, rows int) error { + cw, err := rgw.NextColumn() if err != nil { return err } - for i := range count { - test.I = int32(i) - test.S = strconv.Itoa(i) - err := dataWriter.Write(test) - if err != nil { - return err + //nolint: errcheck + defer cw.Close() + + defLevel := make([]int16, rows) + for i := range rows { + defLevel[i] = 1 + } + + switch w := cw.(type) { + case *file.Int64ColumnChunkWriter: + buf := make([]int64, rows) + for i := range rows { + buf[i] = int64(i) + } + _, err = w.WriteBatch(buf, defLevel, nil) + case *file.Float64ColumnChunkWriter: + buf := make([]float64, rows) + for i := range rows { + buf[i] = float64(i) } + _, err = w.WriteBatch(buf, defLevel, nil) + case *file.ByteArrayColumnChunkWriter: + buf := make([]parquet.ByteArray, rows) + for i := range rows { + s := strconv.Itoa(i) + buf[i] = []byte(s) + } + _, err = w.WriteBatch(buf, defLevel, nil) + default: + return fmt.Errorf("unsupported column type: %T", w) } - err = dataWriter.WriteStop() + + return err +} + +func writeSimpleParquetFile(filePath string, rows int) error { + file, err := os.Create(filePath) if err != nil { return err } - w.Close() + w := &writeWrapper{Writer: file} + + rowNames := []string{"iVal", "s"} + rowTypes := []parquet.Type{parquet.Types.Int64, parquet.Types.ByteArray} + pw := getParquetWriter(w, rowNames, rowTypes) + //nolint: errcheck + defer pw.Close() + // Only one row group for simplicity + rgw := pw.AppendRowGroup() + //nolint: errcheck + defer rgw.Close() + + for range rowNames { + if err := writeColumn(rgw, rows); err != nil { + return err + } + } return nil } +var ( + schemaName = flag.String("schema", "test", "Test schema name") + tableName = flag.String("table", "parquet", "Test table name") + chunks = flag.Int("chunk", 10, "Chunk files count") + rowNumbers = flag.Int("rows", 1000, "Row number for each test file") + sourceDir = flag.String("dir", "", "test directory path") +) + func main() { flag.Parse() for i := range *chunks { - name := fmt.Sprintf("%s.%s.%04d.parquet", *schema, *table, i) - err := genParquetFile(*sourceDir, name, *rowNumbers) + name := fmt.Sprintf("%s.%s.%04d.parquet", *schemaName, *tableName, i) + filePath := fmt.Sprintf("%s/%s", *sourceDir, name) + err := writeSimpleParquetFile(filePath, *rowNumbers) if err != nil { log.Fatalf("generate test source failed, name: %s, err: %+v", name, err) }