diff --git a/MODULE.bazel b/MODULE.bazel index 36d50c8..a8655b5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -174,6 +174,7 @@ use_repo( "com_github_vmihailenco_msgpack_v5", "com_github_wasmerio_wasmer_go", "com_google_cloud_go_bigquery", + "com_google_cloud_go_pubsub", "in_gopkg_natefinch_lumberjack_v2", "in_gopkg_yaml_v2", "in_gopkg_yaml_v3", @@ -199,6 +200,7 @@ use_repo( "org_golang_google_protobuf", "org_golang_x_exp", "org_golang_x_net", + "org_golang_x_sync", "org_modernc_mathutil", "org_uber_go_mock", "org_uber_go_zap", diff --git a/common/compress/BUILD.bazel b/common/compress/BUILD.bazel new file mode 100644 index 0000000..a908aa7 --- /dev/null +++ b/common/compress/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "compress", + srcs = ["compress.go"], + importpath = "sentioxyz/sentio-core/common/compress", + visibility = ["//visibility:public"], + deps = ["@com_github_pkg_errors//:errors"], +) + +go_test( + name = "compress_test", + srcs = ["compress_test.go"], + embed = [":compress"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/common/compress/compress.go b/common/compress/compress.go new file mode 100644 index 0000000..795cb07 --- /dev/null +++ b/common/compress/compress.go @@ -0,0 +1,71 @@ +package compress + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "github.com/pkg/errors" + "io" +) + +type compressPayload struct { + CompressMethod string `json:"compress_method,omitempty"` + Data []byte `json:"data,omitempty"` +} + +const ( + compressMethodGZIP = "gzip" +) + +func Load(raw []byte, d any) (err error) { + if len(raw) == 0 { + return nil + } + var payload compressPayload + _ = json.Unmarshal(raw, &payload) + var r io.Reader + switch payload.CompressMethod { + case compressMethodGZIP: + r, err = gzip.NewReader(bytes.NewReader(payload.Data)) + if err != nil { + return errors.Wrapf(err, "try to load as compressed payload failed") + } + default: + r = bytes.NewReader(raw) + } + return json.NewDecoder(r).Decode(d) +} + +func Dump(d any) ([]byte, error) { + return dump(d, compressMethodGZIP) +} + +func dump(d any, compressMethod string) ([]byte, error) { + // prepare WriteCloser by compressMethod + var buf bytes.Buffer + var w io.WriteCloser + switch compressMethod { + case compressMethodGZIP: + w = gzip.NewWriter(&buf) + default: + return nil, errors.Errorf("compress method %s not supported", compressMethod) + } + // json marshal and do compress + err := json.NewEncoder(w).Encode(d) + if err == nil { + err = w.Close() + } + if err != nil { + return nil, errors.Wrapf(err, "dump with compress method %s failed", compressMethod) + } + // build payload and json marshal + var raw []byte + raw, err = json.Marshal(compressPayload{ + CompressMethod: compressMethod, + Data: buf.Bytes(), + }) + if err != nil { + err = errors.Wrapf(err, "dump with compress method %s failed", compressMethod) + } + return raw, err +} diff --git a/common/compress/compress_test.go b/common/compress/compress_test.go new file mode 100644 index 0000000..be88074 --- /dev/null +++ b/common/compress/compress_test.go @@ -0,0 +1,32 @@ +package compress + +import ( + "encoding/json" + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func Test_loadAndDump(t *testing.T) { + const count = 100000 + origin := make([]string, count) + for i := 0; i < count; i++ { + origin[i] = fmt.Sprintf("%08d", i) + } + + d0, _ := json.Marshal(origin) + t.Logf("origin len: %d", len(d0)) + d, err := Dump(origin) + assert.NoError(t, err) + + t.Logf("dump len: %d", len(d)) + t.Logf("dump prefix: %s", string(d[:100])) + t.Logf("dump suffix: %s", string(d[len(d)-100:])) + + var result []string + assert.NoError(t, Load(d, &result)) + assert.Equal(t, origin, result) + + assert.NoError(t, Load(d0, &result)) + assert.Equal(t, origin, result) +} diff --git a/common/contract/BUILD.bazel b/common/contract/BUILD.bazel new file mode 100644 index 0000000..1f29386 --- /dev/null +++ b/common/contract/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "contract", + srcs = ["contract.go"], + importpath = "sentioxyz/sentio-core/common/contract", + visibility = ["//visibility:public"], + deps = [ + "//common/https", + "@com_github_pkg_errors//:errors", + ], +) + +go_test( + name = "contract_test", + srcs = ["contract_test.go"], + embed = [":contract"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/common/contract/contract.go b/common/contract/contract.go new file mode 100644 index 0000000..c3f82db --- /dev/null +++ b/common/contract/contract.go @@ -0,0 +1,148 @@ +package contract + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/https" +) + +type TokenResult struct { + Decimals *int `json:"decimals"` + Logo *string `json:"logo"` + Name string `json:"name"` + Symbol string `json:"symbol"` +} + +type jsonError struct { + Code int `json:"code"` + Message string `json:"message"` + Data interface{} `json:"data,omitempty"` +} + +type JsonrpcMessage struct { + Version string `json:"jsonrpc,omitempty"` + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Error *jsonError `json:"error,omitempty"` + Result *TokenResult `json:"result,omitempty"` +} + +// IsERC20 refers from https://docs.alchemy.com/reference/alchemy-gettokenmetadata +func IsERC20(ctx context.Context, apiEndpoint string, tokenAddress string) (bool, error) { + payload := strings.NewReader( + fmt.Sprintf( + "{\"id\":1,\"jsonrpc\":\"2.0\",\"method\":\"alchemy_getTokenMetadata\",\"params\":[\"%s\"]}", + tokenAddress, + ), + ) + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, apiEndpoint, payload) + + req.Header.Add("accept", "application/json") + req.Header.Add("content-type", "application/json") + + res, err := https.DefaultClient.Do(req) + + if err != nil { + return false, errors.WithStack(err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return false, errors.New("HTTP request return" + res.Status) + } + + body, _ := io.ReadAll(res.Body) + //bodyString := string(body) + + var jsonrpcRes JsonrpcMessage + err = json.Unmarshal(body, &jsonrpcRes) + if err != nil { + return false, err + } + + if jsonrpcRes.Result != nil { + if jsonrpcRes.Result.Decimals != nil { + return true, nil + } + } + + return false, nil +} + +// IsERC20New refers from https://docs.moralis.io/reference/gettokenmetadata +func IsERC20New(ctx context.Context, apiKey string, chainID string, tokenAddress string) (bool, error) { + info, err := getERC20Info(ctx, apiKey, chainID, []string{tokenAddress}) + if err != nil { + return false, err + } + return info[0].Symbol != "" && info[0].Decimals != "", nil +} + +type ERC20Token struct { + Address string `json:"address"` + Symbol string `json:"symbol"` + Name string `json:"name"` + Decimals string `json:"decimals"` + Logo string `json:"logo,omitempty"` + LogoHash string `json:"logo_hash,omitempty"` + Thumbnail string `json:"thumbnail,omitempty"` + BlockNumber string `json:"block_number,omitempty"` + //Validated string `json:"validated,omitempty"` +} + +// https://docs.moralis.io/reference/gettokenmetadata +func getERC20Info(ctx context.Context, apiKey string, chainID string, tokenAddress []string) ([]ERC20Token, error) { + var chain string + switch chainID { + case "1": + chain = "eth" + case "5": + chain = "goerli" + case "56": + chain = "bsc" + // TDDO add more mappings + default: + return nil, errors.New("chainID not supported") + } + + var addresses = "" + for _, address := range tokenAddress { + addresses += "&addresses=" + address + } + + url := "https://deep-index.moralis.io/api/v2/erc20/metadata?&chain=" + chain + addresses + + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + + req.Header.Add("accept", "application/json") + req.Header.Add("X-API-Key", apiKey) + + res, err := https.DefaultClient.Do(req) + + if err != nil { + return nil, errors.WithStack(err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, errors.New("HTTP request return" + res.Status) + } + + body, _ := io.ReadAll(res.Body) + + var tokenInfo []ERC20Token + err = json.Unmarshal(body, &tokenInfo) + if err != nil { + return nil, errors.WithStack(err) + } + + return tokenInfo, nil +} diff --git a/common/contract/contract_test.go b/common/contract/contract_test.go new file mode 100644 index 0000000..389b043 --- /dev/null +++ b/common/contract/contract_test.go @@ -0,0 +1,83 @@ +package contract + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +// FIXME runs into 403 regularly +//func TestIsERC20(t *testing.T) { +// endpoint := "https://eth-mainnet.g.alchemy.com/v2/KLPDGUUQGKmScCbSdPd0iUNO_JufXdg9" +// +// ctx := context.Background() +// +// var res bool +// var err error +// +// //// abtc +// //res, err = IsERC20(ctx, endpoint, "0xC2fcab14Ec1F2dFA82a23C639c4770345085a50F") +// //assert.NoError(t, err) +// //assert.False(t, res) +// +// // weth address +// res, err = IsERC20(ctx, endpoint, "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2") +// assert.NoError(t, err) +// assert.True(t, res) +// +// // invalid address +// res, err = IsERC20(ctx, endpoint, "0x0000000000000000000000000000000000000000") +// assert.NoError(t, err) +// assert.False(t, res) +// +// // token address but does not implement ERC20 +// res, err = IsERC20(ctx, endpoint, "0xa6794DEc66Df7d8B69752956df1b28cA93f77CD7") +// assert.NoError(t, err) +// assert.False(t, res) +// +// // USDC +// res, err = IsERC20(ctx, endpoint, "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48") +// assert.NoError(t, err) +// assert.True(t, res) +// +// res, err = IsERC20(ctx, endpoint, "0x6B89B97169a797d94F057F4a0B01E2cA303155e4") +// assert.NoError(t, err) +// assert.True(t, res) +//} + +func TestIsERC20New(t *testing.T) { + //endpoint := "https://eth-mainnet.g.alchemy.com/v2/z1Q-YhcYg60C5sOQPUzsMFqiDJSvqbsK" + endpoint := "oGDQZsjYX3IdnfkenzRf0K5NA7Lsy0NljUFVKFp0nGDhwajEq5ltjHU7aFp3V8lG" + + ctx := context.Background() + + var res bool + var err error + + //// abtc + //res, err = IsERC20New(ctx, endpoint, "1", "0xC2fcab14Ec1F2dFA82a23C639c4770345085a50F") + //assert.NoError(t, err) + //assert.False(t, res) + + // weth address + res, err = IsERC20New(ctx, endpoint, "1", "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2") + assert.NoError(t, err) + assert.True(t, res) + + // invalid address + // res, err = IsERC20New(ctx, endpoint, "1", "0x0000000000000000000000000000000000000000") + // assert.NoError(t, err) + // assert.False(t, res) + + // token address but does not implement ERC20 + _, err = IsERC20New(ctx, endpoint, "1", "0xa6794DEc66Df7d8B69752956df1b28cA93f77CD7") + assert.NoError(t, err) + // Uncomment this line when the issue is fixed. + // assert.False(t, res) + + // USDC + res, err = IsERC20New(ctx, endpoint, "1", "0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48") + assert.NoError(t, err) + assert.True(t, res) +} diff --git a/common/jsonutils/BUILD.bazel b/common/jsonutils/BUILD.bazel new file mode 100644 index 0000000..1990541 --- /dev/null +++ b/common/jsonutils/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "jsonutils", + srcs = ["jsonutils.go"], + importpath = "sentioxyz/sentio-core/common/jsonutils", + visibility = ["//visibility:public"], +) + +go_test( + name = "jsonutils_test", + srcs = ["jsonutils_test.go"], + embed = [":jsonutils"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/common/jsonutils/jsonutils.go b/common/jsonutils/jsonutils.go new file mode 100644 index 0000000..6745700 --- /dev/null +++ b/common/jsonutils/jsonutils.go @@ -0,0 +1,74 @@ +package jsonutils + +import ( + "bytes" + "encoding/json" + "fmt" + "reflect" +) + +type Tracker func(path string, or, pa any) + +func Patch(origin, patch []byte, tracker Tracker) (final []byte, err error) { + var o, p map[string]any + var decoder *json.Decoder + decoder = json.NewDecoder(bytes.NewReader(origin)) + decoder.UseNumber() + if err = decoder.Decode(&o); err != nil { + return + } + decoder = json.NewDecoder(bytes.NewReader(patch)) + decoder.UseNumber() + if err = decoder.Decode(&p); err != nil { + return + } + if err = patchObject("", o, p, tracker); err != nil { + return + } + return json.Marshal(o) +} + +const wildcard = "*" + +func patchObject(path string, origin, patch map[string]any, tracker Tracker) error { + for key, pv := range patch { + if key == wildcard { + for key, ov := range origin { + keyPath := path + "." + key + if reflect.ValueOf(ov).Type().String() != reflect.ValueOf(pv).Type().String() { + continue + } else if reflect.ValueOf(ov).Kind() == reflect.Map { + if err := patchObject(keyPath, ov.(map[string]any), pv.(map[string]any), tracker); err != nil { + return err + } + continue + } + if tracker != nil { + tracker(keyPath, ov, pv) + } + origin[key] = pv + } + continue + } + + ov, has := origin[key] + keyPath := path + "." + key + + if has { + if reflect.ValueOf(ov).Type().String() != reflect.ValueOf(pv).Type().String() { + return fmt.Errorf("cannot patch %s/%T by %T", keyPath, ov, pv) + } else if reflect.ValueOf(ov).Kind() == reflect.Map { + if err := patchObject(keyPath, ov.(map[string]any), pv.(map[string]any), tracker); err != nil { + return err + } + continue + } + } + + if tracker != nil { + tracker(keyPath, ov, pv) + } + origin[key] = pv + } + return nil +} diff --git a/common/jsonutils/jsonutils_test.go b/common/jsonutils/jsonutils_test.go new file mode 100644 index 0000000..9e47740 --- /dev/null +++ b/common/jsonutils/jsonutils_test.go @@ -0,0 +1,103 @@ +package jsonutils + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestPatch_Succeed(t *testing.T) { + testcases := [][]string{ + { + `{}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + }, + { + `{"a":123}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + }, + { + `{"a":456,"e":"xxx"}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg"}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg","e":"xxx"}`, + }, + { + `{"a":456,"e":"xxx","f":{"g":123,"h":"yyy"}}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg","f":{"h":"zzz","i":true}}`, + `{"a":1,"b":true,"c":1.1,"d":"abcdefg","e":"xxx","f":{"g":123,"h":"zzz","i":true}}`, + }, + } + for i, testcase := range testcases { + r, err := Patch([]byte(testcase[0]), []byte(testcase[1]), func(path string, or, pa any) { + fmt.Printf("#%d %s: %v => %v\n", i, path, or, pa) + }) + assert.NoError(t, err) + assert.Equal(t, testcase[2], string(r), fmt.Sprintf("testcase #%d %v", i, testcase)) + } +} + +func TestPatch_WildcardSucceed(t *testing.T) { + testcases := [][]string{ + { + `{}`, + `{"*":1}`, + `{}`, + }, + { + `{"a":123,"b":456,"c":"ccc"}`, + `{"*":1}`, + `{"a":1,"b":1,"c":"ccc"}`, + }, + { + `{"a":{"x":1},"b":{"x":2},"c":3}`, + `{"*":{"x":4}}`, + `{"a":{"x":4},"b":{"x":4},"c":3}`, + }, + { + `{"a":{"x":1},"b":{"y":2},"c":3}`, + `{"*":{"x":4}}`, + `{"a":{"x":4},"b":{"x":4,"y":2},"c":3}`, + }, + } + for i, testcase := range testcases { + r, err := Patch([]byte(testcase[0]), []byte(testcase[1]), func(path string, or, pa any) { + fmt.Printf("#%d %s: %v => %v\n", i, path, or, pa) + }) + assert.NoError(t, err) + assert.Equal(t, testcase[2], string(r), fmt.Sprintf("testcase #%d %v", i, testcase)) + } +} + +func TestPatch_TypeMismatch(t *testing.T) { + var err error + + _, err = Patch([]byte(`{"a":true}`), []byte(`{"a":"abc"}`), nil) + assert.ErrorContains(t, err, "cannot patch .a/bool by string") + + _, err = Patch([]byte(`{"a":123}`), []byte(`{"a":"abc"}`), nil) + assert.ErrorContains(t, err, "cannot patch .a/json.Number by string") + + _, err = Patch([]byte(`{"a":{"b":123}}`), []byte(`{"a":"abc"}`), nil) + assert.ErrorContains(t, err, "cannot patch .a/map[string]interface {} by string") + + _, err = Patch([]byte(`{"a":123}`), []byte(`{"a":{"b":true}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a/json.Number by map[string]interface {}") +} + +func TestPatch_WildcardTypeMismatch(t *testing.T) { + var err error + + _, err = Patch([]byte(`{"a":{"x":true}}`), []byte(`{"*":{"x":"abc"}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a.x/bool by string") + + _, err = Patch([]byte(`{"a":{"x":123}}`), []byte(`{"*":{"x":"abc"}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a.x/json.Number by string") + + _, err = Patch([]byte(`{"a":{"x":{"y":123}}}`), []byte(`{"*":{"x":"abc"}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a.x/map[string]interface {} by string") + + _, err = Patch([]byte(`{"a":{"x":123}}`), []byte(`{"*":{"x":{"y":123}}}`), nil) + assert.ErrorContains(t, err, "cannot patch .a.x/json.Number by map[string]interface {}") +} diff --git a/common/window/BUILD.bazel b/common/window/BUILD.bazel new file mode 100644 index 0000000..e0b2ef5 --- /dev/null +++ b/common/window/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "window", + srcs = ["window.go"], + importpath = "sentioxyz/sentio-core/common/window", + visibility = ["//visibility:public"], + deps = ["@org_golang_x_exp//constraints"], +) + +go_test( + name = "window_test", + srcs = ["window_test.go"], + embed = [":window"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/common/window/window.go b/common/window/window.go new file mode 100644 index 0000000..d9a7e80 --- /dev/null +++ b/common/window/window.go @@ -0,0 +1,70 @@ +package window + +import ( + "context" + "golang.org/x/exp/constraints" + "time" +) + +// FindFirstStartPoint find the first x in [start,end] that timeWinGetter(x-1) < timeWinGetter(x) +func FindFirstStartPoint[N constraints.Signed]( + ctx context.Context, + start, end N, + winGetter func(ctx context.Context, n N) (time.Time, error), +) (*N, error) { + if start > end { + return nil, nil + } + + var t time.Time + p, err := winGetter(ctx, start-1) + if err != nil { + return nil, err + } + + t, err = winGetter(ctx, end) + if err != nil { + return nil, err + } + + if p.Equal(t) { + // all x in [start,end] that timeWinGetter(start-1) == timeWinGetter(x), so no result + return nil, nil + } + + for start < end { + mid := (start + end) / 2 + t, err = winGetter(ctx, mid) + if err != nil { + return nil, err + } + if t.Equal(p) { + start = mid + 1 + } else { + end = mid + } + } + return &start, nil +} + +// FindStartPoints find first limit points in [start,end] that timeWinGetter(x-1) < timeWinGetter(x) +func FindStartPoints[N constraints.Signed]( + ctx context.Context, + start, end N, + limit int, + winGetter func(ctx context.Context, n N) (time.Time, error), +) ([]N, error) { + var result []N + for i := 0; limit <= 0 || i < limit; i++ { + x, err := FindFirstStartPoint(ctx, start, end, winGetter) + if err != nil { + return nil, err + } + if x == nil { + return result, nil + } + result = append(result, *x) + start = *x + 1 + } + return result, nil +} diff --git a/common/window/window_test.go b/common/window/window_test.go new file mode 100644 index 0000000..48dae6c --- /dev/null +++ b/common/window/window_test.go @@ -0,0 +1,40 @@ +package window + +import ( + "context" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func Test_findFirstStartPoint(t *testing.T) { + winGetter := func(ctx context.Context, n int) (time.Time, error) { + return time.Unix(int64(n/100), 0), nil + } + + var r *int + for s := 1; s <= 100; s++ { + for e := 100; e < 199; e++ { + r, _ = FindFirstStartPoint[int](context.Background(), s, e, winGetter) + assert.Equal(t, 100, *r) + } + } + for s := 101; s <= 199; s++ { + for e := s; e < 199; e++ { + r, _ = FindFirstStartPoint[int](context.Background(), s, e, winGetter) + assert.Nil(t, r) + } + } + + var rs []int + rs, _ = FindStartPoints[int](context.Background(), 1, 99, 3, winGetter) + assert.Nil(t, rs) + rs, _ = FindStartPoints[int](context.Background(), 1, 299, 1, winGetter) + assert.Equal(t, []int{100}, rs) + rs, _ = FindStartPoints[int](context.Background(), 1, 299, 2, winGetter) + assert.Equal(t, []int{100, 200}, rs) + rs, _ = FindStartPoints[int](context.Background(), 1, 299, 3, winGetter) + assert.Equal(t, []int{100, 200}, rs) + rs, _ = FindStartPoints[int](context.Background(), 1, 299, -1, winGetter) + assert.Equal(t, []int{100, 200}, rs) +} diff --git a/driver/controller/BUILD.bazel b/driver/controller/BUILD.bazel new file mode 100644 index 0000000..2581913 --- /dev/null +++ b/driver/controller/BUILD.bazel @@ -0,0 +1,55 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "controller", + srcs = [ + "analyse.go", + "block_builder.go", + "checkpoint.go", + "config.go", + "entity.go", + "errors.go", + "handler.go", + "main.go", + "notifier.go", + "quota.go", + "range.go", + "timeseries.go", + "webhook.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller", + visibility = ["//visibility:public"], + deps = [ + "//common/concurrency", + "//common/envconf", + "//common/errgroup", + "//common/log", + "//common/sparsify", + "//common/timer", + "//common/timewin", + "//common/utils", + "//driver/entity/persistent", + "//driver/entity/schema", + "//driver/timeseries", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + ], +) + +go_test( + name = "controller_test", + srcs = [ + "checkpoint_test.go", + "errors_test.go", + "main_test.go", + "range_test.go", + ], + embed = [":controller"], + deps = [ + "//common/errgroup", + "//common/log", + "//common/utils", + "@com_github_pkg_errors//:errors", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/analyse.go b/driver/controller/analyse.go new file mode 100644 index 0000000..af6d9a3 --- /dev/null +++ b/driver/controller/analyse.go @@ -0,0 +1,167 @@ +package controller + +import ( + "time" + + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" +) + +type taskStatWindow struct { + startAt time.Time + taskCount map[string]int + taskUsed map[string]time.Duration + fetchWaitUsed time.Duration + taskPreWaitUsed time.Duration + taskPostWaitUsed time.Duration + makeCheckpointUsed time.Duration +} + +func (w *taskStatWindow) GetStartAt() time.Time { + return w.startAt +} + +func (w *taskStatWindow) Merge(a *taskStatWindow) { + for handlerID, v := range a.taskCount { + w.taskCount[handlerID] += v + } + for handlerID, v := range a.taskUsed { + w.taskUsed[handlerID] += v + } + w.fetchWaitUsed += a.fetchWaitUsed + w.taskPreWaitUsed += a.taskPreWaitUsed + w.taskPostWaitUsed += a.taskPostWaitUsed + w.makeCheckpointUsed += a.makeCheckpointUsed +} + +func (w *taskStatWindow) Snapshot(endAt time.Time) any { + taskStat := make(map[string]map[string]any) + for hid, count := range w.taskCount { + used := w.taskUsed[hid] + taskStat[hid] = map[string]any{ + "count": count, + "avgUsed": (w.taskUsed[hid] / time.Duration(count)).String(), + "totalUsed": used.String(), + } + } + return map[string]any{ + "startAt": w.startAt.String(), + "endAt": endAt.String(), + "duration": endAt.Sub(w.startAt).String(), + "task": taskStat, + "taskTotalUsed": utils.SumMap(w.taskUsed).String(), + // fetchWaitUsed is the time the single producer goroutine blocked inside blockBuilder.Next() + // waiting for the data fetcher. It is NOT covered by taskPreWaitUsed (which only starts after + // Next() returns), so without it a fetch-bound pipeline looks idle on the consumer side. + "fetchWaitUsed": w.fetchWaitUsed.String(), + "taskPreWaitUsed": w.taskPreWaitUsed.String(), + "taskPostWaitUsed": w.taskPostWaitUsed.String(), + "makeCheckpointUsed": w.makeCheckpointUsed.String(), + } +} + +type analyser struct { + *timewin.TimeWindowsManager[*taskStatWindow] +} + +func newAnalyser() analyser { + return analyser{ + TimeWindowsManager: timewin.NewTimeWindowsManager[*taskStatWindow](time.Minute), + } +} + +func (a *analyser) fetchWait(used time.Duration) { + a.Append(&taskStatWindow{ + startAt: time.Now(), + taskCount: make(map[string]int), + taskUsed: make(map[string]time.Duration), + fetchWaitUsed: used, + }) +} + +func (a *analyser) taskSent(preWait time.Duration) { + a.Append(&taskStatWindow{ + startAt: time.Now(), + taskCount: make(map[string]int), + taskUsed: make(map[string]time.Duration), + taskPreWaitUsed: preWait, + taskPostWaitUsed: 0, + makeCheckpointUsed: 0, + }) +} + +func (a *analyser) taskComplete(handlerID string, used, postWait time.Duration) { + a.Append(&taskStatWindow{ + startAt: time.Now(), + taskCount: map[string]int{handlerID: 1}, + taskUsed: map[string]time.Duration{handlerID: used}, + taskPreWaitUsed: 0, + taskPostWaitUsed: postWait, + makeCheckpointUsed: 0, + }) +} + +func (a *analyser) makeCheckpoint(used time.Duration) { + a.Append(&taskStatWindow{ + startAt: time.Now(), + taskCount: make(map[string]int), + taskUsed: make(map[string]time.Duration), + taskPreWaitUsed: 0, + taskPostWaitUsed: 0, + makeCheckpointUsed: used, + }) +} + +type checkpointStatWindow struct { + startAt time.Time + checkOverQuotaUsed time.Duration + commitTimeSeriesUsed time.Duration + commitEntityUsed time.Duration + commitWebhookUsed time.Duration + saveUsageUsed time.Duration + saveCheckpointUsed time.Duration + totalBinding uint64 + failedCount int + count int +} + +func (w *checkpointStatWindow) GetStartAt() time.Time { + return w.startAt +} + +func (w *checkpointStatWindow) Merge(a *checkpointStatWindow) { + w.checkOverQuotaUsed += a.checkOverQuotaUsed + w.commitTimeSeriesUsed += a.commitTimeSeriesUsed + w.commitEntityUsed += a.commitEntityUsed + w.commitWebhookUsed += a.commitWebhookUsed + w.saveUsageUsed += a.saveUsageUsed + w.saveCheckpointUsed += a.saveCheckpointUsed + w.totalBinding += a.totalBinding + w.failedCount += a.failedCount + w.count += a.count +} + +func (w *checkpointStatWindow) Snapshot(endAt time.Time) any { + dur := endAt.Sub(w.startAt) + total := w.checkOverQuotaUsed + + w.commitTimeSeriesUsed + + w.commitEntityUsed + + w.commitWebhookUsed + + w.saveUsageUsed + + w.saveCheckpointUsed + return map[string]any{ + "startAt": w.startAt.String(), + "endAt": endAt.String(), + "duration": dur.String(), + "checkOverQuotaUsed": w.checkOverQuotaUsed.String(), + "commitTimeSeriesUsed": w.commitTimeSeriesUsed.String(), + "commitEntityUsed": w.commitEntityUsed.String(), + "commitWebhookUsed": w.commitWebhookUsed.String(), + "saveUsageUsed": w.saveUsageUsed.String(), + "saveCheckpointUsed": w.saveCheckpointUsed.String(), + "totalBinding": w.totalBinding, + "failedCount": w.failedCount, + "count": w.count, + "pressure": float64(total) / float64(dur), + } +} diff --git a/driver/controller/block_builder.go b/driver/controller/block_builder.go new file mode 100644 index 0000000..c8c48ae --- /dev/null +++ b/driver/controller/block_builder.go @@ -0,0 +1,332 @@ +package controller + +import ( + "bytes" + "context" + "fmt" + "strconv" + "sync" + "time" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/sparsify" + "sentioxyz/sentio-core/common/utils" +) + +type BlockHeader interface { + GetBlockNumber() uint64 + GetBlockParentHash() string + GetBlockHash() string + GetBlockTime() time.Time +} + +func GetBlockSummary(b BlockHeader) string { + const hashPreviewLen = 8 + hash := b.GetBlockHash() + if len(hash) == 0 { + return fmt.Sprintf("%d", b.GetBlockNumber()) + } else if len(hash) < hashPreviewLen { + return fmt.Sprintf("%d/%s", b.GetBlockNumber(), hash) + } else { + return fmt.Sprintf("%d/%s", b.GetBlockNumber(), hash[:hashPreviewLen]) + } +} + +func GetBlockFullText[H BlockHeader](b H) string { + var bf bytes.Buffer + bf.WriteString(strconv.FormatUint(b.GetBlockNumber(), 10)) + if hash := b.GetBlockHash(); len(hash) > 0 { + bf.WriteString("/") + bf.WriteString(hash) + } + bf.WriteString("/") + bf.WriteString(b.GetBlockTime().Format(time.RFC3339Nano)) + if hash := b.GetBlockParentHash(); len(hash) > 0 { + bf.WriteString("->") + bf.WriteString(hash) + } + return bf.String() +} + +type BlockData interface { + BlockHeader + + GetTaskList() []Task // BlockData without task may be not a empty BlockData + CheckpointData() map[string]string // at least should return a empty map + DataSource() string + Size() int +} + +type TaskIndex struct { + Global uint64 + InBlock int + TotalInBlock int +} + +type Task interface { + BlockHeader + + Summary() string + GetHandlerID() HandlerID + Init(ctx context.Context, index TaskIndex, progressbar ProgressBar) + Exec(ctx context.Context, checkpointCtrl CheckpointController) *ExternalError +} + +type ProgressBar struct { + LatestBlock BlockHeader + FullBlockRange BlockRange +} + +type BlockBuilder interface { + // Start Reset the templates, then reconstruct all HandlerAgent, start constructing blockData + // after checkpoint.BlockNumber, and return them one by one through Next. + // If checkpoint is nil, it means starting from the beginning, and the first block to be processed will be + // obtained through all HandlerAgent. + Start( + ctx context.Context, + checkpoint *Checkpoint, + templates map[uint64][]TemplateInstance, + ) (agentStat map[string]int, extErr *ExternalError) + + // Next Get the block data of the current block. + // may be waiting for the Fetcher to fetch data, or waiting for the latest block. + // If reorg is detected, the returned reorg will not be nil, and its value indicates that all data from + // that block are invalid. + Next(ctx context.Context) (blockNumber uint64, blockData BlockData, progressBar ProgressBar, reorg *uint64, err error) + + // Finish End the traversal of the block + Finish() + + Snapshot() map[string]any +} + +type Client interface { + // GetLatest return err may be ErrInternalNeedUpgrade + GetLatest(ctx context.Context) (latest BlockHeader, first uint64, err error) + // Subscribe broken in callback may be ErrInternalNeedUpgrade + Subscribe(ctx context.Context, from BlockHeader, callback func(latest BlockHeader, broken error)) + // GetHeaderIgnoreCache get the block header without cache, used to check reorg + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (BlockHeader, error) + // ResetCache should remove the cached data from the target block, used when reorg detected or progress passed + ResetCache(r BlockRange) + Snapshot() any +} + +type FetchTarget interface { + Size() int +} + +type Fetcher[T FetchTarget] interface { + GetName() string + GetFullRange() BlockRange + Snapshot() any + KeepFetch(ctx context.Context) + Get(ctx context.Context, blockNumber uint64) (data T, has bool, latest BlockHeader, err error) + UpdateLatest(latest BlockHeader) + Broken(err error) + MoveStart(start uint64) +} + +type blockBuilder struct { + handlerCtrl HandlerController + client Client + checkLink bool + + dataFetcher Fetcher[BlockData] + + fetchCancel context.CancelFunc + fetchDone sync.WaitGroup + + mu sync.RWMutex + + fullBlockRange BlockRange + currentBlockNumber uint64 + + headerList []BlockHeader +} + +func NewBlockBuilder( + handlerCtrl HandlerController, + client Client, + checkLink bool, +) BlockBuilder { + return &blockBuilder{ + handlerCtrl: handlerCtrl, + client: client, + checkLink: checkLink, + } +} + +func (b *blockBuilder) Start( + ctx context.Context, + checkpoint *Checkpoint, + templates map[uint64][]TemplateInstance, +) (map[string]int, *ExternalError) { + latest, first, err := b.client.GetLatest(ctx) + if err != nil { + if errors.Is(err, ErrInternalNeedUpgrade) { + return nil, NewExternalError(ErrCodeNeedUpgrade, err) + } + return nil, NewExternalError(ErrCodeFetchDataFailed, err) + } + if checkpoint != nil && checkpoint.BlockNumber < first { + first = checkpoint.BlockNumber + } + if extErr := b.handlerCtrl.Prologue(ctx, checkpoint, templates, first, latest); extErr != nil { + return nil, extErr + } + if len(b.handlerCtrl.GetAgentStat()) == 0 { + return nil, NewExternalError(ErrCodeUnexpectedProcessorConfig, errors.Errorf("no handler")) + } + + b.mu.Lock() + defer b.mu.Unlock() + + b.fullBlockRange = b.handlerCtrl.GetBlockRange() + b.currentBlockNumber = b.fullBlockRange.StartBlock + b.headerList = nil + if checkpoint != nil { + b.currentBlockNumber = checkpoint.BlockNumber + 1 + if b.checkLink { + b.headerList = append(b.headerList, checkpoint) + } + } + + b.client.ResetCache(BlockRange{StartBlock: b.currentBlockNumber}) + + _, logger := log.FromContext(ctx) + logger.Infow("will start to fetch data", + "full", b.fullBlockRange.String(), + "current", b.currentBlockNumber, + "latest", GetBlockSummary(latest)) + + b.dataFetcher = b.handlerCtrl.BuildBlockDataFetcher(first, b.currentBlockNumber, latest) + + var fetchCtx context.Context + fetchCtx, b.fetchCancel = context.WithCancel(ctx) + b.fetchDone.Add(2) + go func() { + b.client.Subscribe(fetchCtx, latest, func(latest BlockHeader, broken error) { + if broken != nil { + b.dataFetcher.Broken(broken) + } else { + b.dataFetcher.UpdateLatest(latest) + } + }) + b.fetchDone.Done() + }() + go func() { + b.dataFetcher.KeepFetch(fetchCtx) + b.fetchDone.Done() + }() + return b.handlerCtrl.GetAgentStat(), nil +} + +func (b *blockBuilder) checkReorg(ctx context.Context, cur BlockHeader) (reorg *uint64, err error) { + _, logger := log.FromContext(ctx) + b.mu.RLock() + defer b.mu.RUnlock() + if len(b.headerList) == 0 { + return + } + pre := b.headerList[len(b.headerList)-1] + if pre.GetBlockNumber()+1 == cur.GetBlockNumber() && cur.GetBlockParentHash() == pre.GetBlockHash() { + return + } + for i := len(b.headerList) - 1; i >= 0; i-- { + var h BlockHeader + h, err = b.client.GetHeaderIgnoreCache(ctx, b.headerList[i].GetBlockNumber()) + if err != nil { + return + } + if h.GetBlockHash() == b.headerList[i].GetBlockHash() { + if i < len(b.headerList)-1 { + logger.Warnf("block %s not changed", GetBlockSummary(h)) + // first different block must in [b.headerList[i].GetBlockNumber()+1, b.headerList[i+1].GetBlockNumber()], + // be conservative and clear all data starting from b.headerList[i].GetBlockNumber()+1 + reorg = utils.WrapPointer(h.GetBlockNumber() + 1) + } + return + } + reorg = utils.WrapPointer(b.headerList[i].GetBlockNumber()) + logger.Warnf("block %s changed to %s", GetBlockSummary(b.headerList[i]), GetBlockSummary(h)) + } + return +} + +func (b *blockBuilder) Next(ctx context.Context) ( + blockNumber uint64, + blockData BlockData, + progressBar ProgressBar, + reorg *uint64, + err error, +) { + b.mu.RLock() + blockNumber = b.currentBlockNumber + progressBar.FullBlockRange = b.fullBlockRange + b.mu.RUnlock() + + if !b.fullBlockRange.Contains(blockNumber) { + if blockNumber < b.fullBlockRange.StartBlock { + // unreachable + panic(errors.Errorf("current block number %d is to the left of full block range %s", + blockNumber, b.fullBlockRange)) + } + // out of range, just return + return + } + + var has bool + blockData, has, progressBar.LatestBlock, err = b.dataFetcher.Get(ctx, blockNumber) + if err != nil { + return + } + + // detect reorg + if has && b.checkLink && progressBar.LatestBlock.GetBlockTime().Sub(blockData.GetBlockTime()) < WatchingDelay { + if reorg, err = b.checkReorg(ctx, blockData); reorg != nil || err != nil { + return + } + } + + // progress will go to b.currentBlockNumber + 1, so all cached data in [0,b.currentBlockNumber] are useless now. + if has && progressBar.LatestBlock.GetBlockTime().Sub(blockData.GetBlockTime()) < WatchingDelay { + b.client.ResetCache(BlockRange{EndBlock: &b.currentBlockNumber}) + } + + b.mu.Lock() + if b.checkLink && has { + b.headerList = append(b.headerList, blockData) + b.headerList = sparsify.Sparsify(b.headerList, BlockHeader.GetBlockNumber) + } + b.currentBlockNumber++ + b.dataFetcher.MoveStart(b.currentBlockNumber) + b.mu.Unlock() + return +} + +func (b *blockBuilder) Finish() { + b.fetchCancel() + b.fetchDone.Wait() + b.handlerCtrl.Epilogue() +} + +func (b *blockBuilder) Snapshot() map[string]any { + b.mu.RLock() + defer b.mu.RUnlock() + sn := map[string]any{ + "client": b.client.Snapshot(), + "checkLink": b.checkLink, + "handlerController": b.handlerCtrl.Snapshot(), + } + if b.dataFetcher != nil { + // already started + sn["fetcher"] = b.dataFetcher.Snapshot() + sn["fullRange"] = b.fullBlockRange.String() + sn["currentBlockNumber"] = b.currentBlockNumber + sn["headerList"] = utils.MapSliceNoError(b.headerList, GetBlockSummary) + } + return sn +} diff --git a/driver/controller/checkpoint.go b/driver/controller/checkpoint.go new file mode 100644 index 0000000..1d7abe2 --- /dev/null +++ b/driver/controller/checkpoint.go @@ -0,0 +1,1001 @@ +package controller + +import ( + "context" + "fmt" + "math" + "strconv" + "strings" + "sync" + "time" + + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/sparsify" + "sentioxyz/sentio-core/common/timer" + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" + "sentioxyz/sentio-core/driver/timeseries" + + "github.com/pkg/errors" +) + +type TemplateInstance struct { + TemplateID int32 + TemplateName string + Address string + Labels string + + Removed bool + + BlockRange +} + +func (t TemplateInstance) UniqID() string { + if t.Labels == "" { + return fmt.Sprintf("%d/%s", t.TemplateID, t.Address) + } + return fmt.Sprintf("%d/%s/%s", t.TemplateID, t.Address, t.Labels) +} + +func (t TemplateInstance) String() string { + firstPart := strconv.FormatInt(int64(t.TemplateID), 10) + if t.TemplateName != "" { + firstPart += "/" + t.TemplateName + } + var labelPart string + if t.Labels != "" { + labelPart = "::" + t.Labels + } + onOffPart := utils.Select(t.Removed, "OFF", "ON") + return fmt.Sprintf("%s::%s%s::%s-%s", firstPart, t.Address, labelPart, onOffPart, t.BlockRange.String()) +} + +type TplWithCreated struct { + TemplateInstance + + CreatedBlock uint64 +} + +func (t TplWithCreated) String() string { + return fmt.Sprintf("%d#%s", t.CreatedBlock, t.TemplateInstance.String()) +} + +func CountTemplatesByID(orig map[uint64][]TemplateInstance) map[int32]int { + sum := make(map[int32]int) + for _, templates := range orig { + for _, tpl := range templates { + sum[tpl.TemplateID] += 1 + } + } + return sum +} + +type CheckpointData struct { +} + +type Checkpoint struct { + BlockNumber uint64 + BlockHash string `json:"BlockHash,omitempty"` + BlockParentHash string `json:"BlockParentHash,omitempty"` + BlockTime time.Time + + TotalBindings uint64 + + LatestBlockNumber uint64 + LatestBlockHash string `json:"LatestBlockHash,omitempty"` + LatestBlockParentHash string `json:"LatestBlockParentHash,omitempty"` + LatestBlockTime time.Time + + FullBlockRange BlockRange + + Data map[string]string `json:"Data,omitempty"` +} + +func (c Checkpoint) GetBlockNumber() uint64 { + return c.BlockNumber +} + +func (c Checkpoint) GetBlockParentHash() string { + return c.BlockParentHash +} + +func (c Checkpoint) GetBlockHash() string { + return c.BlockHash +} + +func (c Checkpoint) GetBlockTime() time.Time { + return c.BlockTime +} + +func (c Checkpoint) InWatching() bool { + return c.LatestBlockTime.Sub(c.BlockTime) < WatchingDelay +} + +func (c Checkpoint) AllDone() bool { + return c.FullBlockRange.EndBlock != nil && *c.FullBlockRange.EndBlock == c.BlockNumber +} + +func (c Checkpoint) CurrentLastBlockNumber() uint64 { + if c.FullBlockRange.EndBlock != nil && *c.FullBlockRange.EndBlock < c.LatestBlockNumber { + return *c.FullBlockRange.EndBlock + } + return c.LatestBlockNumber +} + +func (c Checkpoint) Rate() float64 { + return float64(c.BlockNumber-c.FullBlockRange.StartBlock+1) / + float64(c.CurrentLastBlockNumber()-c.FullBlockRange.StartBlock+1) +} + +func (c Checkpoint) RateOrDelay() string { + if c.InWatching() { + return fmt.Sprintf("[Delay:%s]", time.Since(c.BlockTime).String()) + } else { + return fmt.Sprintf("[%.1f%%]", c.Rate()*100) + } +} + +func (c Checkpoint) Snapshot() any { + latest := Checkpoint{ + BlockNumber: c.LatestBlockNumber, + BlockHash: c.LatestBlockHash, + BlockParentHash: c.LatestBlockParentHash, + BlockTime: c.LatestBlockTime, + } + return map[string]any{ + "fullBlockRange": c.FullBlockRange.String(), + "current": GetBlockFullText(c), + "latest": GetBlockFullText(&latest), + "totalBindings": c.TotalBindings, + "data": utils.MapMapNoError(c.Data, utils.StringSummaryV2), + } +} + +func (c Checkpoint) String() string { + return GetBlockFullText(c) +} + +type CheckpointController interface { + // GetLatestCheckpoint get the latest checkpoint + GetLatestCheckpoint() *Checkpoint + + // GetSavedLatestCheckpoint get the latest checkpoint + GetSavedLatestCheckpoint() *Checkpoint + + // GetTemplates return all templates + // There may be template instances later than the checkpoint, which are generated when the binding data of the + // later block is processed, but these are only temporary. + GetTemplates() map[uint64][]TemplateInstance + + // Ready Clean up invalid time series data and entity data by checkpoint, executed at the start of a round + Ready(ctx context.Context, agentStat map[string]int) *ExternalError + + // CleanCheckpoint Delete checkpoints and templates greater than or equal to blockNumberGE, executed after reorg + CleanCheckpoint(ctx context.Context, curBlockNumber, blockNumberGE uint64) *ExternalError + + // MakeCheckpoint Try to construct a checkpoint at the blockData block, indicating that all bindings of this block + // have been processed. + MakeCheckpoint( + ctx context.Context, + blockData BlockDataSummary, + progressBar ProgressBar, + ) (hasNewTemplate bool, err *ExternalError) + + // Save Actually try to save checkpoints + Save(ctx context.Context, saveAll bool) *ExternalError + + // KeepSave Continuously try to save checkpoints, + // if all checkpoint was made, allMade will be closed, and KeepSave will return nil after all Checkpoint was saved. + KeepSave(ctx context.Context, allMade chan struct{}) error + + // SaveError An error occurred during processing, save the error information + SaveError(ctx context.Context, err *ExternalError) error + + // NewTemplateInstance Declare a new template instance in the specified task + NewTemplateInstance(ctx context.Context, task Task, templates []TemplateInstance) *ExternalError + + // InsertTimeSeriesData Insert time series data into the specified block + InsertTimeSeriesData(blockNumber uint64, taskIndex TaskIndex, data []timeseries.Dataset) + + // InsertWebhookData Insert webhook data in the specified block + InsertWebhookData(blockNumber uint64, taskIndex TaskIndex, messages []WebhookMessage) + + // GetEntityOrInterfaceType Get entity or interface declaration by name + GetEntityOrInterfaceType(entity string) schema.EntityOrInterface + // GetEntityType Get entity declaration by name + GetEntityType(entity string) *schema.Entity + // GetEntity get entity + GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, + ) (box *persistent.EntityBox, err *ExternalError) + // GetEntityInBlock get entity + GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, + ) (box *persistent.EntityBox, err *ExternalError) + // ListEntity list entity + ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, + ) (boxes []*persistent.EntityBox, next *string, err *ExternalError) + // ListRelated list related entity to / with the relationship definition in . + ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, + ) ([]*persistent.EntityBox, schema.EntityOrInterface, *ExternalError) + + // SetEntity set or delete entity + SetEntity(ctx context.Context, entityType *schema.Entity, box persistent.UncommittedEntityBox) *ExternalError + + Snapshot() map[string]any +} + +type CheckpointStore interface { + Load(ctx context.Context) ([]Checkpoint, map[uint64][]TemplateInstance, error) + Save( + ctx context.Context, + checkpoints []Checkpoint, + templates map[uint64][]TemplateInstance, + agentStat map[string]int, + ) error + SaveError(ctx context.Context, err *ExternalError) error +} + +var ErrCheckpointsTooBig = errors.New("checkpoints too big") + +var _ CheckpointController = &checkpointController{} + +type checkpointController struct { + chainID string + saveDelay time.Duration + saveInterval time.Duration + maxKeepCheckpointCount uint64 + + checkpointStore CheckpointStore + checkpoints []Checkpoint + savedCheckpoints int // checkpoints[:savedCheckpoints] are all saved in checkpointStore + checkpointSparsifyMultiplier uint64 + + // lastSavedAt and lastSavedCheckpoint are used to estimate Watching needed, + lastSavedAt time.Time + lastSavedCheckpoint *Checkpoint + actuallyLastSavedAt time.Time + + templates map[uint64][]TemplateInstance // key is blockNumber + unsavedTemplates map[uint64][]TemplateInstance // key is blockNumber + + agentStat map[string]int + + quotaService QuotaService + + commitCtxBuilder func(ctx context.Context, chainID string, cur Checkpoint) context.Context + + timeSeriesCtrl TimeSeriesController + entityCtrl EntityController + webhookCtrl WebhookController + + stopped bool + printProcessedExecutor *timer.MinimumIntervalExecutor + + stat *timewin.TimeWindowsManager[*checkpointStatWindow] + + mu sync.Mutex +} + +func NewCheckpointController( + ctx context.Context, + chainID string, + saveDelay time.Duration, + saveInterval time.Duration, + maxKeepCheckpointCount uint64, + checkpointStore CheckpointStore, + quotaService QuotaService, + timeSeriesCtrl TimeSeriesController, + entityCtrl EntityController, + webhookCtrl WebhookController, + commitCtxBuilder func(ctx context.Context, chainID string, cur Checkpoint) context.Context, +) (CheckpointController, error) { + c := checkpointController{ + chainID: chainID, + saveDelay: saveDelay, + saveInterval: saveInterval, + maxKeepCheckpointCount: maxKeepCheckpointCount, + checkpointStore: checkpointStore, + quotaService: quotaService, + commitCtxBuilder: commitCtxBuilder, + timeSeriesCtrl: timeSeriesCtrl, + entityCtrl: entityCtrl, + webhookCtrl: webhookCtrl, + printProcessedExecutor: timer.NewMinimumIntervalExecutor(PrintProcessedInterval), + stat: timewin.NewTimeWindowsManager[*checkpointStatWindow](time.Minute), + } + checkpoints, templates, err := checkpointStore.Load(ctx) + if err != nil { + return nil, err + } + c.checkpoints = checkpoints + c.savedCheckpoints = len(checkpoints) + if templates != nil { + c.templates = templates + } else { + c.templates = make(map[uint64][]TemplateInstance) + } + c.unsavedTemplates = make(map[uint64][]TemplateInstance) + return &c, nil +} + +func (c *checkpointController) findCheckpoint(blockNumberLE uint64) int { + var p int + for p < len(c.checkpoints) && c.checkpoints[p].BlockNumber <= blockNumberLE { + p++ + } + return p - 1 +} + +func (c *checkpointController) getLatestCheckpoint(saved bool) *Checkpoint { + cc := len(c.checkpoints) + if saved { + cc = c.savedCheckpoints + } + if cc == 0 { + return nil + } + return &c.checkpoints[cc-1] +} + +func (c *checkpointController) GetLatestCheckpoint() *Checkpoint { + c.mu.Lock() + defer c.mu.Unlock() + return c.getLatestCheckpoint(false) +} + +func (c *checkpointController) GetSavedLatestCheckpoint() *Checkpoint { + c.mu.Lock() + defer c.mu.Unlock() + return c.getLatestCheckpoint(true) +} + +func (c *checkpointController) getTemplates(savedOnly bool) map[uint64][]TemplateInstance { + if savedOnly { + if c.savedCheckpoints == 0 { + return make(map[uint64][]TemplateInstance) + } + savedBlockNumber := c.checkpoints[c.savedCheckpoints-1].BlockNumber + return utils.FilterMap(c.templates, func(u uint64) bool { + return u <= savedBlockNumber + }) + } + return c.templates +} + +func (c *checkpointController) GetTemplates() map[uint64][]TemplateInstance { + c.mu.Lock() + defer c.mu.Unlock() + return c.getTemplates(false) +} + +func (c *checkpointController) Ready(ctx context.Context, agentStat map[string]int) *ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + c.agentStat = agentStat + var checkpoint *Checkpoint + if len(c.checkpoints) > 0 { + checkpoint = &c.checkpoints[len(c.checkpoints)-1] + } + _, logger := log.FromContext(ctx, "checkpoint", utils.NullOrToString(checkpoint)) + logger.Debug("will clean data") + g, gctx := errgroup.WithContext(ctx) + g.Go(func() error { + if extErr := c.entityCtrl.Reset(gctx, checkpoint); extErr != nil { + logger.Errorfe(extErr, "clean data in entity controller failed") + return extErr + } + return nil + }) + g.Go(func() error { + if extErr := c.timeSeriesCtrl.Reset(gctx, checkpoint); extErr != nil { + logger.Errorfe(extErr, "clean data in time series controller failed") + return extErr + } + return nil + }) + g.Go(func() error { + if extErr := c.webhookCtrl.Reset(gctx, checkpoint); extErr != nil { + logger.Errorfe(extErr, "clean data in webhook controller failed") + return extErr + } + return nil + }) + if err := g.Wait(); err != nil { + var extErr *ExternalError + if errors.As(err, &extErr) { + return extErr + } + // unreachable + return NewExternalError(ErrCodeSystem, err) + } + c.stopped = false + return nil +} + +func (c *checkpointController) CleanCheckpoint( + ctx context.Context, + curBlockNumber, + blockNumberGE uint64, +) *ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + _, logger := log.FromContext(ctx) + logger = logger.UserVisible() + detectedMsg := fmt.Sprintf("Reorg detected when processing block %d, all blocks from block %d are invalid", + curBlockNumber, blockNumberGE) + c.stopped = true + var cc int + for cc < len(c.checkpoints) && c.checkpoints[cc].BlockNumber < blockNumberGE { + cc++ + } + // c.checkpoints[cc:] need to delete + if cc > 0 { + // we have at least one checkpoint + tcc := len(c.checkpoints) + c.checkpoints = c.checkpoints[:cc] + backMsg := fmt.Sprintf("progress will back to %s", c.checkpoints[cc-1].String()) + if cc < c.savedCheckpoints { + logger.Warnf("%s, will remove all %d unsaved checkpoints and %d saved checkpoints in checkpoint store, %s", + detectedMsg, tcc-c.savedCheckpoints, c.savedCheckpoints-cc, backMsg) + // saved checkpoint should rollback + if err := c.checkpointStore.Save(ctx, c.checkpoints, c.getTemplates(false), c.agentStat); err != nil { + return NewExternalError(ErrCodeSaveCheckpointFailed, err) + } + c.savedCheckpoints = len(c.checkpoints) + } else if cc == c.savedCheckpoints { + logger.Warnf("%s, will remove all %d unsaved checkpoints, %s", detectedMsg, tcc-cc, backMsg) + } else { + logger.Warnf("%s, will remove %d unsaved checkpoints, %s", detectedMsg, tcc-cc, backMsg) + } + deleteFilter := func(bn uint64) bool { + return bn > c.checkpoints[cc-1].BlockNumber + } + utils.MapDelete(c.templates, deleteFilter) + utils.MapDelete(c.unsavedTemplates, deleteFilter) + } else { + logger.Warnf("%s, will remove all checkpoints", detectedMsg) + // all checkpoints are invalid, it means all data are useless + if err := c.checkpointStore.Save(ctx, nil, nil, c.agentStat); err != nil { + return NewExternalError(ErrCodeSaveCheckpointFailed, err) + } + c.checkpoints = nil + c.savedCheckpoints = 0 + c.templates = make(map[uint64][]TemplateInstance) + c.unsavedTemplates = make(map[uint64][]TemplateInstance) + } + return nil +} + +func (c *checkpointController) MakeCheckpoint( + ctx context.Context, + blockData BlockDataSummary, + progressBar ProgressBar, +) (templatesChanged bool, extErr *ExternalError) { + _, logger := log.FromContext(ctx, + "current", GetBlockSummary(blockData), + "latest", GetBlockSummary(progressBar.LatestBlock)) + + c.mu.Lock() + defer c.mu.Unlock() + + if c.stopped { + logger.Warnf("try to make checkpoint after checkpoint controller stopped") + return + } + + ck := Checkpoint{ + BlockNumber: blockData.GetBlockNumber(), + BlockHash: blockData.GetBlockHash(), + BlockParentHash: blockData.GetBlockParentHash(), + BlockTime: blockData.GetBlockTime(), + TotalBindings: uint64(blockData.TaskCount), + LatestBlockNumber: progressBar.LatestBlock.GetBlockNumber(), + LatestBlockHash: progressBar.LatestBlock.GetBlockHash(), + LatestBlockParentHash: progressBar.LatestBlock.GetBlockParentHash(), + LatestBlockTime: progressBar.LatestBlock.GetBlockTime(), + FullBlockRange: progressBar.FullBlockRange, + Data: blockData.CheckpointData, + } + processedMsg := fmt.Sprintf("Processed %s[%d/%s/%d] with %d bindings", + ck.RateOrDelay(), + ck.FullBlockRange.StartBlock, + GetBlockSummary(blockData), + ck.CurrentLastBlockNumber(), + ck.TotalBindings) + + var templates []TemplateInstance + if templates, templatesChanged = c.unsavedTemplates[blockData.GetBlockNumber()]; templatesChanged { + // If there are new template instances, temporarily confirm these template instances, + // then return ErrInternalHasNewTemplate and wait for the next time. + c.templates[blockData.GetBlockNumber()] = append(c.templates[blockData.GetBlockNumber()], templates...) + // New template instances that come later need to be ignored because the current block may have new data + // at the beginning, and the generation process of those new template instances may change as a result. + c.unsavedTemplates = make(map[uint64][]TemplateInstance) + // No more templates will be accepted in this run. + c.stopped = true + var minStartBlock uint64 = math.MaxUint64 + for _, tpl := range templates { + minStartBlock = min(minStartBlock, tpl.StartBlock) + } + logger = logger.UserVisible() + processedMsg += fmt.Sprintf(" and %d new templates [%s]", + len(templates), strings.Join(utils.MapSliceNoError(templates, TemplateInstance.String), ",")) + if minStartBlock == ck.BlockNumber { + logger.Warn(processedMsg + ", but this block will be re-processed because has new template from this block") + } else { + // The checkpoint of the current block is still acceptable, but because a new template is created later, + // the BlockBuilder need to be reset and restarted. + c.checkpoints = append(c.checkpoints, ck) + logger.Info(processedMsg) + } + return + } + c.checkpoints = append(c.checkpoints, ck) + + printProcessed := func() { + logger.UserVisible().Info(processedMsg) + } + if ck.InWatching() || ck.TotalBindings > 0 { + printProcessed() + } else { + c.printProcessedExecutor.ExecSimple(printProcessed) + } + + if c.saveDelay == 0 && ck.InWatching() { + // realtime mode + extErr = c.save(ctx, false, true) + } else if uint64(len(c.checkpoints)) >= c.maxKeepCheckpointCount { + logger.Info("will try to save checkpoint because there are too many checkpoints") + extErr = c.save(ctx, false, false) + } else if c.webhookCtrl.CachedTooMuch(ck.BlockNumber) { + logger.Info("will try to save checkpoint because there are too many uncommitted webhook message") + extErr = c.save(ctx, false, false) + } else if c.timeSeriesCtrl.CachedTooMuch(ck.BlockNumber) { + logger.Info("will try to save checkpoint because there are too many uncommitted time series data") + extErr = c.save(ctx, false, false) + } else if c.entityCtrl.CachedTooMuch(ck.BlockNumber) { + logger.Info("will try to save checkpoint because there are too many uncommitted entity changes") + extErr = c.save(ctx, false, false) + } + return +} + +func (c *checkpointController) Save(ctx context.Context, saveAll bool) *ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + return c.save(ctx, saveAll, true) +} + +func (c *checkpointController) estimateWatchingNeed(cur *Checkpoint) string { + if cur == nil { + return "" + } + if cur.InWatching() { + return "0" + } + passed := time.Since(c.lastSavedAt) + if c.lastSavedCheckpoint == nil || passed < time.Minute { + return "" + } + last := *c.lastSavedCheckpoint + growthSpeed := float64(cur.CurrentLastBlockNumber()-last.CurrentLastBlockNumber()) / passed.Seconds() + processSpeed := float64(cur.BlockNumber-last.BlockNumber) / passed.Seconds() + if processSpeed <= growthSpeed { + return "INF" + } + eta := time.Second * time.Duration(float64(cur.CurrentLastBlockNumber()-cur.BlockNumber)/(processSpeed-growthSpeed)) + return eta.String() +} + +func (c *checkpointController) save(ctx context.Context, saveAll bool, checkInterval bool) (extErr *ExternalError) { + _, logger := log.FromContext(ctx) + + if c.stopped { + return + } + + if !saveAll && checkInterval && time.Since(c.actuallyLastSavedAt) < c.saveInterval/2 { + // This is not the final save, and not long enough since the last save, just ignore + return + } + + tm := timer.NewTimer() + startTm := tm.Start("ALL") + + // Find the checkpoints should be saved + var cc int + if saveAll { + cc = len(c.checkpoints) + } else { + cc = c.savedCheckpoints + for cc < len(c.checkpoints) && time.Since(c.checkpoints[cc].BlockTime) > c.saveDelay { + cc++ + } + } + if cc == 0 || cc == c.savedCheckpoints { + // No new checkpoints or all new checkpoints are too close to the current time, so there is nothing to save. + return + } + + cur := c.checkpoints[cc-1] + pre := cur.FullBlockRange.StartBlock + if c.savedCheckpoints > 0 { + pre = c.checkpoints[c.savedCheckpoints-1].BlockNumber + 1 + } + var totalBindings uint64 + for i := c.savedCheckpoints; i < cc; i++ { + totalBindings += c.checkpoints[i].TotalBindings + } + if cc > c.savedCheckpoints { // if c.allDone() is true, cc may be equal to c.savedCheckpoints + // checkpoints[:cc] should be saved. + // However, considering that checkpoint data is only generated when each data store commits, + // so only c.checkpoints[cc-1] should to save, + // so unsaved checkpoints in c.checkpoints[c.savedCheckpoints:cc-1] should be removed + c.checkpoints = utils.RemoveSubSeq(c.checkpoints, c.savedCheckpoints, cc-1-c.savedCheckpoints) + cc = c.savedCheckpoints + 1 + } + + win := checkpointStatWindow{startAt: time.Now(), count: 1} + defer func() { + if extErr == nil { + win.totalBinding = totalBindings + } else { + win.failedCount = 1 + } + c.stat.Append(&win) + }() + + checkOverQuotaTm := tm.Start("O") + over, err := c.quotaService.CheckOverQuota(ctx) + win.checkOverQuotaUsed = checkOverQuotaTm.End() + if err != nil { + return NewExternalError(ErrCodeQuotaServiceError, err) + } else if over != nil { + logger.UserVisible().Errorf("Over quota: %s", over.Detail) + return NewExternalError(ErrCodeOverQuota, errors.Errorf("over quota: %s", over.Msg)) + } + + var usage Usage + + defer func() { + startTm.End() + logger = logger.UserVisible().With("used", tm.ReportDistribution("ALL", "*")) + progress := fmt.Sprintf("%s[%d/%d-%d/%d] with %d bindings", + cur.RateOrDelay(), + cur.FullBlockRange.StartBlock, + pre, + cur.BlockNumber, + cur.CurrentLastBlockNumber(), + totalBindings) + if extErr != nil { + // commits below may be completed partly, so all unsaved checkpoint should be clean, and need to re-init + c.checkpoints = c.checkpoints[:c.savedCheckpoints] + c.templates = c.getTemplates(true) + c.unsavedTemplates = make(map[uint64][]TemplateInstance) + c.stopped = true + if extErr.IsUserError() { + logger.Warnf("Save %s failed: %s", progress, extErr.Error()) + } else { + logger.Warnf("Save %s failed, progress will back to %s", progress, c.getLatestCheckpoint(true)) + } + } else { + // succeed + logger.Infof("Saved %s and %s", progress, usage.String()) + c.actuallyLastSavedAt = time.Now() + if c.lastSavedCheckpoint == nil || time.Since(c.lastSavedAt) > time.Minute*5 { + c.lastSavedAt, c.lastSavedCheckpoint = c.actuallyLastSavedAt, &cur + } + } + }() + + if c.commitCtxBuilder != nil { + ctx = c.commitCtxBuilder(ctx, c.chainID, cur) + } + + // Save the time series data up to cur.BlockNumber + commitTimeSeriesTm := tm.Start("T") + usage.TimeSeries, extErr = c.timeSeriesCtrl.Commit(ctx, cur.BlockNumber, cur.BlockTime) + win.commitTimeSeriesUsed = commitTimeSeriesTm.End() + if extErr != nil { + return extErr + } + + // Save entity data up to cur.BlockNumber + commitEntityTm := tm.Start("E") + usage.EntityCreated, usage.EntityUpdated, extErr = c.entityCtrl.Commit(ctx, cur.BlockNumber, cur.BlockTime) + win.commitEntityUsed = commitEntityTm.End() + if extErr != nil { + return extErr + } + + // Save webhook data up to cur.BlockNumber + commitWebhookTm := tm.Start("W") + usage.Export, extErr = c.webhookCtrl.Commit(ctx, cur.BlockNumber, cur.BlockTime) + win.commitWebhookUsed = commitWebhookTm.End() + if extErr != nil { + return extErr + } + + // Save usage + saveUsageTm := tm.Start("U") + err = c.quotaService.SaveUsage(ctx, usage, cur.InWatching()) + win.saveUsageUsed = saveUsageTm.End() + if err != nil { + return NewExternalError(ErrCodeQuotaServiceError, err) + } + + // Sparse these cc checkpoints and save them + for { + multiplier := max(c.checkpointSparsifyMultiplier, 2) + saveCheckpointTm := tm.Start("S") + remove := sparsify.RemoveWithMultiplier(c.checkpoints[:cc], Checkpoint.GetBlockNumber, multiplier/2, multiplier) + c.checkpoints = utils.RemoveByIndex(c.checkpoints, remove) + cc -= len(remove) + c.savedCheckpoints = cc + err = c.checkpointStore.Save(ctx, c.checkpoints[:c.savedCheckpoints], c.getTemplates(true), c.agentStat) + win.saveCheckpointUsed = saveCheckpointTm.End() + if err != nil { + if errors.Is(err, ErrCheckpointsTooBig) { + logger.With("sparsifyMultiplier", multiplier).Warnfe(err, "checkpoints too big, will reduce it") + c.checkpointSparsifyMultiplier = multiplier * 2 + continue + } + return NewExternalError(ErrCodeSaveCheckpointFailed, err) + } + return nil + } +} + +func (c *checkpointController) KeepSave(ctx context.Context, allMade chan struct{}) error { + _, logger := log.FromContext(ctx) + logger.Info("keep save checkpoint started") + defer func() { + logger.Info("keep save checkpoint finished") + }() + ticker := time.NewTicker(c.saveInterval) + defer ticker.Stop() + for { + if extErr := c.Save(ctx, false); extErr != nil { + return extErr + } + select { + case <-allMade: + // last save + if extErr := c.Save(ctx, true); extErr != nil { + return extErr + } + return nil + case <-ticker.C: + case <-ctx.Done(): + return ctx.Err() + } + } +} + +func (c *checkpointController) SaveError(ctx context.Context, err *ExternalError) error { + return c.checkpointStore.SaveError(ctx, err) +} + +func (c *checkpointController) NewTemplateInstance( + ctx context.Context, + task Task, + templates []TemplateInstance, +) *ExternalError { + blockNumber := task.GetBlockNumber() + c.mu.Lock() + defer c.mu.Unlock() + _, logger := log.FromContext(ctx) + if c.stopped { + logger.Warnw("try to make new template instance after checkpoint controller stopped", + "blockNumber", blockNumber, + "templates", utils.MapSliceNoError(templates, TemplateInstance.String)) + return nil + } + exists := make(map[string][]TplWithCreated) // key is /
/ + for _, bn := range utils.GetOrderedMapKeys(c.templates) { + for _, tpl := range c.templates[bn] { + exists[tpl.UniqID()] = append(exists[tpl.UniqID()], TplWithCreated{TemplateInstance: tpl, CreatedBlock: bn}) + } + } + for _, bn := range utils.GetOrderedMapKeys(c.unsavedTemplates) { + if bn > blockNumber { + break + } + for _, tpl := range c.unsavedTemplates[bn] { + exists[tpl.UniqID()] = append(exists[tpl.UniqID()], TplWithCreated{TemplateInstance: tpl, CreatedBlock: bn}) + } + } + for _, tpl := range templates { + if tpl.StartBlock < blockNumber && !tpl.Removed { + logger.Warnf("start block of the template instance %s less then the block %d created it, will be reset to %d", + tpl, blockNumber, blockNumber) + tpl.StartBlock = blockNumber + } + if exist := exists[tpl.UniqID()]; len(exist) > 0 { + existText := strings.Join(utils.MapSliceNoError(exist, TplWithCreated.String), ",") + on := EmptyBlockRangeSet + for _, ex := range exist { + if ex.Removed { + on = on.Remove(ex.BlockRange) + } else { + on = on.Union(ex.BlockRange) + } + } + on = on.Intersection(BlockRange{StartBlock: blockNumber}) + intersection := on.Intersection(tpl.BlockRange) + if tpl.Removed && intersection.IsEmpty() || !tpl.Removed && intersection.Include(tpl.BlockRange) { + logger.Warnf("try to create new template instance %s at %s, but already created %s, will be ignored", + tpl, task.Summary(), existText) + continue + } + if tpl.Removed { + on = on.Remove(tpl.BlockRange) + } else { + on = on.Union(tpl.BlockRange) + } + if len(on.Holes) > 0 { + return NewExternalError(ErrCodeCreateTemplateFailed, errors.Errorf( + "new template instance %s at %s is invalid, already created %s, "+ + "after created the enable block range will have hole", tpl, task.Summary(), existText)) + } + logger.Infof("has new template %s at %s, and already created %s", tpl, task.Summary(), existText) + } else { + logger.Infof("has new template %s at %s", tpl, task.Summary()) + } + c.unsavedTemplates[blockNumber] = append(c.unsavedTemplates[blockNumber], tpl) + } + return nil +} + +func (c *checkpointController) InsertTimeSeriesData( + blockNumber uint64, + taskIndex TaskIndex, + data []timeseries.Dataset, +) { + c.mu.Lock() + defer c.mu.Unlock() + c.timeSeriesCtrl.Insert(blockNumber, taskIndex, data) +} + +func (c *checkpointController) InsertWebhookData(blockNumber uint64, taskIndex TaskIndex, messages []WebhookMessage) { + c.mu.Lock() + defer c.mu.Unlock() + c.webhookCtrl.Insert(blockNumber, taskIndex, messages) +} + +func (c *checkpointController) GetEntityOrInterfaceType(entity string) schema.EntityOrInterface { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.GetEntityOrInterfaceType(entity) +} + +func (c *checkpointController) GetEntityType(entity string) *schema.Entity { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.GetEntityType(entity) +} + +func (c *checkpointController) GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (box *persistent.EntityBox, err *ExternalError) { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.GetEntity(ctx, typ, id, blockNumber) +} + +func (c *checkpointController) GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (box *persistent.EntityBox, err *ExternalError) { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.GetEntityInBlock(ctx, typ, id, blockNumber) +} + +func (c *checkpointController) ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, +) (boxes []*persistent.EntityBox, next *string, err *ExternalError) { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.ListEntity(ctx, entityType, filters, cursor, limit, blockNumber) +} + +func (c *checkpointController) ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, +) ([]*persistent.EntityBox, schema.EntityOrInterface, *ExternalError) { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.ListRelated(ctx, entityType, id, fieldName, blockNumber) +} + +func (c *checkpointController) SetEntity( + ctx context.Context, + entityType *schema.Entity, + box persistent.UncommittedEntityBox, +) *ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + return c.entityCtrl.SetEntity(ctx, entityType, box) +} + +func (c *checkpointController) unsavedSnapshot() any { + count := len(c.checkpoints) - c.savedCheckpoints + sn := map[string]any{ + "checkpointCount": count, + } + if count > 0 { + var tb uint64 + for i := c.savedCheckpoints; i < len(c.checkpoints); i++ { + tb += c.checkpoints[i].TotalBindings + } + sn["totalBindings"] = tb + sn["latestCheckpoint"] = c.checkpoints[len(c.checkpoints)-1].Snapshot() + } + return sn +} + +func (c *checkpointController) Snapshot() map[string]any { + c.mu.Lock() + defer c.mu.Unlock() + lastSaved := c.getLatestCheckpoint(true) + watching := lastSaved != nil && lastSaved.InWatching() + return map[string]any{ + "chainID": c.chainID, + "config": map[string]any{ + "saveDelay": c.saveDelay.String(), + "saveInterval": c.saveInterval.String(), + "maxKeepCheckpointCount": c.maxKeepCheckpointCount, + "watchingDelay": WatchingDelay.String(), + }, + "checkpointCount": len(c.checkpoints), + "stores": map[string]any{ + "timeSeries": c.timeSeriesCtrl.Snapshot(), + "entity": c.entityCtrl.Snapshot(), + "webhook": c.webhookCtrl.Snapshot(), + }, + "saved": map[string]any{ + "checkpoints": utils.MapSliceNoError(c.checkpoints[:c.savedCheckpoints], Checkpoint.Snapshot), + "inWatching": watching, + "estimateWatchingNeed": c.estimateWatchingNeed(lastSaved), + "checkpointSparsifyMultiplier": c.checkpointSparsifyMultiplier, + }, + "unsaved": c.unsavedSnapshot(), + "templates": c.getTemplates(false), + "statistics": c.stat.Snapshot(), + } +} diff --git a/driver/controller/checkpoint_test.go b/driver/controller/checkpoint_test.go new file mode 100644 index 0000000..2721a38 --- /dev/null +++ b/driver/controller/checkpoint_test.go @@ -0,0 +1,183 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type testCheckpointStore struct { + checkpoints []Checkpoint + templates map[uint64][]TemplateInstance + err *ExternalError +} + +func (cs *testCheckpointStore) Load(ctx context.Context) ([]Checkpoint, map[uint64][]TemplateInstance, error) { + return cs.checkpoints, cs.templates, nil +} + +func (cs *testCheckpointStore) Save( + ctx context.Context, + checkpoints []Checkpoint, + templates map[uint64][]TemplateInstance, + agentStat map[string]int, +) error { + cs.checkpoints = checkpoints + cs.templates = templates + return nil +} + +func (cs *testCheckpointStore) SaveError(ctx context.Context, err *ExternalError) error { + cs.err = err + return nil +} + +type testBlockData struct { + BlockHeader + tasks []Task +} + +func (b testBlockData) DataSource() string { + return "test-data-source" +} + +func (b testBlockData) GetTaskList() []Task { + return b.tasks +} + +func (b testBlockData) CheckpointData() map[string]string { + return nil +} + +func (b testBlockData) Size() int { + return 1 +} + +func newSimpleTestBlockData(blockNumber uint64) testBlockData { + return newTestBlockData(newTestBlockHeader(blockNumber, "", "")) +} + +func newSimpleTestBlockDataSummary(blockNumber uint64) BlockDataSummary { + h := newTestBlockHeader(blockNumber, "", "") + return BlockDataSummary{ + BlockNumber: h.GetBlockNumber(), + BlockParentHash: h.GetBlockParentHash(), + BlockHash: h.GetBlockHash(), + BlockTime: h.GetBlockTime(), + } +} + +func newTestBlockData(header BlockHeader, tasks ...Task) testBlockData { + return testBlockData{ + BlockHeader: header, + tasks: tasks, + } +} + +func Test_save(t *testing.T) { + cs := &testCheckpointStore{} + ctx := context.Background() + cc, err := NewCheckpointController( + context.Background(), + "1", + 0, + time.Hour, + 10000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + assert.NoError(t, err) + + progressBar := ProgressBar{ + LatestBlock: newSimpleTestBlockData(1000), + } + makeCheckpoints := func(bs ...uint64) []Checkpoint { + r := make([]Checkpoint, len(bs)) + for i, bn := range bs { + r[i] = Checkpoint{ + BlockNumber: bn, + BlockTime: newSimpleTestBlockData(bn).GetBlockTime(), + LatestBlockNumber: progressBar.LatestBlock.GetBlockNumber(), + LatestBlockTime: progressBar.LatestBlock.GetBlockTime(), + } + } + return r + } + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(0), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(1), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 1), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(2), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 1, 2), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(3), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 2, 3), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(4), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 2, 3, 4), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(5), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 2, 4, 5), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(6), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 5, 6), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(7), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 6, 7), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(8), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 6, 7, 8), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(9), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 6, 8, 9), cs.checkpoints) + + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(10), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + assert.Equal(t, makeCheckpoints(0, 4, 8, 9, 10), cs.checkpoints) + + for i := uint64(11); i <= 100; i++ { + // all even number has non-empty checkpoint + if i%2 == 0 { + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(i), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + } + } + assert.Equal(t, makeCheckpoints(0, 64, 80, 88, 92, 96, 98, 100), cs.checkpoints) + + for i := uint64(101); i <= 200; i++ { + _, err = cc.MakeCheckpoint(ctx, newSimpleTestBlockDataSummary(i), progressBar) + assert.Nil(t, err) + assert.Nil(t, cc.Save(ctx, true)) + } + assert.Equal(t, makeCheckpoints(0, 128, 160, 176, 192, 196, 198, 199, 200), cs.checkpoints) +} diff --git a/driver/controller/config.go b/driver/controller/config.go new file mode 100644 index 0000000..91ef736 --- /dev/null +++ b/driver/controller/config.go @@ -0,0 +1,31 @@ +package controller + +import ( + "time" + + "sentioxyz/sentio-core/common/envconf" +) + +var ( + ProcessConcurrency = envconf.LoadUInt64("PROCESS_CONCURRENCY", 20, + envconf.WithMax(200), envconf.WithMin(1)) + SaveCheckpointDelay = envconf.LoadDuration("SENTIO_SAVE_CHECKPOINT_DELAY", 0, + envconf.WithMinDuration(0), envconf.WithMaxDuration(time.Minute*10)) + SaveCheckpointInterval = envconf.LoadDuration("SENTIO_SAVE_CHECKPOINT_INTERVAL", time.Second*20, + envconf.WithMinDuration(time.Second)) + MaxKeepCheckpointCount = envconf.LoadUInt64("SENTIO_KEEP_CHECKPOINT_COUNT", 1000000, + envconf.WithMin(10000)) + SubscribeMinWatchInterval = envconf.LoadDuration("SENTIO_SUBSCRIBE_MIN_WATCH_INTERVAL", time.Second) + ClientMaxConcurrency = envconf.LoadUInt64("SENTIO_CLIENT_MAX_CONCURRENCY", 100, envconf.WithMin(10)) + PrintProcessedInterval = envconf.LoadDuration("SENTIO_PRINT_PROCESSED_INTERVAL", time.Second) + SkipStartBlockValidation = envconf.LoadBool("SENTIO_SKIP_START_BLOCK_VALIDATION", false) +) + +const ( + // WatchingDelay If the difference between the time of a processed block and the latest block is less than this value, + // then the block is considered to be in the watching state. + WatchingDelay = time.Minute * 5 + + // RunWaiting If a round got an external error that can be retried, wait this long before starting the next round + RunWaiting = time.Second * 30 +) diff --git a/driver/controller/config/BUILD.bazel b/driver/controller/config/BUILD.bazel new file mode 100644 index 0000000..2d499cf --- /dev/null +++ b/driver/controller/config/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "config", + srcs = ["config.go"], + importpath = "sentioxyz/sentio-core/driver/controller/config", + visibility = ["//visibility:public"], + deps = [ + "//common/jsonutils", + "//common/log", + "//service/processor/models", + ], +) diff --git a/driver/controller/config/config.go b/driver/controller/config/config.go new file mode 100644 index 0000000..5fc5492 --- /dev/null +++ b/driver/controller/config/config.go @@ -0,0 +1,65 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "sentioxyz/sentio-core/common/jsonutils" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/service/processor/models" +) + +// ChainConfig is the per-chain configuration consumed by the streaming +// (driver v3/v4) controller. The legacy driver v2 configuration (chain.Config) +// stays in the sentio repository. +type ChainConfig struct { + ChainID string + Endpoint string + StartBlockOverride int64 + ProcessingDelayBlocks uint64 + KeepSuiEventTypePackage bool + SkipStartBlockValidation bool + IsCustomizedEndpoint bool +} + +// PatchChainsConfigEnv is the env var that, when set, carries a JSON patch +// applied on top of the chains config file before it is parsed. +const PatchChainsConfigEnv = "CHAIN_CONFIG_JSON_PATCH" + +func LoadChainsConfig( + path string, + patchEnv string, + networkOverrides []models.NetworkOverride, +) (map[string]*ChainConfig, error) { + var file []byte + var err error + if file, err = os.ReadFile(path); err != nil { + return nil, err + } + if patch := strings.TrimSpace(os.Getenv(patchEnv)); patchEnv != "" && patch != "" { + file, err = jsonutils.Patch(file, []byte(patch), func(path string, or, pa any) { + log.Infof("patch chains config %s %v => %v", path, or, pa) + }) + if err != nil { + return nil, fmt.Errorf("patch chain config from env %s failed: %w", patchEnv, err) + } + } + var chainsConfig map[string]*ChainConfig + if err = json.Unmarshal(file, &chainsConfig); err != nil { + return nil, err + } + for _, no := range networkOverrides { + chainsConfig[no.Chain] = &ChainConfig{ChainID: no.Chain, Endpoint: no.Host, IsCustomizedEndpoint: true} + log.Infof("will use customized host %q in chain %s", no.Host, no.Chain) + } + return chainsConfig, nil +} + +func NewCustomizedChainConfig(chainID, endpoint string) *ChainConfig { + return &ChainConfig{ + ChainID: chainID, + Endpoint: endpoint, + } +} diff --git a/driver/controller/data/BUILD.bazel b/driver/controller/data/BUILD.bazel new file mode 100644 index 0000000..872dff7 --- /dev/null +++ b/driver/controller/data/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "data", + srcs = [ + "block_cache.go", + "contract.go", + "errors.go", + "interval.go", + "statistics.go", + "subscribe.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data", + visibility = ["//visibility:public"], + deps = [ + "//common/envconf", + "//common/log", + "//common/queue", + "//common/timehist", + "//common/timewin", + "//common/utils", + "//common/window", + "//driver/controller", + "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_golang_lru//:golang-lru", + "@org_golang_x_sync//singleflight", + ], +) + +go_test( + name = "data_test", + srcs = [ + "block_cache_test.go", + "contract_test.go", + "statistics_test.go", + ], + embed = [":data"], + deps = [ + "@com_github_pkg_errors//:errors", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/driver/controller/data/aptos/BUILD.bazel b/driver/controller/data/aptos/BUILD.bazel new file mode 100644 index 0000000..5096af5 --- /dev/null +++ b/driver/controller/data/aptos/BUILD.bazel @@ -0,0 +1,41 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "aptos", + srcs = [ + "block.go", + "change.go", + "client.go", + "resource.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/aptos", + visibility = ["//visibility:public"], + deps = [ + "//chain/aptos", + "//common/concurrency", + "//common/errgroup", + "//common/https", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_aptos_labs_aptos_go_sdk//:aptos-go-sdk", + "@com_github_aptos_labs_aptos_go_sdk//api", + "@com_github_ethereum_go_ethereum//common", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_pkg_errors//:errors", + ], +) + +go_test( + name = "aptos_test", + srcs = ["transaction_test.go"], + embed = [":aptos"], + deps = [ + "//chain/aptos", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/data/aptos/block.go b/driver/controller/data/aptos/block.go new file mode 100644 index 0000000..d12e0c7 --- /dev/null +++ b/driver/controller/data/aptos/block.go @@ -0,0 +1,128 @@ +package aptos + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type BlockMainData struct { + SimpleTxn *aptos.MinimalistTransaction + Txn *aptos.Transaction + Changes []Change + Intervals []data.IntervalConfig +} + +type DataRequirement struct { + Interval []data.IntervalRequirement + Changes []ChangeRequirement + Txn []TransactionRequirement +} + +func (d BlockMainData) IsEmpty() bool { + return d.Size() == 0 +} + +func (d BlockMainData) Size() int { + return len(d.Changes) + len(d.Intervals) + utils.Select(d.Txn == nil, 0, 10) +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetMinimalistTransaction(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 10000, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{ + Intervals: []data.IntervalConfig{req.IntervalConfig}, + } + } + return result, nil + }, + ) +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.Changes = MergeChangeRequirements(currentBlockNumber, req.Changes) + req.Txn = MergeTxnRequirements(currentBlockNumber, req.Txn) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.Changes { + fetchers = append(fetchers, BuildChangeFetcher( + namePrefix+fmt.Sprintf("ChangeFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Txn { + fetchers = append(fetchers, BuildTxnFetcher( + namePrefix+fmt.Sprintf("TxnFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(_ uint64, from []BlockMainData) (data BlockMainData, has bool, _ error) { + has = len(from) > 0 + // Changes and Txn never be repeated, because a range will only have one fetcher with data. + for _, box := range from { + if box.Txn != nil { + data.Txn = box.Txn + } + if box.SimpleTxn != nil { + data.SimpleTxn = box.SimpleTxn + } + data.Changes = append(data.Changes, box.Changes...) + data.Intervals = append(data.Intervals, box.Intervals...) + } + return + }) +} diff --git a/driver/controller/data/aptos/change.go b/driver/controller/data/aptos/change.go new file mode 100644 index 0000000..9e05af2 --- /dev/null +++ b/driver/controller/data/aptos/change.go @@ -0,0 +1,110 @@ +package aptos + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type Change struct { + Raw string + + *aptos.WriteSetChange +} + +func (e *Change) UnmarshalJSON(raw []byte) error { + if err := json.Unmarshal(raw, &e.WriteSetChange); err != nil { + return err + } + e.Raw = string(raw) + return nil +} + +type MinimalistTransactionWithChanges struct { + aptos.MinimalistTransaction `json:",inline"` + + Changes []Change `json:"changes"` +} + +type ChangeRequirement struct { + controller.BlockRange + aptos.ChangeFilter +} + +func (r ChangeRequirement) String() string { + return fmt.Sprintf("ChangeRequirement[%s]%s", r.ChangeFilter.String(), r.BlockRange.String()) +} + +func (r ChangeRequirement) Snapshot() any { + return map[string]any{ + "filter": r.ChangeFilter, + "range": r.BlockRange.String(), + } +} + +// MergeChangeRequirements it can be guaranteed that all the item ranges of the result must be disjoint, +// and each range has at most one filter +func MergeChangeRequirements(current uint64, reqs []ChangeRequirement) (result []ChangeRequirement) { + rs := controller.CutRangeSet(current, utils.MapSliceNoError(reqs, func(r ChangeRequirement) controller.BlockRange { + return r.BlockRange + })) + for _, r := range rs { + var filters []aptos.ChangeFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.ChangeFilter) + } + } + if len(filters) == 0 { + continue + } + result = append(result, ChangeRequirement{ + ChangeFilter: aptos.MergeChangeFilters(filters...), + BlockRange: r, + }) + } + return result +} + +func BuildChangeFetcher( + name string, + req ChangeRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1000, + 1000000, + 100000, + 10000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txs, err := client.GetChanges(ctx, start, end, req.ChangeFilter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData, len(txs)) + for _, tx := range txs { + result[tx.Version] = BlockMainData{SimpleTxn: &tx.MinimalistTransaction, Changes: tx.Changes} + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/aptos/client.go b/driver/controller/data/aptos/client.go new file mode 100644 index 0000000..68344be --- /dev/null +++ b/driver/controller/data/aptos/client.go @@ -0,0 +1,362 @@ +package aptos + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + + aptossdk "github.com/aptos-labs/aptos-go-sdk" + "github.com/aptos-labs/aptos-go-sdk/api" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" + "github.com/pkg/errors" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetTransaction(ctx context.Context, txnVersion uint64) (aptos.Transaction, error) + GetMinimalistTransaction(ctx context.Context, txnVersion uint64) (MinimalistTransaction, error) + GetHeaderIgnoreCache(ctx context.Context, txnVersion uint64) (controller.BlockHeader, error) + + GetChanges( + ctx context.Context, + startTxnVersion, endTxnVersion uint64, + filter aptos.ChangeFilter, + ) ([]MinimalistTransactionWithChanges, error) + GetTransactions( + ctx context.Context, + startTxnVersion, endTxnVersion uint64, + filter aptos.TransactionFilter, + fetchConfig aptos.TransactionFetchConfig, + ) ([]aptos.Transaction, error) + // GetAccountResources key of requirement is Address, + // value is the set of ResourceType, empty set means need all Resource of the Account + GetAccountResources( + ctx context.Context, + txnVersion uint64, + requirement map[string][]string, // requirement[] is null means need all resource of + ) ([]AccountResource, error) + + GetAddressStartBlock(ctx context.Context, address string, start, latest uint64) (uint64, bool, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +type client struct { + endpoint string + firstTxnVersion int64 + watchLatestInterval time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + rpcCli *rpc.Client + rawCli *aptossdk.NodeClient + + cachedMinimalistTxn *data.BlockCache[MinimalistTransaction] +} + +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstTxnVersion int64, + watchLatestInterval time.Duration, +) (Client, error) { + cli := &client{ + endpoint: endpoint, + firstTxnVersion: firstTxnVersion, + watchLatestInterval: watchLatestInterval, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + } + var err error + if cli.rpcCli, err = rpc.DialOptions(ctx, endpoint, rpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.rawCli, err = aptossdk.NewNodeClientWithHttpClient( + fmt.Sprintf("%s/v1", strings.TrimRight(endpoint, "/")), + 0, + https.NewClient(https.WithTimeout(time.Minute))) + if err != nil { + return nil, errors.Wrapf(err, "build aptos node client with endpoint %s failed", endpoint) + } + cli.cachedMinimalistTxn, _ = data.NewBlockCache[MinimalistTransaction](100000) + return cli, nil +} + +func (c *client) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) (err error) { + startAt := time.Now() + // waiting concurrency control token + var release func() + release, err = c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + waitEndAt := time.Now() + defer func() { + c.stat.Called(method, args, err, startAt, waitEndAt) + }() + if err != nil { + return err // always be context.Canceled or context.DeadlineExceeded + } + defer func() { + if err != nil { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + }() + defer release() + // actually call + switch method { + case "raw_getAccountResourcesAll": + txnVersion := args[0].(uint64) + address := args[1].(string) + addr := aptossdk.AccountAddress(common.HexToHash(address)) + var resources []aptossdk.AccountResourceInfo + resources, err = c.rawCli.AccountResourcesByPages(addr, txnVersion, 0) + if err != nil { + return err + } + r := result.(*[]AccountResource) + (*r), err = utils.MapSlice(resources, func(res aptossdk.AccountResourceInfo) (AccountResource, error) { + raw, marshalErr := json.Marshal(res) + if marshalErr != nil { + return AccountResource{}, marshalErr + } + return AccountResource{ + Raw: string(raw), + Address: address, + Type: res.Type, + }, nil + }) + return err + case "raw_getAccountResource": + txnVersion := args[0].(uint64) + address := args[1].(string) + resourceType := args[2].(string) + addr := aptossdk.AccountAddress(common.HexToHash(address)) + var res map[string]any + res, err = c.rawCli.AccountResource(addr, resourceType, txnVersion) + if err != nil { + return err + } + r := result.(*AccountResource) + if raw, marshalErr := json.Marshal(res); marshalErr != nil { + return marshalErr + } else { + r.Raw = string(raw) + r.Address = address + r.Type = resourceType + } + return nil + case "raw_getTxByVersion": + txnVersion := args[0].(uint64) + var tx *api.CommittedTransaction + tx, err = c.rawCli.TransactionByVersion(txnVersion) + if err != nil { + return err + } + // will return error if not found, so here tx will always be non-null + r := result.(*api.CommittedTransaction) + *r = *tx + return nil + default: + return c.rpcCli.CallContext(ctx, &result, method, args...) + } +} + +func (c *client) GetLatest(ctx context.Context) (controller.BlockHeader, uint64, error) { + var resp aptos.GetLatestMinimalistTransactionResponse + err := c.callContext(ctx, &resp, 0, "aptosV2_getLatestMinimalistTransaction", 0) + if err != nil { + return nil, 0, err + } + if err = resp.CheckAPIVersion(); err != nil { + return nil, 0, errors.Wrapf(controller.ErrInternalNeedUpgrade, err.Error()) + } + latest := MinimalistTransaction(resp.Transaction) + return latest, data.GetFirst(c.firstTxnVersion, latest.Version), err +} + +func (c *client) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, txVersionGt uint64) (latest controller.BlockHeader, broken, err error) { + var resp aptos.GetLatestMinimalistTransactionResponse + err = c.callContext(ctx, &resp, 0, "aptosV2_getLatestMinimalistTransaction", txVersionGt) + if err == nil { + latest, broken = MinimalistTransaction(resp.Transaction), resp.CheckAPIVersion() + } + if broken != nil { + broken = errors.Wrapf(controller.ErrInternalNeedUpgrade, broken.Error()) + } + return + }, + callback) +} + +func (c *client) GetTransaction(ctx context.Context, txnVersion uint64) (aptos.Transaction, error) { + var raw api.CommittedTransaction + if err := c.callContext(ctx, &raw, txnVersion, "raw_getTxByVersion", txnVersion); err != nil { + return aptos.Transaction{}, err + } + return aptos.NewTransaction(&raw), nil +} + +func (c *client) GetMinimalistTransaction(ctx context.Context, txnVersion uint64) (MinimalistTransaction, error) { + // Cache + singleflight: concurrent fetchers asking for the same version share one RPC. + return c.cachedMinimalistTxn.GetOrFetch(txnVersion, func() (MinimalistTransaction, error) { + return c.fetchMinimalistTransaction(ctx, txnVersion) + }) +} + +func (c *client) fetchMinimalistTransaction(ctx context.Context, txnVersion uint64) (MinimalistTransaction, error) { + var txn *MinimalistTransaction + if err := c.callContext(ctx, &txn, txnVersion, "aptosV2_getMinimalistTransaction", txnVersion); err != nil { + return MinimalistTransaction{}, err + } + if txn == nil { + return MinimalistTransaction{}, errors.Errorf("transaction %d not found", txnVersion) + } + return *txn, nil +} + +func (c *client) getMinimalistTransaction(ctx context.Context, txnVersion uint64) (MinimalistTransaction, error) { + txn, err := c.fetchMinimalistTransaction(ctx, txnVersion) + if err == nil { + c.cachedMinimalistTxn.Add(txnVersion, txn) + } + return txn, err +} + +func (c *client) GetHeaderIgnoreCache(ctx context.Context, txnVersion uint64) (controller.BlockHeader, error) { + return c.getMinimalistTransaction(ctx, txnVersion) +} + +func (c *client) GetChanges( + ctx context.Context, + startTxnVersion, endTxnVersion uint64, + filter aptos.ChangeFilter, +) (result []MinimalistTransactionWithChanges, err error) { + args := aptos.GetResourceChangesRequest{ + FromVersion: startTxnVersion, + ToVersion: endTxnVersion, + Filter: filter, + } + err = c.callContext(ctx, &result, startTxnVersion, "aptosV2_getResourceChanges", args) + return +} + +func (c *client) GetTransactions( + ctx context.Context, + startTxnVersion, endTxnVersion uint64, + filter aptos.TransactionFilter, + fetchConfig aptos.TransactionFetchConfig, +) (result []aptos.Transaction, err error) { + args := aptos.GetTransactionsRequest{ + FromVersion: startTxnVersion, + ToVersion: endTxnVersion, + Filter: filter, + FetchConfig: fetchConfig, + } + err = c.callContext(ctx, &result, startTxnVersion, "aptosV2_getTransactions", args) + return +} + +func (c *client) GetAccountResources( + ctx context.Context, + txnVersion uint64, + requirement map[string][]string, +) ([]AccountResource, error) { + if int64(txnVersion) < c.firstTxnVersion || len(requirement) == 0 { + return nil, nil + } + var result utils.SafeSlice[AccountResource] + g, gctx := errgroup.WithContext(ctx) + for address_, types := range requirement { + address := address_ + if types == nil { // need all resources + g.Go(func() error { + var resources []AccountResource + err := c.callContext(gctx, &resources, txnVersion, "raw_getAccountResourcesAll", txnVersion, address) + if err == nil { + result.Append(resources...) + } + return err + }) + } else { + for _, typ_ := range set.New(types...).DumpValues() { + typ := typ_ + g.Go(func() error { + var resource AccountResource + err := c.callContext(gctx, &resource, txnVersion, "raw_getAccountResource", txnVersion, address, typ) + if err == nil { + result.Append(resource) + } + return err + }) + } + } + } + if err := g.Wait(); err != nil { + return nil, err + } + return result.Dump(), nil +} + +func (c *client) GetAddressStartBlock(ctx context.Context, address string, start, latest uint64) (uint64, bool, error) { + var result *uint64 + err := c.callContext(ctx, &result, start, "aptosV2_getAddressStartTxVersion", address, latest) + if err != nil { + return 0, false, err + } + if result == nil { + return 0, false, nil + } + return max(*result, start), true, nil +} + +func (c *client) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedMinimalistTxn.Keys() { + if r.Contains(bn) { + c.cachedMinimalistTxn.Remove(bn) + } + } +} + +func (c *client) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "firstTxnVersion": c.firstTxnVersion, + "watchLatestInterval": c.watchLatestInterval.String(), + }, + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cachedMinimalistTxn": c.cachedMinimalistTxn.Snapshot(10, controller.GetBlockFullText[MinimalistTransaction]), + } +} diff --git a/driver/controller/data/aptos/resource.go b/driver/controller/data/aptos/resource.go new file mode 100644 index 0000000..1c08d76 --- /dev/null +++ b/driver/controller/data/aptos/resource.go @@ -0,0 +1,50 @@ +package aptos + +import ( + "fmt" + + "sentioxyz/sentio-core/common/utils" +) + +type AccountResourceFilter struct { + Address string + + // nil means need all resources of the account, empty means do not need any resource + // only contract move interval handler will use empty ResourceType + ResourceType []string +} + +func (c AccountResourceFilter) NeedNothing() bool { + return c.ResourceType != nil && len(c.ResourceType) == 0 +} + +func (c AccountResourceFilter) String() string { + return fmt.Sprintf("Address:%s,ResourceType:%s", c.Address, utils.ArrSummary(c.ResourceType, 10)) +} + +func (c AccountResourceFilter) Check(ar AccountResource) bool { + return c.Address == ar.Address && (c.ResourceType == nil || utils.IndexOf(c.ResourceType, ar.Type) >= 0) +} + +func MergeAccountResourceFilters(filters []AccountResourceFilter) map[string][]string { + result := make(map[string][]string) + for _, filter := range filters { + if types, has := result[filter.Address]; !has { + result[filter.Address] = filter.ResourceType + } else if types != nil { + if filter.ResourceType == nil { + result[filter.Address] = nil + } else { + result[filter.Address] = append(types, filter.ResourceType...) + } + } + } + return result +} + +type AccountResource struct { + Raw string + + Address string + Type string +} diff --git a/driver/controller/data/aptos/transaction.go b/driver/controller/data/aptos/transaction.go new file mode 100644 index 0000000..e7e88e4 --- /dev/null +++ b/driver/controller/data/aptos/transaction.go @@ -0,0 +1,140 @@ +package aptos + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type MinimalistTransaction aptos.MinimalistTransaction + +func (t MinimalistTransaction) GetBlockNumber() uint64 { + return t.Version +} + +func (t MinimalistTransaction) GetBlockHash() string { + return t.Hash +} + +func (t MinimalistTransaction) GetBlockParentHash() string { + return "" +} + +func (t MinimalistTransaction) GetBlockTime() time.Time { + return time.UnixMicro(t.TimestampMS) +} + +type Transaction aptos.Transaction + +func (t *Transaction) GetBlockNumber() uint64 { + return t.Version() +} + +func (t *Transaction) GetBlockHash() string { + return t.Hash() +} + +func (t *Transaction) GetBlockParentHash() string { + return "" +} + +func (t *Transaction) GetBlockTime() time.Time { + if tm := (*aptos.Transaction)(t).Time(); tm != nil { + return *tm + } + return time.Time{} +} + +type TransactionRequirement struct { + controller.BlockRange + + Filter aptos.TransactionFilter + FetchConfig aptos.TransactionFetchConfig +} + +func (r TransactionRequirement) String() string { + return fmt.Sprintf("TransactionRequirement[Filter:[%s],FetchConfig:[%s]]%s", + r.Filter.String(), r.FetchConfig.String(), r.BlockRange.String()) +} + +func (r TransactionRequirement) Snapshot() any { + return map[string]any{ + "filter": r.Filter, + "fetchConfig": r.FetchConfig, + "range": r.BlockRange.String(), + } +} + +func MergeTxnRequirements(current uint64, reqs []TransactionRequirement) (result []TransactionRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r TransactionRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + rr := TransactionRequirement{BlockRange: r} + first := true + for _, req := range reqs { + if !req.BlockRange.Include(r) { + continue + } + if first { + rr.Filter = req.Filter + rr.FetchConfig = req.FetchConfig + first = false + } else { + rr.Filter = rr.Filter.Merge(req.Filter) + rr.FetchConfig = rr.FetchConfig.Merge(req.FetchConfig) + } + } + if first { + continue + } + result = append(result, rr) + } + return result + +} + +func BuildTxnFetcher( + name string, + req TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 100, + 100000, + 10000, // size of transaction is 10, so will cache 1000 transactions + 5000, // the target is that each query got no more than 500 transactions + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txs, err := client.GetTransactions(ctx, start, end, req.Filter, req.FetchConfig) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for i := range txs { + result[txs[i].Version()] = BlockMainData{Txn: &txs[i]} + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/aptos/transaction_test.go b/driver/controller/data/aptos/transaction_test.go new file mode 100644 index 0000000..b5391a2 --- /dev/null +++ b/driver/controller/data/aptos/transaction_test.go @@ -0,0 +1,44 @@ +package aptos + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + + "sentioxyz/sentio-core/chain/aptos" +) + +func Test_txJsonMarshal(t *testing.T) { + raw := aptos.MinimalistTransaction{ + Version: 123, + Hash: "abc", + TimestampMS: 123, + } + b0, err := json.Marshal(raw) + //t.Logf("%s", string(b0)) + assert.NoError(t, err) + + var tx MinimalistTransaction + assert.NoError(t, json.Unmarshal(b0, &tx)) + assert.Equal(t, raw.Version, tx.Version) + assert.Equal(t, raw.Hash, tx.Hash) + assert.Equal(t, raw.TimestampMS, tx.TimestampMS) + + b1, err := json.Marshal(tx) + //t.Logf("%s", string(b1)) + assert.NoError(t, err) + assert.Equal(t, b0, b1) +} + +func Test_diffBetweenNullAndEmptyArr(t *testing.T) { + var a []int + assert.True(t, a == nil) + assert.True(t, len(a) == 0) + a = []int{} + assert.False(t, a == nil) + assert.True(t, len(a) == 0) + a = []int{1} + assert.False(t, a == nil) + assert.False(t, len(a) == 0) +} diff --git a/driver/controller/data/block_cache.go b/driver/controller/data/block_cache.go new file mode 100644 index 0000000..d1ff989 --- /dev/null +++ b/driver/controller/data/block_cache.go @@ -0,0 +1,84 @@ +package data + +import ( + "strconv" + + "sentioxyz/sentio-core/common/utils" + + lru "github.com/sentioxyz/golang-lru" + "golang.org/x/sync/singleflight" +) + +// BlockCache is an LRU keyed by block number combined with singleflight. Chain clients prefetch +// headers/blocks concurrently from several fetchers, so the same block is frequently requested by +// multiple goroutines at nearly the same time. BlockCache caches the value and collapses concurrent +// misses for the same block into a single fetch, which keeps per-block RPCs (e.g. a header lookup) +// off the hot path and from being duplicated. +type BlockCache[V any] struct { + cache *lru.Cache[uint64, V] + sf singleflight.Group +} + +// NewBlockCache creates a BlockCache holding up to size entries. It errors only when size <= 0. +func NewBlockCache[V any](size int) (*BlockCache[V], error) { + cache, err := lru.New[uint64, V](size) + if err != nil { + return nil, err + } + return &BlockCache[V]{cache: cache}, nil +} + +// Get returns the cached value for blockNumber, if present. +func (c *BlockCache[V]) Get(blockNumber uint64) (V, bool) { + return c.cache.Get(blockNumber) +} + +// Add stores v for blockNumber, overwriting any existing entry. +func (c *BlockCache[V]) Add(blockNumber uint64, v V) { + c.cache.Add(blockNumber, v) +} + +// Remove drops blockNumber from the cache. +func (c *BlockCache[V]) Remove(blockNumber uint64) { + c.cache.Remove(blockNumber) +} + +// Keys returns the currently cached block numbers (used to evict a reorged range). +func (c *BlockCache[V]) Keys() []uint64 { + return c.cache.Keys() +} + +// GetOrFetch returns the cached value for blockNumber, or fetches it via fetch and caches the result. +// Concurrent misses for the same block are coalesced into a single fetch. fetch is not invoked even +// when a caller arrives just after an in-flight fetch finished: the cache is re-checked inside the +// flight, so the just-fetched value is reused rather than fetched again. +func (c *BlockCache[V]) GetOrFetch(blockNumber uint64, fetch func() (V, error)) (V, error) { + // Fast path: avoids the strconv + singleflight bookkeeping when the block is already cached. + if v, ok := c.cache.Get(blockNumber); ok { + return v, nil + } + v, err, _ := c.sf.Do(strconv.FormatUint(blockNumber, 10), func() (any, error) { + // Re-check inside the flight (double-checked): a preceding flight for this block may have + // finished and populated the cache between the miss above and entering Do. + if v, ok := c.cache.Get(blockNumber); ok { + return v, nil + } + v, err := fetch() + if err != nil { + return nil, err + } + c.cache.Add(blockNumber, v) + return v, nil + }) + if err != nil { + var zero V + return zero, err + } + return v.(V), nil +} + +// Snapshot renders up to maxCount entries for the debug tracker, using valuePreview to stringify each +// value. It mirrors utils.CacheSnapshot so callers don't need to reach for the underlying cache. +func (c *BlockCache[V]) Snapshot(maxCount int, valuePreview func(V) string) any { + return utils.CacheSnapshot(c.cache, maxCount, valuePreview) +} diff --git a/driver/controller/data/block_cache_test.go b/driver/controller/data/block_cache_test.go new file mode 100644 index 0000000..b3dde55 --- /dev/null +++ b/driver/controller/data/block_cache_test.go @@ -0,0 +1,67 @@ +package data + +import ( + "errors" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBlockCacheGetOrFetch(t *testing.T) { + c, err := NewBlockCache[uint64](1024) + require.NoError(t, err) + + var fetches atomic.Int64 + fetch := func() (uint64, error) { + fetches.Add(1) + return 70, nil + } + + // Concurrent callers that all miss the same block must collapse into a single fetch. Run under + // -race, this also guards that BlockCache itself has no internal data race. + const concurrent = 50 + results := make([]uint64, concurrent) + errs := make([]error, concurrent) + var wg sync.WaitGroup + for i := 0; i < concurrent; i++ { + i := i + wg.Add(1) + go func() { + defer wg.Done() + results[i], errs[i] = c.GetOrFetch(7, fetch) + }() + } + wg.Wait() + + for i := 0; i < concurrent; i++ { + require.NoError(t, errs[i]) + require.Equal(t, uint64(70), results[i]) + } + require.EqualValues(t, 1, fetches.Load(), "concurrent misses for the same block fetch once") + + // A later call hits the cache without fetching again. + v, err := c.GetOrFetch(7, fetch) + require.NoError(t, err) + require.Equal(t, uint64(70), v) + require.EqualValues(t, 1, fetches.Load()) +} + +func TestBlockCacheGetOrFetchError(t *testing.T) { + c, err := NewBlockCache[uint64](16) + require.NoError(t, err) + + boom := errors.New("boom") + _, err = c.GetOrFetch(1, func() (uint64, error) { return 0, boom }) + require.ErrorIs(t, err, boom) + + // Errors are not cached: a subsequent successful fetch for the same block still runs and caches. + v, err := c.GetOrFetch(1, func() (uint64, error) { return 42, nil }) + require.NoError(t, err) + require.Equal(t, uint64(42), v) + + got, ok := c.Get(1) + require.True(t, ok) + require.Equal(t, uint64(42), got) +} diff --git a/driver/controller/data/contract.go b/driver/controller/data/contract.go new file mode 100644 index 0000000..77912fc --- /dev/null +++ b/driver/controller/data/contract.go @@ -0,0 +1,53 @@ +package data + +import ( + "context" + + "sentioxyz/sentio-core/common/utils" +) + +func BinarySearchContractStart( + ctx context.Context, + start, end uint64, + checker func(context.Context, uint64) (bool, error), +) (uint64, bool, error) { + if has, err := checker(ctx, end); err != nil { + return 0, false, err + } else if !has { + return 0, false, nil + } + low, high := start, end + for p := uint64(1); p > 0; p <<= 1 { + if p <= start { + low = p + } + if p >= end { + high = p + break + } + } + internalChecker := func(ctx context.Context, bn uint64) (bool, error) { + if bn < start { + return false, nil + } + if bn >= end { + return true, nil + } + return checker(ctx, bn) + } + for low < high { + mid := (low + high) / 2 + if has, err := internalChecker(ctx, mid); err != nil { + return 0, false, err + } else if has { + high = mid + } else { + low = mid + 1 + } + } + return low, true, nil +} + +func GetFirst(firstConfig int64, latest uint64) uint64 { + return uint64(max(0, utils.Select(firstConfig < 0, int64(latest)+firstConfig, firstConfig))) +} diff --git a/driver/controller/data/contract_test.go b/driver/controller/data/contract_test.go new file mode 100644 index 0000000..574624a --- /dev/null +++ b/driver/controller/data/contract_test.go @@ -0,0 +1,30 @@ +package data + +import ( + "context" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func Test_BinarySearchContractStart(t *testing.T) { + const startBlock = 500 + checker := func(_ context.Context, bn uint64) (bool, error) { + if bn < 100 || bn > 1000 { + return false, errors.Errorf("out of range") + } + return bn >= startBlock, nil + } + + for s := uint64(100); s <= 1000; s++ { + for e := s; e <= 1000; e++ { + n, has, err := BinarySearchContractStart(context.Background(), s, e, checker) + assert.NoError(t, err) + assert.Equalf(t, startBlock <= e, has, "s=%d,e=%d", s, e) + if has { + assert.Equalf(t, max(startBlock, s), n, "s=%d,e=%d", s, e) + } + } + } +} diff --git a/driver/controller/data/errors.go b/driver/controller/data/errors.go new file mode 100644 index 0000000..e667adf --- /dev/null +++ b/driver/controller/data/errors.go @@ -0,0 +1,46 @@ +package data + +import ( + "errors" + "fmt" +) + +// NewClientRetryableError marks a failure returned by a chain data client constructor (the NewClient +// functions under data/, e.g. data/sol.NewClient) that is transient and worth retrying, +// rather than a permanent misconfiguration. For example, a data endpoint that keeps returning +// HTTP/timeout errors while probing its capabilities may simply be temporarily unavailable; failing +// permanently would strand the processor, whereas retrying (by restarting the pod) gives it a fresh +// chance once the endpoint recovers. +// +// This package intentionally knows nothing about how callers act on the error; it only signals that +// the NewClient failure is retryable. The startup controller (which depends on this package) inspects +// construction errors with IsNewClientRetryable / errors.As and chooses to restart the pod instead of +// failing permanently when it matches. +type NewClientRetryableError struct { + // Reason is a human-readable description of why client construction failed. + Reason string + // Err is the underlying error that triggered the retryable failure, if any. + Err error +} + +// NewClientRetryable wraps err into a NewClientRetryableError with the given reason. +func NewClientRetryable(reason string, err error) *NewClientRetryableError { + return &NewClientRetryableError{Reason: reason, Err: err} +} + +func (e *NewClientRetryableError) Error() string { + if e.Err != nil { + return fmt.Sprintf("retryable NewClient error: %s: %v", e.Reason, e.Err) + } + return fmt.Sprintf("retryable NewClient error: %s", e.Reason) +} + +func (e *NewClientRetryableError) Unwrap() error { + return e.Err +} + +// IsNewClientRetryable reports whether err is, or wraps, a *NewClientRetryableError. +func IsNewClientRetryable(err error) bool { + var target *NewClientRetryableError + return errors.As(err, &target) +} diff --git a/driver/controller/data/evm/BUILD.bazel b/driver/controller/data/evm/BUILD.bazel new file mode 100644 index 0000000..4671065 --- /dev/null +++ b/driver/controller/data/evm/BUILD.bazel @@ -0,0 +1,47 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "evm", + srcs = [ + "block.go", + "block_extend.go", + "block_main.go", + "client.go", + "log.go", + "trace.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/evm", + visibility = ["//visibility:public"], + deps = [ + "//chain/evm", + "//common/concurrency", + "//common/contract", + "//common/envconf", + "//common/errgroup", + "//common/https", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_ethereum_go_ethereum//:go-ethereum", + "@com_github_ethereum_go_ethereum//common/hexutil", + "@com_github_ethereum_go_ethereum//core/types", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_golang_lru//:golang-lru", + ], +) + +go_test( + name = "evm_test", + srcs = ["log_test.go"], + embed = [":evm"], + deps = [ + "//common/set", + "//driver/controller", + "@com_github_ethereum_go_ethereum//core/types", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/data/evm/block.go b/driver/controller/data/evm/block.go new file mode 100644 index 0000000..9da1fc3 --- /dev/null +++ b/driver/controller/data/evm/block.go @@ -0,0 +1,67 @@ +package evm + +import ( + "encoding/json" + "math" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +type BlockHeader struct { + Raw json.RawMessage + + BlockNumber uint64 + BlockHash string + BlockTime time.Time + ParentBlockHash string + TxHashes []string +} + +func (b *BlockHeader) UnmarshalJSON(raw []byte) error { + var payload *struct { + Number hexutil.Uint64 `json:"number"` + Hash string `json:"hash"` + Timestamp hexutil.Uint64 `json:"timestamp"` + ParentHash string `json:"parentHash"` + Transactions []string `json:"transactions"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return err + } + if payload == nil { + b = nil + return nil + } + b.Raw = raw + b.BlockNumber = uint64(payload.Number) + b.BlockHash = payload.Hash + b.ParentBlockHash = payload.ParentHash + b.TxHashes = payload.Transactions + if payload.Timestamp < math.MaxInt32 { + b.BlockTime = time.Unix(int64(payload.Timestamp), 0) + } else if payload.Timestamp < math.MaxInt32*1000 { + b.BlockTime = time.UnixMilli(int64(payload.Timestamp)) + } else if payload.Timestamp < math.MaxInt32*1000000 { + b.BlockTime = time.UnixMicro(int64(payload.Timestamp)) + } else { + b.BlockTime = time.Unix(0, int64(payload.Timestamp)) + } + return nil +} + +func (b BlockHeader) GetBlockNumber() uint64 { + return b.BlockNumber +} + +func (b BlockHeader) GetBlockParentHash() string { + return b.ParentBlockHash +} + +func (b BlockHeader) GetBlockHash() string { + return b.BlockHash +} + +func (b BlockHeader) GetBlockTime() time.Time { + return b.BlockTime +} diff --git a/driver/controller/data/evm/block_extend.go b/driver/controller/data/evm/block_extend.go new file mode 100644 index 0000000..3e0668c --- /dev/null +++ b/driver/controller/data/evm/block_extend.go @@ -0,0 +1,63 @@ +package evm + +import ( + "sentioxyz/sentio-core/chain/evm" + "sentioxyz/sentio-core/common/utils" +) + +type BlockExtendData struct { + Transactions map[string]evm.RPCTransaction + Receipts map[string]evm.ExtendedReceipt + Traces map[string][]Trace +} + +type BlockExtendRequirement struct { + AllTransactions bool + SpecialTransactions []string + + AllTransactionReceipts bool + SpecialTransactionReceipts []string + + AllTransactionReceiptLogs bool + SpecialTransactionReceiptLogs []string + + AllTraces bool +} + +func (r *BlockExtendRequirement) Merge(a BlockExtendRequirement) { + r.AllTransactions = r.AllTransactions || a.AllTransactions + r.AllTransactionReceipts = r.AllTransactionReceipts || a.AllTransactionReceipts + r.AllTransactionReceiptLogs = r.AllTransactionReceiptLogs || a.AllTransactionReceiptLogs + r.AllTraces = r.AllTraces || a.AllTraces + r.SpecialTransactions = append(r.SpecialTransactions, a.SpecialTransactions...) + r.SpecialTransactionReceipts = append(r.SpecialTransactionReceipts, a.SpecialTransactionReceipts...) + r.SpecialTransactionReceiptLogs = append(r.SpecialTransactionReceiptLogs, a.SpecialTransactionReceiptLogs...) + r.Trim() +} + +func (r *BlockExtendRequirement) IsEmpty() bool { + return !r.AllTransactions && len(r.SpecialTransactions) == 0 && + !r.AllTransactionReceipts && len(r.SpecialTransactionReceipts) == 0 && + !r.AllTransactionReceiptLogs && len(r.SpecialTransactionReceiptLogs) == 0 && + !r.AllTraces +} + +func (r *BlockExtendRequirement) Trim() { + if r.AllTransactions { + r.SpecialTransactions = nil + } else { + r.SpecialTransactions = utils.GetMapKeys(utils.BuildSet(r.SpecialTransactions)) + } + + if r.AllTransactionReceipts { + r.SpecialTransactionReceipts = nil + } else { + r.SpecialTransactionReceipts = utils.GetMapKeys(utils.BuildSet(r.SpecialTransactionReceipts)) + } + + if r.AllTransactionReceiptLogs { + r.SpecialTransactionReceiptLogs = nil + } else { + r.SpecialTransactionReceiptLogs = utils.GetMapKeys(utils.BuildSet(r.SpecialTransactionReceiptLogs)) + } +} diff --git a/driver/controller/data/evm/block_main.go b/driver/controller/data/evm/block_main.go new file mode 100644 index 0000000..3e70ed8 --- /dev/null +++ b/driver/controller/data/evm/block_main.go @@ -0,0 +1,127 @@ +package evm + +import ( + "context" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/core/types" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type DataRequirement struct { + Log []LogRequirement + Trace []TraceRequirement + Interval []data.IntervalRequirement + Exact []uint64 +} + +type BlockMainData struct { + Logs []types.Log + Traces []Trace + Intervals []data.IntervalConfig + Exact bool +} + +func (d BlockMainData) IsEmpty() bool { + return d.Size() == 0 && !d.Exact +} + +func (d BlockMainData) Size() int { + return len(d.Logs) + len(d.Traces) + len(d.Intervals) +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetHeader(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 100, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{ + Intervals: []data.IntervalConfig{req.IntervalConfig}, + } + } + return result, nil + }, + ) +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.Log = MergeLogRequirements(currentBlockNumber, req.Log) + req.Trace = MergeTraceRequirement(currentBlockNumber, req.Trace) + req.Interval = data.MergeIntervalRequirements(req.Interval) + exact := set.New(req.Exact...) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.Log { + fetchers = append(fetchers, BuildLogFetcher( + namePrefix+fmt.Sprintf("LogFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Trace { + fetchers = append(fetchers, BuildTraceFetcher( + namePrefix+fmt.Sprintf("TraceFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(bn uint64, from []BlockMainData) (data BlockMainData, has bool, _ error) { + data.Exact = exact.Contains(bn) + has = len(from) > 0 || data.Exact + // Logs and traces will never be repeated, because a range will only have one fetcher with data. + for _, box := range from { + data.Logs = append(data.Logs, box.Logs...) + data.Traces = append(data.Traces, box.Traces...) + data.Intervals = append(data.Intervals, box.Intervals...) + } + return + }) +} diff --git a/driver/controller/data/evm/client.go b/driver/controller/data/evm/client.go new file mode 100644 index 0000000..590577e --- /dev/null +++ b/driver/controller/data/evm/client.go @@ -0,0 +1,584 @@ +package evm + +import ( + "context" + "math/big" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "sentioxyz/sentio-core/chain/evm" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/contract" + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + "github.com/pkg/errors" + lru "github.com/sentioxyz/golang-lru" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetHeader(ctx context.Context, blockNumber uint64) (BlockHeader, error) + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) + + GetBlock( + ctx context.Context, + blockNumber uint64, + req BlockExtendRequirement, + ) (BlockExtendData, error) + + GetLogs( + ctx context.Context, + fromBlock, toBlock uint64, + address []string, + topics [][]string, + ) ([]types.Log, error) + + GetTraces( + ctx context.Context, + fromBlock, toBlock uint64, + address []string, + ) ([]Trace, error) + + GetContractStartBlock(ctx context.Context, address string, start, latest uint64) (uint64, bool, error) + IsERC20Address(ctx context.Context, address string) (bool, error) + GetChainID(ctx context.Context) (uint64, error) + CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +type client struct { + endpoint string + firstBlockNumber int64 + latestDelayBlockNumber uint64 + watchLatestInterval time.Duration + getLatestTimeout time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *rpc.Client + + unsupportedMethods set.Set[string] + + cachedHeaders *data.BlockCache[BlockHeader] + cachedERC20AddrCheckResult *lru.Cache[string, bool] +} + +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + latestDelayBlockNumber uint64, + watchLatestInterval time.Duration, + getLatestTimeout time.Duration, +) (c Client, err error) { + cli := &client{ + endpoint: endpoint, + firstBlockNumber: firstBlockNumber, + latestDelayBlockNumber: latestDelayBlockNumber, + watchLatestInterval: watchLatestInterval, + getLatestTimeout: getLatestTimeout, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + unsupportedMethods: set.NewSafe[string](), + } + if cli.cli, err = rpc.DialOptions(ctx, endpoint, rpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.cachedHeaders, _ = data.NewBlockCache[BlockHeader](10000) + cli.cachedERC20AddrCheckResult, _ = lru.New[string, bool](100000) + return cli, nil +} + +// fullBlockFetchThreshold is the number of special transactions/receipts above which it is +// cheaper to fetch the whole block in a single request than to issue one request per hash. +var fullBlockFetchThreshold = envconf.LoadUInt64("SENTIO_EVM_FULL_BLOCK_FETCH_THRESHOLD", 10) + +var ( + errMethodNotSupported = errors.New("method not supported") + + invalidMethodErrorMatcher = []*regexp.Regexp{ + regexp.MustCompile(`unsupported method`), + regexp.MustCompile("method.*not available"), + regexp.MustCompile(`method.*not support`), + regexp.MustCompile(`method.*not found`), + regexp.MustCompile(`method.*not allowed`), + } +) + +func (c *client) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + // waiting concurrency control token + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always be context.Canceled + } + defer release() + // actually call + callStartAt := time.Now() + err = c.cli.CallContext(ctx, &result, method, args...) + if err != nil { + if utils.MatchAny(strings.ToLower(err.Error()), invalidMethodErrorMatcher) { + err = errors.Wrapf(errMethodNotSupported, err.Error()) + } else { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + } + c.stat.Called(method, args, err, startAt, callStartAt) + return err +} + +func (c *client) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + _, logger := log.FromContext(ctx) + for { + var resp evm.GetLatestBlockNumberResponse + callCtx, cancel := context.WithTimeout(ctx, time.Minute) + err := c.callContext(callCtx, &resp, 0, "eth_getLatestBlockNumber", 0) + cancel() + if err == nil { + break + } + if errors.Is(err, errMethodNotSupported) { + logger.Warn("do not support eth_getLatestBlockNumber, will use polling mode with method eth_blockNumber") + data.SubscribeUsingPolling( + ctx, + c.watchLatestInterval, + c.getLatestTimeout, + from, + func(ctx context.Context) (h controller.BlockHeader, err error) { + h, _, err = c.GetLatest(ctx) + return h, err + }, + callback) + return + } + logger.Warnfe(err, "call eth_getLatestBlockNumber failed, will retry after %s", c.watchLatestInterval.String()) + select { + case <-time.After(c.watchLatestInterval): + case <-ctx.Done(): + return + } + } + // have eth_getLatestBlockNumber method, use wait latest mode + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, blockNumberGt uint64) (latest controller.BlockHeader, broken, err error) { + var resp evm.GetLatestBlockNumberResponse + err = c.callContext(ctx, &resp, 0, "eth_getLatestBlockNumber", blockNumberGt+c.latestDelayBlockNumber) + if err == nil { + if broken = resp.CheckAPIVersion(); broken == nil { + latest, err = c.GetHeaderIgnoreCache(ctx, resp.LatestBlockNumber-c.latestDelayBlockNumber) + } else { + broken = errors.Wrapf(controller.ErrInternalNeedUpgrade, broken.Error()) + } + } + return + }, + callback) +} + +func (c *client) GetLatest(ctx context.Context) (controller.BlockHeader, uint64, error) { + var result hexutil.Uint64 + if err := c.callContext(ctx, &result, 0, "eth_blockNumber"); err != nil { + return BlockHeader{}, 0, err + } + latest := uint64(result) + latest -= min(c.latestDelayBlockNumber, latest) + h, err := c.GetHeaderIgnoreCache(ctx, latest) + return h, data.GetFirst(c.firstBlockNumber, latest), err +} + +func (c *client) GetHeader(ctx context.Context, blockNumber uint64) (BlockHeader, error) { + // Cache + singleflight: concurrent fetchers asking for the same block share one eth_getBlockByNumber. + return c.cachedHeaders.GetOrFetch(blockNumber, func() (BlockHeader, error) { + return c.fetchHeader(ctx, blockNumber) + }) +} + +func (c *client) fetchHeader(ctx context.Context, blockNumber uint64) (BlockHeader, error) { + var h *BlockHeader + err := c.callContext(ctx, &h, blockNumber, "eth_getBlockByNumber", hexutil.Uint64(blockNumber), false) + if err != nil { + return BlockHeader{}, errors.Wrapf(err, "failed to get header of block %d", blockNumber) + } + if h == nil { + return BlockHeader{}, errors.Errorf("block %d not found", blockNumber) + } + return *h, nil +} + +func (c *client) getHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (BlockHeader, error) { + h, err := c.fetchHeader(ctx, blockNumber) + if err == nil { + c.cachedHeaders.Add(blockNumber, h) + } + return h, err +} + +func (c *client) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getHeaderIgnoreCache(ctx, blockNumber) +} + +func (c *client) GetBlock( + ctx context.Context, + blockNumber uint64, + req BlockExtendRequirement, +) (r BlockExtendData, err error) { + if req.IsEmpty() { + return + } + r = BlockExtendData{ + Transactions: make(map[string]evm.RPCTransaction), + Receipts: make(map[string]evm.ExtendedReceipt), + Traces: make(map[string][]Trace), + } + g, gctx := errgroup.WithContext(ctx) + var lock sync.Mutex + // get transactions + // fetchFullBlockTxs fetches the whole block in a single request. When keep is nil every + // transaction is kept (AllTransactions); otherwise only the requested hashes are kept and a + // missing one is treated as an error, matching the per-hash behavior. + fetchFullBlockTxs := func(keep set.Set[string]) func() error { + return func() (err error) { + var block *struct { + Transactions []evm.RPCTransaction `json:"transactions"` + } + err = c.callContext(ctx, &block, blockNumber, "eth_getBlockByNumber", hexutil.Uint64(blockNumber), true) + if err != nil { + return + } + found := set.New[string]() + if block != nil { + for _, tx := range block.Transactions { + hash := tx.Hash.String() + if keep != nil && !keep.Contains(hash) { + continue + } + r.Transactions[hash] = tx + found.Add(hash) + } + } + if keep != nil { + for _, txHash := range req.SpecialTransactions { + if !found.Contains(txHash) { + return errors.Errorf("transaction %d/%s not found", blockNumber, txHash) + } + } + } + return + } + } + switch { + case req.AllTransactions: + g.Go(fetchFullBlockTxs(nil)) + case uint64(len(req.SpecialTransactions)) >= fullBlockFetchThreshold: + // too many special transactions, one full-block request is cheaper than one per hash + g.Go(fetchFullBlockTxs(set.New(req.SpecialTransactions...))) + default: + for _, txHash_ := range req.SpecialTransactions { + txHash := txHash_ + g.Go(func() (err error) { + var tx *evm.RPCTransaction + if err = c.callContext(ctx, &tx, blockNumber, "eth_getTransactionByHash", txHash); err != nil { + return + } + if tx == nil { + return errors.Errorf("transaction %d/%s not found", blockNumber, txHash) + } + lock.Lock() + defer lock.Unlock() + r.Transactions[txHash] = *tx + return + }) + } + } + // get receipts + // fetchFullBlockReceipts fetches all receipts of the block in a single request. When keep is + // nil every receipt is kept (AllTransactionReceipts); otherwise only the requested hashes are + // kept and a missing one is treated as an error, matching the per-hash behavior. + fetchFullBlockReceipts := func(keep set.Set[string]) func() error { + return func() (err error) { + var receipts []evm.ExtendedReceipt + if receipts, err = c.GetBlockReceipts(gctx, blockNumber); err != nil { + return + } + found := set.New[string]() + for _, receipt := range receipts { + hash := receipt.TxHash.String() + if keep != nil && !keep.Contains(hash) { + continue + } + r.Receipts[hash] = receipt + found.Add(hash) + } + if keep != nil { + for _, txHash := range req.SpecialTransactionReceipts { + if !found.Contains(txHash) { + return errors.Errorf("transaction receipt %d/%s not found", blockNumber, txHash) + } + } + } + return + } + } + switch { + case req.AllTransactionReceipts: + g.Go(fetchFullBlockReceipts(nil)) + case uint64(len(req.SpecialTransactionReceipts)) >= fullBlockFetchThreshold && + !c.unsupportedMethods.Contains("eth_getBlockReceipts"): + // too many special receipts, one eth_getBlockReceipts request is cheaper than one per hash. + // only take this path when eth_getBlockReceipts is supported, otherwise GetBlockReceipts + // falls back to one request per transaction in the block, which may be even more requests. + g.Go(fetchFullBlockReceipts(set.New(req.SpecialTransactionReceipts...))) + default: + for _, txHash_ := range req.SpecialTransactionReceipts { + txHash := txHash_ + g.Go(func() (err error) { + var receipt *evm.ExtendedReceipt + if err = c.callContext(gctx, &receipt, blockNumber, "eth_getTransactionReceipt", txHash); err != nil { + return + } + if receipt == nil { + return errors.Errorf("transaction receipt %d/%s not found", blockNumber, txHash) + } + lock.Lock() + defer lock.Unlock() + r.Receipts[txHash] = *receipt + return + }) + } + } + // get traces + if req.AllTraces { + g.Go(func() (err error) { + var traces []Trace + traces, err = c.GetTraces(gctx, blockNumber, blockNumber, nil) + if err != nil { + return + } + r.Traces = utils.Group(traces, func(trace Trace) string { + return trace.TransactionHash + }) + return + }) + } + err = g.Wait() + // check receipt logs + if err == nil && !req.AllTransactionReceiptLogs { + need := set.New(req.SpecialTransactionReceiptLogs...) + for txHash, receipt := range r.Receipts { + if !need.Contains(txHash) { + receipt.Logs = nil + r.Receipts[txHash] = receipt + } + } + } + if err == nil { + _, logger := log.FromContext(ctx) + logger.Debugw("GetBlock succeed", "blockNumber", blockNumber, "req", req, "result", r) + } + return +} + +func (c *client) GetBlockReceipts(ctx context.Context, blockNumber uint64) (r []evm.ExtendedReceipt, err error) { + if !c.unsupportedMethods.Contains("eth_getBlockReceipts") { + err = c.callContext(ctx, &r, blockNumber, "eth_getBlockReceipts", hexutil.Uint64(blockNumber)) + if err == nil || !errors.Is(err, errMethodNotSupported) { + return + } + c.unsupportedMethods.Add("eth_getBlockReceipts") + } + var h BlockHeader + if h, err = c.GetHeader(ctx, blockNumber); err != nil { + return + } + g, gctx := errgroup.WithContext(ctx) + r = make([]evm.ExtendedReceipt, len(h.TxHashes)) + for i_, txHash_ := range h.TxHashes { + i, txHash := i_, txHash_ + g.Go(func() error { + return c.callContext(gctx, &r[i], blockNumber, "eth_getTransactionReceipt", txHash) + }) + } + err = g.Wait() + return +} + +func (c *client) GetLogs( + ctx context.Context, + fromBlock, toBlock uint64, + address []string, + topics [][]string, +) (result []types.Log, err error) { + arg := map[string]any{ + "fromBlock": hexutil.Uint64(fromBlock).String(), + "toBlock": hexutil.Uint64(toBlock).String(), + "address": address, + "topics": topics, + } + err = c.callContext(ctx, &result, fromBlock, "eth_getLogs", arg) + return +} + +func (c *client) GetTraces( + ctx context.Context, + fromBlock, toBlock uint64, + address []string, +) (result []Trace, err error) { + arg := map[string]any{ + "fromBlock": hexutil.Uint64(fromBlock).String(), + "toBlock": hexutil.Uint64(toBlock).String(), + "toAddress": address, + } + err = c.callContext(ctx, &result, fromBlock, "trace_filter", arg) + if err != nil { + return + } + return +} + +func (c *client) HasCode(ctx context.Context, address string, blockNumber uint64) (bool, error) { + var result hexutil.Bytes + err := c.callContext(ctx, &result, blockNumber, "eth_getCode", address, hexutil.Uint64(blockNumber)) + return len(result) > 0, err +} + +func (c *client) GetContractStartBlock( + ctx context.Context, + address string, + start, latest uint64, +) (uint64, bool, error) { + return data.BinarySearchContractStart(ctx, start, latest, func(ctx context.Context, bn uint64) (bool, error) { + return c.HasCode(ctx, address, bn) + }) +} + +func (c *client) IsERC20Address(ctx context.Context, address string) (bool, error) { + if is, has := c.cachedERC20AddrCheckResult.Get(address); has { + return is, nil + } + is, err := c.IsERC20AddressIgnoreCache(ctx, address) + if err != nil { + return false, err + } + c.cachedERC20AddrCheckResult.Add(address, is) + return is, nil +} + +func (c *client) IsERC20AddressIgnoreCache(ctx context.Context, address string) (bool, error) { + // TODO need improvement + const endpoint = "https://eth-mainnet.g.alchemy.com/v2/z1Q-YhcYg60C5sOQPUzsMFqiDJSvqbsK" + res, err := contract.IsERC20(ctx, endpoint, address) + if err != nil { + return false, errors.Wrapf(err, "detect address %s is erc20 failed", address) + } + return res, err +} + +func (c *client) GetChainID(ctx context.Context) (uint64, error) { + var chainID hexutil.Uint64 + if err := c.callContext(ctx, &chainID, 0, "eth_chainId"); err != nil { + return 0, err + } + return uint64(chainID), nil +} + +// CallContract Referenced github.com/ethereum/go-ethereum/ethclient.Client.CallContract +func (c *client) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + if blockNumber.Sign() < 0 { + return nil, errors.New("block number for eth_call cannot use tag") + } + arg := map[string]interface{}{ + "from": msg.From, + "to": msg.To, + } + if len(msg.Data) > 0 { + arg["input"] = hexutil.Bytes(msg.Data) + } + if msg.Value != nil { + arg["value"] = (*hexutil.Big)(msg.Value) + } + if msg.Gas != 0 { + arg["gas"] = hexutil.Uint64(msg.Gas) + } + if msg.GasPrice != nil { + arg["gasPrice"] = (*hexutil.Big)(msg.GasPrice) + } + if msg.GasFeeCap != nil { + arg["maxFeePerGas"] = (*hexutil.Big)(msg.GasFeeCap) + } + if msg.GasTipCap != nil { + arg["maxPriorityFeePerGas"] = (*hexutil.Big)(msg.GasTipCap) + } + if msg.AccessList != nil { + arg["accessList"] = msg.AccessList + } + if msg.BlobGasFeeCap != nil { + arg["maxFeePerBlobGas"] = (*hexutil.Big)(msg.BlobGasFeeCap) + } + if msg.BlobHashes != nil { + arg["blobVersionedHashes"] = msg.BlobHashes + } + var bn = blockNumber.Uint64() + var hex hexutil.Bytes + err := c.callContext(ctx, &hex, bn, "eth_call", arg, hexutil.Uint64(bn).String()) + return hex, err +} + +func (c *client) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedHeaders.Keys() { + if r.Contains(bn) { + c.cachedHeaders.Remove(bn) + } + } +} + +func (c *client) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "firstBlockNumber": c.firstBlockNumber, + "latestDelayBlockNumber": c.latestDelayBlockNumber, + "watchLatestInterval": c.watchLatestInterval.String(), + "getLatestTimeout": c.getLatestTimeout.String(), + }, + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "unsupportedMethods": c.unsupportedMethods.DumpValues(), + "cache": map[string]any{ + "cachedHeaders": c.cachedHeaders.Snapshot(10, controller.GetBlockFullText[BlockHeader]), + "cachedERC20AddrCheckResult": utils.CacheSnapshot(c.cachedERC20AddrCheckResult, 100, strconv.FormatBool), + }, + } +} diff --git a/driver/controller/data/evm/log.go b/driver/controller/data/evm/log.go new file mode 100644 index 0000000..2b19296 --- /dev/null +++ b/driver/controller/data/evm/log.go @@ -0,0 +1,238 @@ +package evm + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/ethereum/go-ethereum/core/types" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +// LogFilter has 2 parts, there are linked by AND +type LogFilter struct { + // topic condition + Topics [][]string + // address condition + Address []string + AddressShouldBeERC20 bool +} + +// FilterLogs all filters is linked by OR +func FilterLogs(ctx context.Context, cli Client, logs []types.Log, filters ...LogFilter) ([]types.Log, error) { + checkers := utils.MapSliceNoError(filters, func(f LogFilter) func(log types.Log) (bool, error) { + return f.BuildChecker(ctx, cli) + }) + return utils.FilterArrWithErr(logs, func(log types.Log) (bool, error) { + for _, ck := range checkers { + if fok, err := ck(log); err != nil { + return false, err + } else if fok { + return true, nil + } + } + return false, nil + }) +} + +func (f LogFilter) BuildChecker(ctx context.Context, cli Client) func(log types.Log) (bool, error) { + addrSet := set.New(f.Address...) + topicsSet := utils.MapSliceNoError(f.Topics, func(ss []string) set.Set[string] { + return set.New(ss...) + }) + return func(log types.Log) (bool, error) { + for i, topic := range log.Topics { + if i < len(topicsSet) && !topicsSet[i].Empty() && !topicsSet[i].Contains(topic.String()) { + return false, nil + } + } + if !addrSet.Empty() && !addrSet.Contains(strings.ToLower(log.Address.String())) { + return false, nil + } + if f.AddressShouldBeERC20 { + return cli.IsERC20Address(ctx, log.Address.String()) + } + return true, nil + } +} + +func (f LogFilter) String() string { + return fmt.Sprintf("Topics:[%s],Addr:[%s],AddrIsERC20:%v", + strings.Join(utils.MapSliceNoError(f.Topics, func(t []string) string { + if len(t) == 0 { + return "nil" + } + return "[" + strings.Join(t, ",") + "]" + }), ","), + utils.ArrSummary(f.Address, 10), + f.AddressShouldBeERC20, + ) +} + +// Merge logs match f always match r, logs match a also always match r. Logs(r) >= Logs(f) + Logs(a) +func (f LogFilter) Merge(a LogFilter) (r LogFilter) { + r.Topics = make([][]string, min(len(f.Topics), len(a.Topics))) + for i := 0; i < len(r.Topics); i++ { + if len(f.Topics[i]) > 0 && len(a.Topics[i]) > 0 { + r.Topics[i] = set.SmartNew[string](f.Topics[i], a.Topics[i]).DumpValues() + } + } + if len(f.Address) > 0 && len(a.Address) > 0 { + r.Address = set.SmartNew[string](f.Address, a.Address).DumpValues() + } + r.AddressShouldBeERC20 = f.AddressShouldBeERC20 && a.AddressShouldBeERC20 + return r +} + +func MergeLogFilers(filters []LogFilter) (r LogFilter) { + if len(filters) == 0 { + panic("filters is empty") + } + // Topics + for i := 0; ; i++ { + miss := false + s := set.New[string]() + for _, f := range filters { + if len(f.Topics) <= i { + miss = true + break + } else if len(f.Topics[i]) == 0 { + s = set.New[string]() + break + } else { + s.Add(f.Topics[i]...) + } + } + if miss { + break + } + r.Topics = append(r.Topics, s.DumpValues()) + } + // Address + s := set.New[string]() + for _, f := range filters { + if len(f.Address) == 0 { + s = set.New[string]() + break + } + s.Add(f.Address...) + } + r.Address = s.DumpValues() + // AddressShouldBeERC20 + r.AddressShouldBeERC20 = true + for _, f := range filters { + if !f.AddressShouldBeERC20 { + r.AddressShouldBeERC20 = false + break + } + } + return r +} + +type LogRequirement struct { + controller.BlockRange + LogFilter +} + +func (r LogRequirement) String() string { + return fmt.Sprintf("LogRequirement[%s]%s", r.LogFilter.String(), r.BlockRange.String()) +} + +func (r LogRequirement) Snapshot() any { + return map[string]any{ + "filter": r.LogFilter, + "range": r.BlockRange.String(), + } +} + +// MergeLogRequirements it can be guaranteed that all the item ranges of the result must be disjoint, +// and each range has at most one filter +func MergeLogRequirements(current uint64, reqs []LogRequirement) (result []LogRequirement) { + rs := controller.CutRangeSet(current, utils.MapSliceNoError(reqs, func(r LogRequirement) controller.BlockRange { + return r.BlockRange + })) + for _, r := range rs { + var filters []LogFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.LogFilter) + } + } + if len(filters) == 0 { + continue + } + + result = append(result, LogRequirement{ + LogFilter: MergeLogFilers(filters), + BlockRange: r, + }) + } + return result +} + +var ( + ethGetLogsMaxAddressLen = envconf.LoadUInt64("SENTIO_ETH_GETLOGS_MAX_ADDR_LEN", 200) + ethGetLogsMaxTopicLen = envconf.LoadUInt64("SENTIO_ETH_GETLOGS_MAX_TOPIC_LEN", 100) +) + +func BuildLogFetcher( + name string, + req LogRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10, + 1000, + 10000, + 2000, // the target is that each query got no more than 2000 logs + time.Second*5, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + address := req.Address + if uint64(len(req.Address)) > ethGetLogsMaxAddressLen { + address = nil // the set is too big + } + topics := make([][]string, len(req.Topics)) + for i, s := range req.Topics { + if uint64(len(s)) > ethGetLogsMaxTopicLen { + topics[i] = nil // the set is too big + } else { + topics[i] = s + } + } + allLogs, err := client.GetLogs(ctx, start, end, address, topics) + if err != nil { + return nil, err + } + allLogs, err = FilterLogs(ctx, client, allLogs, req.LogFilter) + if err != nil { + return nil, err + } + blockLogs := utils.Group(allLogs, func(log types.Log) uint64 { + return log.BlockNumber + }) + result := make(map[uint64]BlockMainData) + for bn, logs := range blockLogs { + result[bn] = BlockMainData{Logs: logs} + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/evm/log_test.go b/driver/controller/data/evm/log_test.go new file mode 100644 index 0000000..8d50a51 --- /dev/null +++ b/driver/controller/data/evm/log_test.go @@ -0,0 +1,90 @@ +package evm + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/stretchr/testify/assert" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/driver/controller" +) + +func Test_logFilter(t *testing.T) { + var ev types.Log + evRaw := `{ + "address": "0xbe9895146f7af43049ca1c1ae358b0541ea49704", + "topics": [ + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x000000000000000000000000fae23c30d383df59d3e031c325a73d454e8721a6" + ], + "data": "0x000000000000000000000000000000000000000000000000000000003b9aca00", + "blockNumber": "0xd8110d", + "transactionHash": "0xa4cc1d25099cc2f8fc18ae8a54e079076a89db38dad54696f7453afa3adcfbe8", + "transactionIndex": "0x8c", + "blockHash": "0x5b22e991900f1e961ff296dd37973279a91dc97ecdb2ae290680e512b3dec7fe", + "logIndex": "0xaf", + "removed": false + }` + assert.NoError(t, json.Unmarshal([]byte(evRaw), &ev)) + f := LogFilter{ + Topics: [][]string{ + { + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + }, + }, + Address: []string{ + "0xbe9895146f7af43049ca1c1ae358b0541ea49704", + }, + } + ok, err := f.BuildChecker(nil, nil)(ev) + assert.NoError(t, err) + assert.True(t, ok) +} + +func Test_MergeLogRequirements(t *testing.T) { + var reqs []LogRequirement + var addrs []string + for i := 0; i < 20000; i++ { + addr := fmt.Sprintf("0x%040x", i) + addrs = append(addrs, addr) + reqs = append(reqs, LogRequirement{ + BlockRange: controller.BlockRange{StartBlock: uint64(i * 1000)}, + LogFilter: LogFilter{ + Topics: [][]string{{ + "0x503ec09c1b4597d114eca849f7013c8c457988802d1dc5da49a0c461b5f88658", + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x53a0f15ecd2c4989821244d3c7363d7de154e125eac5b2a7c9bb008152778de1", + "0xd51a9c61267aa6196961883ecf5ff2da6619c37dac0fa92122513fb32c032d2d", + "0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0", + "0x043a568d47b1a65cdc989ff14c921411b62abf40132dc4b5e78675ba8d0bc9df", + }}, + Address: []string{addr}, + }, + }) + } + r := MergeLogRequirements(199991000, reqs) + assert.Equal(t, 1, len(r)) + assert.Equal(t, set.New(addrs...), set.New(r[0].Address...)) + assert.False(t, r[0].AddressShouldBeERC20) + assert.Equal(t, 1, len(r[0].Topics)) + assert.Equal(t, + set.New( + "0x503ec09c1b4597d114eca849f7013c8c457988802d1dc5da49a0c461b5f88658", + "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + "0x53a0f15ecd2c4989821244d3c7363d7de154e125eac5b2a7c9bb008152778de1", + "0xd51a9c61267aa6196961883ecf5ff2da6619c37dac0fa92122513fb32c032d2d", + "0x8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e0", + "0x043a568d47b1a65cdc989ff14c921411b62abf40132dc4b5e78675ba8d0bc9df"), + set.New(r[0].Topics[0]...), + ) + assert.Equal(t, controller.BlockRange{StartBlock: 199991000}, r[0].BlockRange) + //for i, x := range r { + // fmt.Printf("[%d]: %s\n", i, x.String()) + //} + //fmt.Printf("used1: %s\n", used1.String()) + //fmt.Printf("used2: %s\n", used2.String()) +} diff --git a/driver/controller/data/evm/trace.go b/driver/controller/data/evm/trace.go new file mode 100644 index 0000000..301930b --- /dev/null +++ b/driver/controller/data/evm/trace.go @@ -0,0 +1,200 @@ +package evm + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type Trace struct { + Raw json.RawMessage + + BlockNumber uint64 + BlockHash string + TransactionHash string + TransactionIndex int32 + + Error string + Address string + Signature string +} + +func (t *Trace) UnmarshalJSON(raw []byte) error { + var payload *struct { + Action struct { + Input string `json:"input"` + To string `json:"to"` + } `json:"action"` + BlockHash string `json:"blockHash"` + BlockNumber uint64 `json:"blockNumber"` + TransactionPosition int32 `json:"transactionPosition"` + TransactionHash string `json:"transactionHash"` + Error string `json:"error"` + } + if err := json.Unmarshal(raw, &payload); err != nil { + return err + } + if payload == nil { + t = nil + return nil + } + t.Raw = raw + t.BlockNumber = payload.BlockNumber + t.BlockHash = payload.BlockHash + t.TransactionHash = payload.TransactionHash + t.TransactionIndex = payload.TransactionPosition + t.Error = payload.Error + t.Address = payload.Action.To + if len(payload.Action.Input) >= 10 { + t.Signature = payload.Action.Input[:10] + } + return nil +} + +// TraceFilter has 2 parts, there are linked by AND +type TraceFilter struct { + Signature []string + Address []string +} + +func (f TraceFilter) Check(trace Trace) bool { + if len(trace.Error) > 0 { + return false + } + if len(f.Address) > 0 && utils.IndexOf(f.Address, strings.ToLower(trace.Address)) < 0 { + return false + } + if len(f.Signature) > 0 && utils.IndexOf(f.Signature, trace.Signature) < 0 { + return false + } + return true +} + +func (t TraceFilter) String() string { + return fmt.Sprintf("Sig:[%s],Addr:%s", utils.ArrSummary(t.Signature, 10), utils.ArrSummary(t.Address, 10)) +} + +// Merge traces match t always match r, traces match a also always match r. Traces(r) >= Traces(t) + Traces(a) +func (t TraceFilter) Merge(a TraceFilter) (r TraceFilter) { + if len(t.Signature) > 0 && len(a.Signature) > 0 { + r.Signature = set.SmartNew[string](t.Signature, a.Signature).DumpValues() + } + if len(t.Address) > 0 && len(a.Address) > 0 { + r.Address = set.SmartNew[string](t.Address, a.Address).DumpValues() + } + return r +} + +func MergeTraceFilters(filters ...TraceFilter) TraceFilter { + if len(filters) == 0 { + panic("filters is empty") + } + signatures := set.New[string]() + for _, filter := range filters { + if len(filter.Signature) == 0 { + signatures = set.New[string]() + break + } + signatures.Add(filter.Signature...) + } + addresses := set.New[string]() + for _, filter := range filters { + if len(filter.Address) == 0 { + addresses = set.New[string]() + break + } + addresses.Add(filter.Address...) + } + return TraceFilter{ + Signature: signatures.DumpValues(), + Address: addresses.DumpValues(), + } +} + +type TraceRequirement struct { + controller.BlockRange + TraceFilter +} + +func (r TraceRequirement) String() string { + return fmt.Sprintf("TraceRequirement[%s]%s", r.TraceFilter.String(), r.BlockRange.String()) +} + +func (r TraceRequirement) Snapshot() any { + return map[string]any{ + "filter": r.TraceFilter, + "range": r.BlockRange.String(), + } +} + +// MergeTraceRequirement it can be guaranteed that all the item ranges of the result must be disjoint, +// and each range has at most one filter +func MergeTraceRequirement(current uint64, reqs []TraceRequirement) (result []TraceRequirement) { + rs := controller.CutRangeSet(current, utils.MapSliceNoError(reqs, func(r TraceRequirement) controller.BlockRange { + return r.BlockRange + })) + for _, r := range rs { + var filters []TraceFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.TraceFilter) + } + } + if len(filters) == 0 { + continue + } + result = append(result, TraceRequirement{ + TraceFilter: MergeTraceFilters(filters...), + BlockRange: r, + }) + } + return result +} + +func BuildTraceFetcher( + name string, + req TraceRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1, + 100, + 10000, + 2000, // the target is that each query got no more than 2000 traces + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + allTraces, err := client.GetTraces(ctx, start, end, req.TraceFilter.Address) + if err != nil { + return nil, err + } + allTraces = utils.FilterArr(allTraces, req.TraceFilter.Check) + blockTraces := utils.Group(allTraces, func(trace Trace) uint64 { + return trace.BlockNumber + }) + result := make(map[uint64]BlockMainData) + for bn, traces := range blockTraces { + result[bn] = BlockMainData{Traces: traces} + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/fuel/BUILD.bazel b/driver/controller/data/fuel/BUILD.bazel new file mode 100644 index 0000000..6180c64 --- /dev/null +++ b/driver/controller/data/fuel/BUILD.bazel @@ -0,0 +1,26 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "fuel", + srcs = [ + "block.go", + "block_main.go", + "client.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/fuel", + visibility = ["//visibility:public"], + deps = [ + "//chain/fuel", + "//common/concurrency", + "//common/https", + "//common/log", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_fuel_go//types", + ], +) diff --git a/driver/controller/data/fuel/block.go b/driver/controller/data/fuel/block.go new file mode 100644 index 0000000..9968983 --- /dev/null +++ b/driver/controller/data/fuel/block.go @@ -0,0 +1,27 @@ +package fuel + +import ( + "time" + + "github.com/sentioxyz/fuel-go/types" +) + +type Block struct { + types.Header +} + +func (b Block) GetBlockNumber() uint64 { + return uint64(b.Height) +} + +func (b Block) GetBlockParentHash() string { + return "" +} + +func (b Block) GetBlockHash() string { + return b.Id.String() +} + +func (b Block) GetBlockTime() time.Time { + return b.Time.Time +} diff --git a/driver/controller/data/fuel/block_main.go b/driver/controller/data/fuel/block_main.go new file mode 100644 index 0000000..40887eb --- /dev/null +++ b/driver/controller/data/fuel/block_main.go @@ -0,0 +1,113 @@ +package fuel + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type DataRequirement struct { + Tx []TransactionRequirement + Interval []data.IntervalRequirement +} + +type BlockMainData struct { + Txs []fuel.WrappedTransaction + Intervals []data.IntervalConfig +} + +func (d BlockMainData) Size() int { + return len(d.Txs) + len(d.Intervals) +} + +func (d BlockMainData) IsEmpty() bool { + return d.Size() == 0 +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetBlock(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 10000, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{ + Intervals: []data.IntervalConfig{req.IntervalConfig}, + } + } + return result, nil + }, + ) +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.Tx = MergeTxRequirements(currentBlockNumber, req.Tx) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.Tx { + fetchers = append(fetchers, BuildTxFetcher( + namePrefix+fmt.Sprintf("TxFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(bn uint64, from []BlockMainData) (data BlockMainData, has bool, _ error) { + has = len(from) > 0 + // Txs will never be repeated, because a range will only have one fetcher with data. + for _, box := range from { + data.Txs = append(data.Txs, box.Txs...) + data.Intervals = append(data.Intervals, box.Intervals...) + } + return + }) +} diff --git a/driver/controller/data/fuel/client.go b/driver/controller/data/fuel/client.go new file mode 100644 index 0000000..ec39a7f --- /dev/null +++ b/driver/controller/data/fuel/client.go @@ -0,0 +1,196 @@ +package fuel + +import ( + "context" + "time" + + "github.com/ethereum/go-ethereum/rpc" + "github.com/pkg/errors" + "github.com/sentioxyz/fuel-go/types" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) + + GetBlock(ctx context.Context, blockNumber uint64) (Block, error) + GetTransactions(ctx context.Context, param fuel.GetTransactionsParam) ([]fuel.WrappedTransaction, error) + GetContractCreateBlockHeight(ctx context.Context, contractID string, startBlock uint64) (uint64, bool, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +type client struct { + endpoint string + firstBlockNumber int64 + watchLatestInterval time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *rpc.Client + + cachedHeaders *data.BlockCache[Block] +} + +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + watchLatestInterval time.Duration, +) (c Client, err error) { + cli := &client{ + endpoint: endpoint, + firstBlockNumber: firstBlockNumber, + watchLatestInterval: watchLatestInterval, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + } + if cli.cli, err = rpc.DialOptions(ctx, endpoint, rpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.cachedHeaders, _ = data.NewBlockCache[Block](100000) + return cli, nil +} + +func (c *client) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + // waiting concurrency control token + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always be context.Canceled + } + defer release() + // actually call + callStartAt := time.Now() + err = c.cli.CallContext(ctx, &result, method, args...) + if err != nil { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + c.stat.Called(method, args, err, startAt, callStartAt) + return err +} + +func (c *client) GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) { + var resp fuel.GetLatestBlockResponse + if err = c.callContext(ctx, &resp, 0, "fuel_getLatestHeader", 0); err != nil { + return nil, 0, err + } + if err = resp.CheckAPIVersion(); err != nil { + return nil, 0, errors.Wrapf(controller.ErrInternalNeedUpgrade, err.Error()) + } + latest = Block{Header: resp.Header} + return latest, data.GetFirst(c.firstBlockNumber, latest.GetBlockNumber()), err +} + +func (c *client) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, blockHeightGt uint64) (latest controller.BlockHeader, broken, err error) { + var resp fuel.GetLatestBlockResponse + err = c.callContext(ctx, &resp, 0, "fuel_getLatestHeader", blockHeightGt) + if err == nil { + latest, broken = Block{Header: resp.Header}, resp.CheckAPIVersion() + } + if broken != nil { + broken = errors.Wrapf(controller.ErrInternalNeedUpgrade, broken.Error()) + } + return + }, + callback) +} + +func (c *client) fetchBlock(ctx context.Context, blockNumber uint64) (Block, error) { + var header types.Header + if err := c.callContext(ctx, &header, blockNumber, "fuel_getBlockHeader", blockNumber); err != nil { + return Block{}, err + } + return Block{Header: header}, nil +} + +func (c *client) getHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (Block, error) { + block, err := c.fetchBlock(ctx, blockNumber) + if err == nil { + c.cachedHeaders.Add(blockNumber, block) + } + return block, err +} + +func (c *client) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getHeaderIgnoreCache(ctx, blockNumber) +} + +func (c *client) GetBlock(ctx context.Context, blockNumber uint64) (Block, error) { + // Cache + singleflight: concurrent fetchers asking for the same block share one fuel_getBlockHeader. + return c.cachedHeaders.GetOrFetch(blockNumber, func() (Block, error) { + return c.fetchBlock(ctx, blockNumber) + }) +} + +func (c *client) GetTransactions(ctx context.Context, param fuel.GetTransactionsParam) ([]fuel.WrappedTransaction, error) { + var txs []fuel.WrappedTransaction + err := c.callContext(ctx, &txs, param.StartHeight, "fuel_getTransactions", param) + return txs, err +} + +func (c *client) GetContractCreateBlockHeight( + ctx context.Context, + contractID string, + startBlock uint64, +) (blockNumber uint64, has bool, err error) { + var tx *fuel.WrappedTransaction + if err = c.callContext(ctx, &tx, 0, "fuel_getContractCreateTransaction", contractID); err != nil { + return 0, false, err + } + if tx == nil { + return 0, false, nil + } + return max(tx.BlockHeight, startBlock), true, nil +} + +func (c *client) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedHeaders.Keys() { + if r.Contains(bn) { + c.cachedHeaders.Remove(bn) + } + } +} + +func (c *client) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "firstBlockNumber": c.firstBlockNumber, + "watchLatestInterval": c.watchLatestInterval.String(), + }, + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cache": map[string]any{ + "cachedHeaders": c.cachedHeaders.Snapshot(10, controller.GetBlockFullText[Block]), + }, + } +} diff --git a/driver/controller/data/fuel/transaction.go b/driver/controller/data/fuel/transaction.go new file mode 100644 index 0000000..cf68bb9 --- /dev/null +++ b/driver/controller/data/fuel/transaction.go @@ -0,0 +1,94 @@ +package fuel + +import ( + "context" + "time" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type TransactionRequirement struct { + controller.BlockRange + + Filters []fuel.TransactionFilter +} + +func (r TransactionRequirement) Snapshot() any { + return map[string]any{ + "filters": r.Filters, + "range": r.BlockRange.String(), + } +} + +// MergeTxRequirements it can be guaranteed that all the item ranges of the result must be disjoint, +// and each range has at most one filter +func MergeTxRequirements(current uint64, reqs []TransactionRequirement) (result []TransactionRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r TransactionRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + var filters []fuel.TransactionFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.Filters...) + } + } + if len(filters) == 0 { + continue + } + result = append(result, TransactionRequirement{ + Filters: filters, + BlockRange: r, + }) + } + return result +} + +func BuildTxFetcher( + name string, + req TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1, + 100, + 100000, + 1000, // the target is that each query got no more than 1000 transactions + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txs, err := client.GetTransactions(ctx, fuel.GetTransactionsParam{ + StartHeight: start, + EndHeight: end, + Filters: req.Filters, + }) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, tx := range txs { + bd, _ := result[tx.BlockHeight] + bd.Txs = append(bd.Txs, tx) + result[tx.BlockHeight] = bd + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/interval.go b/driver/controller/data/interval.go new file mode 100644 index 0000000..4c4536d --- /dev/null +++ b/driver/controller/data/interval.go @@ -0,0 +1,200 @@ +package data + +import ( + "context" + "fmt" + "sort" + "time" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/common/window" + "sentioxyz/sentio-core/driver/controller" +) + +type BlockInterval struct { + Backfill uint64 + Watching uint64 +} + +type TimeInterval struct { + Backfill time.Duration + Watching time.Duration +} + +type IntervalConfig struct { + BlockInterval *BlockInterval + TimeInterval *TimeInterval +} + +func (c IntervalConfig) String() string { + if c.BlockInterval != nil { + if c.BlockInterval.Backfill == c.BlockInterval.Watching { + return fmt.Sprintf("Per:%d", c.BlockInterval.Watching) + } else { + return fmt.Sprintf("Backfill:%d,Watching:%d", c.BlockInterval.Backfill, c.BlockInterval.Watching) + } + } + if c.TimeInterval != nil { + if c.TimeInterval.Backfill == c.TimeInterval.Watching { + return fmt.Sprintf("Per:%s", c.TimeInterval.Watching) + } else { + return fmt.Sprintf("Backfill:%s,Watching:%s", c.TimeInterval.Backfill, c.TimeInterval.Watching) + } + } + return "empty" +} + +// watchingDiffers reports whether the finer watching window differs from the backfill window, i.e. +// whether the watching pass can produce points the backfill pass does not. +func (c IntervalConfig) watchingDiffers() bool { + if c.BlockInterval != nil { + return c.BlockInterval.Watching != c.BlockInterval.Backfill + } + if c.TimeInterval != nil { + return c.TimeInterval.Watching != c.TimeInterval.Backfill + } + return false +} + +func (c IntervalConfig) Equal(a IntervalConfig) bool { + if c.BlockInterval != nil { + return a.BlockInterval != nil && + a.BlockInterval.Backfill == c.BlockInterval.Backfill && + a.BlockInterval.Watching == c.BlockInterval.Watching + } + if c.TimeInterval != nil { + return a.TimeInterval != nil && + a.TimeInterval.Backfill == c.TimeInterval.Backfill && + a.TimeInterval.Watching == c.TimeInterval.Watching + } + return false +} + +func ContainsInterval(list []IntervalConfig, target IntervalConfig) bool { + for _, item := range list { + if item.Equal(target) { + return true + } + } + return false +} + +type IntervalRequirement struct { + controller.BlockRange + IntervalConfig +} + +func (r IntervalRequirement) String() string { + return fmt.Sprintf("IntervalRequirement[%s]%s", r.IntervalConfig.String(), r.BlockRange.String()) +} + +func (r IntervalRequirement) Snapshot() any { + return map[string]any{ + "config": r.IntervalConfig.String(), + "range": r.BlockRange.String(), + } +} + +// MergeIntervalRequirements Will remove the requirement of being fully included +func MergeIntervalRequirements(reqs []IntervalRequirement) (result []IntervalRequirement) { + index := func(r IntervalRequirement) int { + if r.BlockInterval != nil { + return 0 + } + return 1 + } + // It will ensure that all block intervals are in front of the time interval + sort.Slice(reqs, func(i, j int) bool { + if ix, jx := index(reqs[i]), index(reqs[j]); ix != jx { + return ix < jx + } + if reqs[i].StartBlock != reqs[j].StartBlock { + return reqs[i].StartBlock < reqs[j].StartBlock + } + return reqs[i].BlockRange.Include(reqs[j].BlockRange) + }) + for i, req := range reqs { + var included bool + for j := i - 1; j >= 0 && index(req) == index(reqs[j]); j-- { + pre := reqs[j] + if pre.BlockRange.Include(req.BlockRange) && pre.IntervalConfig.Equal(req.IntervalConfig) { + included = true + break + } + } + if !included { + result = append(result, req) + } + } + return +} + +func QueryInterval( + ctx context.Context, + start, end uint64, + first uint64, + latest controller.BlockHeader, + req IntervalRequirement, + timeGetter func(ctx context.Context, blockNumber uint64) (time.Time, error), +) ([]uint64, error) { + itv := req.IntervalConfig + // inWatching is only used when the watching window differs from the backfill window; when they + // are equal the watching scan/points just duplicate the backfill ones, so there is no need to + // even fetch end's block time to decide it. + var inWatching bool + if itv.watchingDiffers() { + endTime, err := timeGetter(ctx, end) + if err != nil { + return nil, err + } + inWatching = latest.GetBlockTime().Sub(endTime) < controller.WatchingDelay + } + var blockNumbers []uint64 + if itv.BlockInterval != nil { + for n := start; n <= end; n++ { + if n%itv.BlockInterval.Backfill == 0 || (inWatching && n%itv.BlockInterval.Watching == 0) { + blockNumbers = append(blockNumbers, n) + } + } + } else if itv.TimeInterval != nil { + bns, err := window.FindStartPoints[int64]( + ctx, int64(start), int64(end), 0, + func(ctx context.Context, n int64) (time.Time, error) { + // start may equal to first and window.FindStartPoints will query the time of start-1, + // so here need to check whether n is less than first + if n < int64(first) { + return time.Time{}, nil + } + t, getErr := timeGetter(ctx, uint64(n)) + if getErr != nil { + return time.Time{}, getErr + } + return t.Truncate(itv.TimeInterval.Backfill), nil + }) + if err != nil { + return nil, err + } + ns := utils.BuildSet(bns) + // inWatching is already false when the watching window equals the backfill window, so this + // extra scan only runs when it can add new points. + if inWatching { + bns, err = window.FindStartPoints[int64]( + ctx, int64(start), int64(end), 0, + func(ctx context.Context, n int64) (time.Time, error) { + t, getErr := timeGetter(ctx, uint64(n)) + if getErr != nil { + return time.Time{}, getErr + } + return t.Truncate(itv.TimeInterval.Watching), nil + }) + if err != nil { + return nil, err + } + utils.MergeInto(ns, bns) + } + for n := range ns { + blockNumbers = append(blockNumbers, uint64(n)) + } + } + return blockNumbers, nil +} diff --git a/driver/controller/data/sol/BUILD.bazel b/driver/controller/data/sol/BUILD.bazel new file mode 100644 index 0000000..3b1b901 --- /dev/null +++ b/driver/controller/data/sol/BUILD.bazel @@ -0,0 +1,40 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sol", + srcs = [ + "block.go", + "block_main.go", + "client.go", + "native_client.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/sol", + visibility = ["//visibility:public"], + deps = [ + "//chain/sol", + "//common/concurrency", + "//common/https", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_cenkalti_backoff_v4//:backoff", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_gagliardetto_solana_go//:solana-go", + "@com_github_gagliardetto_solana_go//rpc", + "@com_github_pkg_errors//:errors", + ], +) + +go_test( + name = "sol_test", + srcs = ["client_test.go"], + embed = [":sol"], + deps = [ + "//driver/controller/data", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/data/sol/block.go b/driver/controller/data/sol/block.go new file mode 100644 index 0000000..6ec432f --- /dev/null +++ b/driver/controller/data/sol/block.go @@ -0,0 +1,32 @@ +package sol + +import ( + "time" + + "github.com/gagliardetto/solana-go/rpc" +) + +type Block struct { + Slot uint64 + *rpc.GetBlockResult +} + +func (b Block) Skipped() bool { + return b.GetBlockResult == nil +} + +func (b Block) GetBlockNumber() uint64 { + return b.Slot +} + +func (b Block) GetBlockParentHash() string { + return b.PreviousBlockhash.String() +} + +func (b Block) GetBlockHash() string { + return b.Blockhash.String() +} + +func (b Block) GetBlockTime() time.Time { + return b.BlockTime.Time() +} diff --git a/driver/controller/data/sol/block_main.go b/driver/controller/data/sol/block_main.go new file mode 100644 index 0000000..0b8e102 --- /dev/null +++ b/driver/controller/data/sol/block_main.go @@ -0,0 +1,249 @@ +package sol + +import ( + "context" + "fmt" + "time" + + "github.com/gagliardetto/solana-go" + + solcore "sentioxyz/sentio-core/chain/sol" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +const ( + // targetKeepBytes is the buffered-data backpressure target for the main-data fetchers. + // BlockMainData.Size() estimates real memory use (full transactions can be large), so this is + // sized in bytes, not in block/transaction counts. + targetKeepBytes = 100 * 1024 * 1024 // 100MB + + // targetQueryBytes is the per-query data-size target the fetcher grows a range toward, so each + // query fetches roughly this much data. + targetQueryBytes = 10 * 1024 * 1024 // 10MB + + // intervalMinQuerySize is the smallest interval fetch range. It matches the super node's + // per-query block cap: GetBlocksByInterval returns at most one block per slot, so a range this + // size can always fit under the cap, letting the fetcher shrink down to it on an over-cap error. + intervalMinQuerySize = 500 +) + +// DataRequirement is the data demand of all handlers of one processor. +type DataRequirement struct { + // Tx is the instruction handlers' requirements (one per handler, merged into disjoint ranges + // when building the fetchers). + Tx []TransactionRequirement + // Interval is the interval handlers' requirements. + Interval []data.IntervalRequirement +} + +func (r DataRequirement) Snapshot() any { + txs := make([]any, len(r.Tx)) + for i, t := range r.Tx { + txs[i] = t.Snapshot() + } + intervals := make([]any, len(r.Interval)) + for i, iv := range r.Interval { + intervals[i] = iv.Snapshot() + } + return map[string]any{ + "tx": txs, + "intervals": intervals, + } +} + +// BlockMainData is the per-block data produced by the main-data fetchers and consumed directly by +// the handlers; it already carries everything needed to build the binding data, so the block-data +// phase no longer fetches anything. +type BlockMainData struct { + // Slot / Blockhash / PreviousBlockhash / BlockTime are the block header, always set when data + // is present, so the block data can be built without a separate getBlock call. + Slot uint64 + Blockhash string + PreviousBlockhash string + BlockTime *solana.UnixTimeSeconds + // Intervals are the interval configs this block is a target of (interval handler). + Intervals []data.IntervalConfig + // Block is the block header with signatures (interval handler), nil when not an interval target. + Block *Block + // Transactions are the full transactions invoking the requested programs (instruction handler). + Transactions []solcore.WrappedTransaction +} + +func (d BlockMainData) IsEmpty() bool { + return len(d.Intervals) == 0 && d.Block == nil && len(d.Transactions) == 0 +} + +func (d BlockMainData) Size() int { + size := 0 + if d.Block != nil && d.Block.GetBlockResult != nil { + size += len(d.Block.Signatures) * 90 + } + for _, tx := range d.Transactions { + size += estimateTransactionSize(tx) + } + return size +} + +func estimateTransactionSize(tx solcore.WrappedTransaction) int { + size := 256 // base overhead for signature/version/envelope + if tx.Transaction != nil { + for _, in := range tx.Transaction.Message.Instructions { + size += len(in.Data) + len(in.Accounts)*45 + } + } + if tx.Meta != nil { + for _, inner := range tx.Meta.InnerInstructions { + for _, in := range inner.Instructions { + size += len(in.Data) + len(in.Accounts)*45 + } + } + } + return size +} + +func windowsOf(cfg data.IntervalConfig) (backfill, watching solcore.IntervalWindow) { + if cfg.BlockInterval != nil { + return solcore.IntervalWindow{BlockWindow: cfg.BlockInterval.Backfill}, + solcore.IntervalWindow{BlockWindow: cfg.BlockInterval.Watching} + } + return solcore.IntervalWindow{TimeWindow: cfg.TimeInterval.Backfill}, + solcore.IntervalWindow{TimeWindow: cfg.TimeInterval.Watching} +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + backfill, watching := windowsOf(req.IntervalConfig) + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + // minQuerySize is intervalMinQuerySize (not maxQuerySize) so the fetcher can shrink down to + // a range the super node's block cap can always satisfy when it returns an over-cap error. + intervalMinQuerySize, + 10000, + targetKeepBytes, + targetQueryBytes, + time.Second*15, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + result := make(map[uint64]BlockMainData) + // Builds the interval handler's binding data from each window's first block. Only block + // header fields are used (slot, blockhash, parent hash, time). NOTE: blocks served from + // the BigQuery archival tier (slots below the ClickHouse range) carry NO transaction + // signatures — a deliberate BigQuery cost optimization (see GetBlocksByInterval and + // the archival store). That is fine here because interval binding data is header-only; do not + // add a dependency on b's transaction signatures in this path. + add := func(blocks []Block) { + for i := range blocks { + b := blocks[i] + if b.Skipped() { + continue + } + bd := result[b.Slot] + if !data.ContainsInterval(bd.Intervals, req.IntervalConfig) { + bd.Intervals = append(bd.Intervals, req.IntervalConfig) + } + bd.Slot = b.Slot + bd.Blockhash = b.GetBlockHash() + bd.PreviousBlockhash = b.GetBlockParentHash() + bd.BlockTime = b.BlockTime + bd.Block = &b + result[b.Slot] = bd + } + } + // The super node caps the result and errors when exceeded; that error propagates and the + // fetcher halves the range, so no client-side saturation check is needed. + backfillBlocks, err := client.GetBlocksByInterval(ctx, start, end, backfill) + if err != nil { + return nil, err + } + add(backfillBlocks) + // The finer "watching" window only applies near the head. Like data.QueryInterval, decide + // by time: the range is "in watching" when end's block time is within WatchingDelay of the + // latest. end may itself be skipped, so look up the nearest non-skipped block at or before + // end (slot < end+1) for its time; if there is none we conservatively run the watching + // query and rely on the per-block time filter below. + if watching != backfill { + inWatching := true + endBlock, getErr := client.GetPreviousUnskippedBlock(ctx, end+1) + if getErr != nil { + return nil, getErr + } + if endBlock.Found && endBlock.BlockTime != nil { + inWatching = latest.GetBlockTime().Sub(endBlock.BlockTime.Time()) < controller.WatchingDelay + } + if inWatching { + watchingBlocks, err := client.GetBlocksByInterval(ctx, start, end, watching) + if err != nil { + return nil, err + } + // Keep only blocks whose own time is within WatchingDelay of the latest. + cutoff := latest.GetBlockTime().Add(-controller.WatchingDelay) + inWatch := make([]Block, 0, len(watchingBlocks)) + for _, b := range watchingBlocks { + if !b.Skipped() && b.BlockTime != nil && b.BlockTime.Time().After(cutoff) { + inWatch = append(inWatch, b) + } + } + add(inWatch) + } + } + return result, nil + }, + ) +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.Tx = MergeTxRequirements(currentBlockNumber, req.Tx) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.Tx { + fetchers = append(fetchers, BuildTxFetcher( + namePrefix+fmt.Sprintf("TxFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(bn uint64, from []BlockMainData) (result BlockMainData, has bool, _ error) { + result.Slot = bn + for _, box := range from { + if box.Blockhash != "" { + result.Blockhash = box.Blockhash + result.PreviousBlockhash = box.PreviousBlockhash + } + if box.BlockTime != nil { + result.BlockTime = box.BlockTime + } + result.Intervals = append(result.Intervals, box.Intervals...) + if box.Block != nil { + result.Block = box.Block + } + result.Transactions = append(result.Transactions, box.Transactions...) + } + return result, !result.IsEmpty(), nil + }) +} diff --git a/driver/controller/data/sol/client.go b/driver/controller/data/sol/client.go new file mode 100644 index 0000000..08ca63a --- /dev/null +++ b/driver/controller/data/sol/client.go @@ -0,0 +1,385 @@ +package sol + +import ( + "context" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/cenkalti/backoff/v4" + ethrpc "github.com/ethereum/go-ethereum/rpc" + "github.com/gagliardetto/solana-go" + "github.com/pkg/errors" + + solcore "sentioxyz/sentio-core/chain/sol" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) + // GetBlock returns the block header (no transactions). The result may be skipped. + GetBlock(ctx context.Context, blockNumber uint64) (Block, error) + + // GetBlocksByInterval returns the first non-skipped block of each window in [from, to], for the + // interval handler. The super node caps the result and errors when exceeded. + // + // Signatures: blocks within the ClickHouse range carry their transaction signatures; blocks + // served from the BigQuery archival tier (slots below the ClickHouse range) carry headers only, + // with NO signatures — a deliberate BigQuery cost optimization (see the archival store). The interval + // handler only uses block headers (slot/hash/time), so this is fine; callers must not rely on + // per-block signatures from this method. + GetBlocksByInterval( + ctx context.Context, + from, to uint64, + window solcore.IntervalWindow, + ) ([]Block, error) + // FindTransactions returns, grouped by block, the full transactions in [from, to] invoking any + // of the given programs, for the instruction handler. The super node caps the result and errors + // when exceeded (except for a single-block range). + FindTransactions( + ctx context.Context, + from, to uint64, + programs []solana.PublicKey, + ) ([]solcore.BlockTransactions, error) + GetContractStartBlock(ctx context.Context, address solana.PublicKey, start, latest uint64) (uint64, bool, error) + // GetPreviousUnskippedBlock returns the nearest non-skipped block with slot < beforeSlot (pass + // slot+1 to include the slot itself), used to learn the chain time around a possibly-skipped slot. + GetPreviousUnskippedBlock(ctx context.Context, beforeSlot uint64) (solcore.PreviousUnskippedBlock, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +// client talks to the Solana super node over JSON-RPC. The super node answers the sol_* methods +// from its latest-slot cache and ClickHouse, so the driver no longer talks to a node directly. +type supernodeClient struct { + endpoint string + watchLatestInterval time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *ethrpc.Client + + cachedHeaders *data.BlockCache[Block] + + savedLatestBlockNumber atomic.Uint64 +} + +// NewClient selects the data client for a sol chain. It uses the native Solana RPC client when the +// processor is old (DriverVersion < 2) or the endpoint is not a super node (no sol_getLatestHeader), +// and the super-node client otherwise. Only sol_mainnet runs a super node (ClickHouse + BigQuery); +// other sol chains fall back to native RPC. +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + watchLatestInterval time.Duration, + driverVersion int32, +) (Client, error) { + _, logger := log.FromContext(ctx) + if driverVersion < 2 { + logger.Infof("sol: driver version %d < 2, using native Solana RPC at %s", driverVersion, endpoint) + return newNativeClient(endpoint, maxConcurrency, firstBlockNumber, watchLatestInterval) + } + supported, err := endpointSupportsSuperNode(ctx, endpoint) + if err != nil { + // The probe kept hitting transient/HTTP/timeout errors and never got a definitive answer. + // Surface the retryable NewClient error so the controller restarts the pod and tries again, + // rather than failing permanently (NeverRetry) on what may be a temporary outage. + return nil, err + } + if !supported { + logger.Infof("sol: endpoint %s does not support sol_getLatestHeader, using native Solana RPC", endpoint) + return newNativeClient(endpoint, maxConcurrency, firstBlockNumber, watchLatestInterval) + } + // firstBlockNumber is intentionally not passed: the super node is backed by ClickHouse + BigQuery, + // so it serves the full available history and bounds the start range itself (range store + + // BigQuery retention floor). The driver imposes no client-side first block. + return newSupernodeClient(ctx, endpoint, maxConcurrency, watchLatestInterval) +} + +func newSupernodeClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + watchLatestInterval time.Duration, +) (Client, error) { + cli := &supernodeClient{ + endpoint: endpoint, + watchLatestInterval: watchLatestInterval, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + } + var err error + if cli.cli, err = ethrpc.DialOptions(ctx, endpoint, ethrpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.cachedHeaders, _ = data.NewBlockCache[Block](100000) + return cli, nil +} + +// superNodeProbeCache memoizes endpointSupportsSuperNode by endpoint, so each endpoint is probed at +// most once across controllers. +var superNodeProbeCache sync.Map // endpoint string -> bool + +// superNodeProbeMaxAttempts bounds how many times endpointSupportsSuperNode retries a transient +// (HTTP/timeout/dial) probe failure before giving up with a retryable NewClient error. +const superNodeProbeMaxAttempts = 20 + +// endpointSupportsSuperNode probes whether the endpoint implements the super-node sol_getLatestHeader +// method (gt=0 returns immediately when supported), returning: +// +// - (true, nil) — the endpoint answers sol_getLatestHeader, so it is a super node. +// - (false, nil) — the endpoint returns -32601 / "method not found", so it is a plain Solana RPC +// endpoint and the caller should fall back to native RPC. +// - (false, err) — the probe kept failing with transient errors (HTTP/timeout/dial) across all +// attempts; err is a *data.NewClientRetryableError so the caller can restart the pod and retry, +// instead of incorrectly assuming either kind of endpoint. +// +// Transient failures are retried with exponential backoff up to superNodeProbeMaxAttempts times, each +// logged. Only definitive answers (super node / not a super node) are cached per endpoint. +func endpointSupportsSuperNode(ctx context.Context, endpoint string) (bool, error) { + if v, ok := superNodeProbeCache.Load(endpoint); ok { + return v.(bool), nil + } + _, logger := log.FromContext(ctx) + + var supported bool + attempt := 0 + operation := func() error { + attempt++ + probe, err := ethrpc.DialOptions(ctx, endpoint, ethrpc.WithHTTPClient(https.DefaultClient)) + if err != nil { + logger.Warnf("sol: probe dial to %s failed (attempt %d/%d): %v; will retry", + endpoint, attempt, superNodeProbeMaxAttempts, err) + return err + } + defer probe.Close() + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + var blk Block + err = probe.CallContext(probeCtx, &blk, "sol_getLatestHeader", uint64(0)) + if err == nil { + supported = true + return nil + } + // A -32601 / "method not found" is a definitive negative: the endpoint is a plain Solana RPC + // node. Stop retrying by returning nil with supported left false. + var rpcErr ethrpc.Error + if errors.As(err, &rpcErr) && rpcErr.ErrorCode() == -32601 { + supported = false + return nil + } + if msg := strings.ToLower(err.Error()); strings.Contains(msg, "method not found") || strings.Contains(msg, "does not exist") { + supported = false + return nil + } + // Anything else (HTTP error, timeout, connection reset, ...) is transient: retry. + logger.Warnf("sol: probe sol_getLatestHeader at %s failed (attempt %d/%d): %v; will retry", + endpoint, attempt, superNodeProbeMaxAttempts, err) + return err + } + + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = 200 * time.Millisecond + bo.MaxInterval = 10 * time.Second + // WithMaxRetries(n) allows n retries after the initial attempt → superNodeProbeMaxAttempts total. + retry := backoff.WithContext(backoff.WithMaxRetries(bo, superNodeProbeMaxAttempts-1), ctx) + if err := backoff.Retry(operation, retry); err != nil { + return false, data.NewClientRetryable( + fmt.Sprintf("sol: probing super-node support at %s kept failing after %d attempts", endpoint, superNodeProbeMaxAttempts), + err, + ) + } + superNodeProbeCache.Store(endpoint, supported) + return supported, nil +} + +func (c *supernodeClient) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always be context.Canceled + } + defer release() + callStartAt := time.Now() + err = c.cli.CallContext(ctx, &result, method, args...) + if err != nil { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + c.stat.Called(method, args, err, startAt, callStartAt) + return err +} + +func (c *supernodeClient) fetchBlock(ctx context.Context, blockNumber uint64) (Block, error) { + var blk Block + if err := c.callContext(ctx, &blk, blockNumber, "sol_getBlock", blockNumber); err != nil { + return Block{}, err + } + return blk, nil +} + +func (c *supernodeClient) getBlock(ctx context.Context, blockNumber uint64) (Block, error) { + blk, err := c.fetchBlock(ctx, blockNumber) + if err == nil { + c.cachedHeaders.Add(blockNumber, blk) + } + return blk, err +} + +// getLatestHeader long-polls the super node for the latest non-skipped block header with slot > gt. +// The super node blocks until such a block exists, so the driver never has to poll, and the +// returned header is guaranteed non-skipped (BlockTime set). +func (c *supernodeClient) getLatestHeader(ctx context.Context, gt uint64) (Block, error) { + var blk Block + if err := c.callContext(ctx, &blk, 0, "sol_getLatestHeader", gt); err != nil { + return Block{}, err + } + c.savedLatestBlockNumber.Store(blk.Slot) + return blk, nil +} + +// GetLatest returns the latest non-skipped header. first is 0: the super node serves the full +// available history (ClickHouse + BigQuery) and bounds the start range itself, so the driver imposes +// no client-side first block (the processor's own start / GetContractStartBlock decide it). +func (c *supernodeClient) GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) { + blk, err := c.getLatestHeader(ctx, 0) + if err != nil { + return nil, 0, err + } + return blk, 0, nil +} + +func (c *supernodeClient) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, blockNumberGt uint64) (latest controller.BlockHeader, broken, err error) { + blk, err := c.getLatestHeader(ctx, blockNumberGt) + if err != nil { + return nil, nil, err + } + return blk, nil, nil + }, + callback) +} + +func (c *supernodeClient) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getBlock(ctx, blockNumber) +} + +func (c *supernodeClient) GetBlock(ctx context.Context, blockNumber uint64) (Block, error) { + // Cache + singleflight: concurrent fetchers asking for the same block share one sol_getBlock. + return c.cachedHeaders.GetOrFetch(blockNumber, func() (Block, error) { + return c.fetchBlock(ctx, blockNumber) + }) +} + +func (c *supernodeClient) GetBlocksByInterval( + ctx context.Context, + from, to uint64, + window solcore.IntervalWindow, +) ([]Block, error) { + var blocks []Block + param := solcore.GetBlocksByIntervalParam{From: from, To: to, Window: window} + if err := c.callContext(ctx, &blocks, to, "sol_getBlocksByInterval", param); err != nil { + return nil, errors.Wrapf(err, "get interval blocks in [%d,%d] failed", from, to) + } + return blocks, nil +} + +func (c *supernodeClient) FindTransactions( + ctx context.Context, + from, to uint64, + programs []solana.PublicKey, +) ([]solcore.BlockTransactions, error) { + var result []solcore.BlockTransactions + param := solcore.FindTransactionsParam{From: from, To: to, ProgramIDs: programs} + if err := c.callContext(ctx, &result, to, "sol_findTransactions", param); err != nil { + return nil, errors.Wrapf(err, "find transactions in [%d,%d] for %d programs failed", from, to, len(programs)) + } + return result, nil +} + +// GetContractStartBlock asks the super node for the contract's earliest appearance block, then maps +// it to [start, latest]: an appearance before start clamps to start; an appearance after latest (or +// no appearance) is treated as not yet in range. +func (c *supernodeClient) GetContractStartBlock( + ctx context.Context, + address solana.PublicKey, + start, latest uint64, +) (uint64, bool, error) { + var result solcore.GetContractStartBlockResult + if err := c.callContext(ctx, &result, 0, "sol_getContractStartBlock", address); err != nil { + return 0, false, err + } + if !result.Found || result.Slot > latest { + return 0, false, nil + } + if result.Slot < start { + return start, true, nil + } + return result.Slot, true, nil +} + +func (c *supernodeClient) GetPreviousUnskippedBlock( + ctx context.Context, + beforeSlot uint64, +) (solcore.PreviousUnskippedBlock, error) { + var result solcore.PreviousUnskippedBlock + if err := c.callContext(ctx, &result, beforeSlot, "sol_getPreviousUnskippedBlock", beforeSlot); err != nil { + return solcore.PreviousUnskippedBlock{}, err + } + return result, nil +} + +func (c *supernodeClient) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedHeaders.Keys() { + if r.Contains(bn) { + c.cachedHeaders.Remove(bn) + } + } +} + +func (c *supernodeClient) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "watchLatestInterval": c.watchLatestInterval.String(), + }, + "savedLatestBlockNumber": c.savedLatestBlockNumber.Load(), + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cache": map[string]any{ + "cachedHeaders": c.cachedHeaders.Snapshot(10, func(block Block) string { + if block.Skipped() { + return fmt.Sprintf("%d/", block.Slot) + } + return controller.GetBlockFullText(block) + }), + }, + } +} diff --git a/driver/controller/data/sol/client_test.go b/driver/controller/data/sol/client_test.go new file mode 100644 index 0000000..6ce1a65 --- /dev/null +++ b/driver/controller/data/sol/client_test.go @@ -0,0 +1,84 @@ +package sol + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "sentioxyz/sentio-core/driver/controller/data" +) + +func Test_getSkippedBlock(t *testing.T) { + t.Skip("used external endpoint") + + cli, _ := NewClient( + context.Background(), + "https://solana-rpc.publicnode.com", + 1, + 10000, + time.Second, + 0, // driver version < 2 ⇒ native Solana RPC (this endpoint is a plain node) + ) + + blk, err := cli.GetBlock(context.Background(), 392086804) + assert.NoError(t, err) + assert.False(t, blk.Skipped()) + + blk, err = cli.GetBlock(context.Background(), 392086803) + assert.NoError(t, err) + assert.True(t, blk.Skipped()) +} + +func Test_queryInterval(t *testing.T) { + t.Skip("used external endpoint") + + cli, _ := NewClient( + context.Background(), + "https://solana-rpc.publicnode.com", + 1, + 0, + time.Second, + 0, // driver version < 2 ⇒ native Solana RPC (this endpoint is a plain node) + ) + + ctx := context.Background() + + latest, first, err := cli.GetLatest(ctx) + assert.NoError(t, err) + + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + for n := blockNumber; n >= 0; n-- { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + h, err := cli.GetBlock(getCtx, n) + cancel() + if err != nil { + return time.Time{}, err + } + if !h.Skipped() { + fmt.Printf("!!! got block %d/%d: %s\n", blockNumber, n, h.GetBlockTime().Format(time.RFC3339)) + return h.GetBlockTime(), nil + } + fmt.Printf("!!! got block %d/%d but skipped\n", blockNumber, n) + // Slot n was skipped, try n-1 next + } + // all slot in [0,blockNumber] was skipped, just return zero time + return time.Time{}, nil + } + s := uint64(393722931) + e := uint64(393825051) + + req := data.IntervalRequirement{ + IntervalConfig: data.IntervalConfig{ + TimeInterval: &data.TimeInterval{ + Backfill: time.Hour * 4, + Watching: time.Hour, + }, + }, + } + bns, err := data.QueryInterval(ctx, s, e, first, latest, req, timeGetter) + assert.NoError(t, err) + fmt.Printf("!!! result: %v\n", bns) +} diff --git a/driver/controller/data/sol/native_client.go b/driver/controller/data/sol/native_client.go new file mode 100644 index 0000000..664d7c4 --- /dev/null +++ b/driver/controller/data/sol/native_client.go @@ -0,0 +1,542 @@ +package sol + +import ( + "context" + "fmt" + "regexp" + "sort" + "strings" + "sync/atomic" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/pkg/errors" + + solcore "sentioxyz/sentio-core/chain/sol" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" +) + +// nativeClient implements Client against a plain Solana JSON-RPC endpoint (no super node). It is used +// for sol chains that have no super node (anything but sol_mainnet) and for old processors +// (DriverVersion < 2). It reproduces the super-node sol_* capabilities over native primitives +// (getSlot / getBlock / getBlocks / getSignaturesForAddress), porting the pre-super-node driver +// (commit 87d8d1455). These calls are far more RPC-heavy than the super node — acceptable for the +// low-volume chains / legacy processors that need this path. +type nativeClient struct { + endpoint string + firstBlockNumber int64 + watchLatestInterval time.Duration + getLatestTimeout time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *rpc.Client + + cachedHeaders *data.BlockCache[Block] + + savedLatestBlockNumber atomic.Uint64 + savedFirstBlockNumber atomic.Uint64 +} + +func newNativeClient( + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + watchLatestInterval time.Duration, +) (Client, error) { + c := &nativeClient{ + endpoint: endpoint, + firstBlockNumber: firstBlockNumber, + watchLatestInterval: watchLatestInterval, + getLatestTimeout: 30 * time.Second, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + cli: rpc.New(endpoint), + } + c.cachedHeaders, _ = data.NewBlockCache[Block](100000) + return c, nil +} + +var slotSkippedErrorMatcher = regexp.MustCompile(`slot.*was skipped`) + +func isSlotSkippedError(err error) bool { + return err != nil && slotSkippedErrorMatcher.FindString(strings.ToLower(err.Error())) != "" +} + +// parsedBlockResult is the subset of getBlock's jsonParsed/full response we need to assemble +// BlockTransactions (header + ordered transactions). +type parsedBlockResult struct { + Blockhash solana.Hash `json:"blockhash"` + PreviousBlockhash solana.Hash `json:"previousBlockhash"` + BlockTime *solana.UnixTimeSeconds `json:"blockTime"` + Transactions []parsedTransactionWithMeta `json:"transactions"` +} + +type parsedTransactionWithMeta struct { + Transaction *rpc.ParsedTransaction `json:"transaction"` + Meta *rpc.ParsedTransactionMeta `json:"meta"` +} + +// callContext applies the concurrency token + statistics around a native Solana RPC call, dispatched +// by an internal method name (kept sol_*-style for statistics continuity with the super-node client). +func (c *nativeClient) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always context.Canceled + } + defer release() + callStartAt := time.Now() + switch method { + case "sol_getLatestBlockNumber": + r := result.(*uint64) + *r, err = c.cli.GetSlot(ctx, rpc.CommitmentFinalized) + case "sol_getBlock": + opt := rpc.GetBlockOpts{TransactionDetails: rpc.TransactionDetailsSignatures} + r := result.(*Block) + r.Slot = args[0].(uint64) + r.GetBlockResult, err = c.cli.GetBlockWithOpts(ctx, r.Slot, &opt) + if err != nil && isSlotSkippedError(err) { + r.GetBlockResult, err = nil, nil // skipped slot + } + case "sol_getBlockFull": + // solana-go's GetBlockWithOpts rejects the jsonParsed encoding, so issue the raw getBlock + // call to fetch every transaction's parsed detail (and the header) in one request. + obj := rpc.M{ + "encoding": solana.EncodingJSONParsed, + "transactionDetails": rpc.TransactionDetailsFull, + "maxSupportedTransactionVersion": uint64(0), + "rewards": false, + } + r := result.(*parsedBlockResult) + err = c.cli.RPCCallForInto(ctx, r, "getBlock", []any{args[0].(uint64), obj}) + if err != nil && isSlotSkippedError(err) { + *r, err = parsedBlockResult{}, nil // skipped slot + } + case "sol_getBlocks": + end := args[1].(uint64) + r := result.(*rpc.BlocksResult) + *r, err = c.cli.GetBlocks(ctx, args[0].(uint64), &end, rpc.CommitmentFinalized) + case "sol_getSignaturesForAddress": + limit := args[3].(int) + opt := rpc.GetSignaturesForAddressOpts{ + Until: args[1].(solana.Signature), + Before: args[2].(solana.Signature), + Limit: &limit, + } + r := result.(*[]*rpc.TransactionSignature) + *r, err = c.cli.GetSignaturesForAddressWithOpts(ctx, args[0].(solana.PublicKey), &opt) + default: + panic(errors.Errorf("unsupported method %q", method)) + } + if err != nil { + err = errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + c.stat.Called(method, args, err, startAt, callStartAt) + return err +} + +func (c *nativeClient) getLatestBlockNumber(ctx context.Context) (uint64, error) { + var latest uint64 + if err := c.callContext(ctx, &latest, 0, "sol_getLatestBlockNumber"); err != nil { + return 0, err + } + c.savedLatestBlockNumber.Store(latest) + c.savedFirstBlockNumber.CompareAndSwap(0, data.GetFirst(c.firstBlockNumber, latest)) + return latest, nil +} + +func (c *nativeClient) fetchBlock(ctx context.Context, blockNumber uint64) (Block, error) { + var blk Block + if err := c.callContext(ctx, &blk, blockNumber, "sol_getBlock", blockNumber); err != nil { + return Block{}, err + } + return blk, nil +} + +func (c *nativeClient) getBlock(ctx context.Context, blockNumber uint64) (Block, error) { + blk, err := c.fetchBlock(ctx, blockNumber) + if err == nil { + c.cachedHeaders.Add(blockNumber, blk) + } + return blk, err +} + +func (c *nativeClient) getFullBlock(ctx context.Context, blockNumber uint64) (parsedBlockResult, error) { + var blk parsedBlockResult + if err := c.callContext(ctx, &blk, blockNumber, "sol_getBlockFull", blockNumber); err != nil { + return parsedBlockResult{}, err + } + return blk, nil +} + +// getBlocks returns the existing (non-skipped) slots in [start, end] in ascending order. +func (c *nativeClient) getBlocks(ctx context.Context, start, end uint64) ([]uint64, error) { + var res rpc.BlocksResult + if err := c.callContext(ctx, &res, end, "sol_getBlocks", start, end); err != nil { + return nil, err + } + return res, nil +} + +func (c *nativeClient) GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) { + latestBlockNumber, err := c.getLatestBlockNumber(ctx) + if err != nil { + return nil, 0, err + } + // The finalized slot is never skipped, but its block data may lag a moment; retry with exponential + // backoff (cancelled with ctx). + bo := backoff.NewExponentialBackOff() + bo.InitialInterval = 200 * time.Millisecond + var blk Block + if err = backoff.Retry(func() error { + blk, err = c.getBlock(ctx, latestBlockNumber) + return err + }, backoff.WithContext(backoff.WithMaxRetries(bo, 10), ctx)); err != nil { + return nil, 0, err + } + return blk, c.savedFirstBlockNumber.Load(), nil +} + +func (c *nativeClient) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingPolling( + ctx, + c.watchLatestInterval, + c.getLatestTimeout, + from, + func(ctx context.Context) (controller.BlockHeader, error) { + h, _, err := c.GetLatest(ctx) + return h, err + }, + callback) +} + +func (c *nativeClient) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getBlock(ctx, blockNumber) +} + +func (c *nativeClient) GetBlock(ctx context.Context, blockNumber uint64) (Block, error) { + // Cache + singleflight: concurrent fetchers asking for the same block share one sol_getBlock. + return c.cachedHeaders.GetOrFetch(blockNumber, func() (Block, error) { + return c.fetchBlock(ctx, blockNumber) + }) +} + +// timeGetter returns blockNumber's block time, walking back over skipped slots. +func (c *nativeClient) timeGetter(ctx context.Context, blockNumber uint64) (time.Time, error) { + for n := blockNumber; ; n-- { + h, err := c.GetBlock(ctx, n) + if err != nil { + return time.Time{}, err + } + if !h.Skipped() { + return h.GetBlockTime(), nil + } + if n == 0 { + return time.Time{}, nil // all slots through 0 skipped + } + } +} + +func windowToConfig(w solcore.IntervalWindow) data.IntervalConfig { + if w.IsBlockWindow() { + return data.IntervalConfig{BlockInterval: &data.BlockInterval{Backfill: w.BlockWindow, Watching: w.BlockWindow}} + } + return data.IntervalConfig{TimeInterval: &data.TimeInterval{Backfill: w.TimeWindow, Watching: w.TimeWindow}} +} + +func (c *nativeClient) GetBlocksByInterval( + ctx context.Context, + from, to uint64, + window solcore.IntervalWindow, +) ([]Block, error) { + latest, _, err := c.GetLatest(ctx) + if err != nil { + return nil, err + } + req := data.IntervalRequirement{ + BlockRange: controller.BlockRange{StartBlock: from, EndBlock: &to}, + IntervalConfig: windowToConfig(window), + } + // backfill==watching window ⇒ QueryInterval needs no end-time lookup; timeGetter only matters for + // time windows. + bns, err := data.QueryInterval(ctx, from, to, c.savedFirstBlockNumber.Load(), latest, req, c.timeGetter) + if err != nil { + return nil, errors.Wrapf(err, "query interval blocks in [%d,%d] failed", from, to) + } + blocks := make([]Block, 0, len(bns)) + for _, bn := range bns { + blk, err := c.GetBlock(ctx, bn) + if err != nil { + return nil, err + } + if blk.Skipped() { + continue + } + blocks = append(blocks, blk) + } + return blocks, nil +} + +const findSignaturesPageSize = 1000 + +// findProgramSignatures returns the signatures of transactions in [fromBlock, toBlock] referencing +// address, using getSignaturesForAddress paginated by the surrounding non-skipped blocks' border +// signatures (ported from the pre-super-node driver). Result is in descending slot order. +func (c *nativeClient) findProgramSignatures( + ctx context.Context, + fromBlock, toBlock uint64, + address solana.PublicKey, +) (result []*rpc.TransactionSignature, err error) { + latest := c.savedLatestBlockNumber.Load() + first := c.savedFirstBlockNumber.Load() + var fromTxSig, toTxSig solana.Signature + + // fromTxSig = last signature of the nearest non-skipped block below fromBlock (the `until` bound). + for n := fromBlock - 1; fromBlock > 0 && n >= first; n-- { + blk, gerr := c.GetBlock(ctx, n) + if gerr != nil { + return nil, gerr + } + if !blk.Skipped() && len(blk.Signatures) > 0 { + fromTxSig = blk.Signatures[len(blk.Signatures)-1] + break + } + if n == 0 { + break + } + } + // toTxSig = first signature of the nearest non-skipped block above toBlock (the `before` bound). + for n := toBlock + 1; n <= latest; n++ { + blk, gerr := c.GetBlock(ctx, n) + if gerr != nil { + return nil, gerr + } + if !blk.Skipped() && len(blk.Signatures) > 0 { + toTxSig = blk.Signatures[0] + break + } + } + + limit := findSignaturesPageSize + page, err := c.getSignaturesForAddress(ctx, address, fromTxSig, toTxSig, limit) + if err != nil { + return nil, errors.Wrapf(err, "get signatures for %s in [%d,%d] failed", address, fromBlock, toBlock) + } + // page is DESC. fromTxSig/toTxSig may be empty (open borders), so clip to the block range. + var finished bool + for _, sig := range page { + if sig.Slot > toBlock { + continue + } + if sig.Slot < fromBlock { + finished = true + continue + } + result = append(result, sig) + } + if len(page) >= limit && !finished { + return nil, errors.Errorf("too many signatures for %s in [%d,%d] (page limit %d)", address, fromBlock, toBlock, limit) + } + return result, nil +} + +func (c *nativeClient) getSignaturesForAddress( + ctx context.Context, + address solana.PublicKey, + until, before solana.Signature, + limit int, +) ([]*rpc.TransactionSignature, error) { + var sigs []*rpc.TransactionSignature + if err := c.callContext(ctx, &sigs, 0, "sol_getSignaturesForAddress", address, until, before, limit); err != nil { + return nil, err + } + return sigs, nil +} + +func (c *nativeClient) FindTransactions( + ctx context.Context, + from, to uint64, + programs []solana.PublicKey, +) ([]solcore.BlockTransactions, error) { + // Collect the matched transaction signatures per slot, across all programs. + matched := make(map[uint64]map[solana.Signature]struct{}) + for _, p := range programs { + sigs, err := c.findProgramSignatures(ctx, from, to, p) + if err != nil { + return nil, err + } + for _, s := range sigs { + if matched[s.Slot] == nil { + matched[s.Slot] = make(map[solana.Signature]struct{}) + } + matched[s.Slot][s.Signature] = struct{}{} + } + } + if len(matched) == 0 { + return nil, nil + } + slots := make([]uint64, 0, len(matched)) + for s := range matched { + slots = append(slots, s) + } + sort.Slice(slots, func(i, j int) bool { return slots[i] < slots[j] }) + + out := make([]solcore.BlockTransactions, 0, len(slots)) + for _, slot := range slots { + full, err := c.getFullBlock(ctx, slot) + if err != nil { + return nil, err + } + want := matched[slot] + var wts []solcore.WrappedTransaction + for i, tx := range full.Transactions { + if tx.Transaction == nil || len(tx.Transaction.Signatures) == 0 { + continue + } + sig := tx.Transaction.Signatures[0] + if _, ok := want[sig]; !ok { + continue + } + wts = append(wts, solcore.WrappedTransaction{ + TransactionIndex: uint32(i), + Signature: sig, + // Native getBlock(jsonParsed) does not surface the version cheaply; default legacy + // (same as the BigQuery store). Only the version label is affected. + Version: rpc.LegacyTransactionVersion, + Transaction: tx.Transaction, + Meta: tx.Meta, + }) + } + if len(wts) == 0 { + continue + } + out = append(out, solcore.BlockTransactions{ + Slot: slot, + Blockhash: full.Blockhash, + PreviousBlockhash: full.PreviousBlockhash, + BlockTime: full.BlockTime, + Transactions: wts, + }) + } + return out, nil +} + +func (c *nativeClient) GetContractStartBlock( + ctx context.Context, + address solana.PublicKey, + start, latest uint64, +) (uint64, bool, error) { + return data.BinarySearchContractStart(ctx, start, latest, func(ctx context.Context, bn uint64) (bool, error) { + // Does address appear at or after slot bn? Page back from the top border until a signature at + // slot <= bn is found (present) or signatures run out (absent). + var toTxSig solana.Signature + for n := bn + 1; n <= latest; n++ { + blk, err := c.GetBlock(ctx, n) + if err != nil { + return false, err + } + if !blk.Skipped() && len(blk.Signatures) > 0 { + toTxSig = blk.Signatures[0] + break + } + } + for { + sigs, err := c.getSignaturesForAddress(ctx, address, solana.Signature{}, toTxSig, 1) + if err != nil { + return false, err + } + if len(sigs) == 0 { + return false, nil + } + if sigs[0].Slot <= bn { + return true, nil + } + toTxSig = sigs[0].Signature + } + }) +} + +func (c *nativeClient) GetPreviousUnskippedBlock( + ctx context.Context, + beforeSlot uint64, +) (solcore.PreviousUnskippedBlock, error) { + if beforeSlot == 0 { + return solcore.PreviousUnskippedBlock{}, nil + } + first := c.savedFirstBlockNumber.Load() + const chunk = 1000 + hi := beforeSlot - 1 + for { + lo := first + if hi >= chunk && hi-chunk+1 > first { + lo = hi - chunk + 1 + } + slots, err := c.getBlocks(ctx, lo, hi) + if err != nil { + return solcore.PreviousUnskippedBlock{}, err + } + if len(slots) > 0 { + s := slots[len(slots)-1] // nearest below beforeSlot + blk, err := c.GetBlock(ctx, s) + if err != nil { + return solcore.PreviousUnskippedBlock{}, err + } + res := solcore.PreviousUnskippedBlock{Slot: s, Found: true} + if !blk.Skipped() { + res.BlockTime = blk.BlockTime + } + return res, nil + } + if lo <= first { + return solcore.PreviousUnskippedBlock{}, nil // none in [first, beforeSlot) + } + hi = lo - 1 + } +} + +func (c *nativeClient) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedHeaders.Keys() { + if r.Contains(bn) { + c.cachedHeaders.Remove(bn) + } + } +} + +func (c *nativeClient) Snapshot() any { + return map[string]any{ + "mode": "native", + "config": map[string]any{ + "endpoint": c.endpoint, + "firstBlockNumber": c.firstBlockNumber, + "watchLatestInterval": c.watchLatestInterval.String(), + "getLatestTimeout": c.getLatestTimeout.String(), + }, + "savedFirstBlockNumber": c.savedFirstBlockNumber.Load(), + "savedLatestBlockNumber": c.savedLatestBlockNumber.Load(), + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cache": map[string]any{ + "cachedHeaders": c.cachedHeaders.Snapshot(10, func(block Block) string { + if block.Skipped() { + return fmt.Sprintf("%d/", block.Slot) + } + return controller.GetBlockFullText(block) + }), + }, + } +} diff --git a/driver/controller/data/sol/transaction.go b/driver/controller/data/sol/transaction.go new file mode 100644 index 0000000..51e1ac8 --- /dev/null +++ b/driver/controller/data/sol/transaction.go @@ -0,0 +1,108 @@ +package sol + +import ( + "context" + "time" + + "github.com/gagliardetto/solana-go" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +// TransactionRequirement is one instruction handler's demand: the programs it indexes over a block +// range. +type TransactionRequirement struct { + controller.BlockRange + + Programs []solana.PublicKey +} + +func (r TransactionRequirement) Snapshot() any { + programs := make([]string, len(r.Programs)) + for i, p := range r.Programs { + programs[i] = p.String() + } + return map[string]any{ + "programs": programs, + "range": r.BlockRange.String(), + } +} + +// MergeTxRequirements cuts the requirements (from currentBlockNumber onward) into disjoint ranges, +// each carrying only the programs whose range covers it. This avoids fetching a program's +// transactions over sub-ranges it does not need (e.g. a program whose start is far ahead of the +// current progress is not queried over the gap before its start). +func MergeTxRequirements(current uint64, reqs []TransactionRequirement) (result []TransactionRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r TransactionRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + seen := set.New[solana.PublicKey]() + for _, req := range reqs { + if req.BlockRange.Include(r) { + seen.Add(req.Programs...) + } + } + programs := seen.DumpValues() + if len(programs) == 0 { + continue + } + result = append(result, TransactionRequirement{BlockRange: r, Programs: programs}) + } + return result +} + +// BuildTxFetcher builds the fetcher for one disjoint range and its program set, fetching the full +// matching transactions per block via sol_findTransactions. +func BuildTxFetcher( + name string, + req TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + // minQuerySize 1: the super node errors when a multi-block range exceeds its transaction cap, + // so the fetcher must be able to shrink to a single block (where the cap no longer applies). + 1, + 10000, + targetKeepBytes, + targetQueryBytes, + time.Second*15, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + // The super node caps the result and errors when exceeded (except for a single block); + // that error propagates and the fetcher halves the range, so no client-side check here. + blocks, err := client.FindTransactions(ctx, start, end, req.Programs) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData, len(blocks)) + for _, b := range blocks { + result[b.Slot] = BlockMainData{ + Slot: b.Slot, + Blockhash: b.Blockhash.String(), + PreviousBlockhash: b.PreviousBlockhash.String(), + BlockTime: b.BlockTime, + Transactions: b.Transactions, + } + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/statistics.go b/driver/controller/data/statistics.go new file mode 100644 index 0000000..d6bf6be --- /dev/null +++ b/driver/controller/data/statistics.go @@ -0,0 +1,164 @@ +package data + +import ( + "fmt" + "time" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/queue" + "sentioxyz/sentio-core/common/timehist" + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" +) + +func getCaller() string { + return fmt.Sprintf("%+v", errors.Errorf("")) +} + +type call struct { + Caller string + Method string + Params string + Err error + WaitUsed time.Duration + Used time.Duration + EndAt time.Time +} + +func (c call) Snapshot() any { + r := map[string]any{ + "endAt": c.EndAt.String(), + "waitUsed": c.WaitUsed.String(), + "used": c.Used.String(), + "method": c.Method, + "caller": c.Caller, + } + if len(c.Params) > 0 { + r["params"] = c.Params + } + if c.Err != nil { + r["err"] = c.Err.Error() + } + return r +} + +type statWindow struct { + StartAt time.Time + Count map[string]int + FailedCount map[string]int + TotalUsed map[string]time.Duration + TotalWaitUsed map[string]time.Duration + Used map[string]timehist.Histogram + WaitUsed map[string]timehist.Histogram +} + +func newStatWindow(c call) *statWindow { + w := &statWindow{ + StartAt: c.EndAt, + Count: make(map[string]int), + FailedCount: make(map[string]int), + TotalUsed: make(map[string]time.Duration), + TotalWaitUsed: make(map[string]time.Duration), + Used: make(map[string]timehist.Histogram), + WaitUsed: make(map[string]timehist.Histogram), + } + w.Count[c.Method] += 1 + if c.Err != nil { + w.FailedCount[c.Method] += 1 + } + w.TotalUsed[c.Method] += c.Used + w.Used[c.Method] = w.Used[c.Method].Incr(c.Used) + w.TotalWaitUsed[c.Method] += c.WaitUsed + w.WaitUsed[c.Method] = w.WaitUsed[c.Method].Incr(c.WaitUsed) + return w +} + +func (w *statWindow) GetStartAt() time.Time { + return w.StartAt +} + +func (w *statWindow) Merge(a *statWindow) { + for method, v := range a.Count { + w.Count[method] += v + } + for method, v := range a.FailedCount { + w.FailedCount[method] += v + } + for method, v := range a.TotalUsed { + w.TotalUsed[method] += v + } + for method, v := range a.TotalWaitUsed { + w.TotalWaitUsed[method] += v + } + for method, th := range a.Used { + w.Used[method] = w.Used[method].Add(th) + } + for method, th := range a.WaitUsed { + w.WaitUsed[method] = w.WaitUsed[method].Add(th) + } +} + +func (w *statWindow) Snapshot(endAt time.Time) any { + return map[string]any{ + "startAt": w.StartAt.String(), + "endAt": endAt.String(), + "duration": endAt.Sub(w.StartAt).String(), + "count": w.Count, + "failed": w.FailedCount, + "totalUsed": utils.MapMapNoError(w.TotalUsed, time.Duration.String), + "used": utils.MapMapNoError(w.Used, timehist.Histogram.String), + // waitUsed is the time spent waiting for a resource-manager concurrency token before + // the actual RPC; (totalUsed - totalWaitUsed) is therefore the real on-the-wire latency. + // Surfacing it lets us tell network-bound calls apart from token-contention-bound ones. + "totalWaitUsed": utils.MapMapNoError(w.TotalWaitUsed, time.Duration.String), + "waitUsed": utils.MapMapNoError(w.WaitUsed, timehist.Histogram.String), + } +} + +type CallStatistics struct { + latest queue.Circular[call] + stat *timewin.TimeWindowsManager[*statWindow] +} + +var ( + clientKeepRecentRequestCount = envconf.LoadUInt64("SENTIO_CLIENT_KEEP_RECENT_REQUEST_COUNT", + 1000, envconf.WithMin(10)) + clientStatTimeWindowWidth = envconf.LoadDuration("SENTIO_CLIENT_STAT_TIME_WINDOW_WIDTH", + time.Minute, envconf.WithMinDuration(time.Second*30)) +) + +func NewDefaultCallStatistics() *CallStatistics { + return NewCallStatistics(int(clientKeepRecentRequestCount), clientStatTimeWindowWidth) +} + +func NewCallStatistics(latestNum int, winWidth time.Duration) *CallStatistics { + return &CallStatistics{ + latest: queue.NewSafeCircular[call](latestNum), + stat: timewin.NewTimeWindowsManager[*statWindow](winWidth), + } +} + +func (s *CallStatistics) Called(method string, args []any, err error, startAt, waitEndAt time.Time) { + c := call{ + Caller: getCaller(), + Method: method, + Err: err, + EndAt: time.Now(), + } + c.Used = c.EndAt.Sub(startAt) + c.WaitUsed = waitEndAt.Sub(startAt) + if len(args) > 0 { + c.Params = utils.MustJSONMarshal(args) + } + s.latest.Push(c) + s.stat.Append(newStatWindow(c)) +} + +func (s *CallStatistics) Snapshot() any { + return map[string]any{ + "recent": utils.MapSliceNoError(s.latest.Dump(true), call.Snapshot), + "statistics": s.stat.Snapshot(), + } +} diff --git a/driver/controller/data/statistics_test.go b/driver/controller/data/statistics_test.go new file mode 100644 index 0000000..b6c8cf8 --- /dev/null +++ b/driver/controller/data/statistics_test.go @@ -0,0 +1,31 @@ +package data + +import ( + "encoding/json" + "math/rand" + "sync" + "testing" + "time" +) + +func Test_CallStatistics(t *testing.T) { + cs := NewCallStatistics(5, time.Millisecond*100) + + var w sync.WaitGroup + for i := 0; i < 1000; i++ { + time.Sleep(time.Millisecond) + w.Add(1) + go func(i int) { + defer w.Done() + startAt := time.Now() + time.Sleep(time.Millisecond * time.Duration(rand.Int63n(10))) + waitEndAt := time.Now() + time.Sleep(time.Millisecond * time.Duration(rand.Int63n(10))) + cs.Called("foo", []any{i}, nil, startAt, waitEndAt) + }(i) + } + w.Wait() + + b, _ := json.MarshalIndent(cs.Snapshot(), "", " ") + t.Logf(string(b)) +} diff --git a/driver/controller/data/subscribe.go b/driver/controller/data/subscribe.go new file mode 100644 index 0000000..983116a --- /dev/null +++ b/driver/controller/data/subscribe.go @@ -0,0 +1,114 @@ +package data + +import ( + "context" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + + "github.com/cenkalti/backoff/v4" + "github.com/pkg/errors" +) + +func SubscribeUsingPolling( + ctx context.Context, + minWatchInterval time.Duration, + timeout time.Duration, + from controller.BlockHeader, + latestGetter func(context.Context) (controller.BlockHeader, error), + callback func(controller.BlockHeader, error), +) { + const maxWaiting = time.Minute * 3 + _, logger := log.FromContext(ctx) + watchInterval := minWatchInterval // estimated block interval + waiting := watchInterval + for { + getCtx, cancel := context.WithTimeout(ctx, timeout) + latest, err := latestGetter(getCtx) + cancel() + if err != nil { + waiting = min(waiting*2, maxWaiting) + logger.Warnfe(err, "get latest for subscribe failed, will retry after %s", waiting.String()) + } else if latest.GetBlockNumber() < from.GetBlockNumber() { + waiting = min(waiting*2, maxWaiting) + logger.Warnf("latest from %s back to %s, will be ignored, and will retry after %s", + controller.GetBlockSummary(from), controller.GetBlockSummary(latest), waiting.String()) + } else if latest.GetBlockNumber() == from.GetBlockNumber() { + passed := time.Since(from.GetBlockTime()) + if passed < time.Minute { + waiting = watchInterval + logger.Debugf("latest stay at %s, and passed %s, will retry after %s", + controller.GetBlockSummary(from), passed, waiting.String()) + } else { + waiting = min(waiting*2, maxWaiting) + logger.Warnf("latest stay at %s, and passed %s, will retry after %s", + controller.GetBlockSummary(from), passed, waiting.String()) + } + } else { + if latest.GetBlockTime().After(from.GetBlockTime()) { + timeDelta := latest.GetBlockTime().Sub(from.GetBlockTime()) + blockDelta := latest.GetBlockNumber() - from.GetBlockNumber() + watchInterval = max(timeDelta/time.Duration(blockDelta), minWatchInterval) + } else { + watchInterval = minWatchInterval + } + waiting = watchInterval + logger.Debugf("latest growth from %s to %s, will get latest again after %s", + controller.GetBlockSummary(from), controller.GetBlockSummary(latest), waiting.String()) + callback(latest, nil) + from = latest + } + select { + case <-time.After(waiting): + case <-ctx.Done(): + return + } + } +} + +func SubscribeUsingWaiting( + ctx context.Context, + queryInterval time.Duration, + from controller.BlockHeader, + waitLatest func(ctx context.Context, blockNumberGt uint64) (latest controller.BlockHeader, broken, err error), + callback func(controller.BlockHeader, error), +) { + _, logger := log.FromContext(ctx) + var broken error + for broken == nil { + if queryInterval > 0 { + select { + case <-time.After(queryInterval): + case <-ctx.Done(): + return + } + } + fromText := controller.GetBlockSummary(from) + broken = backoff.RetryNotify( + func() error { + callCtx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + latest, brokenErr, err := waitLatest(callCtx, from.GetBlockNumber()) + if err != nil { + return err + } else if brokenErr != nil { + logger.Errorfe(broken, "wait latest from %s for subscribe failed", fromText) + callback(latest, brokenErr) + return backoff.Permanent(brokenErr) + } else if latest.GetBlockNumber() < from.GetBlockNumber() { + return errors.Errorf("latest from %s back to %s", fromText, controller.GetBlockSummary(latest)) + } else if latest.GetBlockNumber() == from.GetBlockNumber() { + return errors.Errorf("latest stay at %s", fromText) + } + logger.Debugf("latest growth from %s to %s", fromText, controller.GetBlockSummary(latest)) + callback(latest, nil) + from = latest + return nil + }, + backoff.WithContext(backoff.NewExponentialBackOff(backoff.WithMaxElapsedTime(0)), ctx), + func(err error, duration time.Duration) { + logger.Warnfe(err, "wait latest from %s for subscribe failed, will retry after %s", fromText, duration.String()) + }) + } +} diff --git a/driver/controller/data/sui/BUILD.bazel b/driver/controller/data/sui/BUILD.bazel new file mode 100644 index 0000000..178026e --- /dev/null +++ b/driver/controller/data/sui/BUILD.bazel @@ -0,0 +1,37 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sui", + srcs = [ + "block.go", + "client.go", + "object_change.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/sui", + visibility = ["//visibility:public"], + deps = [ + "//chain/sui", + "//chain/sui/types", + "//common/concurrency", + "//common/errgroup", + "//common/https", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/fetcher", + "@com_github_ethereum_go_ethereum//rpc", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_golang_lru//:golang-lru", + "@com_github_sentioxyz_sui_apis//sui/rpc/v2:rpc", + ], +) + +go_test( + name = "sui_test", + srcs = ["block_test.go"], + embed = [":sui"], + deps = ["@com_github_stretchr_testify//require"], +) diff --git a/driver/controller/data/sui/block.go b/driver/controller/data/sui/block.go new file mode 100644 index 0000000..f3fc76a --- /dev/null +++ b/driver/controller/data/sui/block.go @@ -0,0 +1,198 @@ +package sui + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type SimpleBlock sui.SimpleCheckpoint + +func (b SimpleBlock) GetBlockNumber() uint64 { + return b.Checkpoint +} + +func (b SimpleBlock) GetBlockParentHash() string { + return "" +} + +func (b SimpleBlock) GetBlockHash() string { + return b.Digest +} + +func (b SimpleBlock) GetBlockTime() time.Time { + return time.UnixMilli(int64(b.TimestampMS)) +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetSimpleBlock(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 10000, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{ + Intervals: []data.IntervalConfig{req.IntervalConfig}, + } + } + // timeGetter above already warmed GetSimpleBlock for these blocks, so this mostly hits cache. + if err := attachSimpleBlocks(ctx, client, result); err != nil { + return nil, err + } + return result, nil + }, + ) +} + +type BlockMainData struct { + Txs []types.TransactionResponseV1 + ObjectChanges []types.ObjectChangeExtend + Intervals []data.IntervalConfig + // SimpleBlock is the checkpoint header, prefetched concurrently by the data fetchers alongside the + // block's data. The strictly block-ordered transfer step (handler.go) needs the header for every + // non-empty block; fetching it here (off the serial path) keeps sui_getSimpleCheckpoint — which is + // order-independent — from becoming the throughput bottleneck. nil means it wasn't prefetched, in + // which case the transfer step falls back to a (serial) Client.GetSimpleBlock. + SimpleBlock *SimpleBlock +} + +func (b BlockMainData) IsEmpty() bool { + return b.Size() == 0 +} + +// Size intentionally ignores SimpleBlock: it is header metadata that only ever rides along with real +// data, and must not on its own make an otherwise-empty block look non-empty. +func (b BlockMainData) Size() int { + return len(b.Intervals) + len(b.ObjectChanges) + len(b.Txs)*10 +} + +// attachSimpleBlocks fetches the checkpoint header for every block that already carries data and stores +// it on the BlockMainData, so the downstream block-ordered transfer can avoid a serial RPC. It runs in +// the (concurrent, read-ahead) fetcher goroutines, and GetSimpleBlock is cached + singleflighted, so +// overlapping fetchers requesting the same block don't produce duplicate sui_getSimpleCheckpoint calls. +func attachSimpleBlocks(ctx context.Context, client Client, result map[uint64]BlockMainData) error { + if len(result) == 0 { + return nil + } + // Snapshot the keys first: the worker goroutines start immediately on Go(), so ranging the map here + // while they write back into it would be a concurrent map iteration + write (a runtime panic). Each + // worker writes its own headers[i] (distinct index, no lock needed); the map is mutated only after + // Wait(), back on this goroutine. + bns := make([]uint64, 0, len(result)) + for bn := range result { + bns = append(bns, bn) + } + headers := make([]SimpleBlock, len(bns)) + // No concurrency limit here: GetSimpleBlock goes through the client's resource manager, which + // already bounds in-flight RPCs. + g, gctx := errgroup.WithContext(ctx) + for i, bn := range bns { + i, bn := i, bn // nogo's loopclosure analyzer predates Go 1.22 per-iteration loop vars + g.Go(func() error { + sb, err := client.GetSimpleBlock(gctx, bn) + if err != nil { + return err + } + headers[i] = sb + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + for i, bn := range bns { + d := result[bn] + d.SimpleBlock = &headers[i] + result[bn] = d + } + return nil +} + +type DataRequirement struct { + Interval []data.IntervalRequirement + ObjectChanges []ObjectChangeRequirement + Txn []TransactionRequirement +} + +func BuildBlockMainDataFetcher( + namePrefix string, + req DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + req.ObjectChanges = MergeObjectChangeRequirements(currentBlockNumber, req.ObjectChanges) + req.Txn = MergeTxnRequirements(currentBlockNumber, req.Txn) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.ObjectChanges { + fetchers = append(fetchers, BuildObjectChangeFetcher( + namePrefix+fmt.Sprintf("ObjectChangeFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Txn { + fetchers = append(fetchers, BuildTxnFetcher( + namePrefix+fmt.Sprintf("TxnFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(_ uint64, from []BlockMainData) (data BlockMainData, has bool, _ error) { + has = len(from) > 0 + // ObjectChanges and Txn never be repeated, because a range will only have one fetcher with data. + for _, box := range from { + data.Txs = append(data.Txs, box.Txs...) + data.ObjectChanges = append(data.ObjectChanges, box.ObjectChanges...) + data.Intervals = append(data.Intervals, box.Intervals...) + if data.SimpleBlock == nil && box.SimpleBlock != nil { + data.SimpleBlock = box.SimpleBlock + } + } + return + }) +} diff --git a/driver/controller/data/sui/block_test.go b/driver/controller/data/sui/block_test.go new file mode 100644 index 0000000..3fd0a26 --- /dev/null +++ b/driver/controller/data/sui/block_test.go @@ -0,0 +1,48 @@ +package sui + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +// fakeSimpleBlockClient implements just the one method attachSimpleBlocks needs. The embedded nil +// Client interface makes any other call panic, keeping the fake honest about what it exercises. +type fakeSimpleBlockClient struct { + Client + calls atomic.Int64 +} + +func (f *fakeSimpleBlockClient) GetSimpleBlock(_ context.Context, bn uint64) (SimpleBlock, error) { + f.calls.Add(1) + return SimpleBlock{Checkpoint: bn}, nil +} + +// TestAttachSimpleBlocks guards the concurrency contract: it must fetch every block's header exactly +// once and pair each header with the right block, with no concurrent access to the result map. Run +// under -race, the many worker goroutines would trip the detector if the map were touched off the +// main goroutine, or the per-block pairing would break if the index bookkeeping were wrong. +func TestAttachSimpleBlocks(t *testing.T) { + const n = 1000 + result := make(map[uint64]BlockMainData, n) + for i := uint64(1); i <= n; i++ { + result[i] = BlockMainData{} + } + client := &fakeSimpleBlockClient{} + + require.NoError(t, attachSimpleBlocks(context.Background(), client, result)) + + require.EqualValues(t, n, client.calls.Load(), "each block fetched exactly once") + for bn, d := range result { + require.NotNilf(t, d.SimpleBlock, "block %d missing prefetched header", bn) + require.Equalf(t, bn, d.SimpleBlock.Checkpoint, "block %d paired with wrong header", bn) + } +} + +func TestAttachSimpleBlocksEmpty(t *testing.T) { + client := &fakeSimpleBlockClient{} + require.NoError(t, attachSimpleBlocks(context.Background(), client, map[uint64]BlockMainData{})) + require.Zero(t, client.calls.Load(), "no fetch for an empty result set") +} diff --git a/driver/controller/data/sui/client.go b/driver/controller/data/sui/client.go new file mode 100644 index 0000000..1979360 --- /dev/null +++ b/driver/controller/data/sui/client.go @@ -0,0 +1,496 @@ +package sui + +import ( + "context" + "math" + "strings" + "time" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/https" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + + "github.com/ethereum/go-ethereum/rpc" + "github.com/pkg/errors" + lru "github.com/sentioxyz/golang-lru" + rpcv2 "github.com/sentioxyz/sui-apis/sui/rpc/v2" +) + +type Client interface { + GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) + Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), + ) + GetSimpleBlock(ctx context.Context, blockNumber uint64) (SimpleBlock, error) + GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) + + GetObjectChanges( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.ObjectChangeFilter, + ) (map[uint64][]types.ObjectChangeExtend, error) + GetTransactions( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.TransactionFilter, + fetchConfig sui.TransactionFetchConfig, + ) (map[uint64][]types.TransactionResponseV1, error) + + // grpc-format counterparts (super node DriverVersion[2]); used by the suigrpc handler path. + GetGrpcTransactions( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.TransactionFilter, + fetchConfig sui.TransactionFetchConfig, + ) (map[uint64][]*sui.ExtendedGrpcTransaction, error) + GetGrpcObjectChanges( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.ObjectChangeFilter, + ) (map[uint64][]*sui.ExtendedGrpcChangedObject, error) + GetGrpcObjects( + ctx context.Context, + reqs []*rpcv2.GetObjectRequest, + concurrency, batchSize int, + ) ([]*rpcv2.GetObjectResult, error) + + TryMultiGetPastObjects( + ctx context.Context, + requests []types.SuiGetPastObjectRequest, + options types.SuiObjectDataOptions, + ) ([]types.SuiPastObjectResponse, error) + MultiGetTransactionBlocks( + ctx context.Context, + txDigests []string, + options map[string]any, + ) ([]types.TransactionResponseV1, error) + GetObjectStat(ctx context.Context, fromBlock, toBlock uint64, objectID string) (sui.ObjectStat, error) + GetObjectsStat(ctx context.Context, fromBlock, toBlock uint64, objectIDList []string) ([]sui.ObjectStat, error) + GetObjectVersionHistory(ctx context.Context, objectID string) ([]types.ObjectChangeExtend, error) + + GetPackageHistory(ctx context.Context, pkgID string) ([]string, error) + GetObjectCreation(ctx context.Context, objectID string, start uint64) (uint64, bool, error) + + ResetCache(r controller.BlockRange) + Snapshot() any +} + +type client struct { + endpoint string + firstBlockNumber int64 + watchLatestInterval time.Duration + + resMgr *concurrency.ResourceManager + stat *data.CallStatistics + + cli *rpc.Client + + cachedSimpleBlock *data.BlockCache[SimpleBlock] + cachedPackageHistory *lru.Cache[string, []string] +} + +func NewClient( + ctx context.Context, + endpoint string, + maxConcurrency int, + firstBlockNumber int64, + watchLatestInterval time.Duration, +) (c Client, err error) { + cli := &client{ + endpoint: endpoint, + firstBlockNumber: firstBlockNumber, + watchLatestInterval: watchLatestInterval, + resMgr: concurrency.NewResourceManager(maxConcurrency), + stat: data.NewDefaultCallStatistics(), + } + if cli.cli, err = rpc.DialOptions(ctx, endpoint, rpc.WithHTTPClient(https.DefaultClient)); err != nil { + return nil, errors.Wrapf(err, "dial to %s failed", endpoint) + } + cli.cachedSimpleBlock, _ = data.NewBlockCache[SimpleBlock](100000) + cli.cachedPackageHistory, _ = lru.New[string, []string](10000) + return cli, nil +} + +func (c *client) callContext(ctx context.Context, result any, priority uint64, method string, args ...any) error { + startAt := time.Now() + // waiting concurrency control token + release, err := c.resMgr.Apply(ctx, int64(priority), 1, time.Minute, func(waited time.Duration) { + _, logger := log.FromContext(ctx, "priority", priority, "args", utils.MustJSONMarshal(args)) + logger.Warnf("call method %s waited %s", method, waited.String()) + }) + if err != nil { + return err // always be context.Canceled + } + defer release() + // actually call + callStartAt := time.Now() + if err = c.cli.CallContext(ctx, &result, method, args...); err != nil { + return errors.Wrapf(err, "call method %s with args %s failed", method, utils.MustJSONMarshal(args)) + } + c.stat.Called(method, args, err, startAt, callStartAt) + return nil +} + +func (c *client) Subscribe( + ctx context.Context, + from controller.BlockHeader, + callback func(latest controller.BlockHeader, broken error), +) { + data.SubscribeUsingWaiting( + ctx, + c.watchLatestInterval, + from, + func(ctx context.Context, blockNumberGt uint64) (latest controller.BlockHeader, broken, err error) { + var resp sui.GetLatestSimpleCheckpointResponse + err = c.callContext(ctx, &resp, 0, "sui_getLatestSimpleCheckpoint", blockNumberGt) + if err == nil { + latest, broken = SimpleBlock(resp.Checkpoint), resp.CheckAPIVersion() + } + if broken != nil { + broken = errors.Wrapf(controller.ErrInternalNeedUpgrade, broken.Error()) + } + return + }, + callback) +} + +func (c *client) GetLatest(ctx context.Context) (latest controller.BlockHeader, first uint64, err error) { + var resp sui.GetLatestSimpleCheckpointResponse + if err = c.callContext(ctx, &resp, 0, "sui_getLatestSimpleCheckpoint", 0); err != nil { + return nil, 0, err + } + if err = resp.CheckAPIVersion(); err != nil { + return nil, 0, errors.Wrapf(controller.ErrInternalNeedUpgrade, err.Error()) + } + latest = SimpleBlock(resp.Checkpoint) + return latest, data.GetFirst(c.firstBlockNumber, latest.GetBlockNumber()), err +} + +func (c *client) fetchSimpleBlock(ctx context.Context, blockNumber uint64) (sc SimpleBlock, err error) { + err = c.callContext(ctx, &sc, blockNumber, "sui_getSimpleCheckpoint", blockNumber) + return +} + +func (c *client) getSimpleBlockIgnoreCache(ctx context.Context, blockNumber uint64) (SimpleBlock, error) { + sc, err := c.fetchSimpleBlock(ctx, blockNumber) + if err == nil { + c.cachedSimpleBlock.Add(blockNumber, sc) + } + return sc, err +} + +func (c *client) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (controller.BlockHeader, error) { + return c.getSimpleBlockIgnoreCache(ctx, blockNumber) +} + +func (c *client) GetSimpleBlock(ctx context.Context, blockNumber uint64) (SimpleBlock, error) { + // The cache + singleflight (in BlockCache) collapses the concurrent prefetch requests for the same + // checkpoint — made by the object-change / txn / interval fetchers — into a single RPC. + return c.cachedSimpleBlock.GetOrFetch(blockNumber, func() (SimpleBlock, error) { + return c.fetchSimpleBlock(ctx, blockNumber) + }) +} + +func (c *client) GetObjectChanges( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.ObjectChangeFilter, +) (map[uint64][]types.ObjectChangeExtend, error) { + var result []types.ObjectChangeExtend + err := c.callContext(ctx, &result, 0, "sui_filterObjectChangesV2", fromBlock, toBlock, filter) + return utils.Group(result, func(oc types.ObjectChangeExtend) uint64 { + return oc.Checkpoint.Uint64() + }), err +} + +func (c *client) GetTransactions( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.TransactionFilter, + fetchConfig sui.TransactionFetchConfig, +) (map[uint64][]types.TransactionResponseV1, error) { + var result []types.TransactionResponseV1 + err := c.callContext(ctx, &result, 0, "sui_getTransactionsV2", fromBlock, toBlock, filter, fetchConfig) + return utils.Group(result, func(oc types.TransactionResponseV1) uint64 { + return oc.Checkpoint.Uint64() + }), err +} + +func (c *client) GetGrpcTransactions( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.TransactionFilter, + fetchConfig sui.TransactionFetchConfig, +) (map[uint64][]*sui.ExtendedGrpcTransaction, error) { + var result []*sui.ExtendedGrpcTransaction + err := c.callContext(ctx, &result, 0, "sui_getGrpcTransactions", fromBlock, toBlock, filter, fetchConfig) + return utils.Group(result, func(tx *sui.ExtendedGrpcTransaction) uint64 { + return tx.Checkpoint + }), err +} + +func (c *client) GetGrpcObjectChanges( + ctx context.Context, + fromBlock, toBlock uint64, + filter sui.ObjectChangeFilter, +) (map[uint64][]*sui.ExtendedGrpcChangedObject, error) { + var result []*sui.ExtendedGrpcChangedObject + err := c.callContext(ctx, &result, 0, "sui_filterGrpcChangedObjects", fromBlock, toBlock, filter) + return utils.Group(result, func(oc *sui.ExtendedGrpcChangedObject) uint64 { + return oc.Checkpoint + }), err +} + +// GetGrpcObjects fetches objects by id+version in grpc format. The super node batches/parallelizes +// server-side per the concurrency/batchSize args, so no client-side paging is needed. +func (c *client) GetGrpcObjects( + ctx context.Context, + reqs []*rpcv2.GetObjectRequest, + concurrency, batchSize int, +) ([]*rpcv2.GetObjectResult, error) { + // GetObjectResult carries a protobuf oneof, which the JSON-RPC transport's + // encoding/json can't round-trip; decode into the protojson-backed wrapper + // (sui.GrpcObjectResult) and unwrap back to the raw proto for callers. + var wrapped []*sui.GrpcObjectResult + if err := c.callContext(ctx, &wrapped, 0, "sui_getGrpcObjects", reqs, concurrency, batchSize); err != nil { + return nil, err + } + return sui.UnwrapGrpcObjectResults(wrapped), nil +} + +const QueryObjectsPageSize = 50 // Max number of objects to request in a single call + +func (c *client) TryMultiGetPastObjects( + ctx context.Context, + requests []types.SuiGetPastObjectRequest, + options types.SuiObjectDataOptions, +) ([]types.SuiPastObjectResponse, error) { + var result []types.SuiPastObjectResponse + for len(requests) > 0 { + query := requests + if len(requests) > QueryObjectsPageSize { + query = requests[:QueryObjectsPageSize] + requests = requests[QueryObjectsPageSize:] + } else { + requests = requests[:0] + } + var pageResult []types.SuiPastObjectResponse + err := c.callContext(ctx, &pageResult, 0, "sui_tryMultiGetPastObjects", query, options) + if err != nil { + return nil, err + } + if len(pageResult) != len(query) { + return nil, errors.Errorf("call sui_tryMultiGetPastObjects failed: unexpected number of results: %d", len(pageResult)) + } + result = append(result, pageResult...) + } + return result, nil +} + +const QueryTxPageSize = 50 // Max number of txs to request in a single call + +func (c *client) MultiGetTransactionBlocks( + ctx context.Context, + txDigests []string, + options map[string]any, +) ([]types.TransactionResponseV1, error) { + txList := make([]types.TransactionResponseV1, 0, len(txDigests)) + for len(txDigests) > 0 { + query := txDigests + if len(txDigests) > QueryTxPageSize { + query = txDigests[:QueryTxPageSize] + txDigests = txDigests[QueryTxPageSize:] + } else { + txDigests = txDigests[:0] + } + var result []types.TransactionResponseV1 + err := c.callContext(ctx, &result, 0, "sui_multiGetTransactionBlocks", query, options) + if err != nil { + return nil, err + } + txList = append(txList, result...) + } + return txList, nil +} + +func (c *client) GetObjectStat(ctx context.Context, fromBlock, toBlock uint64, objectID string) (sui.ObjectStat, error) { + var result sui.ObjectStat + err := c.callContext(ctx, &result, fromBlock, "sui_getObjectStat", fromBlock, toBlock, objectID) + return result, err +} + +func (c *client) getObjectsStat( + ctx context.Context, + fromBlock, toBlock uint64, + objectIDList []string, +) ([]sui.ObjectStat, error) { + var dict map[string]sui.ObjectStat + err := c.callContext(ctx, &dict, fromBlock, "sui_getObjectsStat", fromBlock, toBlock, objectIDList) + result := make([]sui.ObjectStat, len(objectIDList)) + for i, id := range objectIDList { + result[i] = dict[id] + } + return result, err +} + +func (c *client) GetObjectsStat( + ctx context.Context, + fromBlock, toBlock uint64, + objectIDList []string, +) ([]sui.ObjectStat, error) { + const maxPageSize = 200 + const maxConcurrency = 20 + return concurrency.TraverseByPage(ctx, maxConcurrency, maxPageSize, objectIDList, + func(ctx context.Context, page concurrency.Page, ids []string) ([]sui.ObjectStat, error) { + return c.getObjectsStat(ctx, fromBlock, toBlock, ids) + }, + ) +} + +func (c *client) GetObjectVersionHistory(ctx context.Context, objectID string) ([]types.ObjectChangeExtend, error) { + stat, err := c.GetObjectStat(ctx, 0, math.MaxUint64, objectID) + if err != nil { + return nil, err + } + if stat.Count == 0 { + return nil, nil + } + filter := sui.ObjectChangeFilter{ + ObjectIDIn: set.New(objectID), + } + var changes map[uint64][]types.ObjectChangeExtend + if changes, err = c.GetObjectChanges(ctx, stat.MinCheckpoint, stat.MaxCheckpoint, filter); err != nil { + return nil, err + } + return utils.MergeArr(utils.GetMapValuesOrderByKey(changes)...), nil +} + +func (c *client) GetPackageHistory(ctx context.Context, pkgID string) (history []string, err error) { + var has bool + if history, has = c.cachedPackageHistory.Get(pkgID); has { + return history, nil + } + defer func() { + if err == nil { + c.cachedPackageHistory.Add(pkgID, history) + } + }() + // step-1: package object is immutable, so only have one version, just use sui_getObject to fetch it + type getObjectResponse struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + Data struct { + PreviousTransaction string `json:"previousTransaction"` + } `json:"data"` + } + var getObjectResp getObjectResponse + getObjectOpt := types.SuiObjectDataOptions{ShowPreviousTransaction: true} + if err = c.callContext(ctx, &getObjectResp, 0, "sui_getObject", pkgID, getObjectOpt); err != nil { + return nil, errors.Wrapf(err, "get package object %s failed", pkgID) + } else if getObjectResp.Error.Code != "" { + return nil, errors.Errorf("get package object %s failed: %s", pkgID, getObjectResp.Error.Code) + } + // step-2: the tx getObjectResp.Data.PreviousTransaction is the creating tx of the package, + // which contains the creating or update record of the upgrade cap object, + // use sui_getTransactionBlock to fetch the object changes in this tx + var pkgCreatingTxDigest = types.StrToDigestMust(getObjectResp.Data.PreviousTransaction) + var pkgCreateTx *types.TransactionResponseV1 + var getTxOpt = map[string]any{"showObjectChanges": true} + if err = c.callContext(ctx, &pkgCreateTx, 0, "sui_getTransactionBlock", pkgCreatingTxDigest, getTxOpt); err != nil { + err = errors.Wrapf(err, "get creation trransaction %s for package %s failed", pkgCreatingTxDigest.String(), pkgID) + return nil, err + } + const upgradeCapObjectType = "0x2::package::UpgradeCap" + var upgradeCapID string + for _, objectChange := range pkgCreateTx.ObjectChanges { + if utils.EmptyStringIfNil(utils.NullOrToString(objectChange.ObjectType)) == upgradeCapObjectType { + upgradeCapID = objectChange.GetObjectID() + break + } + } + if upgradeCapID == "" { + return []string{pkgID}, nil + } + // step-3: now have the upgrade cap object id, we should find the change history of it, + // which contains all upgraded package creating record. + // use super node to get all change records of the upgrade cap object, + // which contains all package creating tx digest. + var upgradeCapHistory []types.ObjectChangeExtend + upgradeCapHistory, err = c.GetObjectVersionHistory(ctx, upgradeCapID) + if err != nil { + err = errors.Wrapf(err, "get the first tx of the upgrade cap object %s for package %s failed", upgradeCapID, pkgID) + return nil, err + } + if len(upgradeCapHistory) == 0 { + err = errors.Errorf("the history of the upgrade cap object %s for package %s is not found", upgradeCapID, pkgID) + return nil, err + } + historyTxDigest := utils.MapSliceNoError(upgradeCapHistory, func(oc types.ObjectChangeExtend) string { + return oc.TxDigest.String() + }) + // step-4: now get the object changes of all history tx, all the package id is in it + var historyTxList []types.TransactionResponseV1 + historyTxList, err = c.MultiGetTransactionBlocks(ctx, historyTxDigest, getTxOpt) + if err != nil { + err = errors.Wrapf(err, "get history tx of the upgrade cap object %s for package %s failed", upgradeCapID, pkgID) + return nil, err + } + for _, tx := range historyTxList { + for _, change := range tx.ObjectChanges { + if change.Type == types.ObjectChangeTypePublished { + history = append(history, change.GetObjectID()) + } + } + } + if utils.IndexOf(history, pkgID) < 0 { + history = append(history, pkgID) + } + return history, nil +} + +func (c *client) GetObjectCreation(ctx context.Context, objectID string, start uint64) (uint64, bool, error) { + var creation *sui.ObjectCreation + err := c.callContext(ctx, &creation, start, "sui_getObjectCreation", objectID) + if err != nil { + return 0, false, err + } + if creation == nil { + return 0, false, nil + } + return max(creation.Checkpoint, start), true, nil +} + +func (c *client) ResetCache(r controller.BlockRange) { + for _, bn := range c.cachedSimpleBlock.Keys() { + if r.Contains(bn) { + c.cachedSimpleBlock.Remove(bn) + } + } +} + +func (c *client) Snapshot() any { + return map[string]any{ + "config": map[string]any{ + "endpoint": c.endpoint, + "firstBlockNumber": c.firstBlockNumber, + "watchLatestInterval": c.watchLatestInterval.String(), + }, + "resourceManager": c.resMgr.Snapshot(), + "statistics": c.stat.Snapshot(), + "cache": map[string]any{ + "cachedSimpleBlock": c.cachedSimpleBlock.Snapshot(10, controller.GetBlockFullText[SimpleBlock]), + "cachedPackageHistory": utils.CacheSnapshot(c.cachedPackageHistory, 100, func(history []string) string { + return strings.Join(history, ",") + }), + }, + } +} diff --git a/driver/controller/data/sui/grpc/BUILD.bazel b/driver/controller/data/sui/grpc/BUILD.bazel new file mode 100644 index 0000000..368b46b --- /dev/null +++ b/driver/controller/data/sui/grpc/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "grpc", + srcs = [ + "block.go", + "object_change.go", + "transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/data/sui/grpc", + visibility = ["//visibility:public"], + deps = [ + "//chain/sui", + "//common/errgroup", + "//driver/controller", + "//driver/controller/data", + "//driver/controller/data/sui", + "//driver/controller/fetcher", + ], +) diff --git a/driver/controller/data/sui/grpc/block.go b/driver/controller/data/sui/grpc/block.go new file mode 100644 index 0000000..ab0187f --- /dev/null +++ b/driver/controller/data/sui/grpc/block.go @@ -0,0 +1,169 @@ +// Package suigrpc is the grpc-data twin of data/sui: same fetch/requirement +// machinery, but the block data carries grpc-format transactions / object +// changes (sentio-core ExtendedGrpc*), fetched from the super node's +// sui_getGrpcTransactions / sui_filterGrpcChangedObjects methods. The +// format-agnostic pieces (Client, SimpleBlock, the *Requirement types and their +// Merge helpers, DataRequirement) are reused from data/sui. +package grpc + +import ( + "context" + "fmt" + "time" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +// BlockMainData is the grpc-format counterpart of suidata.BlockMainData. +type BlockMainData struct { + Txs []*chainsui.ExtendedGrpcTransaction + ObjectChanges []*chainsui.ExtendedGrpcChangedObject + Intervals []data.IntervalConfig + // SimpleBlock is the checkpoint header, prefetched concurrently alongside the block's data so the + // strictly block-ordered transfer step doesn't need a serial RPC. nil means not prefetched. + SimpleBlock *suidata.SimpleBlock +} + +func (b BlockMainData) IsEmpty() bool { + return b.Size() == 0 +} + +// Size intentionally ignores SimpleBlock (header metadata only rides along with real data). +func (b BlockMainData) Size() int { + return len(b.Intervals) + len(b.ObjectChanges) + len(b.Txs)*10 +} + +// attachSimpleBlocks fetches the checkpoint header for every block and stores it on the BlockMainData. +// Only the interval fetcher needs it: those blocks carry no tx / object-change data to derive the header +// from (the txn and object-change fetchers read the header off the grpc data via GetSimpleCheckpoint). +// GetSimpleBlock is cached + singleflighted. +func attachSimpleBlocks(ctx context.Context, client suidata.Client, result map[uint64]BlockMainData) error { + if len(result) == 0 { + return nil + } + bns := make([]uint64, 0, len(result)) + for bn := range result { + bns = append(bns, bn) + } + headers := make([]suidata.SimpleBlock, len(bns)) + g, gctx := errgroup.WithContext(ctx) + for i, bn := range bns { + i, bn := i, bn + g.Go(func() error { + sb, err := client.GetSimpleBlock(gctx, bn) + if err != nil { + return err + } + headers[i] = sb + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + for i, bn := range bns { + d := result[bn] + d.SimpleBlock = &headers[i] + result[bn] = d + } + return nil +} + +func BuildIntervalFetcher( + name string, + req data.IntervalRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client suidata.Client, +) controller.Fetcher[BlockMainData] { + timeGetter := func(ctx context.Context, blockNumber uint64) (time.Time, error) { + getCtx, cancel := context.WithTimeout(ctx, time.Second*3) + defer cancel() + h, err := client.GetSimpleBlock(getCtx, blockNumber) + if err != nil { + return time.Time{}, err + } + return h.GetBlockTime(), nil + } + return fetcher.NewFetcher[BlockMainData]( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 10000, + 10000, + 10000, + 1000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + bns, err := data.QueryInterval(ctx, start, end, firstBlockNumber, latest, req, timeGetter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for _, bn := range bns { + result[bn] = BlockMainData{Intervals: []data.IntervalConfig{req.IntervalConfig}} + } + if err := attachSimpleBlocks(ctx, client, result); err != nil { + return nil, err + } + return result, nil + }, + ) +} + +// BuildBlockMainDataFetcher mirrors suidata.BuildBlockMainDataFetcher but builds grpc fetchers. It +// reuses suidata.DataRequirement and the Merge* helpers (all format-agnostic). +func BuildBlockMainDataFetcher( + namePrefix string, + req suidata.DataRequirement, + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, + client suidata.Client, +) controller.Fetcher[BlockMainData] { + req.ObjectChanges = suidata.MergeObjectChangeRequirements(currentBlockNumber, req.ObjectChanges) + req.Txn = suidata.MergeTxnRequirements(currentBlockNumber, req.Txn) + req.Interval = data.MergeIntervalRequirements(req.Interval) + var fetchers []controller.Fetcher[BlockMainData] + for i, r := range req.ObjectChanges { + fetchers = append(fetchers, BuildObjectChangeFetcher( + namePrefix+fmt.Sprintf("ObjectChangeFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Txn { + fetchers = append(fetchers, BuildTxnFetcher( + namePrefix+fmt.Sprintf("TxnFetcher#%d", i), r, currentBlockNumber, latest, client)) + } + for i, r := range req.Interval { + fetchers = append(fetchers, BuildIntervalFetcher( + namePrefix+fmt.Sprintf("IntervalFetcher#%d", i), r, firstBlockNumber, currentBlockNumber, latest, client)) + } + return fetcher.MergeIsomorphicFetchers( + namePrefix+"MainDataFetcher", + req, + fetchers, + func(_ uint64, from []BlockMainData) (out BlockMainData, has bool, _ error) { + has = len(from) > 0 + for _, box := range from { + out.Txs = append(out.Txs, box.Txs...) + out.ObjectChanges = append(out.ObjectChanges, box.ObjectChanges...) + out.Intervals = append(out.Intervals, box.Intervals...) + if out.SimpleBlock == nil && box.SimpleBlock != nil { + out.SimpleBlock = box.SimpleBlock + } + } + return + }) +} diff --git a/driver/controller/data/sui/grpc/object_change.go b/driver/controller/data/sui/grpc/object_change.go new file mode 100644 index 0000000..da95745 --- /dev/null +++ b/driver/controller/data/sui/grpc/object_change.go @@ -0,0 +1,53 @@ +package grpc + +import ( + "context" + "time" + + "sentioxyz/sentio-core/driver/controller" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +func BuildObjectChangeFetcher( + name string, + req suidata.ObjectChangeRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client suidata.Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1000, + 10000, + 100000, + 10000, + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + changes, err := client.GetGrpcObjectChanges(ctx, start, end, req.Filter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for bn, cs := range changes { + if len(cs) > 0 { + // each grpc object change already carries its checkpoint header — no extra RPC needed + result[bn] = BlockMainData{ + ObjectChanges: cs, + SimpleBlock: new(suidata.SimpleBlock(cs[0].GetSimpleCheckpoint())), + } + } + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/sui/grpc/transaction.go b/driver/controller/data/sui/grpc/transaction.go new file mode 100644 index 0000000..516a139 --- /dev/null +++ b/driver/controller/data/sui/grpc/transaction.go @@ -0,0 +1,54 @@ +package grpc + +import ( + "context" + "time" + + "sentioxyz/sentio-core/driver/controller" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +func BuildTxnFetcher( + name string, + req suidata.TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client suidata.Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 100, + 10000, + 10000, + 5000, + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txGroups, err := client.GetGrpcTransactions(ctx, start, end, req.Filter, req.FetchConfig) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for bn, txs := range txGroups { + + if len(txs) > 0 { + // each grpc tx already carries its checkpoint header — no extra GetSimpleBlock RPC needed + result[bn] = BlockMainData{ + Txs: txs, + SimpleBlock: new(suidata.SimpleBlock(txs[0].GetSimpleCheckpoint())), + } + } + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/sui/object_change.go b/driver/controller/data/sui/object_change.go new file mode 100644 index 0000000..da1fee0 --- /dev/null +++ b/driver/controller/data/sui/object_change.go @@ -0,0 +1,89 @@ +package sui + +import ( + "context" + "time" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type ObjectChangeRequirement struct { + controller.BlockRange + + Filter sui.ObjectChangeFilter +} + +func (r ObjectChangeRequirement) Snapshot() any { + return map[string]any{ + "filter": r.Filter, + "range": r.BlockRange.String(), + } +} + +func MergeObjectChangeRequirements(current uint64, reqs []ObjectChangeRequirement) (result []ObjectChangeRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r ObjectChangeRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + var filters []sui.ObjectChangeFilter + for _, req := range reqs { + if req.BlockRange.Include(r) { + filters = append(filters, req.Filter) + } + } + if len(filters) == 0 { + continue + } + result = append(result, ObjectChangeRequirement{ + Filter: utils.Reduce(filters, sui.ObjectChangeFilter.Merge), + BlockRange: r, + }) + } + return result +} + +func BuildObjectChangeFetcher( + name string, + req ObjectChangeRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 1000, + 10000, + 100000, + 10000, // the target is that each query got no more than 1000 object change records + time.Minute, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + changes, err := client.GetObjectChanges(ctx, start, end, req.Filter) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for bn, cs := range changes { + result[bn] = BlockMainData{ObjectChanges: cs} + } + if err := attachSimpleBlocks(ctx, client, result); err != nil { + return nil, err + } + return result, nil + }, + ) +} diff --git a/driver/controller/data/sui/transaction.go b/driver/controller/data/sui/transaction.go new file mode 100644 index 0000000..b8e7b48 --- /dev/null +++ b/driver/controller/data/sui/transaction.go @@ -0,0 +1,97 @@ +package sui + +import ( + "context" + "time" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/fetcher" +) + +type TransactionRequirement struct { + controller.BlockRange + + Filter sui.TransactionFilter + FetchConfig sui.TransactionFetchConfig +} + +func (r TransactionRequirement) Snapshot() any { + return map[string]any{ + "filter": r.Filter, + "fetchConfig": r.FetchConfig, + "range": r.BlockRange.String(), + } +} + +func MergeTxnRequirements(current uint64, reqs []TransactionRequirement) (result []TransactionRequirement) { + rs := controller.CutRangeSet( + current, + utils.MapSliceNoError(reqs, func(r TransactionRequirement) controller.BlockRange { + return r.BlockRange + }), + ) + for _, r := range rs { + rr := TransactionRequirement{BlockRange: r} + first := true + for _, req := range reqs { + if !req.BlockRange.Include(r) { + continue + } + if first { + rr.Filter = req.Filter + rr.FetchConfig = req.FetchConfig + first = false + } else { + rr.Filter = rr.Filter.Merge(req.Filter) + rr.FetchConfig = rr.FetchConfig.Merge(req.FetchConfig) + } + } + if first { + continue + } + result = append(result, rr) + } + return result +} + +func BuildTxnFetcher( + name string, + req TransactionRequirement, + currentBlockNumber uint64, + latest controller.BlockHeader, + client Client, +) controller.Fetcher[BlockMainData] { + return fetcher.NewFetcher( + name, + req, + controller.BlockRange{ + StartBlock: max(currentBlockNumber, req.StartBlock), + EndBlock: req.EndBlock, + }, + latest, + 100, + 10000, + 10000, // size of transaction is 10, so will cache 1000 transactions + 5000, // the target is that each query got no more than 500 transactions + time.Second*10, + 20, + time.Second, + 1.5, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]BlockMainData, error) { + txGroups, err := client.GetTransactions(ctx, start, end, req.Filter, req.FetchConfig) + if err != nil { + return nil, err + } + result := make(map[uint64]BlockMainData) + for bn, txs := range txGroups { + result[bn] = BlockMainData{Txs: txs} + } + if err := attachSimpleBlocks(ctx, client, result); err != nil { + return nil, err + } + return result, nil + }, + ) +} diff --git a/driver/controller/entity.go b/driver/controller/entity.go new file mode 100644 index 0000000..ea90449 --- /dev/null +++ b/driver/controller/entity.go @@ -0,0 +1,125 @@ +package controller + +import ( + "context" + "time" + + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" +) + +type EntityController interface { + Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError + CachedTooMuch(blockNumber uint64) bool + Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, + ) (created, updated map[string]int, err *ExternalError) + + GetEntityOrInterfaceType(entity string) schema.EntityOrInterface + GetEntityType(entity string) *schema.Entity + GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, + ) (box *persistent.EntityBox, err *ExternalError) + GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, + ) (box *persistent.EntityBox, err *ExternalError) + ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, + ) (boxes []*persistent.EntityBox, next *string, err *ExternalError) + ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, + ) ([]*persistent.EntityBox, schema.EntityOrInterface, *ExternalError) + SetEntity(ctx context.Context, entityType *schema.Entity, box persistent.UncommittedEntityBox) *ExternalError + + Snapshot() any +} + +type EmptyEntityController struct{} + +func (c EmptyEntityController) Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError { + return nil +} + +func (c EmptyEntityController) CachedTooMuch(blockNumber uint64) bool { + return false +} + +func (c EmptyEntityController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (map[string]int, map[string]int, *ExternalError) { + return nil, nil, nil +} + +func (c EmptyEntityController) GetEntityOrInterfaceType(entity string) schema.EntityOrInterface { + return nil +} + +func (c EmptyEntityController) GetEntityType(entity string) *schema.Entity { + return nil // all are unknown entity type +} + +func (c EmptyEntityController) GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (box *persistent.EntityBox, err *ExternalError) { + return nil, nil +} + +func (c EmptyEntityController) GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (box *persistent.EntityBox, err *ExternalError) { + return nil, nil +} + +func (c EmptyEntityController) ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, +) (boxes []*persistent.EntityBox, next *string, err *ExternalError) { + return nil, nil, nil +} + +func (c EmptyEntityController) ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, +) ([]*persistent.EntityBox, schema.EntityOrInterface, *ExternalError) { + return nil, nil, nil +} + +func (c EmptyEntityController) SetEntity(context.Context, *schema.Entity, persistent.UncommittedEntityBox) *ExternalError { + return nil +} + +func (c EmptyEntityController) Snapshot() any { + return nil +} diff --git a/driver/controller/errors.go b/driver/controller/errors.go new file mode 100644 index 0000000..9214600 --- /dev/null +++ b/driver/controller/errors.go @@ -0,0 +1,152 @@ +package controller + +import ( + "fmt" + "io" + + "github.com/pkg/errors" +) + +var ( + ErrInternalNeedUpgrade = errors.New("need upgrade") + ErrInternalHasNewTemplate = errors.New("has new template") + ErrInternalReorgDetected = errors.New("reorg detected") +) + +func NewExternalError(code int, err error) *ExternalError { + if err == nil { + panic(errors.Errorf("NewExternalError with code %d and nil err", code)) + } + return &ExternalError{code: code, error: err} +} + +type ExternalError struct { + code int + error +} + +func (e *ExternalError) Code() int { + return e.code +} + +func (e *ExternalError) Wrapped() error { + return e.error +} + +func (e *ExternalError) Wrapf(fmt string, args ...any) *ExternalError { + return NewExternalError(e.code, errors.Wrapf(e.error, fmt, args...)) +} + +func (e *ExternalError) Error() string { + return fmt.Sprintf("ERR%03d: %s", e.code, e.error.Error()) +} + +func (e *ExternalError) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + if s.Flag('+') { + _, _ = fmt.Fprintf(s, "ERR%03d: %+v\n", e.code, e.error) + return + } + fallthrough + case 's', 'q': + _, _ = io.WriteString(s, e.Error()) + } +} + +func (e *ExternalError) IsUserError() bool { + return e.code >= 300 +} + +func (e *ExternalError) IsUserRuntimeError() bool { + switch e.code { + case ErrCodeProcessFailed: + return true + default: + return false + } +} + +func (e *ExternalError) IsSystemError() bool { + return e.code < 200 +} + +func (e *ExternalError) IsDriverError() bool { + return e.code >= 200 && e.code < 300 +} + +func (e *ExternalError) IsBillingError() bool { + return e.code >= 400 +} + +// other error +const ( + ErrCodeSystem = iota + 100 + ErrCodeNeedUpgrade + ErrCodeOOM +) + +// driver error +const ( + ErrCodeCallProcessorFailed = iota + 200 + ErrCodeResetWasmInstanceFailed + ErrCodeWasmError + ErrCodeGetContractStartBlockFailed + ErrCodeFetchDataFailed + ErrCodeSubgraphEthCallFailed + ErrCodeSubgraphIpfsCatFailed + + ErrCodeInvalidCheckpointData + ErrCodeSaveCheckpointFailed + ErrCodeQuotaServiceError + + ErrCodeCleanTimeSeriesDataFailed + ErrCodeSaveTimeSeriesDataFailed + + ErrCodeSendWebhookDataFailed + + ErrCodeInitEntityFailed + ErrCodeCleanEntityDataFailed + ErrCodeSaveEntityDataFailed + ErrCodeGetEntityFromDBFailed + ErrCodeListEntityFromDBFailed + ErrCodeInvalidEntityData +) + +// processor error +const ( + ErrCodeUnexpectedProcessorConfig = iota + 300 + ErrCodeInvalidEntitySchema + ErrCodeProcessorConfigsHasDiff + ErrCodeProcessFailed + ErrCodeCallWasmExportFunctionFailed + ErrCodeWasmInitFailed + ErrCodeWasmStackOverFlow + ErrCodeCreateTemplateFailed + ErrCodeInvalidSubgraphManifest + + ErrCodeGetUnknownEntity + ErrCodeListUnknownEntity + ErrCodeListRelatedEntityWithInvalidField + ErrCodeInvalidListEntityFilter + ErrCodeInvalidUpsertEntityRequest + ErrCodeUpsertUnknownEntity + ErrCodeInvalidUpdateEntityRequest + ErrCodeUpdateUnknownEntity + ErrCodeInvalidDeleteEntityRequest + ErrCodeDeleteUnknownEntity + ErrCodeUpdateImmutableEntity + ErrCodeInvalidEntityFieldValue + + ErrCodeInvalidTimeSeriesData + ErrCodeTimeSeriesDataSchemaChanged + + ErrCodeTooManyWebhookMsgEntity + + ErrCodeSubgraphEthCallWithInvalidParam +) + +// billing error +const ( + ErrCodeOverQuota = iota + 400 +) diff --git a/driver/controller/errors_test.go b/driver/controller/errors_test.go new file mode 100644 index 0000000..a3bc948 --- /dev/null +++ b/driver/controller/errors_test.go @@ -0,0 +1,87 @@ +package controller + +import ( + "context" + "testing" + "time" + + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func Test_externalErrorWrap(t *testing.T) { + err := errors.Errorf("level1") + extErr1 := NewExternalError(ErrCodeSystem, err) + extErr2 := extErr1.Wrapf("level2") + assert.Equal(t, "level1", err.Error()) + assert.Equal(t, "ERR100: level1", extErr1.Error()) + assert.Equal(t, "ERR100: level2: level1", extErr2.Error()) + + var extErr *ExternalError + assert.False(t, errors.As(err, &extErr)) + assert.True(t, errors.As(extErr1, &extErr)) + assert.Equal(t, extErr1, extErr) + assert.True(t, errors.As(extErr2, &extErr)) + assert.Equal(t, extErr2, extErr) + + log.Errorfe(err, "err") + log.Errorfe(extErr1, "extErr1") + log.Errorfe(extErr2, "extErr2") + log.Errorf("err: %+v", err) + log.Errorf("extErr1: %+v", extErr1) + log.Errorf("extErr2: %+v", extErr2) +} + +func Test_errgroup(t *testing.T) { + g, gctx := errgroup.WithContext(context.Background()) + fn := func(ctx context.Context, wait time.Duration, code int) *ExternalError { + select { + case <-ctx.Done(): + return NewExternalError(0, ctx.Err()) + case <-time.After(wait): + if code == 0 { + return nil + } + return NewExternalError(code, errors.Errorf("err")) + } + } + g.Go(func() error { + if err := fn(gctx, time.Millisecond*100, 0); err != nil { // got nil + return err + } + return nil + }) + g.Go(func() error { + if err := fn(gctx, time.Millisecond*200, 2); err != nil { // got code:2 + return err + } + return nil + }) + g.Go(func() error { + if err := fn(gctx, time.Millisecond*300, 3); err != nil { // got code:0 and ignored + return err + } + return nil + }) + + err := g.Wait() + var extErr *ExternalError + assert.True(t, errors.As(err, &extErr)) + assert.Equal(t, 2, extErr.code) +} + +func Test_panic(t *testing.T) { + fn := func() (err error) { + defer func() { + if panicErr := recover(); panicErr != nil { + err = errors.Errorf("%v", panicErr) // row2 + } + }() + panic("panic here") // row1 + } + + log.Infof("err: %+v", fn()) // row3, log include all row1-3 +} diff --git a/driver/controller/fetcher/BUILD.bazel b/driver/controller/fetcher/BUILD.bazel new file mode 100644 index 0000000..93295e6 --- /dev/null +++ b/driver/controller/fetcher/BUILD.bazel @@ -0,0 +1,40 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "fetcher", + srcs = [ + "fetcher.go", + "merge.go", + "retry.go", + "stat.go", + "transfer.go", + "utils.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/fetcher", + visibility = ["//visibility:public"], + deps = [ + "//common/log", + "//common/timehist", + "//common/timewin", + "//common/utils", + "//driver/controller", + "@com_github_pkg_errors//:errors", + "@org_golang_x_exp//constraints", + ], +) + +go_test( + name = "fetcher_test", + srcs = [ + "fetcher_test.go", + "transfer_test.go", + ], + embed = [":fetcher"], + deps = [ + "//common/log", + "//driver/controller", + "@com_github_pkg_errors//:errors", + "@com_github_stretchr_testify//assert", + "@org_uber_go_zap//:zap", + ], +) diff --git a/driver/controller/fetcher/fetcher.go b/driver/controller/fetcher/fetcher.go new file mode 100644 index 0000000..0f4ba7b --- /dev/null +++ b/driver/controller/fetcher/fetcher.go @@ -0,0 +1,385 @@ +package fetcher + +import ( + "context" + "fmt" + "math" + "sync" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + + "github.com/pkg/errors" +) + +type Requirement interface { + Snapshot() any +} + +type fetcher[T controller.FetchTarget] struct { + name string + queryFunc func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]T, error) + + requirement Requirement + minQuerySize uint64 + maxQuerySize uint64 + targetKeepDataSize int + targetQueryDataSize int + maxQueryTime time.Duration + maxRetry int + queryRetryInterval time.Duration + querySizeMultiplier float64 + + // protect the data below + mu sync.Mutex + + full controller.BlockRange // full is the whole data range + fetchingStart uint64 // [fetchingStart,fetchingEnd] is the range where data is fetching + fetchingEnd uint64 // so [full.StartBlock, fetchingStart) is the range where data is ready + fetchingDone chan struct{} // if the fetching task has a result, this chan will be closed + fetchingFailed error // has error, the fetching process will be aborted + + brokenErr error + + latest controller.BlockHeader + + data map[uint64]T // the ready data in [full.StartBlock, fetchingStart) will be here，key is blockNumber + totalSize int // is the total size in data，every time data is changed, it will be changed automatically + + // when [full.StartBlock,latest] changed, this chan will be closed to trigger growth, + // and after growth is completed, this chan will be re-build + winChanged chan struct{} + + stat *timewin.TimeWindowsManager[*processStat] +} + +func NewFetcher[T controller.FetchTarget]( + name string, + requirement Requirement, + full controller.BlockRange, + latest controller.BlockHeader, + minQuerySize uint64, + maxQuerySize uint64, + targetKeepDataSize int, + targetQueryDataSize int, + maxQueryTime time.Duration, + maxRetry int, + queryRetryInterval time.Duration, + querySizeMultiplier float64, + queryFunc func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]T, error), +) controller.Fetcher[T] { + if querySizeMultiplier < 1 { + panic("querySizeMultiplier cannot less than 1") + } + return &fetcher[T]{ + name: name, + requirement: requirement, + queryFunc: queryFunc, + minQuerySize: minQuerySize, + maxQuerySize: maxQuerySize, + targetKeepDataSize: targetKeepDataSize, + targetQueryDataSize: targetQueryDataSize, + maxQueryTime: maxQueryTime, + maxRetry: maxRetry, + queryRetryInterval: queryRetryInterval, + querySizeMultiplier: querySizeMultiplier, + full: full, + fetchingStart: full.StartBlock, + fetchingEnd: _min(min(full.StartBlock+minQuerySize-1, latest.GetBlockNumber()), full.EndBlock), + fetchingDone: make(chan struct{}), + latest: latest, + data: make(map[uint64]T), + winChanged: make(chan struct{}), + stat: timewin.NewTimeWindowsManager[*processStat](time.Minute), + } +} + +func (f *fetcher[T]) GetName() string { + return f.name +} + +func (f *fetcher[T]) Snapshot() any { + f.mu.Lock() + defer f.mu.Unlock() + return map[string]any{ + "name": f.name, + "type": fmt.Sprintf("%T", f), + "config": map[string]any{ + "requirement": f.requirement.Snapshot(), + "minQuerySize": f.minQuerySize, + "maxQuerySize": f.maxQuerySize, + "targetKeepDataSize": f.targetKeepDataSize, + "targetQueryDataSize": f.targetQueryDataSize, + "maxQueryTime": f.maxQueryTime.String(), + "maxRetry": f.maxRetry, + "queryRetryInterval": f.queryRetryInterval.String(), + "querySizeMultiplier": f.querySizeMultiplier, + }, + "fullRange": f.full.String(), + "readyRange": controller.BlockRange{ + StartBlock: f.full.StartBlock, + EndBlock: utils.WrapPointer(f.fetchingStart - 1), + }.String(), + "fetchingRange": controller.BlockRange{ + StartBlock: f.fetchingStart, + EndBlock: utils.WrapPointer(f.fetchingEnd), + }.String(), + "fetchingFailed": f.fetchingFailed, + "brokenErr": fmt.Sprintf("%+v", f.brokenErr), + "latest": controller.GetBlockFullText(f.latest), + "dataSize": f.totalSize, + "dataBlockCount": len(f.data), + "statistics": f.stat.Snapshot(), + } +} + +func (f *fetcher[T]) GetFullRange() controller.BlockRange { + f.mu.Lock() + defer f.mu.Unlock() + return f.full +} + +func (f *fetcher[T]) nextFetchSize(current uint64, got int, used time.Duration) uint64 { + if f.targetQueryDataSize > 0 && int(float64(got)*f.querySizeMultiplier) >= f.targetQueryDataSize { + return current // data got is big enough + } + if f.maxQueryTime > 0 && time.Duration(float64(used)*f.querySizeMultiplier) > f.maxQueryTime { + return current // time used is big enough + } + // increase the fetch size, use ceil to make sure newFetchSize not always equal to oldFetchSize + next := uint64(math.Ceil(float64(current) * f.querySizeMultiplier)) + return min(max(next, f.minQuerySize), f.maxQuerySize) +} + +func (f *fetcher[T]) growth(ctx context.Context) (pause bool, reject bool, changed chan struct{}) { + growthStartAt := time.Now() + _, logger := log.FromContext(ctx) + f.mu.Lock() + defer f.mu.Unlock() + if f.fetchingFailed != nil { + // already failed + return true, true, nil + } + if f.full.EndBlock != nil && *f.full.EndBlock < f.fetchingStart { + // no more data need to fetch + return true, true, nil + } + if f.totalSize >= f.targetKeepDataSize { + // total size of f.data is more than the limit, continue pause to wait consume + f.winChanged = make(chan struct{}) + return true, false, f.winChanged + } + start, end, latest := f.fetchingStart, f.fetchingEnd, f.latest + if end < start { + // last growth reached the right side + if start <= latest.GetBlockNumber() { + // latest may be increased, so now we can extend fetching range + end = _min(min(start+f.minQuerySize-1, latest.GetBlockNumber()), f.full.EndBlock) + } else { + // latest not increased, just rebuild f.winChanged + f.winChanged = make(chan struct{}) + return true, false, f.winChanged + } + } + done := f.fetchingDone + f.mu.Unlock() + + var result map[uint64]T + var err error + for retry := f.maxRetry; retry >= 0; { + if f.queryRetryInterval > 0 && err != nil { + select { + case <-ctx.Done(): + f.mu.Lock() + return true, true, nil + case <-time.After(f.queryRetryInterval): + } + } + startAt := time.Now() + if f.maxQueryTime > 0 { + queryCtx, cancel := context.WithTimeout(ctx, f.maxQueryTime) + result, err = f.queryFunc(queryCtx, start, end, latest) + cancel() + } else { + result, err = f.queryFunc(ctx, start, end, latest) + } + used := time.Since(startAt) + var size int + if err == nil { + size = sumSize(result) + } + st := &processStat{startAt: startAt} + st.fetchComplete(used, err == nil, end-start+1, size) + f.stat.Append(st) + tryLogger := logger.With( + "start", start, + "end", end, + "latest", controller.GetBlockSummary(latest), + "used", used.String(), + "retry", retry) + if err != nil { + var pe *PermanentError + if errors.As(err, &pe) { + err = pe.Err + retry = -1 + } + if errors.Is(err, context.Canceled) { + tryLogger.Debug("fetch canceled") + f.mu.Lock() + return true, true, nil + } + tryLogger.Warnfe(err, "fetch failed") + if fetchSize := end - start + 1; fetchSize > f.minQuerySize { + fetchSize = max(f.minQuerySize, fetchSize/2) + end = start + fetchSize - 1 + } else { + retry-- + } + } else { + // got the data in [start,end] + tryLogger.Debugw("fetch succeed", "size", size) + f.mu.Lock() + f.totalSize += size + f.data = utils.MergeMap(f.data, result) + f.fetchingStart = end + 1 + newFetchSize := f.nextFetchSize(end-start+1, size, used) + f.fetchingEnd = _min(min(f.fetchingStart+newFetchSize-1, f.latest.GetBlockNumber()), f.full.EndBlock) + f.fetchingDone = make(chan struct{}) + close(done) + f.winChanged = make(chan struct{}) + logger.Debugw("growth succeed", "used", time.Since(growthStartAt).String(), "current", f.current()) + return f.totalSize >= f.targetKeepDataSize, false, f.winChanged + } + } + // fetch the data in [start,end], keep reducing the range size, and keep retrying, + // but it still fails and can no longer continue + f.mu.Lock() + f.fetchingEnd = end + f.fetchingFailed = err + close(done) + logger.With("current", f.current()).Errore(err, "growth failed") + return true, true, nil +} + +func (f *fetcher[T]) current() string { + var ready string + if f.fetchingStart <= f.full.StartBlock { + ready = "[empty]" + } else { + ready = fmt.Sprintf("[%d,%d]", f.full.StartBlock, f.fetchingStart-1) + } + return fmt.Sprintf("FULL%s,READY%s,LATEST:%d,DATA:%d/%d", + f.full.String(), ready, f.latest.GetBlockNumber(), len(f.data), f.totalSize) +} + +func (f *fetcher[T]) KeepFetch(ctx context.Context) { + _, logger := log.FromContext(ctx, "fetcher", f.name, "requirement", f.requirement) + logger.Info("keep fetch start") + defer logger.Info("keep fetch end") + for round := 0; ; round++ { + roundCtx, _ := log.FromContext(ctx, "fetcher", f.name, "requirement", f.requirement, "fetchRound", round) + if pause, reject, changed := f.growth(roundCtx); !pause && !reject { + continue + } else if reject { + return + } else { + select { + case <-changed: + case <-ctx.Done(): + return + } + } + } +} + +func (f *fetcher[T]) Get(ctx context.Context, blockNumber uint64) ( + data T, + has bool, + latest controller.BlockHeader, + err error, +) { + startAt := time.Now() + defer func() { + st := &processStat{startAt: time.Now()} + st.getComplete(time.Since(startAt), has) + f.stat.Append(st) + }() + f.mu.Lock() + latest = f.latest + if !f.full.Contains(blockNumber) { + f.mu.Unlock() + return + } + for { + latest = f.latest + if f.brokenErr != nil { + // broken, just return the broken error + err = f.brokenErr + f.mu.Unlock() + return + } else if f.fetchingStart <= blockNumber { + // the required data is not yet ready + if f.fetchingFailed != nil { + // the fetch has failed and cannot continue, an error is returned directly + err = f.fetchingFailed + f.mu.Unlock() + return + } + done := f.fetchingDone + f.mu.Unlock() + + // waiting fetch completed + select { + case <-done: + case <-ctx.Done(): + err = ctx.Err() + return + } + f.mu.Lock() + } else { + // the needed data is ready，we can return data now + data, has = f.data[blockNumber] + f.mu.Unlock() + return + } + } +} + +func (f *fetcher[T]) UpdateLatest(latest controller.BlockHeader) { + f.mu.Lock() + defer f.mu.Unlock() + if latest.GetBlockNumber() < f.latest.GetBlockNumber() { + panic(errors.Errorf("try update latest from %d back to %d", f.latest.GetBlockNumber(), latest.GetBlockNumber())) + } + f.latest = latest + utils.TryCloseChan(f.winChanged) +} + +func (f *fetcher[T]) Broken(err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.brokenErr = err + utils.TryCloseChan(f.winChanged) +} + +func (f *fetcher[T]) MoveStart(start uint64) { + f.mu.Lock() + defer f.mu.Unlock() + if start < f.full.StartBlock { + return + } + if start > f.fetchingStart { + start = f.fetchingStart + } + for n := f.full.StartBlock; n < start; n++ { + if item, has := f.data[n]; has { + f.totalSize -= item.Size() + delete(f.data, n) + } + } + f.full.StartBlock = start + utils.TryCloseChan(f.winChanged) +} diff --git a/driver/controller/fetcher/fetcher_test.go b/driver/controller/fetcher/fetcher_test.go new file mode 100644 index 0000000..1aa30ba --- /dev/null +++ b/driver/controller/fetcher/fetcher_test.go @@ -0,0 +1,261 @@ +package fetcher + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +type testBlockHeader struct { + BlockNumber uint64 + BlockTime time.Time +} + +func (b testBlockHeader) GetBlockNumber() uint64 { + return b.BlockNumber +} + +func (b testBlockHeader) GetBlockParentHash() string { + return "" +} + +func (b testBlockHeader) GetBlockHash() string { + return "" +} + +func (b testBlockHeader) GetBlockTime() time.Time { + return b.BlockTime +} + +func newTestBlockHeader(blockNumber uint64) testBlockHeader { + zeroTime, _ := time.Parse(time.DateTime, "2025-07-01 00:00:00") + return testBlockHeader{ + BlockNumber: blockNumber, + BlockTime: zeroTime.Add(time.Second * time.Duration(blockNumber)), + } +} + +type testData []string + +func (t testData) Size() int { return len(t) } + +func buildTestData(bn uint64) (r testData) { + for j := uint64(0); j < bn%3; j++ { + r = append(r, fmt.Sprintf("%d-%d", bn, j)) + } + return +} + +func Test_Fetcher(t *testing.T) { + log.ManuallySetLevel(zap.DebugLevel) + log.BindFlag() + + fr := NewFetcher[testData]( + "testFetcher", + nil, + controller.BlockRange{StartBlock: 10}, + newTestBlockHeader(38), + 3, + 10, + 20, + 20, + time.Second, + 3, + time.Second, + 1.2, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]testData, error) { + r := make(map[uint64]testData) + for i := start; i <= end; i++ { + br := buildTestData(i) + if len(br) > 0 { + r[i] = br + } + } + return r, nil + }, + ) + f := fr.(*fetcher[testData]) + + var g sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + g.Wait() + }() + + // ============================================================================================================== + // ### staging 0, init state + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // S L + // -------------------------------------------------------------------------------------------------------------- + // FS FE + // -------------------------------------------------------------------------------------------------------------- + // count = 0 + // ============================================================================================================== + // ### staging 1, after growth 6 times + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // S L + // -------------------------------------------------------------------------------------------------------------- + // R0 FS FE + // R1 FS FE + // R2 FS FE + // R3 FS FE + // R4 FS FE + // R5 *FS *FE + // -------------------------------------------------------------------------------------------------------------- + // count = 27 + // ============================================================================================================== + // ### staging 2, after pop 7 times + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // *S L + // -------------------------------------------------------------------------------------------------------------- + // FS FE + // -------------------------------------------------------------------------------------------------------------- + // count = 20 + // ============================================================================================================== + // ### staging 3, after pop 4 time and growth 1 time + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // *S L + // -------------------------------------------------------------------------------------------------------------- + // R0 FS FE + // R1 *FE*FS + // -------------------------------------------------------------------------------------------------------------- + // count = 18 + // ============================================================================================================== + // ### staging 4, after update latest + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // S *L + // -------------------------------------------------------------------------------------------------------------- + // R0 FE FS + // R1 *FE*FS + // -------------------------------------------------------------------------------------------------------------- + // count = 19 + // ============================================================================================================== + // ### staging 5, after pop 20 times + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // L *S + // -------------------------------------------------------------------------------------------------------------- + // FE FS + // -------------------------------------------------------------------------------------------------------------- + // count = 0 + // ============================================================================================================== + // ### staging 6, after update latest and growth 1 time and pop 1 time + // -------------------------------------------------------------------------------------------------------------- + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // -------------------------------------------------------------------------------------------------------------- + // *L *S + // -------------------------------------------------------------------------------------------------------------- + // R0 FE FS + // R1 *FE*FS + // -------------------------------------------------------------------------------------------------------------- + // count = 0 + + // ### staging 0 + assert.Equal(t, uint64(10), f.full.StartBlock) + assert.Equal(t, uint64(10), f.fetchingStart) + assert.Equal(t, uint64(12), f.fetchingEnd) + assert.Equal(t, 0, f.totalSize) + + g.Add(1) + go func() { + defer g.Done() + f.KeepFetch(ctx) + }() + + // ### staging 1 + time.Sleep(time.Second) + assert.Equal(t, uint64(10), f.full.StartBlock) + assert.Equal(t, uint64(36), f.fetchingStart) + assert.Equal(t, uint64(38), f.fetchingEnd) + assert.Equal(t, 27, f.totalSize) + + // ### staging 2 + for n := uint64(10); n <= 16; n++ { + r, _, _, err := f.Get(ctx, n) + assert.Equal(t, buildTestData(n), r) + assert.Nil(t, err) + } + f.MoveStart(17) + time.Sleep(time.Second) + assert.Equal(t, uint64(17), f.full.StartBlock) + assert.Equal(t, uint64(36), f.fetchingStart) + assert.Equal(t, uint64(38), f.fetchingEnd) + assert.Equal(t, 20, f.totalSize) + + // ### staging 3 + for n := uint64(17); n <= 20; n++ { + r, _, _, err := f.Get(ctx, n) + assert.Equal(t, buildTestData(n), r) + assert.Nil(t, err) + } + f.MoveStart(21) + time.Sleep(time.Second) + assert.Equal(t, uint64(21), f.full.StartBlock) + assert.Equal(t, uint64(39), f.fetchingStart) + assert.Equal(t, uint64(38), f.fetchingEnd) + assert.Equal(t, 18, f.totalSize) + + // ### staging 4 + f.UpdateLatest(newTestBlockHeader(40)) + time.Sleep(time.Second) + assert.Equal(t, uint64(21), f.full.StartBlock) + assert.Equal(t, uint64(41), f.fetchingStart) + assert.Equal(t, uint64(40), f.fetchingEnd) + assert.Equal(t, 19, f.totalSize) + + // ### staging 5 + for n := uint64(21); n <= 40; n++ { + r, _, _, err := f.Get(ctx, n) + assert.Equal(t, buildTestData(n), r) + assert.Nil(t, err) + } + f.MoveStart(42) // will be automatically reduced to 41 because fetchingStart is still at 41 + time.Sleep(time.Second) + assert.Equal(t, uint64(41), f.full.StartBlock) + assert.Equal(t, uint64(41), f.fetchingStart) + assert.Equal(t, uint64(40), f.fetchingEnd) + assert.Equal(t, 0, f.totalSize) + + // ### staging 6 + go func() { + time.Sleep(time.Second) + f.UpdateLatest(newTestBlockHeader(41)) + }() + r, _, _, err := f.Get(ctx, 41) + assert.Equal(t, buildTestData(41), r) + assert.Nil(t, err) + f.MoveStart(42) + time.Sleep(time.Second) + assert.Equal(t, uint64(42), f.full.StartBlock) + assert.Equal(t, uint64(42), f.fetchingStart) + assert.Equal(t, uint64(41), f.fetchingEnd) + assert.Equal(t, 0, f.totalSize) + +} diff --git a/driver/controller/fetcher/merge.go b/driver/controller/fetcher/merge.go new file mode 100644 index 0000000..65fc5db --- /dev/null +++ b/driver/controller/fetcher/merge.go @@ -0,0 +1,125 @@ +package fetcher + +import ( + "context" + "fmt" + "sync" + "time" + + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" +) + +func MergeIsomorphicFetchers[F controller.FetchTarget, T controller.FetchTarget]( + name string, + config any, + fetchers []controller.Fetcher[F], + mergeFunc func(blockNumber uint64, from []F) (data T, has bool, err error), +) controller.Fetcher[T] { + fetchers = utils.FilterArr(fetchers, func(f controller.Fetcher[F]) bool { + return !f.GetFullRange().IsEmpty() + }) + return &mergedIsomorphicFetchers[F, T]{ + name: name, + config: config, + fetchers: fetchers, + mergeFunc: mergeFunc, + stat: timewin.NewTimeWindowsManager[*processStat](time.Minute), + } +} + +type mergedIsomorphicFetchers[F controller.FetchTarget, T controller.FetchTarget] struct { + name string + config any + fetchers []controller.Fetcher[F] + mergeFunc func(blockNumber uint64, from []F) (data T, has bool, err error) + stat *timewin.TimeWindowsManager[*processStat] +} + +func (f *mergedIsomorphicFetchers[F, T]) GetName() string { + return f.name +} + +func (f *mergedIsomorphicFetchers[F, T]) GetFullRange() controller.BlockRange { + full := controller.EmptyBlockRange + for _, up := range f.fetchers { + full = full.Cover(up.GetFullRange()) + } + return full +} + +func (f *mergedIsomorphicFetchers[F, T]) Snapshot() any { + upstreams := make([]any, len(f.fetchers)) + for i, up := range f.fetchers { + upstreams[i] = up.Snapshot() + } + return map[string]any{ + "name": f.name, + "config": f.config, + "type": fmt.Sprintf("%T", f), + "upstreams": upstreams, + "statistics": f.stat.Snapshot(), + } +} + +func (f *mergedIsomorphicFetchers[F, T]) KeepFetch(ctx context.Context) { + var g sync.WaitGroup + g.Add(len(f.fetchers)) + for _, up := range f.fetchers { + go func(up controller.Fetcher[F]) { + defer g.Done() + up.KeepFetch(ctx) + }(up) + } + g.Wait() +} + +func (f *mergedIsomorphicFetchers[F, T]) Get(ctx context.Context, blockNumber uint64) ( + data T, + has bool, + latest controller.BlockHeader, + err error, +) { + startAt := time.Now() + var from []F + for _, up := range f.fetchers { + var d F + d, has, latest, err = up.Get(ctx, blockNumber) + if err != nil { + return + } + if has { + from = append(from, d) + } + } + mergeStartAt := time.Now() + data, has, err = f.mergeFunc(blockNumber, from) + var dataSize int + if err == nil && has { + dataSize = data.Size() + } + st := &processStat{startAt: time.Now()} + st.fetchComplete(time.Since(mergeStartAt), err == nil, 1, dataSize) + st.getComplete(time.Since(startAt), has) + f.stat.Append(st) + return +} + +func (f *mergedIsomorphicFetchers[F, T]) Broken(err error) { + for _, up := range f.fetchers { + up.Broken(err) + } +} + +func (f *mergedIsomorphicFetchers[F, T]) UpdateLatest(latest controller.BlockHeader) { + for _, up := range f.fetchers { + up.UpdateLatest(latest) + } +} + +func (f *mergedIsomorphicFetchers[F, T]) MoveStart(start uint64) { + for _, up := range f.fetchers { + up.MoveStart(start) + } +} diff --git a/driver/controller/fetcher/retry.go b/driver/controller/fetcher/retry.go new file mode 100644 index 0000000..677806b --- /dev/null +++ b/driver/controller/fetcher/retry.go @@ -0,0 +1,15 @@ +package fetcher + +type PermanentError struct { + Err error +} + +func (e *PermanentError) Error() string { + return e.Err.Error() +} + +func Permanent(err error) error { + return &PermanentError{ + Err: err, + } +} diff --git a/driver/controller/fetcher/stat.go b/driver/controller/fetcher/stat.go new file mode 100644 index 0000000..32fcf4e --- /dev/null +++ b/driver/controller/fetcher/stat.go @@ -0,0 +1,87 @@ +package fetcher + +import ( + "time" + + "sentioxyz/sentio-core/common/timehist" +) + +type processStat struct { + startAt time.Time + + fetchUsed timehist.Histogram + fetchTotalUsed time.Duration + fetchFailedCount int + fetchQueryBlockRange uint64 + fetchGotDataSize int + + getUsed timehist.Histogram + getTotalUsed time.Duration + getEmptyCount int +} + +func (s *processStat) GetStartAt() time.Time { + return s.startAt +} + +func (s *processStat) Merge(a *processStat) { + s.fetchUsed = s.fetchUsed.Add(a.fetchUsed) + s.fetchTotalUsed += a.fetchTotalUsed + s.fetchFailedCount += a.fetchFailedCount + s.fetchQueryBlockRange += a.fetchQueryBlockRange + s.fetchGotDataSize += a.fetchGotDataSize + + s.getEmptyCount += a.getEmptyCount + s.getUsed = s.getUsed.Add(a.getUsed) + s.getTotalUsed += a.getTotalUsed +} + +func (s *processStat) Snapshot(endAt time.Time) any { + sn := map[string]any{ + "startAt": s.startAt.String(), + "endAt": endAt.String(), + "duration": endAt.Sub(s.startAt).String(), + } + if fetchCount := s.fetchUsed.Sum(); fetchCount > 0 { + sn["fetch"] = map[string]any{ + "count": fetchCount, + "failedCount": s.fetchFailedCount, + "totalUsed": s.fetchTotalUsed.String(), + "used": s.fetchUsed.String(), + "avgUsed": (s.fetchTotalUsed / time.Duration(fetchCount)).String(), + "totalGotDataSize": s.fetchGotDataSize, + "avgGotDataSize": s.fetchGotDataSize / fetchCount, + "totalQueryBlockRange": s.fetchQueryBlockRange, + "avgQueryBlockRange": s.fetchQueryBlockRange / uint64(fetchCount), + "pressure": float64(s.fetchTotalUsed) / float64(endAt.Sub(s.startAt)), + } + } + if getCount := s.getUsed.Sum(); getCount > 0 { + sn["get"] = map[string]any{ + "count": getCount, + "emptyCount": s.getEmptyCount, + "totalUsed": s.getTotalUsed, + "used": s.getUsed.String(), + "avgUsed": (s.getTotalUsed / time.Duration(getCount)).String(), + } + } + return sn +} + +func (s *processStat) fetchComplete(used time.Duration, succeed bool, queryRange uint64, dataSize int) { + s.fetchUsed = timehist.Histogram{}.Incr(used) + s.fetchTotalUsed = used + if !succeed { + s.fetchFailedCount = 1 + } + s.fetchQueryBlockRange = queryRange + s.fetchGotDataSize = dataSize +} + +func (s *processStat) getComplete(used time.Duration, has bool) { + s.getUsed = timehist.Histogram{}.Incr(used) + s.getTotalUsed = used + if !has { + s.getEmptyCount = 1 + } +} diff --git a/driver/controller/fetcher/transfer.go b/driver/controller/fetcher/transfer.go new file mode 100644 index 0000000..5636883 --- /dev/null +++ b/driver/controller/fetcher/transfer.go @@ -0,0 +1,418 @@ +package fetcher + +import ( + "context" + "fmt" + "sync" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/timewin" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + + "github.com/pkg/errors" +) + +func TransferFetcher[F controller.FetchTarget, T controller.FetchTarget]( + name string, + upstream controller.Fetcher[F], + latest controller.BlockHeader, + maxConcurrency uint64, + maxKeepDataSize int, + maxKeepDataNum int, + transferTimeout time.Duration, + maxRetry int, + retryInterval time.Duration, + transferFunc func(ctx context.Context, blockNumber uint64, from F) (T, bool, error), +) controller.Fetcher[T] { + full := upstream.GetFullRange() + return &transferFetcher[F, T]{ + name: name, + transferFunc: transferFunc, + maxConcurrency: maxConcurrency, + maxKeepDataSize: maxKeepDataSize, + maxKeepDataNum: maxKeepDataNum, + transferTimeout: transferTimeout, + maxRetry: maxRetry, + retryInterval: retryInterval, + upstream: upstream, + + full: full, + readyEnd: full.StartBlock, + fetchStart: full.StartBlock, + fetched: make(map[uint64]struct{}), + readyEndMoved: make(chan struct{}), + latest: latest, + data: make(map[uint64]T), + tryGrowth: make(chan struct{}), + broken: make(chan struct{}), + stat: timewin.NewTimeWindowsManager[*processStat](time.Minute), + } +} + +type transferFetcher[F controller.FetchTarget, T controller.FetchTarget] struct { + name string + transferFunc func(ctx context.Context, blockNumber uint64, from F) (T, bool, error) + + maxConcurrency uint64 + maxKeepDataSize int + maxKeepDataNum int + transferTimeout time.Duration + maxRetry int + retryInterval time.Duration + + upstream controller.Fetcher[F] + + // protect the data below + mu sync.Mutex + g sync.WaitGroup + + full controller.BlockRange // full is the whole data range + readyEnd uint64 // [full.StartBlock,readyEnd) is the range where data is ready + fetchStart uint64 // [readyEnd,fetchStart) is the range where data is fetching + fetched map[uint64]struct{} + readyEndMoved chan struct{} + fetchFailed error + fetchFailedAt uint64 + + latest controller.BlockHeader + + data map[uint64]T // the ready data in [full.StartBlock, fetchingStart) will be here，key is blockNumber + totalSize int // is the total size in data，every time data is changed, it will be changed automatically + + // when [full.StartBlock,readyEnd) changed or latest changed, this chan will be closed to trigger growth, + // and after growth is completed, this chan will be re-build + tryGrowth chan struct{} + + // will be closed when Broken called + broken chan struct{} + brokenErr error + + stat *timewin.TimeWindowsManager[*processStat] +} + +func (f *transferFetcher[F, T]) GetName() string { + return f.name +} + +func (f *transferFetcher[F, T]) GetFullRange() controller.BlockRange { + f.mu.Lock() + defer f.mu.Unlock() + return f.full +} + +func (f *transferFetcher[F, T]) Snapshot() any { + f.mu.Lock() + defer f.mu.Unlock() + + return map[string]any{ + "name": f.name, + "type": fmt.Sprintf("%T", f), + "config": map[string]any{ + "maxConcurrency": f.maxConcurrency, + "targetKeepDataSize": f.maxKeepDataSize, + "targetKeepDataNum": f.maxKeepDataNum, + "transferTimeout": f.transferTimeout.String(), + "maxRetry": f.maxRetry, + "retryInterval": f.retryInterval.String(), + }, + "fullRange": f.full.String(), + "readyRange": controller.BlockRange{ + StartBlock: f.full.StartBlock, + EndBlock: utils.WrapPointer(f.readyEnd - 1), + }.String(), + "fetchingRange": controller.BlockRange{ + StartBlock: f.readyEnd, + EndBlock: utils.WrapPointer(f.fetchStart - 1), + }.String(), + "fetchFailed": f.fetchFailed, + "fetchFailedAt": f.fetchFailedAt, + "dataSize": f.totalSize, + // dataBlockCount only counts ready blocks that carried data; empty blocks advance readyEnd + // without entering f.data, so fetchedAhead (the ready-but-unconsumed prefetch depth) can be larger. + "dataBlockCount": len(f.data), + // fetchedAhead is the ready prefetch depth: blocks already prepared and waiting to be consumed. + // (MoveStart clamps full.StartBlock to readyEnd, so this never underflows.) + "fetchedAhead": f.readyEnd - f.full.StartBlock, + // fetchedUnreadyCount is the number of out-of-order-completed blocks held back because an + // earlier block in [readyEnd, fetchStart) is not done yet. Always 0 when maxConcurrency==1. + "fetchedUnreadyCount": len(f.fetched), + "upstream": f.upstream.Snapshot(), + "statistics": f.stat.Snapshot(), + } +} + +// current is used as log argument, if not print debug log, current.String will not be called +type current struct { + full controller.BlockRange + readyEnd uint64 + fetchStart uint64 + dataBlockNum int + dataTotalSize int +} + +func (c current) String() string { + return fmt.Sprintf("FULL%s,READY[%d,%d),FETCHING[%d,%d),DATA:%d/%d", + c.full.String(), c.full.StartBlock, c.readyEnd, c.readyEnd, c.fetchStart, c.dataBlockNum, c.dataTotalSize) +} + +func (f *transferFetcher[F, T]) current() current { + return current{ + full: f.full, + readyEnd: f.readyEnd, + fetchStart: f.fetchStart, + dataBlockNum: len(f.data), + dataTotalSize: f.totalSize, + } +} + +func (f *transferFetcher[F, T]) setFetchFailed(blockNumber uint64, err error) { + if f.fetchFailed == nil || f.fetchFailedAt > blockNumber { + f.fetchFailed = err + f.fetchFailedAt = blockNumber + } +} + +func (f *transferFetcher[F, T]) transfer(ctx context.Context, blockNumber uint64) { + _, logger := log.FromContext(ctx) + from, has, _, err := f.upstream.Get(ctx, blockNumber) + var result T + var stage string + if err != nil { + stage = "fetch from upstream" + } else if has { + for retry := f.maxRetry; retry >= 0; { + if f.retryInterval > 0 && err != nil { + select { + case <-ctx.Done(): + return + case <-time.After(f.retryInterval): + } + } + startAt := time.Now() + if f.transferTimeout > 0 { + transferCtx, cancel := context.WithTimeout(ctx, f.transferTimeout) + result, has, err = f.transferFunc(transferCtx, blockNumber, from) + cancel() + } else { + result, has, err = f.transferFunc(ctx, blockNumber, from) + } + used := time.Since(startAt) + var dataSize int + if err == nil && has { + dataSize = result.Size() + } + st := &processStat{startAt: time.Now()} + st.fetchComplete(used, err == nil, 1, dataSize) + f.stat.Append(st) + if err == nil { + break + } + var pe *PermanentError + if errors.As(err, &pe) { + err = pe.Err + retry = -1 + } + tryLogger := logger.With("blockNumber", blockNumber, "used", used.String(), "retry", retry) + if errors.Is(err, context.Canceled) { + tryLogger.Debug("fetch canceled") + return + } + tryLogger.Warnfe(err, "fetch failed") + retry-- + } + if err != nil { + stage = "transfer" + } + } + + f.mu.Lock() + f.fetched[blockNumber] = struct{}{} + if err != nil { + f.setFetchFailed(blockNumber, err) + if errors.Is(err, context.Canceled) { + logger.With("blockNumber", blockNumber).Warnfe(err, "fetch failed because %s canceled", stage) + } else { + logger.With("blockNumber", blockNumber).Errorfe(err, "fetch failed because %s failed", stage) + } + } else if has { + f.data[blockNumber] = result + f.totalSize += result.Size() + logger.Debugw("fetch succeed", "blockNumber", blockNumber) + } else { + logger.Debugw("fetch succeed and no data", "blockNumber", blockNumber) + } + // move f.readyEnd + var readyEnd uint64 + for readyEnd = f.readyEnd; readyEnd < f.fetchStart; readyEnd++ { + if _, ready := f.fetched[readyEnd]; ready { + delete(f.fetched, readyEnd) + } else { + break + } + } + if f.readyEnd < readyEnd { + f.readyEnd = readyEnd + logger.Debugw("growth succeed", "current", f.current()) + utils.TryCloseChan(f.tryGrowth) + utils.TryCloseChan(f.readyEndMoved) + f.readyEndMoved = make(chan struct{}) + } + f.mu.Unlock() +} + +func (f *transferFetcher[F, T]) growth(ctx context.Context) (reject bool, changed chan struct{}) { + f.mu.Lock() + defer f.mu.Unlock() + if f.fetchFailed != nil { + // already failed + return true, nil + } + if f.full.EndBlock != nil && *f.full.EndBlock < f.fetchStart { + // no more data need to fetch + return true, nil + } + if f.totalSize < f.maxKeepDataSize && len(f.data) < f.maxKeepDataNum { + // fetch more + maxFetchBlock := min(f.readyEnd+f.maxConcurrency-1, f.latest.GetBlockNumber()) + if f.full.EndBlock != nil { + maxFetchBlock = min(maxFetchBlock, *f.full.EndBlock) + } + for ; f.fetchStart <= maxFetchBlock; f.fetchStart++ { + f.g.Add(1) + go func(bn uint64) { + f.transfer(ctx, bn) + f.g.Done() + }(f.fetchStart) + } + } + f.tryGrowth = make(chan struct{}) + return false, f.tryGrowth +} + +func (f *transferFetcher[F, T]) keepFetch(ctx context.Context) { + _, logger := log.FromContext(ctx, "fetcher", f.name) + logger.Info("keep fetch start") + defer logger.Info("keep fetch end") + defer func() { + f.g.Wait() + }() + for round := 0; ; round++ { + roundCtx, _ := log.FromContext(ctx, "fetcher", f.name, "round", round) + if reject, changed := f.growth(roundCtx); reject { + return + } else { + select { + case <-changed: + case <-ctx.Done(): + return + } + } + } +} + +func (f *transferFetcher[F, T]) KeepFetch(ctx context.Context) { + var g sync.WaitGroup + g.Add(2) + go func() { + defer g.Done() + f.upstream.KeepFetch(ctx) + }() + go func() { + defer g.Done() + f.keepFetch(ctx) + }() + g.Wait() +} + +func (f *transferFetcher[F, T]) Get(ctx context.Context, blockNumber uint64) ( + data T, + has bool, + latest controller.BlockHeader, + err error, +) { + startAt := time.Now() + defer func() { + st := &processStat{startAt: time.Now()} + st.getComplete(time.Since(startAt), has) + f.stat.Append(st) + }() + f.mu.Lock() + for { + latest = f.latest + if !f.full.Contains(blockNumber) { + // not in full, no data + f.mu.Unlock() + return + } + if f.brokenErr != nil { + err = f.brokenErr + f.mu.Unlock() + return + } + if f.readyEnd <= blockNumber { + // data not ready, wait readyEnd moved + readyEndMoved, broken := f.readyEndMoved, f.broken + f.mu.Unlock() + select { + case <-readyEndMoved: + case <-broken: + case <-ctx.Done(): + err = ctx.Err() + return + } + f.mu.Lock() + continue + } + // data is ready + if f.fetchFailed != nil && f.fetchFailedAt <= blockNumber { + err = f.fetchFailed + } else { + data, has = f.data[blockNumber] + } + f.mu.Unlock() + return + } +} + +func (f *transferFetcher[F, T]) UpdateLatest(latest controller.BlockHeader) { + f.upstream.UpdateLatest(latest) + + f.mu.Lock() + f.latest = latest + utils.TryCloseChan(f.tryGrowth) + f.mu.Unlock() +} + +func (f *transferFetcher[F, T]) Broken(err error) { + f.upstream.Broken(err) + + f.mu.Lock() + defer f.mu.Unlock() + if f.brokenErr == nil { + f.brokenErr = err + utils.TryCloseChan(f.broken) + } +} + +func (f *transferFetcher[F, T]) MoveStart(start uint64) { + f.upstream.MoveStart(start) + + f.mu.Lock() + defer f.mu.Unlock() + if start < f.full.StartBlock { + return + } + if start > f.readyEnd { + start = f.readyEnd + } + for n := f.full.StartBlock; n < start; n++ { + if item, has := f.data[n]; has { + f.totalSize -= item.Size() + delete(f.data, n) + } + } + f.full.StartBlock = start + utils.TryCloseChan(f.tryGrowth) +} diff --git a/driver/controller/fetcher/transfer_test.go b/driver/controller/fetcher/transfer_test.go new file mode 100644 index 0000000..43c1999 --- /dev/null +++ b/driver/controller/fetcher/transfer_test.go @@ -0,0 +1,183 @@ +package fetcher + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func Test_transfer(t *testing.T) { + log.ManuallySetLevel(zap.DebugLevel) + log.BindFlag() + + upstream := NewFetcher[testData]( + "testFetcher", + nil, + controller.BlockRange{StartBlock: 10}, + newTestBlockHeader(25), + 3, + 10, + 20, + 20, + time.Second, + 3, + time.Second, + 1.2, + func(ctx context.Context, start, end uint64, latest controller.BlockHeader) (map[uint64]testData, error) { + r := make(map[uint64]testData) + for i := start; i <= end; i++ { + br := buildTestData(i) + if len(br) > 0 { + r[i] = br + } + } + return r, nil + }, + ) + fr := TransferFetcher[testData]( + "transferFetcher", + upstream, + newTestBlockHeader(25), + 2, + 5, + 5, + 0, + 10, + 0, + func(ctx context.Context, blockNumber uint64, from testData) (testData, bool, error) { + if len(from) == 0 { + return testData{}, false, nil + } + select { + case <-ctx.Done(): + return testData{}, false, ctx.Err() + case <-time.After(time.Millisecond * 100): + //if rand.Int()%2 == 0 { + // return testData{}, false, errors.Errorf("transfer failed sadly") + //} + return testData{strings.Join(from, ",")}, true, nil + } + }) + f := fr.(*transferFetcher[testData, testData]) + + var g sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + defer func() { + cancel() + g.Wait() + }() + + g.Add(1) + go func() { + defer g.Done() + f.KeepFetch(ctx) + }() + + mergedTestData := func(n uint64) testData { + src := buildTestData(n) + if len(src) == 0 { + return nil + } + return testData{strings.Join(src, ",")} + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // FS RE LA + time.Sleep(time.Second) + assert.Equal(t, uint64(10), f.full.StartBlock) + assert.Equal(t, uint64(18), f.readyEnd) + assert.Equal(t, uint64(18), f.fetchStart) + assert.Equal(t, 6, f.totalSize) + for n := uint64(10); n < 18; n++ { + d, has, latest, err := f.Get(ctx, n) + assert.NoError(t, err) + assert.Equalf(t, uint64(25), latest.GetBlockNumber(), "n = %d", n) + assert.Equalf(t, len(buildTestData(n)) > 0, has, "n = %d", n) + assert.Equalf(t, mergedTestData(n), d, "n = %d", n) + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // FS RE LA + f.MoveStart(12) + time.Sleep(time.Second) + assert.Equal(t, uint64(12), f.full.StartBlock) + assert.Equal(t, uint64(21), f.readyEnd) + assert.Equal(t, uint64(21), f.fetchStart) + assert.Equal(t, 6, f.totalSize) + for n := uint64(12); n < 21; n++ { + d, has, latest, err := f.Get(ctx, n) + assert.NoError(t, err) + assert.Equalf(t, uint64(25), latest.GetBlockNumber(), "n = %d", n) + assert.Equalf(t, len(buildTestData(n)) > 0, has, "n = %d", n) + assert.Equalf(t, mergedTestData(n), d, "n = %d", n) + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // FS LA RE + f.MoveStart(21) + time.Sleep(time.Second) + assert.Equal(t, uint64(21), f.full.StartBlock) + assert.Equal(t, uint64(26), f.readyEnd) + assert.Equal(t, uint64(26), f.fetchStart) // latest is 25, so no block is transferring + assert.Equal(t, 3, f.totalSize) + for n := uint64(21); n < 25; n++ { + d, has, latest, err := f.Get(ctx, n) + assert.NoError(t, err) + assert.Equalf(t, uint64(25), latest.GetBlockNumber(), "n = %d", n) + assert.Equalf(t, len(buildTestData(n)) > 0, has, "n = %d", n) + assert.Equalf(t, mergedTestData(n), d, "n = %d", n) + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // FS LA RE + f.UpdateLatest(newTestBlockHeader(26)) + time.Sleep(time.Second) + assert.Equal(t, uint64(21), f.full.StartBlock) + assert.Equal(t, uint64(27), f.readyEnd) + assert.Equal(t, uint64(27), f.fetchStart) // latest is 26, so no block is transferring + assert.Equal(t, 4, f.totalSize) + for n := uint64(21); n < 26; n++ { + d, has, latest, err := f.Get(ctx, n) + assert.NoError(t, err) + assert.Equalf(t, uint64(26), latest.GetBlockNumber(), "n = %d", n) + assert.Equalf(t, len(buildTestData(n)) > 0, has, "n = %d", n) + assert.Equalf(t, mergedTestData(n), d, "n = %d", n) + } + + // 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 + // 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 + // LA RE + // FA + f.MoveStart(27) + time.Sleep(time.Second) + assert.Equal(t, uint64(27), f.full.StartBlock) + assert.Equal(t, uint64(27), f.readyEnd) + assert.Equal(t, uint64(27), f.fetchStart) + assert.Equal(t, 0, f.totalSize) + brokenErr := errors.New("broken") + go func() { + time.Sleep(time.Second) + f.Broken(brokenErr) + }() + { + d, has, latest, err := f.Get(ctx, 27) + assert.ErrorIs(t, err, brokenErr) + assert.Equal(t, uint64(26), latest.GetBlockNumber()) + assert.False(t, has) + assert.Nil(t, d) + } + +} diff --git a/driver/controller/fetcher/utils.go b/driver/controller/fetcher/utils.go new file mode 100644 index 0000000..4d57f6d --- /dev/null +++ b/driver/controller/fetcher/utils.go @@ -0,0 +1,21 @@ +package fetcher + +import ( + "golang.org/x/exp/constraints" + + "sentioxyz/sentio-core/driver/controller" +) + +func sumSize[T controller.FetchTarget](dict map[uint64]T) (size int) { + for _, item := range dict { + size += item.Size() + } + return +} + +func _min[V constraints.Integer](a V, b *V) V { + if b == nil { + return a + } + return min(a, *b) +} diff --git a/driver/controller/handler.go b/driver/controller/handler.go new file mode 100644 index 0000000..a192a1e --- /dev/null +++ b/driver/controller/handler.go @@ -0,0 +1,102 @@ +package controller + +import ( + "context" + "fmt" +) + +type HandlerID struct { + DataSource string + DataSourceID int + Type string + Name string + ID int32 +} + +func (h HandlerID) String() string { + return fmt.Sprintf("%d#%s/%s/%s/%d", h.DataSourceID, h.DataSource, h.Type, h.Name, h.ID) +} + +type HandlerConfig interface { + GetHandlerName() string + GetHandlerId() int32 +} + +type SimpleHandlerConfig struct { + Name string + ID int32 +} + +func (h SimpleHandlerConfig) GetHandlerName() string { + return h.Name +} + +func (h SimpleHandlerConfig) GetHandlerId() int32 { + return h.ID +} + +func BuildHandlerID(dataSource string, dataSourceID int, handlerType string, handlerConfig HandlerConfig) HandlerID { + return HandlerID{ + DataSource: dataSource, + DataSourceID: dataSourceID, + Type: handlerType, + Name: handlerConfig.GetHandlerName(), + ID: handlerConfig.GetHandlerId(), + } +} + +type HandlerAgent interface { + GetHandlerID() HandlerID + GetRange() BlockRange + Snapshot() any +} + +func GetHandleAgentsBlockRange[HA HandlerAgent](agents []HA) BlockRange { + r := EmptyBlockRange + for _, ag := range agents { + r = r.Cover(ag.GetRange()) + } + return r +} + +type BaseHandlerAgent struct { + HandlerID HandlerID + Range BlockRange +} + +func (h BaseHandlerAgent) GetHandlerID() HandlerID { + return h.HandlerID +} + +func (h BaseHandlerAgent) GetRange() BlockRange { + return h.Range +} + +func NewBaseHandlerAgent( + dataSource string, + dataSourceID int, + handlerType string, + handlerConfig HandlerConfig, + blockRange BlockRange, +) BaseHandlerAgent { + return BaseHandlerAgent{ + HandlerID: BuildHandlerID(dataSource, dataSourceID, handlerType, handlerConfig), + Range: blockRange, + } +} + +type HandlerController interface { + Prologue( + ctx context.Context, + checkpoint *Checkpoint, + templates map[uint64][]TemplateInstance, + first uint64, + latest BlockHeader, + ) *ExternalError + GetBlockRange() BlockRange + GetAgentStat() map[string]int + BuildBlockDataFetcher(firstBlockNumber uint64, currentBlockNumber uint64, latest BlockHeader) Fetcher[BlockData] + Epilogue() + + Snapshot() any +} diff --git a/driver/controller/main.go b/driver/controller/main.go new file mode 100644 index 0000000..c32a00a --- /dev/null +++ b/driver/controller/main.go @@ -0,0 +1,354 @@ +package controller + +import ( + "context" + "sync/atomic" + "time" + + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" +) + +type MainController struct { + seqMode bool + + blockBuilder BlockBuilder + checkpointCtrl CheckpointController + processor *models.Processor + chainID string + + bindingIndex atomic.Uint64 + + analyser +} + +func NewMainController( + blockBuilder BlockBuilder, + checkpointCtrl CheckpointController, + seqMode bool, + processor *models.Processor, + chainID string, +) *MainController { + N.DriverCreated(processor, chainID, func() (int64, bool) { + if cc := checkpointCtrl.GetSavedLatestCheckpoint(); cc != nil { + return int64(cc.BlockNumber), true + } + return 0, false + }) + return &MainController{ + seqMode: seqMode, + blockBuilder: blockBuilder, + checkpointCtrl: checkpointCtrl, + processor: processor, + chainID: chainID, + analyser: newAnalyser(), + } +} + +func (c *MainController) Main(ctx context.Context) error { + const maxDupErrRetryTimes = 10 + lastExtErrCode, lastExtErrRound := 0, 0 + for round := 0; ; round++ { + runCtx, logger := log.FromContext(ctx, "runID", round) + err := c.run(runCtx) + if err == nil { + logger.UserVisible().Info("chain is done") + return nil + } + switch { + case errors.Is(err, ErrInternalReorgDetected): + continue + case errors.Is(err, ErrInternalHasNewTemplate): + continue + } + var extErr *ExternalError + if errors.As(err, &extErr) { + logger.Errorf("run got external error: %+v", extErr) + saveErr := c.checkpointCtrl.SaveError(ctx, extErr) + if saveErr != nil { + logger.Errore(saveErr, "save chain error failed") + } + if extErr.IsDriverError() || extErr.IsUserRuntimeError() || saveErr != nil { + if saveErr == nil && extErr.Code() == lastExtErrCode { + if dupTimes := round - lastExtErrRound; dupTimes >= maxDupErrRetryTimes { + logger.Warnf("same error dupped %d times, will exit now", dupTimes) + return err + } else { + logger.Warnf("same error dupped %d times", round-lastExtErrRound) + } + } else { + lastExtErrCode, lastExtErrRound = extErr.Code(), round + } + logger.Infof("will retry after %s", RunWaiting.String()) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(RunWaiting): + continue + } + } + } else { + logger.Errorf("run got error: %+v", err) + } + return err + } +} + +func (c *MainController) Snapshot() any { + // Mirror the taskProcessConcurrency computed in run(): in sequential mode tasks run on a single + // worker, otherwise on ProcessConcurrency workers. Surfacing both (plus seqMode) lets a reader tell + // "handlers are saturated" apart from "handlers are starved" without reverse-engineering the config. + taskProcessConcurrency := ProcessConcurrency + if c.seqMode { + taskProcessConcurrency = 1 + } + return map[string]any{ + "seqMode": c.seqMode, + "taskProcessConcurrency": taskProcessConcurrency, + "bindingIndex": c.bindingIndex.Load(), + "blockBuilder": c.blockBuilder.Snapshot(), + "checkpointController": c.checkpointCtrl.Snapshot(), + "statistics": c.analyser.Snapshot(), + } +} + +type BlockDataSummary struct { + BlockNumber uint64 + BlockParentHash string + BlockHash string + BlockTime time.Time + + TaskCount int + CheckpointData map[string]string +} + +func (s BlockDataSummary) GetBlockNumber() uint64 { + return s.BlockNumber +} + +func (s BlockDataSummary) GetBlockParentHash() string { + return s.BlockParentHash +} + +func (s BlockDataSummary) GetBlockHash() string { + return s.BlockHash +} + +func (s BlockDataSummary) GetBlockTime() time.Time { + return s.BlockTime +} + +type BlockPanel struct { + BlockNumber uint64 + DataSummary *BlockDataSummary + TaskCount int + + ProgressBar +} + +type ProgressMessage struct { + BlockStart *BlockPanel + BlockTaskDone uint64 + BlockAllDone *uint64 +} + +func (c *MainController) run(ctx context.Context) error { + _, logger := log.FromContext(ctx) + logger.Info("run started") + defer func() { + logger.Info("run finished") + }() + + checkpoint, templates := c.checkpointCtrl.GetLatestCheckpoint(), c.checkpointCtrl.GetTemplates() + agentStat, extErr := c.blockBuilder.Start(ctx, checkpoint, templates) + if extErr != nil { + return extErr + } + defer c.blockBuilder.Finish() + if extErr = c.checkpointCtrl.Ready(ctx, agentStat); extErr != nil { + return extErr + } + taskProcessConcurrency := utils.Select(c.seqMode, 1, int(ProcessConcurrency)) + logger.Infow("main stream is ready", + "checkpoint", utils.NullOrToString(checkpoint), + "templates", utils.CountMap(templates), + "taskProcessConcurrency", taskProcessConcurrency) + + N.DriverStarted(ctx, c.processor, c.chainID, CountTemplatesByID(templates)) + + g, gctx := errgroup.WithContext(ctx) + // More capacity to ensure the goroutines that execute tasks and build blockData are less likely to be blocked by this + progressNotice := make(chan ProgressMessage, 10000) + // Once all the data for a block is ready, an element will be push to this chan. + // When a block's checkpoint is constructed, an element will be pop from this chan. + // So its capacity is the number of blocks that can be processed simultaneously. + waitingBlocks := make(chan struct{}, 1000) + concurrency.RunWithProducer( + g, + gctx, + taskProcessConcurrency, + func(ctx context.Context, taskChan chan<- Task) error { + logger.Info("keep build block data started") + defer func() { + logger.Info("keep build block data finished") + }() + for { + // fetch BlockData, may be waiting some fetcher, may be waiting latest block + fetchStartAt := time.Now() + blockNumber, blockData, progressBar, reorg, getErr := c.blockBuilder.Next(ctx) + c.analyser.fetchWait(time.Since(fetchStartAt)) + if getErr != nil { + if errors.Is(getErr, ErrInternalNeedUpgrade) { + return NewExternalError(ErrCodeNeedUpgrade, getErr) + } + // even the endpoint was override by user, the fetch data failed error should also be driver error + return NewExternalError(ErrCodeFetchDataFailed, getErr) + } + if reorg != nil { + if cleanErr := c.checkpointCtrl.CleanCheckpoint(ctx, blockNumber, *reorg); cleanErr != nil { + return cleanErr + } + N.ReorgDetected(ctx, c.processor, c.chainID) + return ErrInternalReorgDetected + } + if !progressBar.FullBlockRange.Contains(blockNumber) { + // no more data, build block data can finish now, + // blockData and progressBar.LatestBlock will always be nil here. + logger.Infow("no more block data", "blockNumber", blockNumber, "full", progressBar.FullBlockRange.String()) + select { + case progressNotice <- ProgressMessage{BlockAllDone: &blockNumber}: + case <-ctx.Done(): + } + return nil + } + startAt := time.Now() + // Using `dataSummary` instead of `blockData` is to release the reference to `blockData`, allowing its memory + // to be released promptly and preventing subsequent tasks from having their references to `blockData` + // continuously attached due to a task running for too long. + var dataSummary *BlockDataSummary + var taskList []Task + if blockData != nil { + taskList = blockData.GetTaskList() + dataSummary = &BlockDataSummary{ + BlockNumber: blockData.GetBlockNumber(), + BlockParentHash: blockData.GetBlockParentHash(), + BlockHash: blockData.GetBlockHash(), + BlockTime: blockData.GetBlockTime(), + TaskCount: len(taskList), + CheckpointData: blockData.CheckpointData(), + } + } + // If waitingBlocks is full, it means that the checkpoint goroutine is stuck. + // This could be because making checkpoint is too time-consuming, or because a task is taking too long. + select { + case waitingBlocks <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + } + select { + case progressNotice <- ProgressMessage{ + BlockStart: &BlockPanel{ + BlockNumber: blockNumber, + DataSummary: dataSummary, + ProgressBar: progressBar, + TaskCount: len(taskList), + }, + }: + case <-ctx.Done(): + return ctx.Err() + } + for i, task := range taskList { + index := TaskIndex{ + Global: c.bindingIndex.Add(1), + InBlock: i, + TotalInBlock: len(taskList), + } + task.Init(ctx, index, progressBar) + select { + case taskChan <- task: + case <-ctx.Done(): + return ctx.Err() + } + } + c.analyser.taskSent(time.Since(startAt)) + } + }, + func(ctx context.Context, task Task) error { + startAt := time.Now() + if taskErr := task.Exec(ctx, c.checkpointCtrl); taskErr != nil { + return taskErr + } + completeAt := time.Now() + select { + case progressNotice <- ProgressMessage{BlockTaskDone: task.GetBlockNumber()}: + case <-ctx.Done(): + } + c.analyser.taskComplete(task.GetHandlerID().String(), time.Since(startAt), time.Since(completeAt)) + return nil + }) + + makeCheckpointDone := make(chan struct{}) + g.Go(func() error { + logger.Info("keep make checkpoint started") + defer func() { + logger.Info("keep make checkpoint finished") + }() + waiting := make(map[uint64]*BlockPanel) // size will always less than cap(waitingBlocks) + for { + var bn uint64 + select { + case <-gctx.Done(): + return gctx.Err() + case pm := <-progressNotice: + if pm.BlockStart != nil { + bn = pm.BlockStart.BlockNumber + waiting[bn] = pm.BlockStart + } else if pm.BlockAllDone != nil { + bn = *pm.BlockAllDone + waiting[bn] = nil // bn is the finalize block, will out of full block range + } else { + bn = pm.BlockTaskDone + waiting[bn].TaskCount -= 1 + } + } + if bn > 0 && waiting[bn-1] != nil { + continue + } + startAt := time.Now() + for { + r, has := waiting[bn] + if has && r == nil { + // no more checkpoint need to be make now + close(makeCheckpointDone) + return nil + } + if !has || r.TaskCount > 0 { + break + } + if r.DataSummary != nil { + hasNewTpl, makeErr := c.checkpointCtrl.MakeCheckpoint(gctx, *r.DataSummary, r.ProgressBar) + if makeErr != nil { + return makeErr + } else if hasNewTpl { + return ErrInternalHasNewTemplate + } + } + delete(waiting, bn) + <-waitingBlocks + bn++ + } + c.analyser.makeCheckpoint(time.Since(startAt)) + } + }) + + g.Go(func() error { + return c.checkpointCtrl.KeepSave(gctx, makeCheckpointDone) + }) + + return g.Wait() +} diff --git a/driver/controller/main_test.go b/driver/controller/main_test.go new file mode 100644 index 0000000..50cdf01 --- /dev/null +++ b/driver/controller/main_test.go @@ -0,0 +1,659 @@ +package controller + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +type testBlockHeader struct { + BlockNumber uint64 + BlockHash string + BlockParentHash string + BlockTime time.Time +} + +func (b testBlockHeader) GetBlockNumber() uint64 { + return b.BlockNumber +} + +func (b testBlockHeader) GetBlockParentHash() string { + return b.BlockParentHash +} + +func (b testBlockHeader) GetBlockHash() string { + return b.BlockHash +} + +func (b testBlockHeader) GetBlockTime() time.Time { + return b.BlockTime +} + +func newTestBlockHeader(blockNumber uint64, blockHash, blockParentHash string) testBlockHeader { + zeroTime, _ := time.Parse(time.DateTime, "2025-07-01 00:00:00") + return testBlockHeader{ + BlockNumber: blockNumber, + BlockHash: blockHash, + BlockParentHash: blockParentHash, + BlockTime: zeroTime.Add(time.Second * time.Duration(blockNumber)), + } +} + +type testClient struct { + mu sync.Mutex + + first uint64 + latest uint64 + subscribeCallBack func(latest BlockHeader, broken error) + + fork []uint64 + + broken error +} + +func newTestClient(first uint64, latest uint64) *testClient { + return &testClient{first: first, latest: latest} +} + +func (c *testClient) GetLatest(ctx context.Context) (latest BlockHeader, first uint64, err error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.buildHeader(c.latest), c.first, nil +} + +func (c *testClient) Subscribe(ctx context.Context, from BlockHeader, callback func(latest BlockHeader, broken error)) { + c.mu.Lock() + c.subscribeCallBack = callback + c.mu.Unlock() + <-ctx.Done() +} + +func (c *testClient) GetHeaderIgnoreCache(ctx context.Context, blockNumber uint64) (BlockHeader, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.buildHeader(blockNumber), nil +} + +func (c *testClient) ResetCache(r BlockRange) { +} + +func (c *testClient) Snapshot() any { + return nil +} + +func (c *testClient) buildBlockHash(bn uint64) string { + var ver int + for _, fk := range c.fork { + if fk <= bn { + ver++ + } + } + return fmt.Sprintf("h%d-%d", bn, ver) +} + +func (c *testClient) buildHeader(bn uint64) testBlockHeader { + if bn == 0 { + return newTestBlockHeader(bn, c.buildBlockHash(0), "") + } + return newTestBlockHeader(bn, c.buildBlockHash(bn), c.buildBlockHash(bn-1)) +} + +func (c *testClient) Fork(bn uint64) { + c.mu.Lock() + defer c.mu.Unlock() + log.Warnf("fork at %d", bn) + c.fork = append(c.fork, bn) +} + +func (c *testClient) UpdateLatest(bn uint64) { + c.mu.Lock() + defer c.mu.Unlock() + if bn <= c.latest { + return + } + if c.latest < bn { + c.latest = bn + } + c.subscribeCallBack(c.buildHeader(bn), c.broken) +} + +func (c *testClient) Broken(err error) { + c.mu.Lock() + defer c.mu.Unlock() + c.broken = err +} + +type testBlockDataFetcher struct { + client *testClient + taskSleep time.Duration + errTaskIndex uint64 + newTplIndex map[uint64]TemplateInstance + + mu sync.Mutex + latest BlockHeader + latestChanged chan struct{} +} + +func (f *testBlockDataFetcher) GetName() string { + return "testBlockDataFetcher" +} + +func (f *testBlockDataFetcher) GetFullRange() BlockRange { + return BlockRange{} +} + +func (f *testBlockDataFetcher) Snapshot() any { + return nil +} + +func (f *testBlockDataFetcher) KeepFetch(ctx context.Context) { + <-ctx.Done() +} + +func (f *testBlockDataFetcher) Get(ctx context.Context, bn uint64) ( + data BlockData, + has bool, + latest BlockHeader, + err error, +) { + f.mu.Lock() + defer f.mu.Unlock() + for bn > f.latest.GetBlockNumber() { + changed := f.latestChanged + f.mu.Unlock() + select { + case <-changed: + case <-ctx.Done(): + err = ctx.Err() + f.mu.Lock() + return + } + f.mu.Lock() + } + latest = f.latest + + taskLen := bn % 5 + // no data in block 5 15 25 ... + if taskLen == 0 && bn%2 != 0 { + return + } + + header, _ := f.client.GetHeaderIgnoreCache(context.Background(), bn) + tasks := make([]Task, taskLen) + for i := uint64(0); i < taskLen; i++ { + tasks[i] = &testTask{ + BlockHeader: header, + errIndex: f.errTaskIndex, + newTplIndex: f.newTplIndex, + sleep: f.taskSleep, + } + } + data = newTestBlockData(header, tasks...) + has = true + return +} + +func (f *testBlockDataFetcher) UpdateLatest(latest BlockHeader) { + f.mu.Lock() + f.latest = latest + close(f.latestChanged) + f.latestChanged = make(chan struct{}) + f.mu.Unlock() +} + +func (f *testBlockDataFetcher) Broken(err error) { +} + +func (f *testBlockDataFetcher) MoveStart(start uint64) { +} + +type testHandlerController struct { + Client *testClient + TaskSleep time.Duration + ErrTaskIndex uint64 + NewTplIndex map[uint64]TemplateInstance + EndBlock *uint64 + + BlockRange +} + +func (c *testHandlerController) Prologue( + ctx context.Context, + checkpoint *Checkpoint, + templates map[uint64][]TemplateInstance, + first uint64, + latest BlockHeader, +) *ExternalError { + c.BlockRange = BlockRange{StartBlock: first, EndBlock: c.EndBlock} + return nil +} + +func (c *testHandlerController) Epilogue() { +} + +func (c *testHandlerController) GetBlockRange() BlockRange { + return c.BlockRange +} + +func (c *testHandlerController) GetAgentStat() map[string]int { + return map[string]int{"placeholder": 1} +} + +func (c *testHandlerController) BuildBlockDataFetcher(_, _ uint64, latest BlockHeader) Fetcher[BlockData] { + return &testBlockDataFetcher{ + client: c.Client, + taskSleep: c.TaskSleep, + errTaskIndex: c.ErrTaskIndex, + newTplIndex: c.NewTplIndex, + latest: latest, + latestChanged: make(chan struct{}), + } +} + +func (c *testHandlerController) Snapshot() any { + return nil +} + +type testTask struct { + BlockHeader + index TaskIndex + errIndex uint64 + newTplIndex map[uint64]TemplateInstance + sleep time.Duration +} + +func (t *testTask) GetHandlerID() HandlerID { + return HandlerID{} +} + +func (t *testTask) Init(ctx context.Context, index TaskIndex, progressbar ProgressBar) { + t.index = index +} + +func (t *testTask) Summary() string { + return fmt.Sprintf("#%d binding data %d/%d in block %s", + t.index.Global, t.index.InBlock, t.index.TotalInBlock, GetBlockSummary(t)) +} + +func (t *testTask) Exec(ctx context.Context, checkpointCtrl CheckpointController) *ExternalError { + _, logger := log.FromContext(ctx, "block", t.GetBlockNumber(), "index", t.index) + logger.Debug("task start") + if t.errIndex == t.index.Global { + time.Sleep(t.sleep / 2) + logger.Warnf("task failed") + return NewExternalError(ErrCodeCallProcessorFailed, errors.Errorf("task %#v fail", t.index)) + } + if newTpl, has := t.newTplIndex[t.GetBlockNumber()]; has && t.index.InBlock == 0 { + time.Sleep(t.sleep / 2) + logger.Warnf("task has new template %s", newTpl) + return checkpointCtrl.NewTemplateInstance(ctx, t, []TemplateInstance{newTpl}) + } + select { + case <-ctx.Done(): + logger.Warnf("task canceled") + return NewExternalError(ErrCodeCallProcessorFailed, errors.Wrapf(ctx.Err(), "task %#v canceled", t.index)) + case <-time.After(t.sleep): + logger.Debug("task end") + return nil + } +} + +func Test_main_succeed(t *testing.T) { + //log.ManuallySetLevel(zap.DebugLevel) + log.BindFlag() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + NewTplIndex: make(map[uint64]TemplateInstance), + } + bb := NewBlockBuilder(hc, cli, false) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + mc := NewMainController(bb, cc, false, nil, "") + err := mc.run(ctx) + log.Warnf("err: %+v", err) + + last := cs.checkpoints[len(cs.checkpoints)-1] + assert.Equal(t, uint64(100), last.BlockNumber) +} + +func Test_main_succeed_and_end(t *testing.T) { + //log.ManuallySetLevel(zap.DebugLevel) + log.BindFlag() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + NewTplIndex: make(map[uint64]TemplateInstance), + EndBlock: utils.WrapPointer[uint64](80), // 80 will build a empty BlockData and make checkpoint, 75 will not + } + bb := NewBlockBuilder(hc, cli, false) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + mc := NewMainController(bb, cc, false, nil, "") + err := mc.run(ctx) + log.Warnf("err: %+v", err) + assert.NoError(t, err) + + last := cs.checkpoints[len(cs.checkpoints)-1] + assert.Equal(t, uint64(80), last.BlockNumber) + assert.True(t, last.AllDone()) +} + +func Test_main_failed(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + ErrTaskIndex: 50, + NewTplIndex: make(map[uint64]TemplateInstance), + } + bb := NewBlockBuilder(hc, cli, false) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + mc := NewMainController(bb, cc, false, nil, "") + err := mc.run(ctx) + log.Warnf("err: %+v", err) + + var extErr *ExternalError + assert.True(t, errors.As(err, &extErr)) + assert.Equal(t, ErrCodeCallProcessorFailed, extErr.code) + assert.Equal(t, "task controller.TaskIndex{Global:0x32, InBlock:3, TotalInBlock:4} fail", extErr.error.Error()) +} + +func Test_main_reorg(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 30) + hc := &testHandlerController{ + Client: cli, + NewTplIndex: make(map[uint64]TemplateInstance), + } + bb := NewBlockBuilder(hc, cli, true) + + go func() { + time.Sleep(time.Second) + cli.Fork(26) + cli.UpdateLatest(60) + time.Sleep(time.Second * 3) + cli.Fork(58) + cli.UpdateLatest(100) + }() + + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + mc := NewMainController(bb, cc, false, nil, "") + roundCtx, _ := log.FromContext(ctx, "round", 0) + err := mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalReorgDetected) + + roundCtx, _ = log.FromContext(ctx, "round", 1) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalReorgDetected) + + roundCtx, _ = log.FromContext(ctx, "round", 2) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + last := cs.checkpoints[len(cs.checkpoints)-1] + assert.Equal(t, uint64(100), last.BlockNumber) +} + +func Test_main_newTpl(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + NewTplIndex: map[uint64]TemplateInstance{ + 49: { + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }, + 51: { + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }, + 52: { // dup create, will be ignored + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 70}, + }, + }, + } + bb := NewBlockBuilder(hc, cli, true) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + ccc := cc.(*checkpointController) + mc := NewMainController(bb, cc, false, nil, "") + roundCtx, _ := log.FromContext(ctx, "round", 0) + err := mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(48), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 1) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + 51: {{ + TemplateID: 0, + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(51), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 2) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.Equal(t, uint64(100), cs.checkpoints[len(cs.checkpoints)-1].BlockNumber) +} + +func Test_main_removeTpl(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + cs := &testCheckpointStore{} + cli := newTestClient(0, 100) + hc := &testHandlerController{ + Client: cli, + TaskSleep: time.Millisecond * 500, + NewTplIndex: map[uint64]TemplateInstance{ + 49: { + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }, + 51: { + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }, + 52: { // dup create, will be ignored + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 70}, + }, + 53: { // remove 0::0x1111 + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 70}, + Removed: true, + }, + }, + } + bb := NewBlockBuilder(hc, cli, true) + cc, _ := NewCheckpointController( + ctx, + "1", + time.Second, + time.Second*2, + 100000, + cs, + EmptyQuotaService{}, + EmptyTimeSeriesController{}, + EmptyEntityController{}, + EmptyWebhookController{}, + nil, + ) + ccc := cc.(*checkpointController) + mc := NewMainController(bb, cc, false, nil, "") + roundCtx, _ := log.FromContext(ctx, "round", 0) + err := mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(48), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 1) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + 51: {{ + TemplateID: 0, + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(51), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 2) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + assert.ErrorIs(t, err, ErrInternalHasNewTemplate) + assert.Equal(t, map[uint64][]TemplateInstance{ + 49: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 49}, + }}, + 51: {{ + TemplateID: 0, + Address: "0x2222", + BlockRange: BlockRange{StartBlock: 70}, + }}, + 53: {{ + TemplateID: 0, + Address: "0x1111", + BlockRange: BlockRange{StartBlock: 70}, + Removed: true, + }}, + }, ccc.templates) + assert.Equal(t, map[uint64][]TemplateInstance{}, ccc.unsavedTemplates) + assert.Equal(t, uint64(53), ccc.checkpoints[len(ccc.checkpoints)-1].BlockNumber) + + roundCtx, _ = log.FromContext(ctx, "round", 3) + err = mc.run(roundCtx) + log.Warnf("err: %+v", err) + last := cs.checkpoints[len(cs.checkpoints)-1] + assert.Equal(t, uint64(100), last.BlockNumber) +} diff --git a/driver/controller/notifier.go b/driver/controller/notifier.go new file mode 100644 index 0000000..7c71123 --- /dev/null +++ b/driver/controller/notifier.go @@ -0,0 +1,102 @@ +package controller + +import ( + "context" + "time" + + "sentioxyz/sentio-core/service/processor/models" +) + +// TaskInfo identifies the handler task an event is attributed to. +type TaskInfo struct { + Processor *models.Processor + ChainID string + Handler string + Category string + DataSource string +} + +// Notifier is the controller's outbound side channel. Each method describes +// something that happened in the controller; what to do with it (record an +// OpenTelemetry metric, call the processor service, ...) is up to the +// implementation, which lives in the driver binary. This keeps the controller +// free of the binary's metric and service-client packages so it can be hosted +// in sentio-core. +// +// No method returns an error: the controller treats notifications as +// best-effort, so the implementation logs and swallows any failure. +// +// The implementation is installed once at startup via SetNotifier. +type Notifier interface { + // Chain lifecycle. + + // DriverCreated reports that a chain's main controller has been created (once + // per chain). get returns the latest processed block number and whether it is + // available yet, for continuous observation. + DriverCreated(processor *models.Processor, chainID string, get func() (int64, bool)) + // DriverStarted reports that a chain's main stream has (re)started, carrying + // the current instance count of each template keyed by template id. + DriverStarted(ctx context.Context, processor *models.Processor, chainID string, templateInstanceCounts map[int32]int) + // ReorgDetected reports that a chain reorg was detected while fetching blocks. + ReorgDetected(ctx context.Context, processor *models.Processor, chainID string) + // ReorgDone reports that a detected reorg has been applied: the chain state was + // rolled back and persisted. reorgBlocks is the number of blocks rolled back; + // reduceToBlock is the processed block number afterwards (-1 if the rollback + // went below the start of the processed range). + ReorgDone(ctx context.Context, processor *models.Processor, chainID string, reorgBlocks uint64, reduceToBlock int64) + + // Per task. + + // BeforeEntityOperation returns a context tagged with the task, so the entity + // store can attribute the events it reports while serving the task. + BeforeEntityOperation(ctx context.Context, task TaskInfo) context.Context + // TaskDone reports that a handler task finished. + TaskDone(ctx context.Context, task TaskInfo, succeed bool, used time.Duration) + // SubgraphTaskDone reports that a subgraph handler task finished, together with + // the extra resource usage a subgraph task exposes: the time spent in each + // import function and the wasm memory used. + SubgraphTaskDone(ctx context.Context, task TaskInfo, succeed bool, used time.Duration, + importFuncUsed map[string]time.Duration, memoryUsed uint32) + // SubgraphRPCDone reports that an RPC call issued by a subgraph handler finished. + SubgraphRPCDone(ctx context.Context, task TaskInfo, succeed bool, used time.Duration) + // DataEmitted reports data a task produced. dataType/subtype/name describe the + // data point, e.g. ("event", "", name), ("metric", "gauge", name) or + // ("entity", op, name). + DataEmitted(ctx context.Context, task TaskInfo, dataType, subtype, name string, count int64) + + // On commit. + + // DataSaved reports data persisted for a processor on a chain. dataType/subtype/name + // describe the data point as in DataEmitted. + DataSaved(ctx context.Context, processor *models.Processor, chainID, dataType, subtype, name string, count int64) +} + +// N is the process-wide notifier. It defaults to a no-op so that tests and tools +// that build controllers without a driver binary do not panic; the driver binary +// installs the real implementation via SetNotifier before any controller runs. +var N Notifier = noopNotifier{} + +// SetNotifier installs the process-wide notifier. Called once by the driver +// binary at startup. +func SetNotifier(n Notifier) { + if n != nil { + N = n + } +} + +type noopNotifier struct{} + +func (noopNotifier) DriverCreated(*models.Processor, string, func() (int64, bool)) {} +func (noopNotifier) DriverStarted(context.Context, *models.Processor, string, map[int32]int) {} +func (noopNotifier) ReorgDetected(context.Context, *models.Processor, string) {} +func (noopNotifier) ReorgDone(context.Context, *models.Processor, string, uint64, int64) {} +func (noopNotifier) BeforeEntityOperation(ctx context.Context, _ TaskInfo) context.Context { + return ctx +} +func (noopNotifier) TaskDone(context.Context, TaskInfo, bool, time.Duration) {} +func (noopNotifier) SubgraphTaskDone(context.Context, TaskInfo, bool, time.Duration, map[string]time.Duration, uint32) { +} +func (noopNotifier) SubgraphRPCDone(context.Context, TaskInfo, bool, time.Duration) {} +func (noopNotifier) DataEmitted(context.Context, TaskInfo, string, string, string, int64) {} +func (noopNotifier) DataSaved(context.Context, *models.Processor, string, string, string, string, int64) { +} diff --git a/driver/controller/quota.go b/driver/controller/quota.go new file mode 100644 index 0000000..023aec9 --- /dev/null +++ b/driver/controller/quota.go @@ -0,0 +1,60 @@ +package controller + +import ( + "context" + "fmt" + "strings" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/timeseries" +) + +type OverQuota struct { + Msg string + Detail string +} + +type Usage struct { + TimeSeries map[timeseries.MetaType]map[string]int + Export map[string]int + EntityCreated map[string]int + EntityUpdated map[string]int +} + +func (u Usage) String() string { + var parts []string + metric := utils.SumMap(u.TimeSeries[timeseries.MetaTypeCounter]) + + utils.SumMap(u.TimeSeries[timeseries.MetaTypeGauge]) + if metric > 0 { + parts = append(parts, fmt.Sprintf("%d metric points", metric)) + } + if event := utils.SumMap(u.TimeSeries[timeseries.MetaTypeEvent]); event > 0 { + parts = append(parts, fmt.Sprintf("%d events", event)) + } + if entity := utils.SumMap(u.EntityCreated) + utils.SumMap(u.EntityUpdated); entity > 0 { + parts = append(parts, fmt.Sprintf("%d entity upserts", entity)) + } + if export := utils.SumMap(u.Export); export > 0 { + parts = append(parts, fmt.Sprintf("%d export messages", export)) + } + if len(parts) == 0 { + return "0 data" + } + return strings.Join(parts, " ") +} + +type QuotaService interface { + CheckOverQuota(ctx context.Context) (*OverQuota, error) + SaveUsage(ctx context.Context, usage Usage, inWatching bool) error +} + +type EmptyQuotaService struct { +} + +func (u EmptyQuotaService) CheckOverQuota(ctx context.Context) (*OverQuota, error) { + return nil, nil +} + +func (u EmptyQuotaService) SaveUsage(ctx context.Context, usage Usage, inWatching bool) error { + return nil +} diff --git a/driver/controller/range.go b/driver/controller/range.go new file mode 100644 index 0000000..c30f36a --- /dev/null +++ b/driver/controller/range.go @@ -0,0 +1,582 @@ +package controller + +import ( + "bytes" + "fmt" + "math" + + "sentioxyz/sentio-core/common/utils" +) + +type BlockRange struct { + StartBlock uint64 + EndBlock *uint64 +} + +var EmptyBlockRange = BlockRange{ + StartBlock: math.MaxUint64, + EndBlock: utils.WrapPointer[uint64](0), +} + +func EqualNilAsInf(a, b *uint64) bool { + if a == nil && b == nil { + return true + } + if a != nil && b != nil { + return *a == *b + } + return false +} + +func MinNilAsInf(ns ...*uint64) *uint64 { + var r uint64 + var has bool + for _, n := range ns { + if n == nil { + continue + } + if !has { + r, has = *n, true + } else { + r = min(r, *n) + } + } + if !has { + return nil + } + return &r +} + +func MaxNilAsInf(ns ...*uint64) *uint64 { + var r uint64 + for _, n := range ns { + if n == nil { + return nil + } + r = max(r, *n) + } + return &r +} + +func LessNilAsInf(a, b *uint64) bool { + if a == nil { + return false + } + if b == nil { + return true + } + return *a < *b +} + +func LessEqualNilAsInf(a, b *uint64) bool { + if a == nil { + return b == nil + } + if b == nil { + return true + } + return *a <= *b +} + +func (r BlockRange) EndOrZero() uint64 { + if r.EndBlock == nil { + return 0 + } + return *r.EndBlock +} + +func (r BlockRange) Equal(a BlockRange) bool { + return r.Include(a) && a.Include(r) +} + +func (r BlockRange) Contains(n uint64) bool { + if r.IsEmpty() { + return false + } + return r.StartBlock <= n && LessEqualNilAsInf(&n, r.EndBlock) +} + +func (r BlockRange) Include(a BlockRange) bool { + if a.IsEmpty() { + return true + } + if r.IsEmpty() { + return false + } + return r.StartBlock <= a.StartBlock && LessEqualNilAsInf(a.EndBlock, r.EndBlock) +} + +func (r BlockRange) IsEmpty() bool { + return r.EndBlock != nil && *r.EndBlock < r.StartBlock +} + +func (r BlockRange) String() string { + if r.EndBlock == nil { + return fmt.Sprintf("[%d,INF]", r.StartBlock) + } else if r.StartBlock > *r.EndBlock { + return fmt.Sprintf("[%d,%d/EMPTY]", r.StartBlock, *r.EndBlock) + } else { + return fmt.Sprintf("[%d,%d/%d]", r.StartBlock, *r.EndBlock, *r.EndBlock+1-r.StartBlock) + } +} + +func (r BlockRange) Intersection(a BlockRange) BlockRange { + if a.IsEmpty() || r.IsEmpty() { + return EmptyBlockRange + } + return BlockRange{ + StartBlock: max(r.StartBlock, a.StartBlock), + EndBlock: MinNilAsInf(r.EndBlock, a.EndBlock), + } +} + +func (r BlockRange) Remove(a BlockRange) BlockRangeSet { + if r.IsEmpty() { + return EmptyBlockRangeSet + } + if a.IsEmpty() { + return BlockRangeSet{BlockRange: r} + } + if a.StartBlock <= r.StartBlock { + if LessNilAsInf(a.EndBlock, &r.StartBlock) { + return BlockRangeSet{BlockRange: r} // no intersection, a is to the left of r + } + if LessNilAsInf(a.EndBlock, r.EndBlock) { + return BlockRangeSet{BlockRange: BlockRange{ // left part of r removed + StartBlock: *a.EndBlock + 1, + EndBlock: r.EndBlock, + }} + } + return EmptyBlockRangeSet // all removed + } + if LessNilAsInf(r.EndBlock, &a.StartBlock) { + return BlockRangeSet{BlockRange: r} // no intersection, a is to the right of r + } + // now r.StartBlock < a.StartBlock <= r.EndBlock + if LessNilAsInf(a.EndBlock, r.EndBlock) { + return BlockRangeSet{ // a middle part removed, remains are two separate part + BlockRange: r, + Holes: [][2]uint64{{a.StartBlock, *a.EndBlock}}, + } + } + return BlockRangeSet{BlockRange: BlockRange{ // right part of r removed + StartBlock: r.StartBlock, + EndBlock: utils.WrapPointer(a.StartBlock - 1), + }} +} + +// Cover return a minimal BlockRange that both include r and a +func (r BlockRange) Cover(a BlockRange) BlockRange { + if r.IsEmpty() { + return a + } + if a.IsEmpty() { + return r + } + return BlockRange{ + StartBlock: min(r.StartBlock, a.StartBlock), + EndBlock: MaxNilAsInf(r.EndBlock, a.EndBlock), + } +} + +type BlockRangeSet struct { + BlockRange + + // All the holes must be arranged strictly in ascending order, + // with no two holes overlapping or adjacent to each other. + // The left side of the leftmost hole and the right side of the rightmost hole are definitely not empty. + Holes [][2]uint64 +} + +var EmptyBlockRangeSet = BlockRangeSet{ + BlockRange: EmptyBlockRange, +} + +func (rs BlockRangeSet) Equal(a BlockRangeSet) bool { + if !rs.BlockRange.Equal(a.BlockRange) { + return false + } + if len(rs.Holes) != len(a.Holes) { + return false + } + for i := 0; i < len(rs.Holes); i++ { + if rs.Holes[i] != a.Holes[i] { + return false + } + } + return true +} + +func (rs BlockRangeSet) Contains(n uint64) bool { + if !rs.BlockRange.Contains(n) { + return false + } + for i := range rs.Holes { + if rs.Holes[i][0] <= n && n <= rs.Holes[i][1] { + return false + } + } + return true +} + +func (rs BlockRangeSet) Include(a BlockRange) bool { + if !rs.BlockRange.Include(a) { + return false + } + for i := range rs.Holes { + if !a.Intersection(BlockRange{StartBlock: rs.Holes[i][0], EndBlock: &rs.Holes[i][1]}).IsEmpty() { + return false + } + } + return true +} + +func (rs BlockRangeSet) Last() BlockRange { + if rs.IsEmpty() { + return EmptyBlockRange + } + if len(rs.Holes) == 0 { + return rs.BlockRange + } + return BlockRange{ + StartBlock: rs.Holes[len(rs.Holes)-1][1] + 1, + EndBlock: rs.EndBlock, + } +} + +func (rs BlockRangeSet) String() string { + var b bytes.Buffer + total, s := uint64(0), rs.StartBlock + b.WriteString(fmt.Sprintf("[%d,", rs.StartBlock)) + for _, hole := range rs.Holes { + leftLen := hole[0] - s + total += leftLen + s = hole[1] + 1 + b.WriteString(fmt.Sprintf("%d/%d]+[%d,", hole[0]-1, leftLen, s)) + } + if rs.EndBlock == nil { + b.WriteString("INF]") + } else if s > *rs.EndBlock { + b.WriteString(fmt.Sprintf("%d/EMPTY]", *rs.EndBlock)) + } else { + lastLen := *rs.EndBlock + 1 - s + b.WriteString(fmt.Sprintf("%d/%d]", *rs.EndBlock, lastLen)) + if len(rs.Holes) > 0 { + b.WriteString(fmt.Sprintf("/%d", total+lastLen)) + } + } + return b.String() +} + +func (rs BlockRangeSet) Intersection(a BlockRange) BlockRangeSet { + r := BlockRangeSet{ + BlockRange: rs.BlockRange.Intersection(a), + Holes: rs.Holes, + } + if r.IsEmpty() { + return EmptyBlockRangeSet + } + // remove invalid holes to the left + // pl: * x + // r.Holes: ... { } { } ... + // r.BlockRange: [ ... + // r.BlockRange: [ ... + pl := 0 + for pl < len(r.Holes) && r.Holes[pl][1] < r.StartBlock { + pl++ + } + if pl == len(r.Holes) { + r.Holes = nil // all holes are to the left of r + } else if r.StartBlock < r.Holes[pl][0] { + r.Holes = r.Holes[pl:] + } else { + r.StartBlock = r.Holes[pl][1] + 1 + r.Holes = r.Holes[pl+1:] + } + // remove invalid holes to the right + // pr: x * + // r.Holes: ... { } { } ... + // r.BlockRange: ... ] + // r.BlockRange: ... ] + pr := len(r.Holes) - 1 + for pr >= 0 && LessNilAsInf(r.EndBlock, &r.Holes[pr][0]) { + pr-- + } + if pr < 0 { + r.Holes = nil // all holes are to the right of r + } else if LessNilAsInf(&r.Holes[pr][1], r.EndBlock) { + r.Holes = r.Holes[:pr+1] + } else { + r.EndBlock = utils.WrapPointer(r.Holes[pr][0] - 1) + r.Holes = r.Holes[:pr] + } + + if r.IsEmpty() { + return EmptyBlockRangeSet + } + if len(r.Holes) == 0 { + r.Holes = nil + } + return r +} + +func (rs BlockRangeSet) Remove(a BlockRange) (result BlockRangeSet) { + if a.IsEmpty() || rs.IsEmpty() { + return rs + } + if LessNilAsInf(a.EndBlock, &rs.StartBlock) { + // rs: [ ] + // a: [ ] + // no intersection, a is to the left of rs + return rs + } + if LessNilAsInf(rs.EndBlock, &a.StartBlock) { + // rs: [ ] + // a: [ ] + // no intersection, a is to the right of rs + return rs + } + // rs: [ ] + // a: [ ] + // ^ ^ + // left right + left := EmptyBlockRangeSet + if rs.StartBlock < a.StartBlock { + // rs: [ ] + // a: [ ... + // always have left part + for i := 0; i < len(rs.Holes); i++ { + if a.StartBlock < rs.Holes[i][0] { + // rs: ... [ ] (rs.Holes[i]) [ ] [ ] + // a: [ ... + // a: [ ... + left = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: utils.WrapPointer(a.StartBlock - 1), + }, + Holes: rs.Holes[:i], + } + break + } + if a.StartBlock <= rs.Holes[i][1]+1 { + // rs: ... [ ] (rs.Holes[i]) [ ] ... + // a: [ ... + // a: [ ... + left = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: utils.WrapPointer(rs.Holes[i][0] - 1), + }, + Holes: rs.Holes[:i], + } + break + } + } + if left.IsEmpty() { + // rs: ... [ ] [ ] [ ] + // a: [ ... + // a: [ ... + left = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: utils.WrapPointer(a.StartBlock - 1), + }, + Holes: rs.Holes, + } + } + } + right := EmptyBlockRangeSet + if LessNilAsInf(a.EndBlock, rs.EndBlock) { + // rs: [ ] + // a: ... ] + // always have right part + for i := 0; i < len(rs.Holes); i++ { + if *a.EndBlock < rs.Holes[i][0]-1 { + // rs: ... [ ] (rs.Holes[i]) [ ] ... + // a: ... ] + // a: ... ] + right = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: *a.EndBlock + 1, + EndBlock: rs.EndBlock, + }, + Holes: rs.Holes[i:], + } + break + } + if *a.EndBlock <= rs.Holes[i][1] { + // rs: ... [ ] (rs.Holes[i]) [ ] ... + // a: ... ] + // a: ... ] + right = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.Holes[i][1] + 1, + EndBlock: rs.EndBlock, + }, + Holes: rs.Holes[i+1:], + } + break + } + } + if right.IsEmpty() { + // rs: ... [ ] [ ] [ ] + // a: ... ] + // a: ... ] + right = BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: *a.EndBlock + 1, + EndBlock: rs.EndBlock, + }, + } + } + } + if !left.IsEmpty() && !right.IsEmpty() { + result = BlockRangeSet{ + BlockRange: rs.BlockRange, + } + result.Holes = append(result.Holes, left.Holes...) + result.Holes = append(result.Holes, [2]uint64{ + *left.EndBlock + 1, + right.StartBlock - 1, + }) + result.Holes = append(result.Holes, right.Holes...) + } else if left.IsEmpty() { + result = right + } else { + result = left + } + if result.IsEmpty() { + return EmptyBlockRangeSet + } + if len(result.Holes) == 0 { + result.Holes = nil + } + return result +} + +func (rs BlockRangeSet) Union(a BlockRange) BlockRangeSet { + if a.IsEmpty() { + return rs + } + if rs.IsEmpty() { + return BlockRangeSet{BlockRange: a} + } + if LessNilAsInf(rs.EndBlock, &a.StartBlock) { + if *rs.EndBlock+1 == a.StartBlock { + // rs: [ ] + // a: [ ] + return BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: a.EndBlock, + }, + Holes: rs.Holes, + } + } + // rs: [ ] + // a: [ ] + return BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: rs.StartBlock, + EndBlock: a.EndBlock, + }, + Holes: append(rs.Holes, [2]uint64{ + *rs.EndBlock + 1, + a.StartBlock - 1, + }), + } + } + if LessNilAsInf(a.EndBlock, &rs.StartBlock) { + if *a.EndBlock+1 == rs.StartBlock { + // rs: [ ] + // a: [ ] + return BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: a.StartBlock, + EndBlock: rs.EndBlock, + }, + Holes: rs.Holes, + } + } + // rs: [ ] + // a: [ ] + return BlockRangeSet{ + BlockRange: BlockRange{ + StartBlock: a.StartBlock, + EndBlock: rs.EndBlock, + }, + Holes: utils.Prepend(rs.Holes, [2]uint64{ + *a.EndBlock + 1, + rs.StartBlock - 1, + }), + } + } + result := BlockRangeSet{BlockRange: a.Cover(rs.BlockRange)} + for i := 0; i < len(rs.Holes); i++ { + remain := BlockRange{StartBlock: rs.Holes[i][0], EndBlock: &rs.Holes[i][1]}.Remove(a) + if remain.IsEmpty() { + continue // The hole was completely filled. + } + if len(remain.Holes) == 1 { + // The middle part of the hole was filled in, turned into two holes. + // rs: ... [ ] (rs.Holes[i]) [ ] ... + // a: [ ] + // remain: [ ] [ ] + // result: ... [ ] [ ] [ ] ... + result.Holes = append(result.Holes, [2]uint64{ + rs.Holes[i][0], + remain.Holes[0][0] - 1, + }, [2]uint64{ + remain.Holes[0][1] + 1, + rs.Holes[i][1], + }) + // append all remain rs.Holes and then break + result.Holes = append(result.Holes, rs.Holes[i+1:]...) + break + } + // len(remain.Holes) cannot be greater than 1, so here it must be 0, + // it means the hole was either partially filled on the left or right side, or left completely as it was. + result.Holes = append(result.Holes, [2]uint64{ + remain.StartBlock, + *remain.EndBlock, + }) + } + return result +} + +// CutRangeSet divide the entire range into multiple non-overlapping ranges +// using the endpoints of multiple potentially intersecting ranges. +func CutRangeSet(start uint64, rs []BlockRange) []BlockRange { + if len(rs) == 0 { + return nil + } + sbn := make(map[uint64]bool) + var inf bool + for _, r := range rs { + if r.EndBlock != nil && *r.EndBlock < start { + continue + } + sbn[max(r.StartBlock, start)] = true + if r.EndBlock != nil { + sbn[*r.EndBlock+1] = true + } else { + inf = true + } + } + ns := utils.GetOrderedMapKeys(sbn) + var result []BlockRange + for i := 0; i+1 < len(ns); i++ { + end := ns[i+1] - 1 + result = append(result, BlockRange{ + StartBlock: ns[i], + EndBlock: &end, + }) + } + if inf { + result = append(result, BlockRange{ + StartBlock: ns[len(ns)-1], + }) + } + return result +} diff --git a/driver/controller/range_test.go b/driver/controller/range_test.go new file mode 100644 index 0000000..6bd4658 --- /dev/null +++ b/driver/controller/range_test.go @@ -0,0 +1,552 @@ +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "sentioxyz/sentio-core/common/utils" +) + +func newBlockRange(a, b uint64) BlockRange { + return BlockRange{StartBlock: a, EndBlock: &b} +} + +func Test_cmpNilAsInf(t *testing.T) { + one := uint64(1) + two := uint64(2) + testcases := []struct { + a, b, max, min *uint64 + eq, lt, le bool + }{ + {a: nil, b: nil, max: nil, min: nil, eq: true, lt: false, le: true}, + {a: nil, b: &one, max: nil, min: &one, eq: false, lt: false, le: false}, + {a: &one, b: nil, max: nil, min: &one, eq: false, lt: true, le: true}, + {a: &one, b: &two, max: &two, min: &one, eq: false, lt: true, le: true}, + {a: &two, b: &one, max: &two, min: &one, eq: false, lt: false, le: false}, + {a: &two, b: &two, max: &two, min: &two, eq: true, lt: false, le: true}, + } + for i, tc := range testcases { + assert.Equalf(t, tc.max, MaxNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + assert.Equalf(t, tc.min, MinNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + assert.Equalf(t, tc.eq, EqualNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + assert.Equalf(t, tc.lt, LessNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + assert.Equalf(t, tc.le, LessEqualNilAsInf(tc.a, tc.b), "testcase #%d: %v", i, tc) + } +} + +func Test_remove(t *testing.T) { + for si := uint64(0); si <= 10; si++ { + for ei := si - 1; ei <= 10; ei++ { + for sj := uint64(0); sj <= 10; sj++ { + for ej := sj - 1; ej <= 10; ej++ { + ri := newBlockRange(si, ei) + rj := newBlockRange(sj, ej) + rk := ri.Remove(rj) + for x := uint64(0); x <= 11; x++ { + ok := si <= x && x <= ei && !(sj <= x && x <= ej) + //log.Debugf("!!! ri:%s, rj:%s, ri.Remove(rj):%s, x:%d, ok:%v\n", ri, rj, rk, x, ok) + assert.Equalf(t, ok, rk.Contains(x), "invalid result ri:%s, rj:%s, ri.Remove(rj):%s, x:%d, ok:%v", rj, rj, rk, x, ok) + } + } + } + } + } +} + +func Test_equal(t *testing.T) { + assert.Equal(t, true, EmptyBlockRange.Equal(newBlockRange(1, 0))) + assert.Equal(t, true, newBlockRange(1, 0).Equal(EmptyBlockRange)) + + for si := uint64(0); si <= 10; si++ { + for ei := si; ei <= 10; ei++ { + for sj := uint64(0); sj <= 10; sj++ { + for ej := sj; ej <= 10; ej++ { + ri := newBlockRange(si, ei) + rj := newBlockRange(sj, ej) + eq := si == sj && ei == ej + assert.Equalf(t, eq, ri.Equal(rj), "invalid result ri:%s, rj:%s, eq:%v", ri, rj, eq) + } + } + } + } +} + +func Test_include(t *testing.T) { + assert.Equal(t, true, EmptyBlockRange.Include(newBlockRange(1, 0))) + assert.Equal(t, true, newBlockRange(1, 0).Include(EmptyBlockRange)) + assert.Equal(t, false, EmptyBlockRange.Include(BlockRange{StartBlock: 0})) + assert.Equal(t, true, BlockRange{StartBlock: 0}.Include(EmptyBlockRange)) +} + +func Test_setContains(t *testing.T) { + set := BlockRangeSet{ // [1][4-6][10] + BlockRange: newBlockRange(1, 10), + Holes: [][2]uint64{ + {2, 3}, + {7, 9}, + }, + } + contains := []uint64{1, 4, 5, 6, 10} + for x := uint64(0); x <= 11; x++ { + ok := utils.IndexOf(contains, x) >= 0 + assert.Equalf(t, ok, set.Contains(x), "invalid result contains:%v", x) + } +} + +func Test_setInclude(t *testing.T) { + set := BlockRangeSet{ // [1][4-6][10] + BlockRange: newBlockRange(1, 10), + Holes: [][2]uint64{ + {2, 3}, + {7, 9}, + }, + } + okPairs := [][2]uint64{ + {1, 1}, + {4, 4}, + {4, 5}, + {4, 6}, + {5, 5}, + {5, 6}, + {6, 6}, + {10, 10}, + } + for s := uint64(1); s <= 10; s++ { + for e := s; e <= 10; e++ { + ok := utils.IndexOf(okPairs, [2]uint64{s, e}) >= 0 + assert.Equalf(t, ok, set.Include(newBlockRange(s, e)), "invalid testcase: [%s,%s], ok: %v", s, e, ok) + } + } +} + +func Test_setLast(t *testing.T) { + assert.Equal(t, EmptyBlockRange, EmptyBlockRangeSet.Last()) + + set := BlockRangeSet{ + BlockRange: newBlockRange(1, 10), + } + assert.Equal(t, newBlockRange(1, 10), set.Last()) + + set.Holes = [][2]uint64{ + {2, 3}, + {7, 9}, + } + assert.Equal(t, newBlockRange(10, 10), set.Last()) +} + +func Test_setIntersection(t *testing.T) { + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Intersection(EmptyBlockRange)) + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Intersection(BlockRange{StartBlock: 1})) + + // [1][4-6][10] + set := BlockRangeSet{ + BlockRange: newBlockRange(1, 10), + Holes: [][2]uint64{{2, 3}, {7, 9}}, + } + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(EmptyBlockRange)) + assert.Equal(t, set, set.Intersection(BlockRange{StartBlock: 0})) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Intersection(newBlockRange(1, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Intersection(newBlockRange(1, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Intersection(newBlockRange(1, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 4), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 5), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Intersection(newBlockRange(1, 9))) + + assert.Equal(t, set, set.Intersection(newBlockRange(1, 10))) + assert.Equal(t, set, set.Intersection(newBlockRange(1, 11))) + assert.Equal(t, set, set.Intersection(newBlockRange(0, 10))) + assert.Equal(t, set, set.Intersection(newBlockRange(0, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(2, 2))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(2, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 4)}, set.Intersection(newBlockRange(2, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 5)}, set.Intersection(newBlockRange(2, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(2, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(2, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(2, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(2, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(2, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(2, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(3, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 4)}, set.Intersection(newBlockRange(3, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 5)}, set.Intersection(newBlockRange(3, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(3, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(3, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(3, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(3, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(3, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(3, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 4)}, set.Intersection(newBlockRange(4, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 5)}, set.Intersection(newBlockRange(4, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(4, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(4, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(4, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 6)}, set.Intersection(newBlockRange(4, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(4, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(4, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 5)}, set.Intersection(newBlockRange(5, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 6)}, set.Intersection(newBlockRange(5, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 6)}, set.Intersection(newBlockRange(5, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 6)}, set.Intersection(newBlockRange(5, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 6)}, set.Intersection(newBlockRange(5, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(5, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(5, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 6)}, set.Intersection(newBlockRange(6, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 6)}, set.Intersection(newBlockRange(6, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 6)}, set.Intersection(newBlockRange(6, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 6)}, set.Intersection(newBlockRange(6, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(6, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 10), Holes: [][2]uint64{{7, 9}}}, set.Intersection(newBlockRange(6, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(7, 7))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(7, 8))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(7, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(7, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(7, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(8, 8))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(8, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(8, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(8, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(9, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(9, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(9, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(10, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Intersection(newBlockRange(10, 11))) + + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(11, 11))) + assert.Equal(t, EmptyBlockRangeSet, set.Intersection(newBlockRange(0, 0))) +} + +func Test_setRemove(t *testing.T) { + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Remove(EmptyBlockRange)) + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Remove(BlockRange{StartBlock: 1})) + + set := BlockRangeSet{ // [1][4-6][10] + BlockRange: newBlockRange(1, 10), + Holes: [][2]uint64{ + {2, 3}, + {7, 9}, + }, + } + assert.Equal(t, set, set.Remove(EmptyBlockRange)) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(BlockRange{StartBlock: 0})) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(0, 0))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(0, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(0, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(0, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(0, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(0, 9))) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(newBlockRange(0, 10))) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(newBlockRange(0, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(4, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(5, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(6, 10), Holes: [][2]uint64{{7, 9}}}, set.Remove(newBlockRange(1, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(1, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(1, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(1, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(10, 10)}, set.Remove(newBlockRange(1, 9))) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(newBlockRange(1, 10))) + assert.Equal(t, EmptyBlockRangeSet, set.Remove(newBlockRange(1, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(2, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(2, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 4}, {7, 9}}}, set.Remove(newBlockRange(2, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 5}, {7, 9}}}, set.Remove(newBlockRange(2, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(2, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(2, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(2, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(2, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(2, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(2, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(3, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 4}, {7, 9}}}, set.Remove(newBlockRange(3, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 5}, {7, 9}}}, set.Remove(newBlockRange(3, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(3, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(3, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(3, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(3, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(3, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(3, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 4}, {7, 9}}}, set.Remove(newBlockRange(4, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 5}, {7, 9}}}, set.Remove(newBlockRange(4, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(4, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(4, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(4, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 9}}}, set.Remove(newBlockRange(4, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(4, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 1)}, set.Remove(newBlockRange(4, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 5}, {7, 9}}}, set.Remove(newBlockRange(5, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 9}}}, set.Remove(newBlockRange(5, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 9}}}, set.Remove(newBlockRange(5, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 9}}}, set.Remove(newBlockRange(5, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {5, 9}}}, set.Remove(newBlockRange(5, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 4), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(5, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 4), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(5, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {6, 9}}}, set.Remove(newBlockRange(6, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {6, 9}}}, set.Remove(newBlockRange(6, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {6, 9}}}, set.Remove(newBlockRange(6, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {6, 9}}}, set.Remove(newBlockRange(6, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 5), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(6, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 5), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(6, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(7, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(7, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(7, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(7, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(7, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(8, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(8, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(8, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(8, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(9, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(9, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(9, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(10, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 6), Holes: [][2]uint64{{2, 3}}}, set.Remove(newBlockRange(10, 11))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 10), Holes: [][2]uint64{{2, 3}, {7, 9}}}, set.Remove(newBlockRange(11, 11))) +} + +func Test_setUnion(t *testing.T) { + assert.Equal(t, EmptyBlockRangeSet, EmptyBlockRangeSet.Union(EmptyBlockRange)) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 1}}, BlockRangeSet{BlockRange: BlockRange{StartBlock: 1}}.Union(EmptyBlockRange)) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 1}}, EmptyBlockRangeSet.Union(BlockRange{StartBlock: 1})) + + set := BlockRangeSet{ // [2][5-7][11] + BlockRange: newBlockRange(2, 11), + Holes: [][2]uint64{ + {3, 4}, + {8, 10}, + }, + } + assert.Equal(t, set, set.Union(EmptyBlockRange)) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{1, 1}, {3, 4}, {8, 10}}}, set.Union(newBlockRange(0, 0))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(0, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(0, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{4, 4}, {8, 10}}}, set.Union(newBlockRange(0, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(0, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(0, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(0, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(0, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{9, 10}}}, set.Union(newBlockRange(0, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11), Holes: [][2]uint64{{10, 10}}}, set.Union(newBlockRange(0, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11)}, set.Union(newBlockRange(0, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 11)}, set.Union(newBlockRange(0, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(0, 12)}, set.Union(newBlockRange(0, 12))) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 0}}, set.Union(BlockRange{StartBlock: 0})) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(1, 1))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(1, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{4, 4}, {8, 10}}}, set.Union(newBlockRange(1, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(1, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(1, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(1, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(1, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{9, 10}}}, set.Union(newBlockRange(1, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11), Holes: [][2]uint64{{10, 10}}}, set.Union(newBlockRange(1, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11)}, set.Union(newBlockRange(1, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 11)}, set.Union(newBlockRange(1, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(1, 12)}, set.Union(newBlockRange(1, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(2, 2))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{4, 4}, {8, 10}}}, set.Union(newBlockRange(2, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(2, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(2, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(2, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(2, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{9, 10}}}, set.Union(newBlockRange(2, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{10, 10}}}, set.Union(newBlockRange(2, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11)}, set.Union(newBlockRange(2, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11)}, set.Union(newBlockRange(2, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12)}, set.Union(newBlockRange(2, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{4, 4}, {8, 10}}}, set.Union(newBlockRange(3, 3))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(3, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(3, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(3, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{8, 10}}}, set.Union(newBlockRange(3, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{9, 10}}}, set.Union(newBlockRange(3, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{10, 10}}}, set.Union(newBlockRange(3, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11)}, set.Union(newBlockRange(3, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11)}, set.Union(newBlockRange(3, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12)}, set.Union(newBlockRange(3, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {8, 10}}}, set.Union(newBlockRange(4, 4))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {8, 10}}}, set.Union(newBlockRange(4, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {8, 10}}}, set.Union(newBlockRange(4, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {8, 10}}}, set.Union(newBlockRange(4, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {9, 10}}}, set.Union(newBlockRange(4, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}, {10, 10}}}, set.Union(newBlockRange(4, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}}}, set.Union(newBlockRange(4, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 3}}}, set.Union(newBlockRange(4, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 3}}}, set.Union(newBlockRange(4, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(5, 5))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(5, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(5, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {9, 10}}}, set.Union(newBlockRange(5, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {10, 10}}}, set.Union(newBlockRange(5, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(5, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(5, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(5, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(6, 6))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(6, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {9, 10}}}, set.Union(newBlockRange(6, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {10, 10}}}, set.Union(newBlockRange(6, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(6, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(6, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(6, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(7, 7))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {9, 10}}}, set.Union(newBlockRange(7, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {10, 10}}}, set.Union(newBlockRange(7, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(7, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(7, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(7, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {9, 10}}}, set.Union(newBlockRange(8, 8))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {10, 10}}}, set.Union(newBlockRange(8, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(8, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(8, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}}}, set.Union(newBlockRange(8, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 8}, {10, 10}}}, set.Union(newBlockRange(9, 9))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 8}}}, set.Union(newBlockRange(9, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 8}}}, set.Union(newBlockRange(9, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}, {8, 8}}}, set.Union(newBlockRange(9, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 9}}}, set.Union(newBlockRange(10, 10))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 9}}}, set.Union(newBlockRange(10, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}, {8, 9}}}, set.Union(newBlockRange(10, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 11), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(11, 11))) + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(11, 12))) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 12), Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(newBlockRange(12, 12))) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 2}, Holes: [][2]uint64{{3, 4}, {8, 10}}}, set.Union(BlockRange{StartBlock: 12})) + + assert.Equal(t, BlockRangeSet{BlockRange: newBlockRange(2, 13), Holes: [][2]uint64{{3, 4}, {8, 10}, {12, 12}}}, set.Union(newBlockRange(13, 13))) + assert.Equal(t, BlockRangeSet{BlockRange: BlockRange{StartBlock: 2}, Holes: [][2]uint64{{3, 4}, {8, 10}, {12, 12}}}, set.Union(BlockRange{StartBlock: 13})) +} + +func Test_CutRangeSet(t *testing.T) { + assert.Nil(t, CutRangeSet(0, nil)) + assert.Nil(t, CutRangeSet(0, []BlockRange{})) + + src := []BlockRange{ + {StartBlock: 100}, + {StartBlock: 100}, + {StartBlock: 100}, + } + assert.Equal(t, []BlockRange{{StartBlock: 100}}, CutRangeSet(0, src)) + assert.Equal(t, []BlockRange{{StartBlock: 100}}, CutRangeSet(100, src)) + assert.Equal(t, []BlockRange{{StartBlock: 110}}, CutRangeSet(110, src)) + + src = []BlockRange{ + {StartBlock: 100}, + {StartBlock: 200}, + {StartBlock: 300}, + } + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(0, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(100, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 150, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(150, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 199, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(199, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(200, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 250, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(250, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 299, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300}, + }, CutRangeSet(299, src)) + assert.Equal(t, []BlockRange{{StartBlock: 300}}, CutRangeSet(300, src)) + assert.Equal(t, []BlockRange{{StartBlock: 301}}, CutRangeSet(301, src)) + + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](150)}, + {StartBlock: 151, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](350)}, + {StartBlock: 351, EndBlock: utils.WrapPointer[uint64](550)}, + }, CutRangeSet(0, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](150)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](350)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](550)}, + })) + + src = []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}, + } + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}, + }, CutRangeSet(0, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 100, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}, + }, CutRangeSet(100, src)) + assert.Equal(t, []BlockRange{ + {StartBlock: 199, EndBlock: utils.WrapPointer[uint64](199)}, + {StartBlock: 200, EndBlock: utils.WrapPointer[uint64](299)}, + {StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}, + }, CutRangeSet(199, src)) + assert.Equal(t, []BlockRange{{StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(200, src)) + assert.Equal(t, []BlockRange{{StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(299, src)) + assert.Equal(t, []BlockRange{{StartBlock: 300, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(300, src)) + assert.Equal(t, []BlockRange{{StartBlock: 350, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(350, src)) + assert.Equal(t, []BlockRange{{StartBlock: 399, EndBlock: utils.WrapPointer[uint64](399)}}, CutRangeSet(399, src)) + assert.Equal(t, []BlockRange(nil), CutRangeSet(400, src)) + assert.Equal(t, []BlockRange(nil), CutRangeSet(401, src)) +} diff --git a/driver/controller/standard/BUILD.bazel b/driver/controller/standard/BUILD.bazel new file mode 100644 index 0000000..643502b --- /dev/null +++ b/driver/controller/standard/BUILD.bazel @@ -0,0 +1,50 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "standard", + srcs = [ + "binding_data.go", + "config.go", + "convert.go", + "handler.go", + "helper.go", + "interval.go", + "task.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard", + visibility = ["//visibility:public"], + deps = [ + "//common/concurrency", + "//common/envconf", + "//common/log", + "//common/protojson", + "//common/timer", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/entity/persistent", + "//driver/entity/schema", + "//driver/timeseries", + "//processor/protos", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//encoding/gzip", + "@org_golang_google_protobuf//types/known/structpb", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) + +go_test( + name = "standard_test", + srcs = ["convert_test.go"], + embed = [":standard"], + deps = [ + "//driver/controller", + "//processor/protos", + "@com_github_stretchr_testify//assert", + "@org_golang_google_protobuf//proto", + "@org_golang_google_protobuf//types/known/structpb", + ], +) diff --git a/driver/controller/standard/aptos/BUILD.bazel b/driver/controller/standard/aptos/BUILD.bazel new file mode 100644 index 0000000..59bd18c --- /dev/null +++ b/driver/controller/standard/aptos/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "aptos", + srcs = [ + "block_data.go", + "handler.go", + "handler_change.go", + "handler_event.go", + "handler_function.go", + "handler_interval.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/aptos", + visibility = ["//visibility:public"], + deps = [ + "//chain/aptos", + "//chain/move", + "//common/log", + "//common/set", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/aptos", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor/protos", + "//service/processor/models", + "@com_github_aptos_labs_aptos_go_sdk//:aptos-go-sdk", + "@com_github_pkg_errors//:errors", + ], +) diff --git a/driver/controller/standard/aptos/block_data.go b/driver/controller/standard/aptos/block_data.go new file mode 100644 index 0000000..b46ee23 --- /dev/null +++ b/driver/controller/standard/aptos/block_data.go @@ -0,0 +1,81 @@ +package aptos + +import ( + "encoding/json" + "strings" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + aptosdata "sentioxyz/sentio-core/driver/controller/data/aptos" +) + +type BlockData struct { + controller.BlockHeader + + mainData aptosdata.BlockMainData + accountResources []aptosdata.AccountResource + + cachedTxn map[string]string + cachedEvent map[int]string + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (b *BlockData) DataSource() string { + return b.dataSource +} + +func (b *BlockData) CheckpointData() map[string]string { + return b.checkpointData +} + +func (b *BlockData) Size() int { + return b.taskTotalSize +} + +func (b *BlockData) GetTaskList() []controller.Task { + return b.taskList +} + +func (b *BlockData) getRawTxn( + fetchConfig aptos.TransactionFetchConfig, + eventFilters []aptos.EventFilter, +) (string, error) { + key := fetchConfig.String() + "@" + strings.Join(utils.MapSliceNoError(eventFilters, aptos.EventFilter.String), "|") + if b.cachedTxn == nil { + b.cachedTxn = make(map[string]string) + } + if rawTxn, has := b.cachedTxn[key]; has { + return rawTxn, nil + } + txn := fetchConfig.PruneTransaction(*b.mainData.Txn, eventFilters) + raw, err := json.Marshal(txn) + if err != nil { + return "", err + } + rawTxn := string(raw) + b.cachedTxn[key] = rawTxn + return rawTxn, nil +} + +func (b *BlockData) getRawEvent(index int) (string, error) { + if b.cachedEvent == nil { + b.cachedEvent = make(map[int]string) + } + if rawEvent, has := b.cachedEvent[index]; has { + return rawEvent, nil + } + ev := b.mainData.Txn.Events[index] + raw, err := json.Marshal(ev) + if err != nil { + return "", err + } + rawEvent := string(raw) + b.cachedEvent[index] = rawEvent + return rawEvent, nil +} diff --git a/driver/controller/standard/aptos/handler.go b/driver/controller/standard/aptos/handler.go new file mode 100644 index 0000000..4ab9818 --- /dev/null +++ b/driver/controller/standard/aptos/handler.go @@ -0,0 +1,431 @@ +package aptos + +import ( + "context" + "fmt" + "time" + + aptossdk "github.com/aptos-labs/aptos-go-sdk" + "github.com/pkg/errors" + + chainaptos "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/chain/move" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/aptos" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" +) + +type AptosHandlerAgent interface { + standard.HandlerAgent[*BlockData] +} + +type HandlerController struct { + *standard.BaseHandlerController[aptos.Client, *BlockData, AptosHandlerAgent] +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *config.ChainConfig, + client aptos.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[aptos.Client, *BlockData, AptosHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start, latest uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetAddressStartBlock(ctx, address, start, latest) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + return latest + 1, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, latest uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + var err error + + for dataSourceID, accountConfig := range c.Config.AccountConfigs { + accountAddress := standard.AdjustAddress(accountConfig.GetAddress()) + normalizedAccountAddr := accountAddress + var accountAddr aptossdk.AccountAddress + if accountAddress != "" { + if err = accountAddr.ParseStringRelaxed(accountAddress); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, + errors.Wrapf(err, "invalid account address %q", accountAddress)) + } + normalizedAccountAddr = accountAddr.String() + } + dataSource := fmt.Sprintf("APTOS:%s/Account:%s", c.ChainConfig.ChainID, accountAddress) + blockRange := controller.BlockRange{ + StartBlock: max(accountConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(accountConfig.GetEndBlock()), + } + blockRange.StartBlock, err = c.getAddressStart(ctx, normalizedAccountAddr, blockRange.StartBlock, latest) + if err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + for _, intervalConfig := range accountConfig.GetMoveIntervalConfigs() { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig.GetIntervalConfig(), blockRange), + } + if accountAddress == "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("account address cannot be empty for handler %s", agent.GetHandlerID().String())) + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig.GetIntervalConfig()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + agent.FetchConfig = aptos.AccountResourceFilter{ + Address: normalizedAccountAddr, + ResourceType: nil, // default need all resources of the account + } + if resourceType := intervalConfig.GetType(); resourceType != "" { + agent.FetchConfig.ResourceType = []string{resourceType} + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, changeConfig := range accountConfig.GetMoveResourceChangeConfigs() { + resTypes := changeConfig.GetTypes() + agent := HandlerAgentChange{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "change", changeConfig, blockRange), + Filter: chainaptos.ChangeFilter{ + Address: set.New[aptossdk.AccountAddress](), + ResourceTypes: nil, + }, + } + agent.Filter.ResourceTypes, err = utils.MapSlice(resTypes, move.BuildType) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + if accountAddress != "" { + agent.Filter.Address.Add(accountAddr) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + for dataSourceID, contractConfig := range c.Config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + normalizedContractAddr := contractAddress + if contractAddress != "" { + var contractAddr aptossdk.AccountAddress + if err = contractAddr.ParseStringRelaxed(contractAddress); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, + errors.Wrapf(err, "invalid contract address %q", contractAddress)) + } + normalizedContractAddr = contractAddr.String() + } + dataSource := fmt.Sprintf("APTOS:%s/Contract:%s", c.ChainConfig.ChainID, contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + blockRange.StartBlock, err = c.getAddressStart(ctx, normalizedContractAddr, blockRange.StartBlock, latest) + if err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + for _, moveCallConfig := range contractConfig.GetMoveCallConfigs() { + agent := HandlerAgentFunction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "call", moveCallConfig, blockRange), + Filter: chainaptos.TransactionFilter{ + FailedIsOK: moveCallConfig.GetFetchConfig().GetIncludeFailedTransaction(), + MultiSigTxnIsOK: moveCallConfig.GetFetchConfig().GetSupportMultisigFunc(), + }, + } + agent.Filter.FunctionFilters, err = utils.MapSlice( + moveCallConfig.GetFilters(), + func(f *protos.MoveCallFilter) (ff chainaptos.FunctionFilter, err error) { + var funcName string + if f.GetFunction() != "" { + funcName = contractAddress + "::" + f.GetFunction() + } else if contractAddress != "" { + funcName = contractAddress + "::" + contractConfig.GetContract().GetName() + } + ff.FunctionPattern, err = move.BuildType(funcName) + if err != nil { + return ff, errors.Wrapf(err, "invalid call function %q", funcName) + } + if f.GetWithTypeArguments() { + ff.CheckTypeArguments, ff.TypedArguments = true, f.GetTypeArguments() + } + if sender := f.GetFromAndToAddress().GetFrom(); sender != "" { + var senderAddr aptossdk.AccountAddress + if err = senderAddr.ParseStringRelaxed(sender); err != nil { + return ff, errors.Wrapf(err, "invalid sender address %q", sender) + } + ff.Sender = &senderAddr + } + return + }, + ) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "handler %s have invalid filter", agent.GetHandlerID().String())) + } + if len(agent.Filter.FunctionFilters) == 0 { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("no filter in handler %s", agent.GetHandlerID().String())) + } + agent.FetchConfig.NeedAllEvents = moveCallConfig.GetFetchConfig().GetAllEvents() + if moveCallConfig.GetFetchConfig().GetResourceChanges() { + var resType move.Type + resType, err = move.BuildType(moveCallConfig.GetFetchConfig().GetResourceConfig().GetMoveTypePrefix()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s: invalid resouce type prefix %q", + agent.GetHandlerID().String(), + moveCallConfig.GetFetchConfig().GetResourceConfig().GetMoveTypePrefix())) + } + agent.FetchConfig.ChangeResourceTypes = move.TypeSet{resType} + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, eventConfig := range contractConfig.GetMoveEventConfigs() { + agent := HandlerAgentEvent{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "event", eventConfig, blockRange), + Filter: chainaptos.TransactionFilter{ + FailedIsOK: eventConfig.GetFetchConfig().GetIncludeFailedTransaction(), + MultiSigTxnIsOK: eventConfig.GetFetchConfig().GetSupportMultisigFunc(), + }, + } + if contractAddress == "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("contract address cannot be empty for handler %s", agent.GetHandlerID().String())) + } + agent.Filter.EventFilters, err = utils.MapSlice( + eventConfig.GetFilters(), + func(f *protos.MoveEventFilter) (ff chainaptos.EventFilter, err error) { + eventType := contractAddress + "::" + f.GetType() + ff.Type, err = move.BuildType(eventType) + if err != nil { + return ff, errors.Wrapf(err, "invalid event type %q", eventType) + } + if f.GetEventAccount() != "" { + var addr aptossdk.AccountAddress + if err = addr.ParseStringRelaxed(f.GetEventAccount()); err != nil { + return ff, errors.Wrapf(err, "invalid event account %q", f.GetEventAccount()) + } + ff.GuiAccountAddress = &addr + } + if ff.IsEmpty() { + return ff, errors.New("event filter is empty") + } + return ff, nil + }) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "handler %s have invalid filter", agent.GetHandlerID().String())) + } + if len(agent.Filter.EventFilters) == 0 { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("no filter in handler %s", agent.GetHandlerID().String())) + } + agent.FetchConfig.NeedAllEvents = eventConfig.GetFetchConfig().GetAllEvents() + if eventConfig.GetFetchConfig().GetResourceChanges() { + var resType move.Type + resType, err = move.BuildType(eventConfig.GetFetchConfig().GetResourceConfig().GetMoveTypePrefix()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s: invalid resouce type prefix %q", + agent.GetHandlerID().String(), + eventConfig.GetFetchConfig().GetResourceConfig().GetMoveTypePrefix())) + } + agent.FetchConfig.ChangeResourceTypes = move.TypeSet{resType} + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, intervalConfig := range contractConfig.GetMoveIntervalConfigs() { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig.GetIntervalConfig(), blockRange), + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig.GetIntervalConfig()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + if contractAddress == "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("contract address cannot be empty for handler %s", agent.GetHandlerID().String())) + } + agent.FetchConfig = aptos.AccountResourceFilter{ + Address: normalizedContractAddr, + ResourceType: make([]string, 0), // do not need any resource + } + if resourceType := intervalConfig.GetType(); resourceType != "" { + logger.Warnf("type %q in contract move interval config %s will be ignored", resourceType, agent.HandlerID) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + logger.Infof("built %d agents", len(c.Agents)) + return nil +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + + fetchNamePrefix := fmt.Sprintf("APTOS::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + aptos.BuildBlockMainDataFetcher(fetchNamePrefix, req, firstBlockNumber, currentBlockNumber, latest, c.Client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, // 256MB + 1000, + time.Second*10, + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from aptos.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + result := BlockData{mainData: from, checkpointData: make(map[string]string)} + // take the main data and ask the agents what account resources are needed + var accountResourceFilters []aptos.AccountResourceFilter + var needFullTx bool + for _, agent := range c.Agents { + ag, is := agent.(HandlerAgentInterval) + if !is || !data.ContainsInterval(from.Intervals, ag.IntervalConfig) { + continue + } + if ag.FetchConfig.NeedNothing() { + needFullTx = true + } else { + accountResourceFilters = append(accountResourceFilters, ag.FetchConfig) + } + } + // actually get the extended data + result.accountResources, err = c.Client.GetAccountResources( + ctx, blockNumber, aptos.MergeAccountResourceFilters(accountResourceFilters)) + if err != nil { + return nil, false, err + } + // fetch the transaction + if needFullTx { + // Although result.mainData.Txn may already exist, it might be missing some events or changes. + // So need to re-get the tx here + var tx chainaptos.Transaction + if tx, err = c.Client.GetTransaction(ctx, blockNumber); err != nil { + return nil, false, err + } + result.mainData.Txn = &tx + } + // always need header, fill result.BlockHeader here + if result.mainData.Txn != nil { + result.BlockHeader = (*aptos.Transaction)(result.mainData.Txn) + } else if result.mainData.SimpleTxn != nil { + result.BlockHeader = (*aptos.MinimalistTransaction)(result.mainData.SimpleTxn) + } else if result.BlockHeader, err = c.Client.GetMinimalistTransaction(ctx, blockNumber); err != nil { + return nil, false, err + } + // build binding data + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + return nil, false, err + } + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) + +} + +func (c *HandlerController) getDataRequirement() (dr aptos.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentChange: + dr.Changes = append(dr.Changes, aptos.ChangeRequirement{ + BlockRange: ag.Range, + ChangeFilter: ag.Filter, + }) + case HandlerAgentEvent: + dr.Txn = append(dr.Txn, aptos.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentFunction: + dr.Txn = append(dr.Txn, aptos.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } + } + return +} diff --git a/driver/controller/standard/aptos/handler_change.go b/driver/controller/standard/aptos/handler_change.go new file mode 100644 index 0000000..0609134 --- /dev/null +++ b/driver/controller/standard/aptos/handler_change.go @@ -0,0 +1,53 @@ +package aptos + +import ( + "context" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + aptosdata "sentioxyz/sentio-core/driver/controller/data/aptos" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +// HandlerAgentChange only need txn changes, do not need the txn self, so no aptos.FetchTxnConfig in it +type HandlerAgentChange struct { + controller.BaseHandlerAgent + + Filter aptos.ChangeFilter +} + +func (a HandlerAgentChange) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + changes := utils.FilterArr(bd.mainData.Changes, func(c aptosdata.Change) bool { + return a.Filter.Check(c.WriteSetChange) + }) + resources := utils.MapSliceNoError(changes, func(c aptosdata.Change) string { return c.Raw }) + if len(resources) == 0 { + return nil, nil + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_APT_RESOURCE, + Data: &protos.Data{ + Value: &protos.Data_AptResource_{ + AptResource: &protos.Data_AptResource{ + Version: int64(bd.GetBlockNumber()), + RawResources: resources, + TimestampMicros: bd.GetBlockTime().UnixMicro(), + }, + }, + }, + DataSize: utils.StringsLenSum(resources), + }}, nil +} + +func (a HandlerAgentChange) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + } +} diff --git a/driver/controller/standard/aptos/handler_event.go b/driver/controller/standard/aptos/handler_event.go new file mode 100644 index 0000000..0aa20c0 --- /dev/null +++ b/driver/controller/standard/aptos/handler_event.go @@ -0,0 +1,65 @@ +package aptos + +import ( + "context" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentEvent struct { + controller.BaseHandlerAgent + + Filter aptos.TransactionFilter + FetchConfig aptos.TransactionFetchConfig +} + +func (a HandlerAgentEvent) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if bd.mainData.Txn == nil || !a.Filter.Check(*bd.mainData.Txn) { + return nil, nil + } + rawTxn, err := bd.getRawTxn(a.FetchConfig, a.Filter.EventFilters) + if err != nil { + return nil, err + } + var result []standard.BindingDataInner + eventChecker := aptos.BuildEventFilter(a.Filter.EventFilters) + for i, ev := range bd.mainData.Txn.Events { + if !eventChecker(ev) { + continue + } + var rawEvent string + if rawEvent, err = bd.getRawEvent(i); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_APT_EVENT, + TxInnerIndex: int(ev.Index), + Data: &protos.Data{ + Value: &protos.Data_AptEvent_{ + AptEvent: &protos.Data_AptEvent{ + EventIndex: ev.Index, + RawEvent: rawEvent, + RawTransaction: rawTxn, + }, + }, + }, + DataSize: len(rawTxn) + len(rawEvent), + }) + } + return result, nil +} + +func (a HandlerAgentEvent) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/aptos/handler_function.go b/driver/controller/standard/aptos/handler_function.go new file mode 100644 index 0000000..1c5c1de --- /dev/null +++ b/driver/controller/standard/aptos/handler_function.go @@ -0,0 +1,50 @@ +package aptos + +import ( + "context" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentFunction struct { + controller.BaseHandlerAgent + + Filter aptos.TransactionFilter + FetchConfig aptos.TransactionFetchConfig +} + +func (a HandlerAgentFunction) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if bd.mainData.Txn == nil || !a.Filter.Check(*bd.mainData.Txn) { + return nil, nil + } + rawTxn, err := bd.getRawTxn(a.FetchConfig, a.Filter.EventFilters) + if err != nil { + return nil, err + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_APT_CALL, + Data: &protos.Data{ + Value: &protos.Data_AptCall_{ + AptCall: &protos.Data_AptCall{ + RawTransaction: rawTxn, + }, + }, + }, + DataSize: len(rawTxn), + }}, nil +} + +func (a HandlerAgentFunction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/aptos/handler_interval.go b/driver/controller/standard/aptos/handler_interval.go new file mode 100644 index 0000000..0665a1d --- /dev/null +++ b/driver/controller/standard/aptos/handler_interval.go @@ -0,0 +1,78 @@ +package aptos + +import ( + "context" + + "sentioxyz/sentio-core/chain/aptos" + "sentioxyz/sentio-core/chain/move" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + aptosdata "sentioxyz/sentio-core/driver/controller/data/aptos" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + IntervalConfig data.IntervalConfig + FetchConfig aptosdata.AccountResourceFilter +} + +func (a HandlerAgentInterval) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + if a.FetchConfig.NeedNothing() { + // is a contract move interval handler, need a APT_CALL binding data + rawTxn, err := bd.getRawTxn(aptos.TransactionFetchConfig{ + NeedAllEvents: true, + ChangeResourceTypes: move.TypeSet{move.MustBuildType("")}, + }, nil) + if err != nil { + return nil, err + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_APT_CALL, + Data: &protos.Data{ + Value: &protos.Data_AptCall_{ + AptCall: &protos.Data_AptCall{ + RawTransaction: rawTxn, + }, + }, + }, + }}, nil + } + // is a account move interval handler, need a APT_RESOURCE binding data + ars := utils.FilterArr(bd.accountResources, a.FetchConfig.Check) + resources := utils.MapSliceNoError(ars, func(ar aptosdata.AccountResource) string { return ar.Raw }) + if len(resources) == 0 { + return nil, nil + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_APT_RESOURCE, + Data: &protos.Data{ + Value: &protos.Data_AptResource_{ + AptResource: &protos.Data_AptResource{ + Version: int64(bd.GetBlockNumber()), + RawResources: resources, + TimestampMicros: bd.GetBlockTime().UnixMicro(), + }, + }, + }, + DataSize: utils.StringsLenSum(resources), + }}, nil +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/binding_data.go b/driver/controller/standard/binding_data.go new file mode 100644 index 0000000..e70e0c6 --- /dev/null +++ b/driver/controller/standard/binding_data.go @@ -0,0 +1,69 @@ +package standard + +import ( + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/processor/protos" +) + +type bindingData struct { + controller.BlockHeader + + handlerID controller.HandlerID + data *protos.DataBinding + txIndex int + txInnerIndex int +} + +func (b bindingData) Cmp(x bindingData, mode protos.ExecutionConfig_HandlerOrderInsideTransaction) int { + if r := utils.Cmp(b.GetBlockNumber(), x.GetBlockNumber()); r != 0 { + return r + } + if r := utils.Cmp(b.txIndex, x.txIndex); r != 0 { + return r + } + if mode == protos.ExecutionConfig_BY_PROCESSOR_AND_LOG_INDEX { + if r := utils.Cmp(b.handlerID.DataSourceID, x.handlerID.DataSourceID); r != 0 { + return r + } + if r := CmpHandlerType(b.data, x.data); r != 0 { + return r + } + if r := utils.Cmp(b.handlerID.ID, x.handlerID.ID); r != 0 { + return r + } + return utils.Cmp(b.txInnerIndex, x.txInnerIndex) + } else { + if r := CmpHandlerType(b.data, x.data); r != 0 { + return r + } + if r := utils.Cmp(b.txInnerIndex, x.txInnerIndex); r != 0 { + return r + } + return utils.Cmp(b.handlerID.ID, x.handlerID.ID) + } +} + +var executeOrder = map[protos.HandlerType]int{ + protos.HandlerType_ETH_TRACE: 1, + protos.HandlerType_ETH_LOG: 2, + protos.HandlerType_ETH_TRANSACTION: 3, + protos.HandlerType_ETH_BLOCK: 4, + protos.HandlerType_SOL_INSTRUCTION: 1, + protos.HandlerType_APT_CALL: 1, + protos.HandlerType_APT_EVENT: 2, + protos.HandlerType_APT_RESOURCE: 3, + protos.HandlerType_SUI_CALL: 1, + protos.HandlerType_SUI_EVENT: 2, + protos.HandlerType_SUI_OBJECT_CHANGE: 3, + protos.HandlerType_SUI_OBJECT: 4, + protos.HandlerType_FUEL_RECEIPT: 2, + protos.HandlerType_FUEL_TRANSACTION: 3, + protos.HandlerType_FUEL_BLOCK: 4, + protos.HandlerType_COSMOS_CALL: 1, + protos.HandlerType_STARKNET_EVENT: 1, +} + +func CmpHandlerType(a, b *protos.DataBinding) int { + return utils.Cmp(executeOrder[a.GetHandlerType()], executeOrder[b.GetHandlerType()]) +} diff --git a/driver/controller/standard/config.go b/driver/controller/standard/config.go new file mode 100644 index 0000000..e1b30bc --- /dev/null +++ b/driver/controller/standard/config.go @@ -0,0 +1,16 @@ +package standard + +import ( + "time" + + "sentioxyz/sentio-core/common/envconf" +) + +var ( + enableBindingDataPartition = envconf.LoadBool("SENTIO_ENABLE_BINDING_DATA_PARTITION", false) + minBackfillSlotInterval = envconf.LoadUInt64("SENTIO_MIN_BACKFILL_SLOT_INTERVAL", 1000, envconf.WithMin(1)) + minBackfillTimeInterval = envconf.LoadDuration("SENTIO_MIN_BACKFILL_TIME_INTERVAL", time.Hour, + envconf.WithMinDuration(time.Minute)) + disableAgentTypes = envconf.LoadString("SENTIO_DISABLE_AGENT_TYPES", "") + grpcEnableCompress = envconf.LoadBool("SENTIO_GRPC_ENABLE_COMPRESS", false) +) diff --git a/driver/controller/standard/convert.go b/driver/controller/standard/convert.go new file mode 100644 index 0000000..731fea7 --- /dev/null +++ b/driver/controller/standard/convert.go @@ -0,0 +1,140 @@ +package standard + +import ( + "bytes" + "encoding/json" + "sort" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/structpb" + + "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/timeseries" + "sentioxyz/sentio-core/processor/protos" +) + +func (b *task) ConvertTimeSeriesData(data []*protos.TimeseriesResult) ( + []timeseries.Dataset, + *controller.ExternalError, +) { + dss, err := timeseries.Convert( + b.chainID, + b.GetBlockNumber(), + b.GetBlockHash(), + b.GetBlockTime(), + b.metricConfigs, + data) + if err == nil { + return dss, nil + } + if errors.Is(err, timeseries.ErrInvalidMetaDiff) { + return nil, controller.NewExternalError(controller.ErrCodeTimeSeriesDataSchemaChanged, err) + } + if errors.Is(err, timeseries.ErrInvalidMeta) { + return nil, controller.NewExternalError(controller.ErrCodeInvalidTimeSeriesData, err) + } + // unreachable + panic(err) +} + +func (b *task) ConvertExportData(r []*protos.ExportResult) []controller.WebhookMessage { + result := make([]controller.WebhookMessage, len(r)) + for i, er := range r { + result[i] = controller.WebhookMessage{ + Name: er.GetMetadata().GetName(), + BlockTime: b.GetBlockTime(), + Channel: b.webhookChannels[er.GetMetadata().GetName()], + Payload: er.GetPayload(), + } + } + return result +} + +func ConvertTemplateInstance(r []*protos.TemplateInstance, remove bool) []controller.TemplateInstance { + return utils.MapSliceNoError(r, func(t *protos.TemplateInstance) controller.TemplateInstance { + var labels []byte + if len(t.GetBaseLabels().GetFields()) > 0 { + if raw, err := protojson.Marshal(t.GetBaseLabels()); err == nil { + var buf bytes.Buffer + if json.Compact(&buf, raw) == nil { + labels = buf.Bytes() + } + } + } + return controller.TemplateInstance{ + TemplateID: t.GetTemplateId(), + TemplateName: t.GetContract().GetName(), + Address: t.GetContract().GetAddress(), + BlockRange: controller.BlockRange{ + StartBlock: t.GetStartBlock(), + EndBlock: utils.Select(t.GetEndBlock() == 0, nil, utils.WrapPointer(t.GetEndBlock())), + }, + Labels: string(labels), + Removed: remove, + } + }) +} + +func ConvertTemplateInstanceBack( + chainID string, + templates map[uint64][]controller.TemplateInstance, +) []*protos.TemplateInstance { + dict := make(map[string][]controller.TemplateInstance) + for _, bn := range utils.GetOrderedMapKeys(templates) { + for _, tpl := range templates[bn] { + dict[tpl.UniqID()] = append(dict[tpl.UniqID()], tpl) + } + } + var result []controller.TemplateInstance + for _, tpls := range dict { + on := controller.EmptyBlockRangeSet + for _, tpl := range tpls { + if tpl.Removed { + on = on.Remove(tpl.BlockRange) + } else { + on = on.Union(tpl.BlockRange) + } + } + if on.IsEmpty() { + continue + } + result = append(result, controller.TemplateInstance{ + TemplateID: tpls[0].TemplateID, + TemplateName: tpls[0].TemplateName, + Address: tpls[0].Address, + Labels: tpls[0].Labels, + BlockRange: on.Last(), + }) + } + // sort by (StartBlock, EndBlock, TemplateID, Address, Labels) ASC, result always stable + sort.Slice(result, func(i, j int) bool { + if result[i].BlockRange.StartBlock != result[j].BlockRange.StartBlock { + return result[i].BlockRange.StartBlock < result[j].BlockRange.StartBlock + } + if controller.EqualNilAsInf(result[i].BlockRange.EndBlock, result[j].BlockRange.EndBlock) { + return controller.LessNilAsInf(result[i].BlockRange.EndBlock, result[j].BlockRange.EndBlock) + } + if result[i].TemplateID != result[j].TemplateID { + return result[i].TemplateID < result[j].TemplateID + } + if result[i].Address != result[j].Address { + return result[i].Address < result[j].Address + } + return result[i].Labels < result[j].Labels + }) + return utils.MapSliceNoError(result, func(t controller.TemplateInstance) *protos.TemplateInstance { + var labels *structpb.Struct + if t.Labels != "" { + _ = json.Unmarshal([]byte(t.Labels), &labels) + } + return &protos.TemplateInstance{ + Contract: &protos.ContractInfo{Name: t.TemplateName, ChainId: chainID, Address: t.Address}, + StartBlock: t.StartBlock, + EndBlock: t.EndOrZero(), + TemplateId: t.TemplateID, + BaseLabels: labels, + } + }) +} diff --git a/driver/controller/standard/convert_test.go b/driver/controller/standard/convert_test.go new file mode 100644 index 0000000..bff1fac --- /dev/null +++ b/driver/controller/standard/convert_test.go @@ -0,0 +1,265 @@ +package standard + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/processor/protos" +) + +func indexOf(s, substr string) int { return strings.Index(s, substr) } + +func Test_ConvertTemplateInstance(t *testing.T) { + hundred := uint64(100) + + // basic conversion without labels + assert.Equal(t, []controller.TemplateInstance{ + { + TemplateID: 1, + TemplateName: "MyContract", + Address: "0x1111", + Labels: "", + Removed: false, + BlockRange: controller.BlockRange{StartBlock: 10}, + }, + }, ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Name: "MyContract", Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + }, + }, false)) + + // with EndBlock + assert.Equal(t, []controller.TemplateInstance{ + { + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }, + }, ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + EndBlock: 100, + }, + }, false)) + + // with remove=true + assert.Equal(t, []controller.TemplateInstance{ + { + TemplateID: 1, + Address: "0x1111", + Removed: true, + BlockRange: controller.BlockRange{StartBlock: 10}, + }, + }, ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + }, + }, true)) + + // with BaseLabels — should be marshaled to JSON string + labels, err := structpb.NewStruct(map[string]any{"pid": "abc", "foo": "bar"}) + assert.NoError(t, err) + result := ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + BaseLabels: labels, + }, + }, false) + assert.Len(t, result, 1) + assert.Equal(t, `{"foo":"bar","pid":"abc"}`, result[0].Labels) + + // empty BaseLabels — Labels should be empty string + emptyLabels, err := structpb.NewStruct(map[string]any{}) + assert.NoError(t, err) + assert.Equal(t, []controller.TemplateInstance{ + { + TemplateID: 1, + Address: "0x1111", + Labels: "", + BlockRange: controller.BlockRange{StartBlock: 10}, + }, + }, ConvertTemplateInstance([]*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + TemplateId: 1, + StartBlock: 10, + BaseLabels: emptyLabels, + }, + }, false)) +} + +// Test_ConvertTemplateInstance_LabelsMarshalStability verifies that BaseLabels with multiple +// keys always marshals to the same JSON string (keys sorted alphabetically by protojson), +// so the result can be used as a stable identity key in UniqID. +func Test_ConvertTemplateInstance_LabelsMarshalStability(t *testing.T) { + // Build two Struct values with identical content but keys inserted in different orders. + labelsABC, err := structpb.NewStruct(map[string]any{"a": "1", "b": "2", "c": "3"}) + assert.NoError(t, err) + labelsCBA, err := structpb.NewStruct(map[string]any{"c": "3", "b": "2", "a": "1"}) + assert.NoError(t, err) + + convert := func(s *structpb.Struct) string { + result := ConvertTemplateInstance([]*protos.TemplateInstance{ + {Contract: &protos.ContractInfo{Address: "0x1"}, TemplateId: 1, BaseLabels: s}, + }, false) + return result[0].Labels + } + + gotABC := convert(labelsABC) + gotCBA := convert(labelsCBA) + + // Both must produce identical JSON regardless of insertion order. + assert.Equal(t, gotABC, gotCBA) + + // No unnecessary whitespace — compact JSON only. + assert.NotContains(t, gotABC, " ") + + // Keys must be sorted alphabetically (protojson guarantee): "a" before "b" before "c". + assert.Contains(t, gotABC, `"a"`) + assert.Contains(t, gotABC, `"b"`) + assert.Contains(t, gotABC, `"c"`) + assert.Less(t, indexOf(gotABC, `"a"`), indexOf(gotABC, `"b"`)) + assert.Less(t, indexOf(gotABC, `"b"`), indexOf(gotABC, `"c"`)) + + // Run 20 more times to rule out map-iteration luck. + for range 20 { + assert.Equal(t, gotABC, convert(labelsABC)) + assert.Equal(t, gotABC, convert(labelsCBA)) + } +} + +func Test_ConvertTemplateInstanceBack(t *testing.T) { + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 10, + EndBlock: 0, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10}, + }}, + })) + + hundred := uint64(100) + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 10, + EndBlock: 100, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }}, + })) + + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 10, + EndBlock: 19, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }}, + 12: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 20}, + Removed: true, + }}, + })) + + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 10, + EndBlock: 0, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }}, + 12: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 20}, + Removed: true, + }}, + 15: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 15}, + }}, + })) + + assert.Equal(t, []*protos.TemplateInstance{ + { + Contract: &protos.ContractInfo{Address: "0x1111"}, + StartBlock: 30, + EndBlock: 0, + TemplateId: 1, + }, + }, ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 10, EndBlock: &hundred}, + }}, + 12: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 20}, + Removed: true, + }}, + 15: {{ + TemplateID: 1, + Address: "0x1111", + BlockRange: controller.BlockRange{StartBlock: 30}, + }}, + })) + + // BaseLabels round-trip: use proto.Equal to compare protobuf messages correctly. + expectedLabels, err := structpb.NewStruct(map[string]any{"pid": "abc"}) + assert.NoError(t, err) + gotBack := ConvertTemplateInstanceBack("", map[uint64][]controller.TemplateInstance{ + 10: {{ + TemplateID: 1, + Address: "0x1111", + Labels: `{"pid":"abc"}`, + BlockRange: controller.BlockRange{StartBlock: 10}, + }}, + }) + assert.Len(t, gotBack, 1) + assert.Equal(t, int32(1), gotBack[0].TemplateId) + assert.Equal(t, "0x1111", gotBack[0].Contract.GetAddress()) + assert.Equal(t, uint64(10), gotBack[0].StartBlock) + assert.True(t, proto.Equal(expectedLabels, gotBack[0].BaseLabels)) +} diff --git a/driver/controller/standard/evm/BUILD.bazel b/driver/controller/standard/evm/BUILD.bazel new file mode 100644 index 0000000..f8b9b53 --- /dev/null +++ b/driver/controller/standard/evm/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "evm", + srcs = [ + "block_data.go", + "handler.go", + "handler_interval.go", + "handler_log.go", + "handler_trace.go", + "handler_transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/evm", + visibility = ["//visibility:public"], + deps = [ + "//common/log", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/evm", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor/protos", + "//service/processor/models", + "@com_github_ethereum_go_ethereum//core/types", + "@com_github_pkg_errors//:errors", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) diff --git a/driver/controller/standard/evm/block_data.go b/driver/controller/standard/evm/block_data.go new file mode 100644 index 0000000..41ef49d --- /dev/null +++ b/driver/controller/standard/evm/block_data.go @@ -0,0 +1,93 @@ +package evm + +import ( + "encoding/json" + + "sentioxyz/sentio-core/driver/controller" + evmExtend "sentioxyz/sentio-core/driver/controller/data/evm" +) + +type BlockData struct { + evmExtend.BlockHeader + + mainData evmExtend.BlockMainData + extendData evmExtend.BlockExtendData + + headerJSON string + txnJSON map[string]string + receiptJSON map[string]string + receiptWithLogsJSON map[string]string + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (b *BlockData) DataSource() string { + return b.dataSource +} + +func (b *BlockData) CheckpointData() map[string]string { + return b.checkpointData +} + +func (b *BlockData) Size() int { + return b.taskTotalSize +} + +func (b *BlockData) GetTaskList() []controller.Task { + return b.taskList +} + +func (b *BlockData) getHeaderJSON() string { + if b.headerJSON == "" { + b.headerJSON = string(b.BlockHeader.Raw) + } + return b.headerJSON +} + +func (b *BlockData) getTransactionJSON(txHash string) string { + if b.txnJSON == nil { + b.txnJSON = make(map[string]string) + } + if r, has := b.txnJSON[txHash]; has { + return r + } + if tx, has := b.extendData.Transactions[txHash]; !has { + return "" + } else { + r, _ := json.Marshal(tx) + b.txnJSON[txHash] = string(r) + return b.txnJSON[txHash] + } +} + +func (b *BlockData) getReceiptJSON(txHash string, withLogs bool) string { + var cache map[string]string + if withLogs { + if b.receiptWithLogsJSON == nil { + b.receiptWithLogsJSON = make(map[string]string) + } + cache = b.receiptWithLogsJSON + } else { + if b.receiptJSON == nil { + b.receiptJSON = make(map[string]string) + } + cache = b.receiptJSON + } + if pb, has := cache[txHash]; has { + return pb + } + if receipt, has := b.extendData.Receipts[txHash]; !has { + return "" + } else { + if !withLogs { + receipt.Logs = nil + } + r, _ := json.Marshal(receipt) + cache[txHash] = string(r) + return cache[txHash] + } +} diff --git a/driver/controller/standard/evm/handler.go b/driver/controller/standard/evm/handler.go new file mode 100644 index 0000000..16aa0bc --- /dev/null +++ b/driver/controller/standard/evm/handler.go @@ -0,0 +1,313 @@ +package evm + +import ( + "context" + "fmt" + "strings" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" +) + +type EvmHandlerAgent interface { + standard.HandlerAgent[*BlockData] + + GetExtendRequirements(context.Context, *BlockData) (evm.BlockExtendRequirement, error) +} + +type HandlerController struct { + *standard.BaseHandlerController[evm.Client, *BlockData, EvmHandlerAgent] +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *config.ChainConfig, + client evm.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[evm.Client, *BlockData, EvmHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + + fetchNamePrefix := fmt.Sprintf("EVM::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + evm.BuildBlockMainDataFetcher(fetchNamePrefix, req, firstBlockNumber, currentBlockNumber, latest, c.Client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, // 256MB + 100, + time.Second*10, + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from evm.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + _, logger := log.FromContext(ctx) + logger.Debugf("will build block data in block #%d with %d logs %d traces %d intervals in main data", + blockNumber, len(from.Logs), len(from.Traces), len(from.Intervals)) + var err error + result := BlockData{mainData: from, checkpointData: make(map[string]string)} + // always need header + if result.BlockHeader, err = c.Client.GetHeader(ctx, blockNumber); err != nil { + return nil, false, err + } + // check block hash of main data with the header got above + for _, l := range from.Logs { + if l.BlockHash.String() != result.GetBlockHash() { + return nil, false, fetcher.Permanent(errors.Errorf("invalid block hash of the log %s, expected is %s", + l.BlockHash.String(), controller.GetBlockSummary(result.BlockHeader))) + } + } + for _, t := range from.Traces { + if t.BlockHash != result.GetBlockHash() { + return nil, false, fetcher.Permanent(errors.Errorf("invalid block hash of the trace %s, expected is %s", + t.BlockHash, controller.GetBlockSummary(result.BlockHeader))) + } + } + // take the main data and ask the handler controller what extend data is needed + var r evm.BlockExtendRequirement + if r, err = c.getBlockExtendRequirements(ctx, &result); err != nil { + return nil, false, err + } + // actually get the extended data + if result.extendData, err = c.Client.GetBlock(ctx, blockNumber, r); err != nil { + return nil, false, err + } + // build binding data + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + return nil, false, err + } + logger.Debugf("built %d task in block #%d with handlerIDs %v", + len(result.taskList), + blockNumber, + utils.Stat(utils.MapSliceNoError( + utils.MapSliceNoError(result.taskList, controller.Task.GetHandlerID), + controller.HandlerID.String, + ))) + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start, latest uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetContractStartBlock(ctx, address, start, latest) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + return latest + 1, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, latest uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + var err error + + for dataSourceID, accountConfig := range c.Config.AccountConfigs { + accountAddress := standard.AdjustAddress(accountConfig.GetAddress()) + dataSource := standard.BuildDataSource("EVM", c.ChainConfig.ChainID, "Account", accountAddress) + blockRange := controller.BlockRange{ + StartBlock: max(accountConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(accountConfig.GetEndBlock()), + } + for _, logConfig := range accountConfig.GetLogConfigs() { + agent := HandlerAgentLog{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "log", logConfig, blockRange), + Client: c.Client, + FetchConfig: logConfig.GetFetchConfig(), + } + agent.Filters, err = NewLogFilters(logConfig.GetFilters(), true, "") + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + var hasTxnHandler bool + for dataSourceID, contractConfig := range c.Config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + dataSource := standard.BuildDataSource("EVM", c.ChainConfig.ChainID, "Contract", contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + if blockRange.StartBlock, err = c.getAddressStart(ctx, contractAddress, blockRange.StartBlock, latest); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + for _, logConfig := range contractConfig.GetLogConfigs() { + agent := HandlerAgentLog{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "log", logConfig, blockRange), + Client: c.Client, + FetchConfig: logConfig.GetFetchConfig(), + } + agent.Filters, err = NewLogFilters(logConfig.GetFilters(), false, contractAddress) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, traceConfig := range contractConfig.GetTraceConfigs() { + agent := HandlerAgentTrace{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "trace", traceConfig, blockRange), + FetchConfig: traceConfig.GetFetchConfig(), + } + if contractAddress != "" { + agent.Filter.Address = []string{strings.ToLower(contractAddress)} + } + if traceConfig.GetSignature() != "" { + agent.Filter.Signature = []string{traceConfig.GetSignature()} + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + if len(contractConfig.GetTransactionConfig()) > 0 { + if contractAddress != "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("transaction handler only support global processor")) + } + if len(contractConfig.GetTransactionConfig()) > 1 || hasTxnHandler { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("there can only be one transaction handler")) + } + txnConfig := contractConfig.GetTransactionConfig()[0] + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "transaction", txnConfig, blockRange), + FetchConfig: txnConfig.GetFetchConfig(), + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + hasTxnHandler = true + } + + for _, intervalConfig := range contractConfig.GetIntervalConfigs() { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig, blockRange), + FetchConfig: intervalConfig.GetFetchConfig(), + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + logger.Infof("built %d agents", len(c.Agents)) + return nil +} + +func (c *HandlerController) getDataRequirement() (dr evm.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentTransaction: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: data.IntervalConfig{ + BlockInterval: &data.BlockInterval{Backfill: 1, Watching: 1}, + }, + BlockRange: ag.Range, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + case HandlerAgentTrace: + dr.Trace = append(dr.Trace, evm.TraceRequirement{ + TraceFilter: ag.Filter, + BlockRange: ag.Range, + }) + case HandlerAgentLog: + dr.Log = append(dr.Log, evm.LogRequirement{ + LogFilter: utils.Reduce(ag.Filters, evm.LogFilter.Merge), + BlockRange: ag.Range, + }) + } + } + return dr +} + +func (c *HandlerController) getBlockExtendRequirements( + ctx context.Context, + blockData *BlockData, +) (req evm.BlockExtendRequirement, err error) { + var ar evm.BlockExtendRequirement + for _, agent := range c.Agents { + if ar, err = agent.GetExtendRequirements(ctx, blockData); err != nil { + return + } + req.Merge(ar) + } + return +} diff --git a/driver/controller/standard/evm/handler_interval.go b/driver/controller/standard/evm/handler_interval.go new file mode 100644 index 0000000..e782818 --- /dev/null +++ b/driver/controller/standard/evm/handler_interval.go @@ -0,0 +1,132 @@ +package evm + +import ( + "context" + "encoding/json" + "math" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + FetchConfig *protos.EthFetchConfig + IntervalConfig data.IntervalConfig +} + +func (a HandlerAgentInterval) GetExtendRequirements( + _ context.Context, + d *BlockData, +) (evm.BlockExtendRequirement, error) { + var r evm.BlockExtendRequirement + if !a.Range.Contains(d.GetBlockNumber()) { + return r, nil + } + if !a.FetchConfig.GetTransaction() && + !a.FetchConfig.GetTransactionReceipt() && + !a.FetchConfig.GetTransactionReceiptLogs() { + return r, nil + } + if !data.ContainsInterval(d.mainData.Intervals, a.IntervalConfig) { + return r, nil + } + if a.FetchConfig.GetTransaction() { + r.AllTransactions = true + } + if a.FetchConfig.GetTransactionReceipt() { + r.AllTransactionReceipts = true + } + if a.FetchConfig.GetTransactionReceiptLogs() { + r.AllTransactionReceiptLogs = true + } + if a.FetchConfig.GetTrace() { + r.AllTraces = true + } + return r, nil +} + +func (a HandlerAgentInterval) BuildBindingDataList( + _ context.Context, + d *BlockData, +) ([]standard.BindingDataInner, error) { + if !data.ContainsInterval(d.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + rawBlock := d.getHeaderJSON() + if a.FetchConfig.GetTransaction() || a.FetchConfig.GetTransactionReceipt() || a.FetchConfig.GetTrace() { + // Splice transactions / receipts / traces (each already raw JSON) into the header + // object to build the composite raw_block. Header fields are kept as raw bytes. + var block map[string]json.RawMessage + if err := json.Unmarshal(d.BlockHeader.Raw, &block); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal block header %d", d.GetBlockNumber()) + } + if a.FetchConfig.GetTransaction() { + txns := make([]json.RawMessage, 0, len(d.BlockHeader.TxHashes)) + for _, txHash := range d.BlockHeader.TxHashes { + txns = append(txns, json.RawMessage(d.getTransactionJSON(txHash))) + } + raw, err := json.Marshal(txns) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal transactions for block %d", d.GetBlockNumber()) + } + block["transactions"] = raw + } + if a.FetchConfig.GetTransactionReceipt() { + receipts := make([]json.RawMessage, 0, len(d.BlockHeader.TxHashes)) + for _, txHash := range d.BlockHeader.TxHashes { + receipts = append(receipts, json.RawMessage(d.getReceiptJSON(txHash, a.FetchConfig.GetTransactionReceiptLogs()))) + } + raw, err := json.Marshal(receipts) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal transaction receipts for block %d", d.GetBlockNumber()) + } + block["transactionReceipts"] = raw + } + if a.FetchConfig.GetTrace() { + var traces []json.RawMessage + for _, txHash := range d.BlockHeader.TxHashes { + for _, trace := range d.extendData.Traces[txHash] { + traces = append(traces, json.RawMessage(trace.Raw)) + } + } + raw, err := json.Marshal(traces) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal traces for block %d", d.GetBlockNumber()) + } + block["traces"] = raw + } + raw, err := json.Marshal(block) + if err != nil { + return nil, errors.Wrapf(err, "failed to marshal block %d", d.GetBlockNumber()) + } + rawBlock = string(raw) + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_ETH_BLOCK, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_EthBlock_{ + EthBlock: &protos.Data_EthBlock{ + RawBlock: rawBlock, + }, + }, + }, + DataSize: len(rawBlock), + }}, nil +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/evm/handler_log.go b/driver/controller/standard/evm/handler_log.go new file mode 100644 index 0000000..24bdf25 --- /dev/null +++ b/driver/controller/standard/evm/handler_log.go @@ -0,0 +1,158 @@ +package evm + +import ( + "context" + "encoding/json" + "strings" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentLog struct { + controller.BaseHandlerAgent + + Client evm.Client `json:"-"` // used to check address is a ERC20 address + + FetchConfig *protos.EthFetchConfig + Filters []evm.LogFilter // linked by OR +} + +func (a HandlerAgentLog) GetExtendRequirements( + ctx context.Context, + d *BlockData, +) (evm.BlockExtendRequirement, error) { + var r evm.BlockExtendRequirement + if !a.Range.Contains(d.GetBlockNumber()) { + return r, nil + } + if !a.FetchConfig.GetTransaction() && + !a.FetchConfig.GetTransactionReceipt() && + !a.FetchConfig.GetTransactionReceiptLogs() { + return r, nil + } + + logs, err := evm.FilterLogs(ctx, a.Client, d.mainData.Logs, a.Filters...) + if err != nil { + return r, err + } + txnSet := make(map[string]bool) + for _, log := range logs { + txnSet[log.TxHash.String()] = true + } + for txnHash := range txnSet { + if a.FetchConfig.GetTransaction() { + r.SpecialTransactions = append(r.SpecialTransactions, txnHash) + } + if a.FetchConfig.GetTransactionReceipt() { + r.SpecialTransactionReceipts = append(r.SpecialTransactionReceipts, txnHash) + } + if a.FetchConfig.GetTransactionReceiptLogs() { + r.SpecialTransactionReceiptLogs = append(r.SpecialTransactionReceiptLogs, txnHash) + } + } + return r, nil +} + +func (a HandlerAgentLog) BuildBindingDataList( + ctx context.Context, + d *BlockData, +) (r []standard.BindingDataInner, err error) { + var logs []types.Log + logs, err = evm.FilterLogs(ctx, a.Client, d.mainData.Logs, a.Filters...) + if err != nil { + return nil, err + } + for _, log := range logs { + var rawLog string + var raw []byte + if raw, err = json.Marshal(&log); err != nil { + return nil, err + } else { + rawLog = string(raw) + } + size := len(rawLog) + var rawBlock *string + if a.FetchConfig.GetBlock() { + rawBlock = new(d.getHeaderJSON()) + size += len(*rawBlock) + } + var rawTransaction *string + if a.FetchConfig.GetTransaction() { + rawTransaction = new(d.getTransactionJSON(log.TxHash.String())) + size += len(*rawTransaction) + } + var rawReceipt *string + if a.FetchConfig.GetTransactionReceipt() { + rawReceipt = new(d.getReceiptJSON(log.TxHash.String(), a.FetchConfig.GetTransactionReceiptLogs())) + size += len(*rawReceipt) + } + data := standard.BindingDataInner{ + HandlerType: protos.HandlerType_ETH_LOG, + TxIndex: int(log.TxIndex), + TxInnerIndex: int(log.Index), + Data: &protos.Data{ + Value: &protos.Data_EthLog_{ + EthLog: &protos.Data_EthLog{ + Timestamp: timestamppb.New(d.GetBlockTime()), + RawLog: rawLog, + RawBlock: rawBlock, + RawTransaction: rawTransaction, + RawTransactionReceipt: rawReceipt, + }, + }, + }, + DataSize: size, + } + r = append(r, data) + } + return +} + +func (a HandlerAgentLog) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filters": a.Filters, + "FetchConfig": a.FetchConfig, + } +} + +func NewLogFilters(filters []*protos.LogFilter, accountLogFilter bool, contractAddress string) ([]evm.LogFilter, error) { + if len(filters) == 0 { + return nil, errors.Errorf("filters is empty") + } + result := make([]evm.LogFilter, len(filters)) + for i, filter := range filters { + result[i] = evm.LogFilter{ + Topics: utils.MapSliceNoError(filter.Topics, func(t *protos.Topic) []string { + return t.GetHashes() + }), + } + if accountLogFilter { + switch v := filter.AddressOrType.(type) { + case *protos.LogFilter_AddressType: + at := v.AddressType + if at == protos.AddressType_ERC20 { + result[i].AddressShouldBeERC20 = true + } else { + return nil, errors.Errorf("unsupported address type %s", at.String()) + } + case *protos.LogFilter_Address: + result[i].Address = []string{strings.ToLower(v.Address)} + default: + return nil, errors.Errorf("unsupported address type %T", v) + } + } else if contractAddress != "" { + result[i].Address = []string{strings.ToLower(contractAddress)} + } + } + return result, nil +} diff --git a/driver/controller/standard/evm/handler_trace.go b/driver/controller/standard/evm/handler_trace.go new file mode 100644 index 0000000..701c2ec --- /dev/null +++ b/driver/controller/standard/evm/handler_trace.go @@ -0,0 +1,114 @@ +package evm + +import ( + "context" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentTrace struct { + controller.BaseHandlerAgent + + FetchConfig *protos.EthFetchConfig + Filter evm.TraceFilter +} + +func (a HandlerAgentTrace) GetExtendRequirements( + _ context.Context, + d *BlockData, +) (evm.BlockExtendRequirement, error) { + var r evm.BlockExtendRequirement + if !a.Range.Contains(d.GetBlockNumber()) { + return r, nil + } + if !a.FetchConfig.GetTransaction() && + !a.FetchConfig.GetTransactionReceipt() && + !a.FetchConfig.GetTransactionReceiptLogs() { + return r, nil + } + txnSet := make(map[string]bool) + for _, trace := range d.mainData.Traces { + if a.Filter.Check(trace) { + txnSet[trace.TransactionHash] = true + } + } + for txnHash := range txnSet { + if a.FetchConfig.GetTransaction() { + r.SpecialTransactions = append(r.SpecialTransactions, txnHash) + } + if a.FetchConfig.GetTransactionReceipt() { + r.SpecialTransactionReceipts = append(r.SpecialTransactionReceipts, txnHash) + } + if a.FetchConfig.GetTransactionReceiptLogs() { + r.SpecialTransactionReceiptLogs = append(r.SpecialTransactionReceiptLogs, txnHash) + } + } + return r, nil +} + +func (a HandlerAgentTrace) BuildBindingDataList( + ctx context.Context, + d *BlockData, +) (r []standard.BindingDataInner, err error) { + for _, trace := range d.mainData.Traces { + if !a.Filter.Check(trace) { + continue + } + rawTrace := string(trace.Raw) + var rawBlock, rawTransaction, rawReceipt *string + if a.FetchConfig.GetBlock() { + s := d.getHeaderJSON() + rawBlock = &s + } + if a.FetchConfig.GetTransaction() { + s := d.getTransactionJSON(trace.TransactionHash) + rawTransaction = &s + } + if a.FetchConfig.GetTransactionReceipt() { + s := d.getReceiptJSON(trace.TransactionHash, a.FetchConfig.GetTransactionReceiptLogs()) + rawReceipt = &s + } + dataSize := len(rawTrace) + if rawBlock != nil { + dataSize += len(*rawBlock) + } + if rawTransaction != nil { + dataSize += len(*rawTransaction) + } + if rawReceipt != nil { + dataSize += len(*rawReceipt) + } + data := standard.BindingDataInner{ + HandlerType: protos.HandlerType_ETH_TRACE, + TxIndex: int(trace.TransactionIndex), + Data: &protos.Data{ + Value: &protos.Data_EthTrace_{ + EthTrace: &protos.Data_EthTrace{ + Timestamp: timestamppb.New(d.GetBlockTime()), + RawTrace: rawTrace, + RawBlock: rawBlock, + RawTransaction: rawTransaction, + RawTransactionReceipt: rawReceipt, + }, + }, + }, + DataSize: dataSize, + } + r = append(r, data) + } + return +} + +func (a HandlerAgentTrace) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/evm/handler_transaction.go b/driver/controller/standard/evm/handler_transaction.go new file mode 100644 index 0000000..746acf8 --- /dev/null +++ b/driver/controller/standard/evm/handler_transaction.go @@ -0,0 +1,71 @@ +package evm + +import ( + "context" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentTransaction struct { + controller.BaseHandlerAgent + + FetchConfig *protos.EthFetchConfig +} + +func (a HandlerAgentTransaction) GetExtendRequirements( + _ context.Context, + d *BlockData, +) (r evm.BlockExtendRequirement, err error) { + if !a.Range.Contains(d.GetBlockNumber()) { + return r, nil + } + r.AllTransactions = true + r.AllTransactionReceipts = a.FetchConfig.GetTransactionReceipt() + r.AllTransactionReceiptLogs = a.FetchConfig.GetTransactionReceiptLogs() + return r, nil +} + +func (a HandlerAgentTransaction) BuildBindingDataList( + ctx context.Context, + d *BlockData, +) (r []standard.BindingDataInner, err error) { + for txIndex, txHash := range d.BlockHeader.TxHashes { + rawTransaction := d.getTransactionJSON(txHash) + rawBlock := new(d.getHeaderJSON()) + size := len(rawTransaction) + len(*rawBlock) + var rawReceipt *string + if a.FetchConfig.GetTransactionReceipt() { + rawReceipt = new(d.getReceiptJSON(txHash, a.FetchConfig.GetTransactionReceiptLogs())) + size += len(*rawReceipt) + } + data := standard.BindingDataInner{ + HandlerType: protos.HandlerType_ETH_TRANSACTION, + TxIndex: txIndex, + Data: &protos.Data{ + Value: &protos.Data_EthTransaction_{ + EthTransaction: &protos.Data_EthTransaction{ + Timestamp: timestamppb.New(d.GetBlockTime()), + RawTransaction: rawTransaction, + RawBlock: rawBlock, + RawTransactionReceipt: rawReceipt, + }, + }, + }, + DataSize: size, + } + r = append(r, data) + } + return +} +func (a HandlerAgentTransaction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/fuel/BUILD.bazel b/driver/controller/standard/fuel/BUILD.bazel new file mode 100644 index 0000000..ebf84af --- /dev/null +++ b/driver/controller/standard/fuel/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "fuel", + srcs = [ + "block_data.go", + "handler.go", + "handler_interval.go", + "handler_receipt.go", + "handler_transaction.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/fuel", + visibility = ["//visibility:public"], + deps = [ + "//chain/fuel", + "//common/log", + "//common/protojson", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/fuel", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor/protos", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_fuel_go//types", + "@org_golang_google_protobuf//types/known/structpb", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) diff --git a/driver/controller/standard/fuel/block_data.go b/driver/controller/standard/fuel/block_data.go new file mode 100644 index 0000000..f1a453a --- /dev/null +++ b/driver/controller/standard/fuel/block_data.go @@ -0,0 +1,78 @@ +package fuel + +import ( + "encoding/json" + "reflect" + + "github.com/pkg/errors" + "github.com/sentioxyz/fuel-go/types" + "google.golang.org/protobuf/types/known/structpb" + + "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/fuel" +) + +type BlockData struct { + fuel.Block + + mainData fuel.BlockMainData + + blockPb *structpb.Struct + blockPbSize int + transactionPb []*structpb.Struct + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (d *BlockData) GetTaskList() []controller.Task { + return d.taskList +} + +func (d *BlockData) CheckpointData() map[string]string { + return d.checkpointData +} + +func (d *BlockData) DataSource() string { + return d.dataSource +} + +func (d *BlockData) Size() int { + return d.taskTotalSize +} + +func (d *BlockData) getBlockPb() (*structpb.Struct, int, error) { + if d.blockPb == nil { + blockPb := new(structpb.Struct) + j, err := json.Marshal(d.Block) + if err != nil { + return nil, 0, errors.Wrapf(err, "marshal header of block %d failed", d.GetBlockNumber()) + } + err = protojson.Unmarshal(j, blockPb) + if err != nil { + return nil, 0, errors.Wrapf(err, "build structpb of block %d failed", d.GetBlockNumber()) + } + d.blockPb, d.blockPbSize = blockPb, len(j) + } + return d.blockPb, d.blockPbSize, nil +} + +var fuelTxTyp = reflect.TypeOf(types.Transaction{}) + +func (d *BlockData) getTxPb(i int) *structpb.Struct { + if i >= len(d.mainData.Txs) { + panic(errors.Errorf("index %d out of range [0,%d) in BlockData #%d", i, len(d.mainData.Txs), d.GetBlockNumber())) + } + if len(d.transactionPb) == 0 { + d.transactionPb = make([]*structpb.Struct, len(d.mainData.Txs)) + } + if d.transactionPb[i] == nil { + d.transactionPb[i] = utils.ConvertToStructpb(&d.mainData.Txs[i].Transaction, fuelTxTyp) + } + return d.transactionPb[i] +} diff --git a/driver/controller/standard/fuel/handler.go b/driver/controller/standard/fuel/handler.go new file mode 100644 index 0000000..1454157 --- /dev/null +++ b/driver/controller/standard/fuel/handler.go @@ -0,0 +1,277 @@ +package fuel + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + + chainFuel "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/fuel" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" +) + +type FuelHandlerAgent interface { + standard.HandlerAgent[*BlockData] +} + +type HandlerController struct { + *standard.BaseHandlerController[fuel.Client, *BlockData, FuelHandlerAgent] +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *config.ChainConfig, + client fuel.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[fuel.Client, *BlockData, FuelHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if extErr := c.BaseHandlerController.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start, latest uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetContractCreateBlockHeight(ctx, address, start) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + return latest + 1, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, latest uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + var err error + + for dataSourceID, contractConfig := range c.Config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + dataSource := standard.BuildDataSource("FUEL", c.ChainConfig.ChainID, "Contract", contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + if blockRange.StartBlock, err = c.getAddressStart(ctx, contractAddress, blockRange.StartBlock, latest); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + // interval + for _, intervalConfig := range contractConfig.IntervalConfigs { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig, blockRange), + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + // asset transfer + for _, assetConfig := range contractConfig.AssetConfigs { + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "transfer", assetConfig, blockRange), + Filters: make([]chainFuel.TransactionFilter, len(assetConfig.Filters)), + } + if len(assetConfig.Filters) == 0 { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("no filter for handler %s", agent.GetHandlerID().String())) + } + for i, filterConfig := range assetConfig.Filters { + agent.Filters[i] = chainFuel.TransactionFilter{ + TransferFilter: &chainFuel.TransferFilter{ + AssetID: strings.ToLower(filterConfig.GetAssetId()), + From: strings.ToLower(filterConfig.GetFromAddress()), + To: strings.ToLower(filterConfig.GetToAddress()), + }, + ExcludeFailed: true, + } + if contractAddress != "" { + agent.Filters[i].CallFilter = &chainFuel.CallFilter{ + ContractID: contractAddress, + } + } + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + // all transactions + for _, txConfig := range contractConfig.FuelTransactionConfigs { + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "transaction", txConfig, blockRange), + Filters: []chainFuel.TransactionFilter{{ + CallFilter: &chainFuel.CallFilter{ + ContractID: contractAddress, + }, + ExcludeFailed: true, + }}, + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + // receipt + for _, receiptConfig := range contractConfig.FuelReceiptConfigs { + if receiptConfig.GetLog() != nil { + // log + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "log", receiptConfig, blockRange), + Filters: make([]chainFuel.TransactionFilter, len(receiptConfig.GetLog().GetLogIds())), + } + for i, logID := range receiptConfig.GetLog().GetLogIds() { + rb, parseErr := strconv.ParseUint(logID, 0, 64) + if parseErr != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, errors.Wrapf(parseErr, + "parse logId %q to uint failed for handler %s", logID, agent.GetHandlerID().String())) + } + agent.Filters[i] = chainFuel.TransactionFilter{ + CallFilter: &chainFuel.CallFilter{ + ContractID: contractAddress, + }, + LogFilter: &chainFuel.LogFilter{ + ContractID: contractAddress, + LogRb: &rb, + }, + ExcludeFailed: true, + } + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + if receiptConfig.GetTransfer() != nil { + // receipt transfer + agent := HandlerAgentTransaction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "receiptTransfer", receiptConfig, blockRange), + Filters: []chainFuel.TransactionFilter{{ + CallFilter: &chainFuel.CallFilter{ + ContractID: contractAddress, + }, + ReceiptTransferFilter: &chainFuel.ReceiptTransferFilter{ + AssetID: receiptConfig.GetTransfer().GetAssetId(), + From: receiptConfig.GetTransfer().GetFrom(), + To: receiptConfig.GetTransfer().GetTo(), + }, + ExcludeFailed: true, + }}, + } + c.Agents = append(c.Agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + } + return nil +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + + fetchNamePrefix := fmt.Sprintf("FUEL::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + fuel.BuildBlockMainDataFetcher(fetchNamePrefix, req, firstBlockNumber, currentBlockNumber, latest, c.Client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, // 256MB + 100, + time.Second*3, + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from fuel.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + result := BlockData{mainData: from, checkpointData: make(map[string]string)} + // always need header + if result.Block, err = c.Client.GetBlock(ctx, blockNumber); err != nil { + return nil, false, err + } + // build binding data + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + return nil, false, err + } + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) +} + +func (c *HandlerController) getDataRequirement() (dr fuel.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentTransaction: + dr.Tx = append(dr.Tx, fuel.TransactionRequirement{ + Filters: ag.Filters, + BlockRange: ag.Range, + }) + case HandlerAgentReceipt: + dr.Tx = append(dr.Tx, fuel.TransactionRequirement{ + Filters: ag.Filters, + BlockRange: ag.Range, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } + } + return dr +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} diff --git a/driver/controller/standard/fuel/handler_interval.go b/driver/controller/standard/fuel/handler_interval.go new file mode 100644 index 0000000..33e3cad --- /dev/null +++ b/driver/controller/standard/fuel/handler_interval.go @@ -0,0 +1,53 @@ +package fuel + +import ( + "context" + "math" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + IntervalConfig data.IntervalConfig +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + } +} + +func (a HandlerAgentInterval) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + blockPb, size, err := bd.getBlockPb() + if err != nil { + return nil, err + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_FUEL_BLOCK, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_FuelBlock_{ + FuelBlock: &protos.Data_FuelBlock{ + Block: blockPb, + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: size, + }}, nil +} diff --git a/driver/controller/standard/fuel/handler_receipt.go b/driver/controller/standard/fuel/handler_receipt.go new file mode 100644 index 0000000..e62ad45 --- /dev/null +++ b/driver/controller/standard/fuel/handler_receipt.go @@ -0,0 +1,73 @@ +package fuel + +import ( + "context" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentReceipt struct { + controller.BaseHandlerAgent + + Filters []fuel.TransactionFilter +} + +func (a HandlerAgentReceipt) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filters": a.Filters, + } +} + +func (a HandlerAgentReceipt) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for i, tx := range bd.mainData.Txs { + ok := utils.HasAny(a.Filters, func(filter fuel.TransactionFilter) bool { + return filter.Check(tx.Transaction) + }) + if !ok { + continue + } + receiptIndexes := make(map[int]bool) + receipts := fuel.GetTxnReceipt(tx.Status) + for _, filter := range a.Filters { + if filter.LogFilter != nil { + for _, receiptIndex := range filter.LogFilter.Check(receipts) { + receiptIndexes[receiptIndex] = true + } + } + if filter.ReceiptTransferFilter != nil { + for _, receiptIndex := range filter.ReceiptTransferFilter.Check(receipts) { + receiptIndexes[receiptIndex] = true + } + } + } + for _, receiptIndex := range utils.GetOrderedMapKeys(receiptIndexes) { + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_FUEL_RECEIPT, + TxIndex: int(tx.TransactionIndex), + TxInnerIndex: receiptIndex, + Data: &protos.Data{ + Value: &protos.Data_FuelLog{ + FuelLog: &protos.Data_FuelReceipt{ + Transaction: bd.getTxPb(i), + ReceiptIndex: int64(receiptIndex), + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: 1000, + }) + } + } + return result, nil +} diff --git a/driver/controller/standard/fuel/handler_transaction.go b/driver/controller/standard/fuel/handler_transaction.go new file mode 100644 index 0000000..0bf319c --- /dev/null +++ b/driver/controller/standard/fuel/handler_transaction.go @@ -0,0 +1,55 @@ +package fuel + +import ( + "context" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/fuel" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentTransaction struct { + controller.BaseHandlerAgent + + Filters []fuel.TransactionFilter +} + +func (a HandlerAgentTransaction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filters": a.Filters, + } +} + +func (a HandlerAgentTransaction) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for i, tx := range bd.mainData.Txs { + ok := utils.HasAny(a.Filters, func(filter fuel.TransactionFilter) bool { + return filter.Check(tx.Transaction) + }) + if !ok { + continue + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_FUEL_TRANSACTION, + TxIndex: int(tx.TransactionIndex), + Data: &protos.Data{ + Value: &protos.Data_FuelTransaction_{ + FuelTransaction: &protos.Data_FuelTransaction{ + Transaction: bd.getTxPb(i), + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: 1000, + }) + } + return result, nil +} diff --git a/driver/controller/standard/handler.go b/driver/controller/standard/handler.go new file mode 100644 index 0000000..8fe65cc --- /dev/null +++ b/driver/controller/standard/handler.go @@ -0,0 +1,335 @@ +package standard + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/timeseries" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/encoding/gzip" +) + +type BindingDataInner struct { + Data *protos.Data + DataSize int + HandlerType protos.HandlerType + TxIndex int + TxInnerIndex int +} + +type HandlerAgent[BKD controller.BlockHeader] interface { + controller.HandlerAgent + + BuildBindingDataList(ctx context.Context, bd BKD) ([]BindingDataInner, error) +} + +type streamPool chan protos.ProcessorV3_ProcessBindingsStreamClient + +type HandlerConfig struct { + ContractConfigs []*protos.ContractConfig + AccountConfigs []*protos.AccountConfig +} + +func (c HandlerConfig) String() string { + return utils.MustJSONMarshal(c) +} + +type BaseHandlerController[CLI controller.Client, BKD controller.BlockHeader, HA HandlerAgent[BKD]] struct { + Processor *models.Processor + InitResult *protos.InitResponse + ChainConfig *config.ChainConfig + Client CLI + + Config HandlerConfig // result of SetTemplates + Agents []HA // built from Config + + metricConfigs map[timeseries.MetaType]map[string]*protos.MetricConfig // load from InitResult + webhookChannels map[string]string // load from InitResult + + addressStart map[string]uint64 + addressStartData string + + processorClients []protos.ProcessorV3Client + processStreams streamPool + waiter *waiter +} + +func NewBaseHandlerController[CLI controller.Client, BKD controller.BlockHeader, HA HandlerAgent[BKD]]( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *config.ChainConfig, + client CLI, + processorClients []protos.ProcessorV3Client, +) *BaseHandlerController[CLI, BKD, HA] { + webhookChannels := make(map[string]string) + for _, wc := range initResult.GetExportConfigs() { + webhookChannels[wc.GetName()] = wc.GetChannel() + } + return &BaseHandlerController[CLI, BKD, HA]{ + Processor: processor, + InitResult: initResult, + ChainConfig: chainConfig, + Client: client, + metricConfigs: timeseries.BuildMetricConfigs(initResult.GetMetricConfigs()), + webhookChannels: webhookChannels, + processorClients: processorClients, + } +} + +func (c *BaseHandlerController[CLI, BKD, HA]) SetTemplates( + ctx context.Context, + templates map[uint64][]controller.TemplateInstance, +) *controller.ExternalError { + _, logger := log.FromContext(ctx) + req := &protos.UpdateTemplatesRequest{ + ChainId: c.ChainConfig.ChainID, + TemplateInstances: ConvertTemplateInstanceBack(c.ChainConfig.ChainID, templates), + } + + var confText string + for i, cli := range c.processorClients { + // update templates + if _, err := cli.UpdateTemplates(ctx, req); err != nil { + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("update templates failed: %v", err)) + } + // get handler config + var config HandlerConfig + if resp, err := cli.GetConfig(ctx, &protos.ProcessConfigRequest{}); err != nil { + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("get handler config failed: %v", err)) + } else { + config = HandlerConfig{ + ContractConfigs: utils.FilterArr(resp.GetContractConfigs(), func(cc *protos.ContractConfig) bool { + return cc.GetContract().GetChainId() == c.ChainConfig.ChainID + }), + AccountConfigs: utils.FilterArr(resp.GetAccountConfigs(), func(cc *protos.AccountConfig) bool { + return cc.GetChainId() == c.ChainConfig.ChainID + }), + } + } + // check handler config + if i == 0 { + c.Config, confText = config, config.String() + } else if another := config.String(); confText != another { + logger.Errorw("configs from different processor has diff", "config1", confText, "config2", another) + return controller.NewExternalError(controller.ErrCodeProcessorConfigsHasDiff, + errors.Errorf("configs from different processor has diff")) + } + } + + logger.Infow("got config", "config", confText) + return nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) PrepareExecute(ctx context.Context) *controller.ExternalError { + clientCount := uint64(len(c.processorClients)) + streamSize := max(controller.ProcessConcurrency, clientCount) + c.processStreams = make(streamPool, streamSize) + var opts = []grpc.CallOption{ + grpc.UseCompressor(utils.Select[string](grpcEnableCompress, gzip.Name, "")), + } + for i := uint64(0); i < streamSize; i++ { + stream, err := c.processorClients[i%clientCount].ProcessBindingsStream(ctx, opts...) + if err != nil { + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("open stream for process binding failed: %v", err)) + } + c.processStreams <- stream + } + c.waiter = &waiter{ + ready: concurrency.NewResourceWaiter[uint64](), + finish: concurrency.NewResourceWaiter[partitionWithIndex](), + } + return nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) DisableAgents(ctx context.Context) { + if len(strings.TrimSpace(disableAgentTypes)) == 0 { + return + } + _, logger := log.FromContext(ctx) + das := utils.BuildSet(utils.MapSliceNoError(strings.Split(disableAgentTypes, ","), strings.TrimSpace)) + c.Agents = utils.FilterArr(c.Agents, func(a HA) bool { + typ := fmt.Sprintf("%T", a) + if das[typ] { + logger.Warnw("disable agent", "type", typ, "agent", a.Snapshot()) + return false + } + return true + }) +} + +func (c *BaseHandlerController[CLI, BKD, HA]) FinishExecute() { + close(c.processStreams) + for stream := range c.processStreams { + _ = stream.CloseSend() + } +} + +func (c *BaseHandlerController[CLI, BKD, HA]) BuildTaskList( + ctx context.Context, + d BKD, +) ([]controller.Task, int, error) { + var result []bindingData + var totalSize int + for _, agent := range c.Agents { + if !agent.GetRange().Contains(d.GetBlockNumber()) { + continue + } + if inners, err := agent.BuildBindingDataList(ctx, d); err != nil { + return nil, 0, err + } else { + for _, inner := range inners { + result = append(result, bindingData{ + BlockHeader: d, + handlerID: agent.GetHandlerID(), + data: &protos.DataBinding{ + Data: inner.Data, + HandlerType: inner.HandlerType, + HandlerIds: []int32{agent.GetHandlerID().ID}, + ChainId: c.ChainConfig.ChainID, + }, + txIndex: inner.TxIndex, + txInnerIndex: inner.TxInnerIndex, + }) + totalSize += inner.DataSize + } + } + } + // The purpose of using stable sorting is to ensure that tasks of the same handler remain in the order + // in which they were generated. + sort.SliceStable(result, func(i, j int) bool { + return result[i].Cmp(result[j], c.InitResult.ExecutionConfig.GetHandlerOrderInsideTransaction()) < 0 + }) + var r []controller.Task + for _, bd := range result { + r = append(r, &task{ + bindingData: bd, + sp: c.processStreams, + waiter: c.waiter, + chainID: c.ChainConfig.ChainID, + processor: c.Processor, + metricConfigs: c.metricConfigs, + webhookChannels: c.webhookChannels, + }) + } + return r, totalSize, nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) BuildReportRequirements( + currentBlockNumber uint64, +) []data.IntervalRequirement { + endBlock := c.GetBlockRange().EndBlock + // In the backfill phase, at least one non-empty BlockData is generated for every DAY. + // In the watching phase, at least one non-empty BlockData is generated for every MINUTE. + reqs := []data.IntervalRequirement{{ + BlockRange: controller.BlockRange{StartBlock: currentBlockNumber, EndBlock: endBlock}, + IntervalConfig: data.IntervalConfig{TimeInterval: &data.TimeInterval{ + Backfill: time.Hour * 24, + Watching: time.Minute, + }}, + }} + if endBlock != nil { + reqs = append(reqs, data.IntervalRequirement{ + BlockRange: controller.BlockRange{StartBlock: *endBlock, EndBlock: endBlock}, + IntervalConfig: data.IntervalConfig{BlockInterval: &data.BlockInterval{ + Backfill: 1, + Watching: 1, + }}, + }) + } + return reqs +} + +func (c *BaseHandlerController[CLI, BKD, HA]) GetBlockRange() controller.BlockRange { + return controller.GetHandleAgentsBlockRange(c.Agents) +} + +func (c *BaseHandlerController[CLI, BKD, HA]) GetAgentStat() map[string]int { + stat := make(map[string]int) + for _, ag := range c.Agents { + stat[fmt.Sprintf("%T", ag)] += 1 + } + return stat +} + +const checkpointDataKeyAddressStart = "AddressStart" + +func (c *BaseHandlerController[CLI, BKD, HA]) LoadAddressStart(checkpoint *controller.Checkpoint) *controller.ExternalError { + c.addressStart = make(map[string]uint64) + if checkpoint == nil || checkpoint.Data == nil { + return nil + } + raw, has := checkpoint.Data[checkpointDataKeyAddressStart] + if !has { + return nil + } + err := json.Unmarshal([]byte(raw), &c.addressStart) + if err != nil { + return controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "load address start failed")) + } + return nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) GetAddressStart( + address string, + start uint64, + loader func() (uint64, error), +) (uint64, error) { + if c.InitResult.GetExecutionConfig().GetSkipStartBlockValidation() || + c.ChainConfig.SkipStartBlockValidation || controller.SkipStartBlockValidation { + return start, nil + } + if address == "" { + return start, nil + } + var has bool + if start, has = c.addressStart[address]; has { + return start, nil + } + var err error + start, err = loader() + if err != nil { + return 0, errors.Wrapf(err, "get start block for %s failed", address) + } + c.addressStart[address] = start + return start, nil +} + +func (c *BaseHandlerController[CLI, BKD, HA]) AddressStartReady() { + b, _ := json.Marshal(c.addressStart) + c.addressStartData = string(b) +} + +func (c *BaseHandlerController[CLI, BKD, HA]) DumpAddressStart(checkpointData map[string]string) { + checkpointData[checkpointDataKeyAddressStart] = c.addressStartData +} + +func (c *BaseHandlerController[CLI, BKD, HA]) Snapshot() any { + return map[string]any{ + "initResult": c.InitResult, + "chainConfig": c.ChainConfig, + "handlerConfig": c.Config, + "agents": utils.MapSliceNoError(c.Agents, HA.Snapshot), + "agentStat": c.GetAgentStat(), + "addressStart": c.addressStart, + "processorCount": len(c.processorClients), + "streamCount": cap(c.processStreams), + } +} diff --git a/driver/controller/standard/helper.go b/driver/controller/standard/helper.go new file mode 100644 index 0000000..5308a54 --- /dev/null +++ b/driver/controller/standard/helper.go @@ -0,0 +1,29 @@ +package standard + +import ( + "bytes" + + "sentioxyz/sentio-core/common/utils" +) + +func AdjustAddress(address string) string { + return utils.Select(address == "" || address == "*", "", address) +} + +func AdjustEndBlock(endBlock uint64) *uint64 { + return utils.Select(endBlock == 0, nil, &endBlock) +} + +func BuildDataSource(chainType, chainID, srcType, address string) string { + var b bytes.Buffer + b.WriteString(chainType) + b.WriteString(":") + b.WriteString(chainID) + b.WriteString(":") + b.WriteString(srcType) + if address != "" { + b.WriteString(":") + b.WriteString(address) + } + return b.String() +} diff --git a/driver/controller/standard/interval.go b/driver/controller/standard/interval.go new file mode 100644 index 0000000..4f7329b --- /dev/null +++ b/driver/controller/standard/interval.go @@ -0,0 +1,34 @@ +package standard + +import ( + "time" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/processor/protos" +) + +func NewIntervalConfig(c *protos.OnIntervalConfig) (data.IntervalConfig, error) { + if c.GetSlot() > 0 || c.GetSlotInterval() != nil { + interval := data.BlockInterval{ + Backfill: max(uint64(c.GetSlotInterval().GetBackfillInterval()), uint64(c.GetSlot()), minBackfillSlotInterval), + Watching: max(uint64(c.GetSlotInterval().GetRecentInterval()), uint64(c.GetSlot()), 1), + } + return data.IntervalConfig{BlockInterval: &interval}, nil + } else if c.GetMinutes() > 0 || c.GetMinutesInterval() != nil { + interval := data.TimeInterval{ + Backfill: max( + time.Duration(c.GetMinutesInterval().GetBackfillInterval())*time.Minute, + time.Duration(c.GetMinutes())*time.Minute, + minBackfillTimeInterval), + Watching: max( + time.Duration(c.GetMinutesInterval().GetRecentInterval())*time.Minute, + time.Duration(c.GetMinutes())*time.Minute, + time.Minute), + } + return data.IntervalConfig{TimeInterval: &interval}, nil + } else { + return data.IntervalConfig{}, errors.Errorf("invalid interval config %#v", c) + } +} diff --git a/driver/controller/standard/sol/BUILD.bazel b/driver/controller/standard/sol/BUILD.bazel new file mode 100644 index 0000000..8b18287 --- /dev/null +++ b/driver/controller/standard/sol/BUILD.bazel @@ -0,0 +1,30 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "sol", + srcs = [ + "block_data.go", + "handler.go", + "handler_instruction.go", + "handler_interval.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/sol", + visibility = ["//visibility:public"], + deps = [ + "//chain/sol", + "//common/log", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/sol", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor/protos", + "//service/processor/models", + "@com_github_gagliardetto_solana_go//:solana-go", + "@com_github_gagliardetto_solana_go//rpc", + "@com_github_pkg_errors//:errors", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) diff --git a/driver/controller/standard/sol/block_data.go b/driver/controller/standard/sol/block_data.go new file mode 100644 index 0000000..3a1f04f --- /dev/null +++ b/driver/controller/standard/sol/block_data.go @@ -0,0 +1,91 @@ +package sol + +import ( + "encoding/json" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/pkg/errors" + + solcore "sentioxyz/sentio-core/chain/sol" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/sol" +) + +// BlockData is built directly from BlockMainData: the main-data fetchers already returned the block +// header (interval) and the full matching transactions (instruction), so no extra fetch is needed. +type BlockData struct { + mainData sol.BlockMainData + + blockJSON string + txJSONDict map[solana.Signature]string // key is txSig + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (d *BlockData) GetBlockNumber() uint64 { + return d.mainData.Slot +} + +func (d *BlockData) GetBlockHash() string { + return d.mainData.Blockhash +} + +func (d *BlockData) GetBlockParentHash() string { + return d.mainData.PreviousBlockhash +} + +func (d *BlockData) GetBlockTime() time.Time { + if d.mainData.BlockTime != nil { + return d.mainData.BlockTime.Time() + } + return time.Time{} +} + +func (d *BlockData) getBlockJSON() (string, error) { + if d.blockJSON != "" { + return d.blockJSON, nil + } + if d.mainData.Block == nil || d.mainData.Block.GetBlockResult == nil { + return "", errors.Errorf("block %d has no header", d.GetBlockNumber()) + } + b, err := json.Marshal(d.mainData.Block.GetBlockResult) + if err != nil { + return "", errors.Wrapf(err, "marshal block %d failed", d.GetBlockNumber()) + } + d.blockJSON = string(b) + return d.blockJSON, nil +} + +func (d *BlockData) getTxJSON(tx solcore.WrappedTransaction) (string, error) { + if r, has := d.txJSONDict[tx.Signature]; has { + return r, nil + } + b, err := json.Marshal(tx.ToParsedTransactionResult(d.GetBlockNumber(), d.mainData.BlockTime)) + if err != nil { + return "", errors.Wrapf(err, "marshal tx %d/%s failed", d.GetBlockNumber(), tx.Signature) + } + r := string(b) + d.txJSONDict[tx.Signature] = r + return r, nil +} + +func (d *BlockData) GetTaskList() []controller.Task { + return d.taskList +} + +func (d *BlockData) CheckpointData() map[string]string { + return d.checkpointData +} + +func (d *BlockData) DataSource() string { + return d.dataSource +} + +func (d *BlockData) Size() int { + return d.taskTotalSize +} diff --git a/driver/controller/standard/sol/handler.go b/driver/controller/standard/sol/handler.go new file mode 100644 index 0000000..bf78f13 --- /dev/null +++ b/driver/controller/standard/sol/handler.go @@ -0,0 +1,205 @@ +package sol + +import ( + "context" + "fmt" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/sol" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" +) + +type HandlerController struct { + *standard.BaseHandlerController[sol.Client, *BlockData, standard.HandlerAgent[*BlockData]] +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *config.ChainConfig, + client sol.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[sol.Client, *BlockData, standard.HandlerAgent[*BlockData]]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + + fetchNamePrefix := fmt.Sprintf("SOL::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + sol.BuildBlockMainDataFetcher(fetchNamePrefix, req, currentBlockNumber, latest, c.Client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, // 256MB + 100, + time.Second*30, // may need to get a lot of transaction + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from sol.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + _, logger := log.FromContext(ctx) + // The main-data fetchers already returned the block header (interval) and the full + // matching transactions (instruction), so the block data is built without any fetch. + result := BlockData{ + mainData: from, + txJSONDict: make(map[solana.Signature]string), + checkpointData: make(map[string]string), + } + var err error + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + return nil, false, err + } + logger.Debugf("built %d task in block #%d with handlerIDs %v", + len(result.taskList), + blockNumber, + utils.Stat(utils.MapSliceNoError( + utils.MapSliceNoError(result.taskList, controller.Task.GetHandlerID), + controller.HandlerID.String, + ))) + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, latest uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + var err error + + for dataSourceID, contractConfig := range c.Config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + dataSource := standard.BuildDataSource("SOL", c.ChainConfig.ChainID, "Contract", contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + parsedContractAddress, parseContractAddrErr := solana.PublicKeyFromBase58(contractAddress) + blockRange.StartBlock, err = c.GetAddressStart( + contractAddress, + blockRange.StartBlock, + func() (uint64, error) { + ns, has, getErr := c.Client.GetContractStartBlock(ctx, parsedContractAddress, blockRange.StartBlock, latest) + if getErr != nil { + return 0, getErr + } + if has { + return ns, nil + } + return latest + 1, nil + }) + if err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + if config := contractConfig.GetInstructionConfig(); config != nil { + if parseContractAddrErr != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, errors.Wrapf( + parseContractAddrErr, "contract address %q is invalid for the instruction handler", contractAddress)) + } + agent := HandlerAgentInstruction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "instruction", controller.SimpleHandlerConfig{}, blockRange), + Address: parsedContractAddress, + ProcessInnerInstruction: config.GetInnerInstruction(), + ProcessParsedInstruction: config.GetParsedInstruction(), + ProcessRawInstruction: config.GetRawDataInstruction(), + FetchTx: config.GetFetchTx(), + } + if !agent.ProcessParsedInstruction && !agent.ProcessRawInstruction { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, errors.Errorf( + "unexpected config for handler %s: neigher parsed nor raw data is needed", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + } + + for _, intervalConfig := range contractConfig.GetIntervalConfigs() { + if contractAddress != "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("contract %s cannot have interval config", contractAddress)) + } + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig, blockRange), + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + c.Agents = append(c.Agents, agent) + } + } + + logger.Infof("built %d agents", len(c.Agents)) + return nil +} + +func (c *HandlerController) getDataRequirement() (dr sol.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentInstruction: + dr.Tx = append(dr.Tx, sol.TransactionRequirement{ + BlockRange: ag.Range, + Programs: []solana.PublicKey{ag.Address}, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + BlockRange: ag.Range, + IntervalConfig: ag.IntervalConfig, + }) + } + } + return dr +} diff --git a/driver/controller/standard/sol/handler_instruction.go b/driver/controller/standard/sol/handler_instruction.go new file mode 100644 index 0000000..72372fe --- /dev/null +++ b/driver/controller/standard/sol/handler_instruction.go @@ -0,0 +1,133 @@ +package sol + +import ( + "context" + "encoding/json" + + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInstruction struct { + controller.BaseHandlerAgent + + Address solana.PublicKey + + // configures about building binding data + ProcessInnerInstruction bool + ProcessParsedInstruction bool + ProcessRawInstruction bool + FetchTx bool +} + +// BuildBindingDataList builds the per-instruction binding data for this handler's program. +// +// NOTE on the BigQuery data source: when a transaction is served from the BigQuery tier (archival +// history below the ClickHouse range), it carries ONLY the instructions of the program(s) the +// super node was queried for — not the transaction's full instruction set (this is a deliberate +// cost optimization in the archival-tier transaction lookup, which is clustered by program_id). +// That is fine here: this handler only emits binding data for instructions whose +// ProgramId == a.Address (the program it targets), which the BigQuery query always includes; and +// the raw transaction attached below (via getTxJSON) still has complete transaction-level data +// (account keys, balances, token balances, logs, status/err, fee, compute units). A handler must +// not rely on unrelated programs' instructions being present in the raw transaction. +func (a HandlerAgentInstruction) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for _, tx := range bd.mainData.Transactions { + if tx.Transaction == nil { + continue + } + // get instructions + var instructions []*rpc.ParsedInstruction + var indexStartOffset int + if a.ProcessInnerInstruction && tx.Meta != nil { + for _, innerInstruction := range tx.Meta.InnerInstructions { + instructions = append(instructions, innerInstruction.Instructions...) + } + indexStartOffset = -len(instructions) + } + instructions = append(instructions, tx.Transaction.Message.Instructions...) + // build binding data for each instruction + for i, instruction := range instructions { + if instruction.ProgramId != a.Address { + continue + } + if (!a.ProcessParsedInstruction || instruction.Parsed == nil) && !a.ProcessRawInstruction { + continue // no data + } + dataSize := len(instruction.Accounts)*45 + len(instruction.Data) + // get parsed + var rawParsed *string + if a.ProcessParsedInstruction && instruction.Parsed != nil { + b, marshalErr := json.Marshal(instruction.Parsed) + if marshalErr != nil { + return nil, errors.Wrapf(marshalErr, + "marshal #%d instruction parsed in transaction %d/%s failed", i, bd.GetBlockNumber(), tx.Signature) + } + s := string(b) + rawParsed = &s + dataSize += len(b) + } + // get instructionData + var instructionData string + if a.ProcessRawInstruction { + instructionData = instruction.Data.String() + } + // get raw transaction + var rawTx *string + if a.FetchTx { + rawTxJSON, getRawTxErr := bd.getTxJSON(tx) + if getRawTxErr != nil { + return nil, getRawTxErr + } + rawTx = &rawTxJSON + dataSize += len(rawTxJSON) + } + // build binding data + data := &protos.Data{ + Value: &protos.Data_SolInstruction_{ + SolInstruction: &protos.Data_SolInstruction{ + Slot: bd.GetBlockNumber(), + ProgramAccountId: a.Address.String(), + Accounts: utils.MapSliceNoError(instruction.Accounts, solana.PublicKey.String), + InstructionData: instructionData, + RawParsed: rawParsed, + RawTransaction: rawTx, + }, + }, + } + // append result + // calculate the TxInnerIndex of the binding data + // - index of inner instruction is in [-len(InnerInstructions), -1] + // - index of normal instruction is in [0, len(NormalInstructions)-1] + result = append(result, standard.BindingDataInner{ + Data: data, + DataSize: dataSize, + HandlerType: protos.HandlerType_SOL_INSTRUCTION, + TxIndex: int(tx.TransactionIndex), + TxInnerIndex: i + indexStartOffset, + }) + } + } + return result, nil +} + +func (a HandlerAgentInstruction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Address": a.Address.String(), + "ProcessInnerInstruction": a.ProcessInnerInstruction, + "ProcessParsedInstruction": a.ProcessParsedInstruction, + "ProcessRawInstruction": a.ProcessRawInstruction, + "FetchTx": a.FetchTx, + } +} diff --git a/driver/controller/standard/sol/handler_interval.go b/driver/controller/standard/sol/handler_interval.go new file mode 100644 index 0000000..e9a9a65 --- /dev/null +++ b/driver/controller/standard/sol/handler_interval.go @@ -0,0 +1,61 @@ +package sol + +import ( + "context" + "math" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + IntervalConfig data.IntervalConfig +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + } +} + +func (a HandlerAgentInterval) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) ([]standard.BindingDataInner, error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + // NOTE on the BigQuery data source: for an interval handler over the archival range, the block is + // served by the BigQuery tier, whose getBlocksByInterval returns the block HEADER ONLY — no + // transaction signatures (a deliberate BigQuery cost optimization; see the archival store and the + // data-layer GetBlocksByInterval). So rawBlock here has the header fields (slot, hashes, time) + // but an empty transaction/signature list. The SOL_BLOCK interval handler is block-header + // oriented, so this is fine; a processor that needs the block's transactions must not rely on an + // interval handler over the BigQuery range. + rawBlock, err := bd.getBlockJSON() + if err != nil { + return nil, err + } + return []standard.BindingDataInner{{ + HandlerType: protos.HandlerType_SOL_BLOCK, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_SolBlock_{ + SolBlock: &protos.Data_SolBlock{ + RawBlock: rawBlock, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(rawBlock), + }}, nil +} diff --git a/driver/controller/standard/sui/BUILD.bazel b/driver/controller/standard/sui/BUILD.bazel new file mode 100644 index 0000000..862d014 --- /dev/null +++ b/driver/controller/standard/sui/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sui", + srcs = [ + "agents.go", + "block_data.go", + "handler.go", + "handler_change.go", + "handler_event.go", + "handler_function.go", + "handler_interval.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/sui", + visibility = ["//visibility:public"], + deps = [ + "//chain/move", + "//chain/sui", + "//chain/sui/types", + "//common/compress", + "//common/envconf", + "//common/errgroup", + "//common/log", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/sui", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//processor", + "//processor/protos", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) + +go_test( + name = "sui_test", + srcs = ["handler_interval_test.go"], + embed = [":sui"], + deps = ["@com_github_stretchr_testify//assert"], +) diff --git a/driver/controller/standard/sui/agents.go b/driver/controller/standard/sui/agents.go new file mode 100644 index 0000000..fb234a7 --- /dev/null +++ b/driver/controller/standard/sui/agents.go @@ -0,0 +1,291 @@ +package sui + +import ( + "context" + "strings" + + "sentioxyz/sentio-core/chain/move" + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor" + "sentioxyz/sentio-core/processor/protos" + + "github.com/pkg/errors" +) + +// BuildSuiAgents turns the parsed processor config into the SUI handler agents, +// emitting each built agent via emit. It is the format-agnostic core shared by +// the json-rpc handler controller (this package) and the grpc one +// (standard/sui/grpc): both build the SAME agents/filters here, and only differ +// in how the agent's BuildBindingDataList reads data and serializes the binding. +// The grpc controller wraps each emitted agent into a grpc agent that embeds it. +func BuildSuiAgents( + ctx context.Context, + config standard.HandlerConfig, + chainConfig *config.ChainConfig, + sdkVersion string, + client sui.Client, + first uint64, + getAddressStart func(ctx context.Context, address string, start uint64) (uint64, error), + emit func(SuiHandlerAgent), +) *controller.ExternalError { + _, logger := log.FromContext(ctx) + + processorVersion, err := processor.ParseVersion(sdkVersion) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "parse processor sdk version %q failed", sdkVersion)) + } + + for dataSourceID, accountConfig := range config.AccountConfigs { + accountAddress := standard.AdjustAddress(accountConfig.GetAddress()) + dataSource := standard.BuildDataSource("SUI", chainConfig.ChainID, "Account", accountAddress) + blockRange := controller.BlockRange{ + StartBlock: max(accountConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(accountConfig.GetEndBlock()), + } + + if blockRange.StartBlock, err = getAddressStart(ctx, accountAddress, blockRange.StartBlock); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + for _, intervalConfig := range accountConfig.GetMoveIntervalConfigs() { + agent := HandlerAgentInterval{ + BaseHandlerAgent: controller.NewBaseHandlerAgent( + dataSource, dataSourceID, "interval", intervalConfig.GetIntervalConfig(), blockRange), + Client: client, + } + agent.IntervalConfig, err = standard.NewIntervalConfig(intervalConfig.GetIntervalConfig()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + switch intervalConfig.GetOwnerType() { + case protos.MoveOwnerType_ADDRESS, protos.MoveOwnerType_OBJECT, protos.MoveOwnerType_WRAPPED_OBJECT: + if accountAddress == "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("unexpected config for handler %s: address should not be empty because owner type is %s", + agent.GetHandlerID().String(), intervalConfig.GetOwnerType().String())) + } + agent.Filter.OwnerFilter = &chainsui.ObjectChangeOwnerFilter{OwnerID: []string{accountAddress}} + agent.UnwrapDynamicObject = true + switch intervalConfig.GetOwnerType() { + case protos.MoveOwnerType_ADDRESS: + agent.Filter.OwnerFilter.OwnerType = []string{"address"} + case protos.MoveOwnerType_OBJECT: + agent.Filter.OwnerFilter.OwnerType = []string{"object"} + agent.NeedSelf = true + case protos.MoveOwnerType_WRAPPED_OBJECT: + agent.Filter.OwnerFilter.OwnerType = []string{"object"} + } + if !intervalConfig.GetResourceFetchConfig().GetOwned() { + agent.Filter.OwnerFilter.OwnerType = nil + } + case protos.MoveOwnerType_TYPE: + if accountAddress != "" { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, errors.Errorf( + "unexpected config for handler %s: address is %s, it should be empty because owner type is %s", + agent.GetHandlerID().String(), accountAddress, intervalConfig.GetOwnerType().String())) + } + var objectType move.Type + objectType, err = move.BuildType(intervalConfig.GetType()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s: invalid object type %q", + agent.GetHandlerID().String(), intervalConfig.GetType())) + } + agent.Filter.TypePattern = move.TypeSet{objectType} + agent.UnwrapDynamicObject = false + default: + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("unexpected config for handler %s: unknown owner type %s", + agent.GetHandlerID().String(), intervalConfig.GetOwnerType().String())) + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, moveCallConfig := range accountConfig.GetMoveCallConfigs() { + agent, extErr := buildFunctionAgent(dataSource, dataSourceID, accountAddress, moveCallConfig, blockRange) + if extErr != nil { + return extErr + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, changeConfig := range accountConfig.GetMoveResourceChangeConfigs() { + agent := HandlerAgentChange{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "change", changeConfig, blockRange), + } + agent.Filter.TypePattern, err = utils.MapSlice(changeConfig.GetTypes(), move.BuildType) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + for dataSourceID, contractConfig := range config.ContractConfigs { + contractAddress := standard.AdjustAddress(contractConfig.GetContract().GetAddress()) + dataSource := standard.BuildDataSource("SUI", chainConfig.ChainID, "Contract", contractAddress) + blockRange := controller.BlockRange{ + StartBlock: max(contractConfig.GetStartBlock(), first), + EndBlock: standard.AdjustEndBlock(contractConfig.GetEndBlock()), + } + + if blockRange.StartBlock, err = getAddressStart(ctx, contractAddress, blockRange.StartBlock); err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + + var pkgHistory []string // used for search real event type + if contractAddress != "" { + if !chainConfig.KeepSuiEventTypePackage && len(contractConfig.GetMoveEventConfigs()) > 0 { + if pkgHistory, err = client.GetPackageHistory(ctx, contractAddress); err != nil { + return controller.NewExternalError(controller.ErrCodeFetchDataFailed, + errors.Wrapf(err, "get package history for contract %s failed", contractAddress)) + } + } else { + pkgHistory = []string{contractAddress} + } + } + + for _, eventConfig := range contractConfig.GetMoveEventConfigs() { + agent := HandlerAgentEvent{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "event", eventConfig, blockRange), + Filter: chainsui.TransactionFilter{ + FailedIsOK: eventConfig.GetFetchConfig().GetIncludeFailedTransaction(), + }, + FetchConfig: chainsui.TransactionFetchConfig{ + NeedInputs: true, + NeedEffects: true, + NeedAllEvents: eventConfig.GetFetchConfig().GetAllEvents(), + }, + } + // sdk before v2.32, includeInputs and includeResourceChanges always be true + if processorVersion.Major > 2 || (processorVersion.Major == 2 && processorVersion.Minor >= 32) { + agent.FetchConfig.NeedInputs = eventConfig.GetFetchConfig().GetInputs() + agent.FetchConfig.NeedEffects = eventConfig.GetFetchConfig().GetResourceChanges() + } + for _, f := range eventConfig.GetFilters() { + var ff chainsui.EventFilterV2 + if sender := f.GetEventAccount(); sender != "" { + ff.Sender = &sender + } + // Address is needed as the packageId to form a complete event type; the address part of the + // event type maybe the upgraded package id, so iterate the whole package history. + for _, addr := range pkgHistory { + var typ move.Type + typ, err = move.BuildType(addr + "::" + f.GetType()) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s: invalid event type %q", + agent.GetHandlerID().String(), f.GetType())) + } + ff.TypePattern = append(ff.TypePattern, typ) + } + agent.Filter.EventFilters = append(agent.Filter.EventFilters, ff) + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, moveCallConfig := range contractConfig.GetMoveCallConfigs() { + agent, extErr := buildFunctionAgent(dataSource, dataSourceID, contractAddress, moveCallConfig, blockRange) + if extErr != nil { + return extErr + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + + for _, changeConfig := range contractConfig.GetMoveResourceChangeConfigs() { + agent := HandlerAgentChange{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "change", changeConfig, blockRange), + } + agent.Filter.TypePattern, err = utils.MapSlice(changeConfig.GetTypes(), move.BuildType) + if err != nil { + return controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "unexpected config for handler %s", agent.GetHandlerID().String())) + } + emit(agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + + return nil +} + +// buildFunctionAgent builds a move-call (function) agent; shared by both the account and contract loops. +func buildFunctionAgent( + dataSource string, + dataSourceID int, + address string, + moveCallConfig *protos.MoveCallHandlerConfig, + blockRange controller.BlockRange, +) (HandlerAgentFunction, *controller.ExternalError) { + agent := HandlerAgentFunction{ + BaseHandlerAgent: controller.NewBaseHandlerAgent(dataSource, dataSourceID, "call", moveCallConfig, blockRange), + FetchConfig: chainsui.TransactionFetchConfig{ + NeedInputs: true, + NeedEffects: true, + NeedAllEvents: true, + }, + } + for _, f := range moveCallConfig.GetFilters() { + var ff chainsui.FunctionFilter + ff.Kind = utils.WrapPointer("ProgrammableTransaction") + ff.CommandFilter = &chainsui.CommandFilter{} + if address != "" { + packageID, parseErr := types.StrToObjectID(address) + if parseErr != nil { + return agent, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(parseErr, "unexpected config for handler %s: address %q is not a valid object id", + agent.GetHandlerID().String(), address)) + } + ff.CommandFilter.CallPackage = utils.WrapPointer(packageID.String()) + } + callModule, callFunc, _ := strings.Cut(f.GetFunction(), "::") + if callModule != "" && callModule != "*" { + ff.CommandFilter.CallModule = &callModule + } + if callFunc != "" && callFunc != "*" { + ff.CommandFilter.CallFunction = &callFunc + } + if ff.CommandFilter.IsEmpty() { + ff.CommandFilter = nil + } + if prefix := f.GetPublicKeyPrefix(); prefix != "" { + if !strings.HasPrefix(prefix, "0x") { + prefix = "0x" + prefix + } + ff.MultiSigPublicKeyPrefix = &prefix + } + if from := f.GetFromAndToAddress().GetFrom(); from != "" { + ff.Sender = &from + } + if to := f.GetFromAndToAddress().GetTo(); to != "" { + ff.Receiver = &to + } + ff.FailedIsOK = f.GetIncludeFailed() && moveCallConfig.GetFetchConfig().GetIncludeFailedTransaction() + agent.Filter.FunctionFilters = append(agent.Filter.FunctionFilters, ff) + } + agent.Filter.FailedIsOK = moveCallConfig.GetFetchConfig().GetIncludeFailedTransaction() + if len(agent.Filter.FunctionFilters) > 0 { + agent.Filter.FailedIsOK = utils.Reduce( + utils.MapSliceNoError(agent.Filter.FunctionFilters, func(ff chainsui.FunctionFilter) bool { + return ff.FailedIsOK + }), + func(a, b bool) bool { return a || b }, + ) + } + return agent, nil +} diff --git a/driver/controller/standard/sui/block_data.go b/driver/controller/standard/sui/block_data.go new file mode 100644 index 0000000..98c39df --- /dev/null +++ b/driver/controller/standard/sui/block_data.go @@ -0,0 +1,87 @@ +package sui + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/sui" +) + +type BlockData struct { + controller.BlockHeader + + mainData sui.BlockMainData + checkpointData map[string]string + + objMgr *ObjectDictSetManager + cachedTxn map[string]string + cachedChanges map[int]string + + taskList []controller.Task + taskTotalSize int + dataSource string +} + +func (b *BlockData) DataSource() string { + return b.dataSource +} + +func (b *BlockData) CheckpointData() map[string]string { + return b.checkpointData +} + +func (b *BlockData) Size() int { + return b.taskTotalSize +} + +func (b *BlockData) GetTaskList() []controller.Task { + return b.taskList +} + +func (b *BlockData) getTxn( + txIndex int, + eventFilters []chainsui.EventFilterV2, + fetchConfig chainsui.TransactionFetchConfig, +) (string, error) { + key := fmt.Sprintf("%d/%s", txIndex, fetchConfig) + if !fetchConfig.NeedAllEvents { + key = fmt.Sprintf("%s/%v", key, eventFilters) + } + if b.cachedTxn == nil { + b.cachedTxn = make(map[string]string) + } + if str, has := b.cachedTxn[key]; has { + return str, nil + } + tx := b.mainData.Txs[txIndex] + raw, err := json.Marshal(fetchConfig.PruneTransaction(tx, eventFilters)) + if err != nil { + return "", errors.Wrapf(err, "marshal sui tx %s with fetch config %s and event filters %s failed", + tx.Digest.String(), fetchConfig.String(), utils.MustJSONMarshal(eventFilters)) + } + str := string(raw) + b.cachedTxn[key] = str + return str, nil +} + +func (b *BlockData) getChange(index int) (string, error) { + if b.cachedChanges == nil { + b.cachedChanges = make(map[int]string) + } + if str, has := b.cachedChanges[index]; has { + return str, nil + } + raw, err := json.Marshal(b.mainData.ObjectChanges[index]) + if err != nil { + return "", errors.Wrapf(err, "marshal sui object change #%d in block %d failed: %#v ", + index, b.GetBlockNumber(), b.mainData.ObjectChanges[index]) + } + str := string(raw) + b.cachedChanges[index] = str + return str, nil +} diff --git a/driver/controller/standard/sui/grpc/BUILD.bazel b/driver/controller/standard/sui/grpc/BUILD.bazel new file mode 100644 index 0000000..942f20d --- /dev/null +++ b/driver/controller/standard/sui/grpc/BUILD.bazel @@ -0,0 +1,56 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "grpc", + srcs = [ + "block_data.go", + "handler.go", + "handler_change.go", + "handler_event.go", + "handler_function.go", + "handler_interval.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/standard/sui/grpc", + visibility = ["//visibility:public"], + deps = [ + "//chain/sui", + "//chain/sui/types", + "//common/envconf", + "//common/errgroup", + "//common/log", + "//common/protojson", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/sui", + "//driver/controller/data/sui/grpc", + "//driver/controller/fetcher", + "//driver/controller/standard", + "//driver/controller/standard/sui", + "//processor/protos", + "//service/processor/models", + "@com_github_pkg_errors//:errors", + "@com_github_sentioxyz_sui_apis//sui/rpc/v2:rpc", + "@org_golang_google_protobuf//types/known/fieldmaskpb", + "@org_golang_google_protobuf//types/known/structpb", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) + +go_test( + name = "grpc_test", + srcs = ["handler_test.go"], + embed = [":grpc"], + deps = [ + "//chain/sui", + "//common/set", + "//driver/controller/data/sui", + "//driver/controller/data/sui/grpc", + "//driver/controller/standard/sui", + "@com_github_sentioxyz_sui_apis//sui/rpc/v2:rpc", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + "@org_golang_google_protobuf//proto", + ], +) diff --git a/driver/controller/standard/sui/grpc/block_data.go b/driver/controller/standard/sui/grpc/block_data.go new file mode 100644 index 0000000..d4e4846 --- /dev/null +++ b/driver/controller/standard/sui/grpc/block_data.go @@ -0,0 +1,92 @@ +// Package grpc is the grpc-data twin of standard/sui: same handler controller / +// agents / config parsing (reused from the parent via sui.BuildSuiAgents and the +// embedded sui agents), but each agent's BuildBindingDataList reads grpc-format +// block data (data/sui/grpc) and serializes the DataBinding raw_* fields from the +// grpc structs (ExtendedGrpc* / rpcv2.* via protojson). Selected by the launcher +// only for the SUI variation at DriverVersion >= 2. +package grpc + +import ( + "encoding/json" + "fmt" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/driver/controller" + suigrpcdata "sentioxyz/sentio-core/driver/controller/data/sui/grpc" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + + "github.com/pkg/errors" +) + +type BlockData struct { + controller.BlockHeader + + mainData suigrpcdata.BlockMainData + checkpointData map[string]string + + objMgr *suihandler.ObjectDictSetManager + cachedTxn map[string]string + cachedChanges map[int]string + + taskList []controller.Task + taskTotalSize int + dataSource string +} + +func (b *BlockData) DataSource() string { + return b.dataSource +} + +func (b *BlockData) CheckpointData() map[string]string { + return b.checkpointData +} + +func (b *BlockData) Size() int { + return b.taskTotalSize +} + +func (b *BlockData) GetTaskList() []controller.Task { + return b.taskList +} + +// getTxn returns the grpc-format json of the (pruned) transaction at txIndex. +func (b *BlockData) getTxn( + txIndex int, + eventFilters []chainsui.EventFilterV2, + fetchConfig chainsui.TransactionFetchConfig, +) (string, error) { + key := fmt.Sprintf("%d/%s", txIndex, fetchConfig) + if !fetchConfig.NeedAllEvents { + key = fmt.Sprintf("%s/%v", key, eventFilters) + } + if b.cachedTxn == nil { + b.cachedTxn = make(map[string]string) + } + if str, has := b.cachedTxn[key]; has { + return str, nil + } + tx := b.mainData.Txs[txIndex] + raw, err := json.Marshal(fetchConfig.PruneGrpcTransaction(tx, eventFilters)) + if err != nil { + return "", errors.Wrapf(err, "marshal grpc sui tx at index %d in block %d failed", txIndex, b.GetBlockNumber()) + } + str := string(raw) + b.cachedTxn[key] = str + return str, nil +} + +func (b *BlockData) getChange(index int) (string, error) { + if b.cachedChanges == nil { + b.cachedChanges = make(map[int]string) + } + if str, has := b.cachedChanges[index]; has { + return str, nil + } + raw, err := json.Marshal(b.mainData.ObjectChanges[index]) + if err != nil { + return "", errors.Wrapf(err, "marshal grpc sui object change #%d in block %d failed", index, b.GetBlockNumber()) + } + str := string(raw) + b.cachedChanges[index] = str + return str, nil +} diff --git a/driver/controller/standard/sui/grpc/handler.go b/driver/controller/standard/sui/grpc/handler.go new file mode 100644 index 0000000..2ba1d54 --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler.go @@ -0,0 +1,238 @@ +package grpc + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + suigrpcdata "sentioxyz/sentio-core/driver/controller/data/sui/grpc" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" +) + +type GrpcHandlerAgent interface { + standard.HandlerAgent[*BlockData] +} + +type HandlerController struct { + *standard.BaseHandlerController[suidata.Client, *BlockData, GrpcHandlerAgent] + + objMgr suihandler.ObjectDictSetManager +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *config.ChainConfig, + client suidata.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[suidata.Client, *BlockData, GrpcHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if err := c.objMgr.Load(checkpoint); err != nil { + return controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "parse object set from checkpoint data failed")) + } + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetObjectCreation(ctx, address, start) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + return start, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, _ uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + extErr := suihandler.BuildSuiAgents( + ctx, c.Config, c.ChainConfig, c.Processor.SdkVersion, c.Client, first, c.getAddressStart, + func(agent suihandler.SuiHandlerAgent) { + c.Agents = append(c.Agents, wrapAgent(agent)) + }, + ) + if extErr != nil { + return extErr + } + logger.Infof("built %d grpc agents", len(c.Agents)) + return nil +} + +// wrapAgent wraps a json-rpc sui agent (built by the shared BuildSuiAgents) into its grpc twin, which +// reuses the embedded agent's filters but reads grpc data when building bindings. +func wrapAgent(agent suihandler.SuiHandlerAgent) GrpcHandlerAgent { + switch ag := agent.(type) { + case suihandler.HandlerAgentEvent: + return HandlerAgentEvent{HandlerAgentEvent: ag} + case suihandler.HandlerAgentFunction: + return HandlerAgentFunction{HandlerAgentFunction: ag} + case suihandler.HandlerAgentChange: + return HandlerAgentChange{HandlerAgentChange: ag} + case suihandler.HandlerAgentInterval: + return HandlerAgentInterval{HandlerAgentInterval: ag} + default: + panic(errors.Errorf("unknown sui handler agent type %T", agent)) + } +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + fetchNamePrefix := fmt.Sprintf("SUI::%s::grpc::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + suigrpcdata.BuildBlockMainDataFetcher( + fetchNamePrefix, + req, + firstBlockNumber, + currentBlockNumber, + latest, + c.Client, + ), + latest, + 1, + 256*1024*1024, + 1000, + 0, + 20, + time.Second*10, + func(ctx context.Context, blockNumber uint64, from suigrpcdata.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + bd := BlockData{mainData: from, checkpointData: make(map[string]string)} + if from.SimpleBlock != nil { + bd.BlockHeader = *from.SimpleBlock + } else { + bd.BlockHeader, err = c.Client.GetSimpleBlock(ctx, blockNumber) + if err != nil { + return nil, false, err + } + } + if err = c.pushIntervalAgent(ctx, blockNumber, &bd); err != nil { + return nil, false, err + } + if bd.taskList, bd.taskTotalSize, err = c.BuildTaskList(ctx, &bd); err != nil { + return nil, false, err + } + c.DumpAddressStart(bd.checkpointData) + return &bd, true, nil + }, + ) +} + +func (c *HandlerController) pushIntervalAgent(ctx context.Context, blockNumber uint64, blockData *BlockData) error { + g, gctx := errgroup.WithContext(ctx) + for _, agent := range c.Agents { + ag, is := agent.(HandlerAgentInterval) + if !is || !data.ContainsInterval(blockData.mainData.Intervals, ag.IntervalConfig) { + continue + } + key := ag.ObjMgrKey() + g.Go(func() error { + newDict, err := ag.PushObjectLatestVersion(gctx, blockNumber, c.objMgr.Get(key)) + if err != nil { + return err + } + c.objMgr.Put(key, newDict) + return nil + }) + } + if err := g.Wait(); err != nil { + return err + } + blockData.objMgr = &c.objMgr + blockData.checkpointData[suihandler.CheckpointDataKey] = c.objMgr.GetData() + return nil +} + +func (c *HandlerController) getDataRequirement() (dr suidata.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentFunction: + dr.Txn = append(dr.Txn, suidata.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentEvent: + dr.Txn = append(dr.Txn, suidata.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentChange: + dr.ObjectChanges = append(dr.ObjectChanges, suidata.ObjectChangeRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } + } + return +} + +func (c *HandlerController) Snapshot() any { + sp := c.BaseHandlerController.Snapshot().(map[string]any) + sp["objectManager"] = c.objMgr.Snapshot() + return sp +} diff --git a/driver/controller/standard/sui/grpc/handler_change.go b/driver/controller/standard/sui/grpc/handler_change.go new file mode 100644 index 0000000..705b95c --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_change.go @@ -0,0 +1,47 @@ +package grpc + +import ( + "context" + + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentChange struct { + suihandler.HandlerAgentChange +} + +func (a HandlerAgentChange) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + checker := a.Filter.CheckerGrpc() + for i, oc := range bd.mainData.ObjectChanges { + if !checker(oc.ChangedObject) { + continue + } + var rawChange string + if rawChange, err = bd.getChange(i); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT_CHANGE, + TxIndex: int(oc.TxIndex), + Data: &protos.Data{ + Value: &protos.Data_SuiObjectChange_{ + SuiObjectChange: &protos.Data_SuiObjectChange{ + RawChanges: []string{rawChange}, + Slot: bd.GetBlockNumber(), + TxDigest: oc.TxDigest, + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: len(rawChange), + }) + } + return +} diff --git a/driver/controller/standard/sui/grpc/handler_event.go b/driver/controller/standard/sui/grpc/handler_event.go new file mode 100644 index 0000000..7751933 --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_event.go @@ -0,0 +1,64 @@ +package grpc + +import ( + "context" + + chainsui "sentioxyz/sentio-core/chain/sui" + cprotojson "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// HandlerAgentEvent embeds the json-rpc event agent (reusing its Filter / +// FetchConfig / BaseHandlerAgent / Snapshot) and only overrides binding building +// to read grpc transactions and emit grpc-format raw_event / raw_transaction. +type HandlerAgentEvent struct { + suihandler.HandlerAgentEvent +} + +func (a HandlerAgentEvent) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for txIndex, tx := range bd.mainData.Txs { + if !a.Filter.CheckGrpcTx(tx.ExecutedTransaction) { + continue + } + var rawTxn string + if rawTxn, err = bd.getTxn(txIndex, a.Filter.EventFilters, a.FetchConfig); err != nil { + return nil, err + } + eventChecker := chainsui.BuildGrpcEventChecker(a.Filter.EventFilters) + for evIndex, ev := range tx.GetEvents().GetEvents() { + if !eventChecker(ev) { + continue + } + var rawEvent []byte + if rawEvent, err = cprotojson.Marshal(ev); err != nil { + return nil, errors.Wrapf(err, "marshal grpc sui event #%d in tx %d in block %d failed", + evIndex, txIndex, bd.GetBlockNumber()) + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_EVENT, + TxIndex: txIndex, + TxInnerIndex: evIndex, + Data: &protos.Data{ + Value: &protos.Data_SuiEvent_{ + SuiEvent: &protos.Data_SuiEvent{ + RawEvent: string(rawEvent), + RawTransaction: rawTxn, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(rawEvent) + len(rawTxn), + }) + } + } + return +} diff --git a/driver/controller/standard/sui/grpc/handler_function.go b/driver/controller/standard/sui/grpc/handler_function.go new file mode 100644 index 0000000..5b000c1 --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_function.go @@ -0,0 +1,45 @@ +package grpc + +import ( + "context" + + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentFunction struct { + suihandler.HandlerAgentFunction +} + +func (a HandlerAgentFunction) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for txIndex, tx := range bd.mainData.Txs { + if !a.Filter.CheckGrpcTx(tx.ExecutedTransaction) { + continue + } + var rawTxn string + if rawTxn, err = bd.getTxn(txIndex, nil, a.FetchConfig); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + TxIndex: txIndex, + HandlerType: protos.HandlerType_SUI_CALL, + Data: &protos.Data{ + Value: &protos.Data_SuiCall_{ + SuiCall: &protos.Data_SuiCall{ + RawTransaction: rawTxn, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(rawTxn), + }) + } + return +} diff --git a/driver/controller/standard/sui/grpc/handler_interval.go b/driver/controller/standard/sui/grpc/handler_interval.go new file mode 100644 index 0000000..2d54b42 --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_interval.go @@ -0,0 +1,297 @@ +package grpc + +import ( + "context" + "fmt" + "math" + "time" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/log" + cprotojson "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + "sentioxyz/sentio-core/processor/protos" + + "github.com/pkg/errors" + rpcv2 "github.com/sentioxyz/sui-apis/sui/rpc/v2" + "google.golang.org/protobuf/types/known/fieldmaskpb" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type HandlerAgentInterval struct { + suihandler.HandlerAgentInterval +} + +const ( + grpcObjectsConcurrency = 10 + grpcObjectsBatchSize = 50 +) + +// objectReadMask asks the super node for the fields the binding needs (the default mask is only +// object_id,version,digest). Unlike the json-rpc handler — which binds just the parsed content +// (SuiParsedData) — the grpc handler binds the whole rpcv2.Object (see RawSelf below), so the mask +// must include every field that object carries: object_type and has_public_transfer (NOT present in +// the rendered json), owner, and json itself. object_type additionally drives the dynamic-object check. +var objectReadMask = &fieldmaskpb.FieldMask{ + Paths: []string{"object_id", "version", "digest", "object_type", "has_public_transfer", "owner", "json"}, +} + +var ignoreNotExistObject = envconf.LoadBool("SENTIO_SUI_INTERVAL_HANDLER_IGNORE_NON_EXISTENT_OBJECTS", true) + +var dynamicType = types.TypeTagFromStringMust( + "0x2::dynamic_field::Field<0x2::dynamic_object_field::Wrapper, 0x2::object::ID>") + +type grpcObject struct { + Version uint64 + Digest string + Content string // protojson of rpcv2.Object — the grpc-format object json +} + +// PushObjectLatestVersion overrides the embedded json-rpc implementation to track the latest object +// versions from grpc object changes (sui_filterGrpcChangedObjects) instead of the json-rpc +// sui_filterObjectChangesV2, keeping the interval handler on the grpc data source end to end (it later +// fetches those versions via GetGrpcObjects). The version-tracking logic is otherwise identical. +func (a HandlerAgentInterval) PushObjectLatestVersion( + ctx context.Context, + blockNumber uint64, + dict *suihandler.ObjectDict, +) (suihandler.ObjectDict, error) { + _, logger := log.FromContext(ctx, "handler", a.HandlerID.String(), "filter", utils.MustJSONMarshal(a.Filter)) + if dict != nil && dict.BlockNumber == blockNumber { // may be retried, so dict.BlockNumber may be equal to blockNumber + return *dict, nil + } + var from uint64 + result := suihandler.ObjectDict{ + BlockNumber: blockNumber, + ObjectLatestVersion: make(map[string]uint64), + } + if dict != nil { + from = dict.BlockNumber + 1 + result.ObjectLatestVersion = utils.CopyMap(dict.ObjectLatestVersion) + } + logger.Infof("will push grpc object latest version dict in [%d,%d]", from, blockNumber) + for from <= blockNumber { + startAt := time.Now() + end := min(blockNumber, from+suihandler.MaxQueryObjectChangeRangeSize-1) + changes, err := a.Client.GetGrpcObjectChanges(ctx, from, end, a.Filter) + if err != nil { + return suihandler.ObjectDict{}, err + } + beforeSize := len(result.ObjectLatestVersion) + var deleteCount, updateCount, createCount int + for _, bn := range utils.GetOrderedMapKeys(changes) { + for _, oc := range changes[bn] { + objectID, objectVersion := oc.GetObjectId(), oc.GetOutputVersion() + if chainsui.GetChangeType(oc.ChangedObject).IsDeleted() { + delete(result.ObjectLatestVersion, objectID) + deleteCount++ + } else if _, has := result.ObjectLatestVersion[objectID]; has { + result.ObjectLatestVersion[objectID] = objectVersion + updateCount++ + } else { + result.ObjectLatestVersion[objectID] = objectVersion + createCount++ + } + } + } + logger.With("used", time.Since(startAt).String()). + Infof("pushed grpc object latest version dict in [%d,%d], size %d => %d, created %d and updated %d and deleted %d", + from, end, beforeSize, len(result.ObjectLatestVersion), createCount, updateCount, deleteCount) + from = end + 1 + } + if size := len(result.ObjectLatestVersion); size > suihandler.MaxObjectDictLen { + err := errors.Errorf("object latest version dict size for handler %s with filter %s is too big: %d > %d", + a.HandlerID.String(), utils.MustJSONMarshal(a.Filter), size, suihandler.MaxObjectDictLen) + return suihandler.ObjectDict{}, fetcher.Permanent(err) + } + return result, nil +} + +func (a HandlerAgentInterval) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return + } + dict := bd.objMgr.Get(a.ObjMgrKey()) + if dict == nil { + return + } + + _, logger := log.FromContext(ctx) + var requests []*rpcv2.GetObjectRequest + for objectID, version := range dict.ObjectLatestVersion { + requests = append(requests, newObjectRequest(objectID, version)) + } + contents := make(map[string]grpcObject) + for len(requests) > 0 { + var resp []*rpcv2.GetObjectResult + if resp, err = a.Client.GetGrpcObjects(ctx, requests, grpcObjectsConcurrency, grpcObjectsBatchSize); err != nil { + return + } + var wrappedObjectIDList []string + for i, res := range resp { + req := requests[i] + obj := res.GetObject() + if obj == nil { + // deleted / not found — GetObjectResult carries an error instead of an object + message := fmt.Sprintf("object %s version %d not returned: %s", + req.GetObjectId(), req.GetVersion(), res.GetError().GetMessage()) + if !ignoreNotExistObject { + return nil, errors.Errorf(message) + } + logger.Warnf("%s, will be ignored", message) + continue + } + objectType, _ := types.TypeTagFromString(obj.GetObjectType()) + if a.UnwrapDynamicObject && objectType != nil && dynamicType.Include(*objectType) { + wrappedObjectIDList = append(wrappedObjectIDList, wrappedObjectID(obj.GetJson())) + } else { + var content []byte + // Bind the WHOLE rpcv2.Object, intentionally diverging from the json-rpc handler (which + // binds only the parsed content, SuiParsedData). The grpc object's rendered value + // (obj.GetJson()) is leaner than SuiParsedData: it flattens the move struct's fields to + // the top level and drops object_type / has_public_transfer (those live as sibling + // fields on rpcv2.Object, not inside json). Marshaling the whole object preserves that + // info for the processor. Example — sui-mainnet object + // 0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d version 309855314 + // grpc json (fields flattened, no object_type / has_public_transfer): + // { + // "id": "0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d", + // "name": {"name": "0xe859a7ebc84e7573d1e81ef99946f8821aeb0ff67454e579a32dd216da239621"}, + // "value": "0xc0254d60d00d9215c3a878ad2ea020aeebfb336eb143e5919a8209e71db998a0" + // } + // whereas json-rpc content wraps the fields and adds type info: + // { + // "dataType": "moveObject", + // "type": "...", + // "hasPublicTransfer": false, + // "fields": { + // "id": {"id": "0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d"}, + // "name": {...}, + // "value": "0xc0254d60d00d9215c3a878ad2ea020aeebfb336eb143e5919a8209e71db998a0" + // } + // } + if content, err = cprotojson.Marshal(obj); err != nil { + return nil, errors.Wrapf(err, "marshal grpc object %s failed", obj.GetObjectId()) + } + contents[obj.GetObjectId()] = grpcObject{ + Version: obj.GetVersion(), + Digest: obj.GetDigest(), + Content: string(content), + } + } + } + requests = requests[:0] + if len(wrappedObjectIDList) > 0 { + var stat []chainsui.ObjectStat + if stat, err = a.Client.GetObjectsStat(ctx, 0, bd.GetBlockNumber(), wrappedObjectIDList); err != nil { + return nil, err + } + for i, objectID := range wrappedObjectIDList { + if stat[i].Count > 0 { + requests = append(requests, newObjectRequest(objectID, stat[i].MaxObjectVersion)) + } else { + logger.Warnf("Object %s has no history", objectID) + } + } + } + } + + if a.Filter.TypePattern != nil { + for objectID, obj := range contents { + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT, + Data: &protos.Data{ + Value: &protos.Data_SuiObject_{ + SuiObject: &protos.Data_SuiObject{ + RawSelf: utils.WrapPointer(obj.Content), + ObjectId: objectID, + ObjectVersion: obj.Version, + ObjectDigest: obj.Digest, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(obj.Content), + }) + } + } + if a.Filter.OwnerFilter != nil { + var dataSize int + var owned []string + var self *grpcObject + for objectID, obj := range contents { + if utils.IndexOf(a.Filter.OwnerFilter.OwnerID, objectID) < 0 { + owned = append(owned, obj.Content) + dataSize += len(obj.Content) + } else { + self = utils.WrapPointer(contents[objectID]) + } + } + var selfContent *string + var selfVersion uint64 + var selfDigest string + if self != nil { + selfContent = utils.WrapPointer(self.Content) + selfVersion = self.Version + selfDigest = self.Digest + dataSize += len(self.Content) + } else if a.NeedSelf { + return + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_SuiObject_{ + SuiObject: &protos.Data_SuiObject{ + RawSelf: selfContent, + ObjectId: a.Filter.OwnerFilter.OwnerID[0], + ObjectVersion: selfVersion, + ObjectDigest: selfDigest, + RawObjects: owned, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: dataSize, + }) + } + return +} + +func newObjectRequest(objectID string, version uint64) *rpcv2.GetObjectRequest { + return &rpcv2.GetObjectRequest{ + ObjectId: utils.WrapPointer(objectID), + Version: utils.WrapPointer(version), + ReadMask: objectReadMask, + } +} + +// wrappedObjectID extracts the wrapped object id from a dynamic-object-field object's grpc json. The +// json-rpc handler reads content.fields.value, but the grpc rendered json flattens the move struct's +// fields to the top level, so "value" sits directly under the root. Verified against sui-mainnet object +// 0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d version 309855314 (a +// 0x2::dynamic_field::Field, object::ID>), whose grpc json is +// +// { +// "id": "0xc061d544681939544136efac81d212de377e2ff13eb07ef9079404ebd57cad5d", +// "name": {"name": "0xe859a7ebc84e7573d1e81ef99946f8821aeb0ff67454e579a32dd216da239621"}, +// "value": "0xc0254d60d00d9215c3a878ad2ea020aeebfb336eb143e5919a8209e71db998a0" +// } +// +// → value = 0xc0254d60d00d9215c3a878ad2ea020aeebfb336eb143e5919a8209e71db998a0 (the wrapped object id). +func wrappedObjectID(v *structpb.Value) string { + return v.GetStructValue().GetFields()["value"].GetStringValue() +} diff --git a/driver/controller/standard/sui/grpc/handler_test.go b/driver/controller/standard/sui/grpc/handler_test.go new file mode 100644 index 0000000..dba61fc --- /dev/null +++ b/driver/controller/standard/sui/grpc/handler_test.go @@ -0,0 +1,127 @@ +package grpc + +import ( + "context" + "encoding/json" + "testing" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/set" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + suigrpcdata "sentioxyz/sentio-core/driver/controller/data/sui/grpc" + suihandler "sentioxyz/sentio-core/driver/controller/standard/sui" + + rpcv2 "github.com/sentioxyz/sui-apis/sui/rpc/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func grpcTxWithEvent() *chainsui.ExtendedGrpcTransaction { + return &chainsui.ExtendedGrpcTransaction{ + Checkpoint: 100, + CheckpointDigest: "ckpt", + TimestampMs: 1700000000000, + Epoch: 5, + TxIndex: 0, + ExecutedTransaction: &rpcv2.ExecutedTransaction{ + Digest: proto.String("tx1"), + Effects: &rpcv2.TransactionEffects{Status: &rpcv2.ExecutionStatus{Success: proto.Bool(true)}}, + Events: &rpcv2.TransactionEvents{Events: []*rpcv2.Event{{ + PackageId: proto.String("0x0000000000000000000000000000000000000000000000000000000000000002"), + Module: proto.String("m"), + EventType: proto.String("0x2::m::E"), + Sender: proto.String("0xabc"), + }}}, + }, + } +} + +func newBlockData(txs ...*chainsui.ExtendedGrpcTransaction) *BlockData { + bd := &BlockData{mainData: suigrpcdata.BlockMainData{Txs: txs}} + bd.BlockHeader = suidata.SimpleBlock{Checkpoint: 100, Digest: "ckpt", TimestampMS: 1700000000000} + return bd +} + +// matchAnyFilter passes the tx-level CheckGrpcTx (an empty EventFilterV2 matches any event of a +// successful tx) so the binding-serialization is what's under test, not the filtering. +func matchAnyFilter() chainsui.TransactionFilter { + return chainsui.TransactionFilter{EventFilters: []chainsui.EventFilterV2{{}}} +} + +func TestEventHandlerGrpcBinding(t *testing.T) { + agent := HandlerAgentEvent{suihandler.HandlerAgentEvent{ + Filter: matchAnyFilter(), + FetchConfig: chainsui.TransactionFetchConfig{NeedAllEvents: true}, + }} + result, err := agent.BuildBindingDataList(context.Background(), newBlockData(grpcTxWithEvent())) + require.NoError(t, err) + require.Len(t, result, 1) + + se := result[0].Data.GetSuiEvent() + require.NotNil(t, se) + assert.Equal(t, uint64(100), se.GetSlot()) + + // raw_event is protojson of rpcv2.Event (enum/field names in camelCase, not numbers). + var ev map[string]any + require.NoError(t, json.Unmarshal([]byte(se.GetRawEvent()), &ev)) + assert.Equal(t, "0x2::m::E", ev["eventType"]) + assert.Equal(t, "m", ev["module"]) + + // raw_transaction is the grpc ExtendedGrpcTransaction shape: header fields + nested executedTransaction. + var tx map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(se.GetRawTransaction()), &tx)) + assert.Contains(t, tx, "checkpoint") + assert.Contains(t, tx, "executedTransaction") +} + +func TestFunctionHandlerGrpcBinding(t *testing.T) { + agent := HandlerAgentFunction{suihandler.HandlerAgentFunction{ + Filter: matchAnyFilter(), + FetchConfig: chainsui.TransactionFetchConfig{NeedAllEvents: true}, + }} + result, err := agent.BuildBindingDataList(context.Background(), newBlockData(grpcTxWithEvent())) + require.NoError(t, err) + require.Len(t, result, 1) + + sc := result[0].Data.GetSuiCall() + require.NotNil(t, sc) + var tx map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(sc.GetRawTransaction()), &tx)) + assert.Contains(t, tx, "checkpoint") + assert.Contains(t, tx, "executedTransaction") +} + +func TestChangeHandlerGrpcBinding(t *testing.T) { + const objectID = "0x0000000000000000000000000000000000000000000000000000000000000abc" + oc := &chainsui.ExtendedGrpcChangedObject{ + Checkpoint: 100, + TxIndex: 3, + TxDigest: "txd", + ChangedObject: &rpcv2.ChangedObject{ + ObjectId: proto.String(objectID), + ObjectType: proto.String("0x2::coin::Coin"), + }, + } + agent := HandlerAgentChange{suihandler.HandlerAgentChange{ + Filter: chainsui.ObjectChangeFilter{ObjectIDIn: set.New[string](objectID)}, + }} + bd := newBlockData() + bd.mainData.ObjectChanges = []*chainsui.ExtendedGrpcChangedObject{oc} + + result, err := agent.BuildBindingDataList(context.Background(), bd) + require.NoError(t, err) + require.Len(t, result, 1) + + soc := result[0].Data.GetSuiObjectChange() + require.NotNil(t, soc) + assert.Equal(t, "txd", soc.GetTxDigest()) + assert.Equal(t, 3, result[0].TxIndex) + require.Len(t, soc.GetRawChanges(), 1) + + // raw_changes[0] is the ExtendedGrpcChangedObject shape: header fields + nested changedObject. + var change map[string]json.RawMessage + require.NoError(t, json.Unmarshal([]byte(soc.GetRawChanges()[0]), &change)) + assert.Contains(t, change, "checkpoint") + assert.Contains(t, change, "changedObject") +} diff --git a/driver/controller/standard/sui/handler.go b/driver/controller/standard/sui/handler.go new file mode 100644 index 0000000..e82c603 --- /dev/null +++ b/driver/controller/standard/sui/handler.go @@ -0,0 +1,225 @@ +package sui + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" +) + +type SuiHandlerAgent interface { + standard.HandlerAgent[*BlockData] +} + +type HandlerController struct { + *standard.BaseHandlerController[sui.Client, *BlockData, SuiHandlerAgent] + + objMgr ObjectDictSetManager +} + +func NewHandlerController( + processor *models.Processor, + initResult *protos.InitResponse, + chainConfig *config.ChainConfig, + client sui.Client, + processorClients []protos.ProcessorV3Client, +) *HandlerController { + return &HandlerController{ + BaseHandlerController: standard.NewBaseHandlerController[sui.Client, *BlockData, SuiHandlerAgent]( + processor, initResult, chainConfig, client, processorClients), + } +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + if err := c.objMgr.Load(checkpoint); err != nil { + return controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "parse object set from checkpoint data failed")) + } + if extErr := c.SetTemplates(ctx, templates); extErr != nil { + return extErr + } + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber()); extErr != nil { + return extErr + } + c.AddressStartReady() + c.DisableAgents(ctx) + if extErr := c.PrepareExecute(ctx); extErr != nil { + return extErr + } + return nil +} + +func (c *HandlerController) Epilogue() { + c.BaseHandlerController.FinishExecute() +} + +func (c *HandlerController) getAddressStart(ctx context.Context, address string, start uint64) (uint64, error) { + return c.GetAddressStart( + address, + start, + func() (uint64, error) { + newStart, has, getErr := c.Client.GetObjectCreation(ctx, address, start) + if getErr != nil { + return 0, getErr + } + if has { + return newStart, nil + } + // if has is false, object with id `address` may not exist, in this situation keep the user's StartBlock + return start, nil + }) +} + +func (c *HandlerController) buildAgents(ctx context.Context, first, _ uint64) *controller.ExternalError { + _, logger := log.FromContext(ctx) + c.Agents = nil + extErr := BuildSuiAgents( + ctx, c.Config, c.ChainConfig, c.Processor.SdkVersion, c.Client, first, c.getAddressStart, + func(agent SuiHandlerAgent) { + c.Agents = append(c.Agents, agent) + }, + ) + if extErr != nil { + return extErr + } + logger.Infof("built %d agents", len(c.Agents)) + return nil +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.BuildReportRequirements(currentBlockNumber)...) + fetchNamePrefix := fmt.Sprintf("SUI::%s::", c.ChainConfig.ChainID) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + sui.BuildBlockMainDataFetcher( + fetchNamePrefix, + req, + firstBlockNumber, + currentBlockNumber, + latest, + c.Client, + ), + latest, + 1, // The transfer process must be performed strictly in block order，so it cannot be performed concurrently. + 256*1024*1024, + 1000, + 0, // The push process may take a long time, so no timeout is set. + 20, + time.Second*10, + func(ctx context.Context, blockNumber uint64, from sui.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + bd := BlockData{mainData: from, checkpointData: make(map[string]string)} + // Always need the header. Prefer the one the data fetchers prefetched concurrently (off this + // strictly block-ordered path); only fall back to a serial RPC if it's missing, which keeps + // the order-independent sui_getSimpleCheckpoint out of the throughput-limiting path. + if from.SimpleBlock != nil { + bd.BlockHeader = *from.SimpleBlock + } else { + bd.BlockHeader, err = c.Client.GetSimpleBlock(ctx, blockNumber) + if err != nil { + return nil, false, err + } + } + // interval agent should push object latest version + if err = c.pushIntervalAgent(ctx, blockNumber, &bd); err != nil { + return nil, false, err + } + // build binding data + if bd.taskList, bd.taskTotalSize, err = c.BuildTaskList(ctx, &bd); err != nil { + return nil, false, err + } + c.DumpAddressStart(bd.checkpointData) + return &bd, true, nil + }, + ) +} + +func (c *HandlerController) pushIntervalAgent(ctx context.Context, blockNumber uint64, blockData *BlockData) error { + g, gctx := errgroup.WithContext(ctx) + for _, agent := range c.Agents { + ag, is := agent.(HandlerAgentInterval) + if !is || !data.ContainsInterval(blockData.mainData.Intervals, ag.IntervalConfig) { + continue + } + key := ag.ObjMgrKey() + g.Go(func() error { + if newDict, err := ag.PushObjectLatestVersion(gctx, blockNumber, c.objMgr.Get(key)); err != nil { + return err + } else { + c.objMgr.Put(key, newDict) + return nil + } + }) + } + if err := g.Wait(); err != nil { + return err + } + blockData.objMgr = &c.objMgr + blockData.checkpointData[CheckpointDataKey] = c.objMgr.GetData() + return nil +} + +func (c *HandlerController) getDataRequirement() (dr sui.DataRequirement) { + for _, agent := range c.Agents { + switch ag := agent.(type) { + case HandlerAgentFunction: + dr.Txn = append(dr.Txn, sui.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentEvent: + dr.Txn = append(dr.Txn, sui.TransactionRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + FetchConfig: ag.FetchConfig, + }) + case HandlerAgentChange: + dr.ObjectChanges = append(dr.ObjectChanges, sui.ObjectChangeRequirement{ + BlockRange: ag.Range, + Filter: ag.Filter, + }) + case HandlerAgentInterval: + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } + } + return +} + +func (c *HandlerController) Snapshot() any { + sp := c.BaseHandlerController.Snapshot().(map[string]any) + sp["objectManager"] = c.objMgr.Snapshot() + return sp +} diff --git a/driver/controller/standard/sui/handler_change.go b/driver/controller/standard/sui/handler_change.go new file mode 100644 index 0000000..2496068 --- /dev/null +++ b/driver/controller/standard/sui/handler_change.go @@ -0,0 +1,69 @@ +package sui + +import ( + "context" + "math" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentChange struct { + controller.BaseHandlerAgent + + Filter sui.ObjectChangeFilter +} + +func (a HandlerAgentChange) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + checker := a.Filter.Checker() + for i, oc := range bd.mainData.ObjectChanges { + if !checker(oc) { + continue + } + // rawChange is the result of json marshal types.ObjectChange + // the required fields include: + // - objectId + // - digest + // - version + // - previousVersion (may not exist) + // - type (changeType) + // - owner + // - objectType + var rawChange string + if rawChange, err = bd.getChange(i); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT_CHANGE, + TxIndex: utils.Select(oc.TxIndex < 0, math.MaxInt, oc.TxIndex), + Data: &protos.Data{ + Value: &protos.Data_SuiObjectChange_{ + SuiObjectChange: &protos.Data_SuiObjectChange{ + RawChanges: []string{rawChange}, + Slot: bd.GetBlockNumber(), + TxDigest: oc.TxDigest.String(), + Timestamp: timestamppb.New(bd.GetBlockTime()), + }, + }, + }, + DataSize: len(rawChange), + }) + } + return +} + +func (a HandlerAgentChange) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + } +} diff --git a/driver/controller/standard/sui/handler_event.go b/driver/controller/standard/sui/handler_event.go new file mode 100644 index 0000000..db80f19 --- /dev/null +++ b/driver/controller/standard/sui/handler_event.go @@ -0,0 +1,73 @@ +package sui + +import ( + "context" + "encoding/json" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentEvent struct { + controller.BaseHandlerAgent + + Filter sui.TransactionFilter + FetchConfig sui.TransactionFetchConfig +} + +func (a HandlerAgentEvent) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for txIndex, tx := range bd.mainData.Txs { + if !a.Filter.Check(tx) { + continue + } + var rawTxn string + if rawTxn, err = bd.getTxn(txIndex, a.Filter.EventFilters, a.FetchConfig); err != nil { + return nil, err + } + eventChecker := sui.BuildEventChecker(a.Filter.EventFilters) + for evIndex, ev := range tx.Events { + if !eventChecker(ev) { + continue + } + var rawEvent []byte + if rawEvent, err = json.Marshal(ev); err != nil { + return nil, errors.Wrapf(err, "marshal sui event #%d in tx %d/%s in block %d failed", + evIndex, tx.TransactionPosition, tx.Digest.String(), bd.GetBlockNumber()) + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_EVENT, + TxIndex: txIndex, + TxInnerIndex: evIndex, + Data: &protos.Data{ + Value: &protos.Data_SuiEvent_{ + SuiEvent: &protos.Data_SuiEvent{ + RawEvent: string(rawEvent), + RawTransaction: rawTxn, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(rawEvent) + len(rawTxn), + }) + } + } + return +} + +func (a HandlerAgentEvent) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/sui/handler_function.go b/driver/controller/standard/sui/handler_function.go new file mode 100644 index 0000000..b81233c --- /dev/null +++ b/driver/controller/standard/sui/handler_function.go @@ -0,0 +1,58 @@ +package sui + +import ( + "context" + + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type HandlerAgentFunction struct { + controller.BaseHandlerAgent + + Filter sui.TransactionFilter + FetchConfig sui.TransactionFetchConfig +} + +func (a HandlerAgentFunction) BuildBindingDataList( + _ context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + for txIndex, tx := range bd.mainData.Txs { + if !a.Filter.Check(tx) { + continue + } + var rawTxn string + if rawTxn, err = bd.getTxn(txIndex, nil, a.FetchConfig); err != nil { + return nil, err + } + result = append(result, standard.BindingDataInner{ + TxIndex: txIndex, + HandlerType: protos.HandlerType_SUI_CALL, + Data: &protos.Data{ + Value: &protos.Data_SuiCall_{ + SuiCall: &protos.Data_SuiCall{ + RawTransaction: rawTxn, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: tx.Checkpoint.Uint64(), + }, + }, + }, + DataSize: len(rawTxn), + }) + } + return +} + +func (a HandlerAgentFunction) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + "FetchConfig": a.FetchConfig, + } +} diff --git a/driver/controller/standard/sui/handler_interval.go b/driver/controller/standard/sui/handler_interval.go new file mode 100644 index 0000000..377b726 --- /dev/null +++ b/driver/controller/standard/sui/handler_interval.go @@ -0,0 +1,449 @@ +package sui + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" + + chainsui "sentioxyz/sentio-core/chain/sui" + "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/compress" + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/processor/protos" +) + +type ObjectDictSetManager struct { + mu sync.RWMutex + data map[string]ObjectDict + cachedData string + cachedBlockNumber *uint64 +} + +func (m *ObjectDictSetManager) Get(key string) *ObjectDict { + m.mu.RLock() + defer m.mu.RUnlock() + if od, has := m.data[key]; has { + return &od + } + return nil +} + +func (m *ObjectDictSetManager) Put(key string, value ObjectDict) { + m.mu.Lock() + defer m.mu.Unlock() + m.data[key] = value +} + +const CheckpointDataKey = "ObjectDictSetManager" + +func (m *ObjectDictSetManager) Load(checkpoint *controller.Checkpoint) error { + var raw string + if checkpoint != nil { + raw = checkpoint.Data[CheckpointDataKey] + } + return m.load(raw) +} + +// GetData use c.cachedData so the unsaved checkpoint will not dup the same data, can save memory +func (m *ObjectDictSetManager) GetData() string { + m.mu.RLock() + defer m.mu.RUnlock() + if len(m.data) == 0 { + return "" + } + var bn uint64 + for _, dict := range m.data { + bn = max(bn, dict.BlockNumber) + } + if m.cachedBlockNumber == nil || *m.cachedBlockNumber < bn { + m.cachedBlockNumber, m.cachedData = &bn, m.dump() + } + return m.cachedData +} + +func (m *ObjectDictSetManager) dump() string { + var buf bytes.Buffer + for key, dict := range m.data { + if buf.Len() > 0 { + buf.WriteRune('\n') + } + buf.WriteString(key) + buf.WriteRune('@') + buf.WriteString(dict.Dump()) + } + b, _ := compress.Dump(buf.String()) + return string(b) +} + +func (m *ObjectDictSetManager) load(data string) error { + var raw string + if err := compress.Load([]byte(data), &raw); err != nil { + return errors.Wrapf(err, "decompress data failed") + } + result := make(map[string]ObjectDict) + var line string + var blockNumber uint64 + for len(raw) > 0 { + line, raw, _ = strings.Cut(raw, "\n") + key, value, _ := strings.Cut(line, "@") + var dict ObjectDict + if err := dict.Load(value); err != nil { + return errors.Wrapf(err, "load object dictionary for %s failed", key) + } + blockNumber = max(blockNumber, dict.BlockNumber) + result[key] = dict + } + m.mu.Lock() + m.data = result + if len(result) > 0 { + m.cachedBlockNumber, m.cachedData = &blockNumber, data + } else { + m.cachedBlockNumber, m.cachedData = nil, "" + } + m.mu.Unlock() + return nil +} + +var objectDictPreviewCount = envconf.LoadUInt64("SENTIO_SUI_INTERVAL_HANDLER_OBJECT_DICT_PREVIEW_COUNT", 100) + +func (m *ObjectDictSetManager) Snapshot() any { + m.mu.RLock() + defer m.mu.RUnlock() + dataPreview := make(map[string]any) + for key, dict := range m.data { + dataPreview[key] = dict.Snapshot(int(objectDictPreviewCount)) + } + return dataPreview +} + +type ObjectDict struct { + BlockNumber uint64 + ObjectLatestVersion map[string]uint64 +} + +func (d *ObjectDict) Snapshot(previewCount int) any { + preview := make(map[string]uint64) + for objectID, latestVersion := range d.ObjectLatestVersion { + preview[objectID] = latestVersion + if len(preview) >= previewCount { + break + } + } + return map[string]any{ + "blockNumber": d.BlockNumber, + "size": len(d.ObjectLatestVersion), + "preview": preview, + } +} + +func (d *ObjectDict) Dump() string { + var buf bytes.Buffer + buf.WriteString(strconv.FormatUint(d.BlockNumber, 16)) + for _, objectID := range utils.GetOrderedMapKeys(d.ObjectLatestVersion) { + buf.WriteRune('#') + buf.WriteString(objectID) + buf.WriteRune(':') + buf.WriteString(strconv.FormatUint(d.ObjectLatestVersion[objectID], 16)) + } + return buf.String() +} + +func (d *ObjectDict) Load(raw string) (err error) { + var part string + part, raw, _ = strings.Cut(raw, "#") + if d.BlockNumber, err = strconv.ParseUint(part, 16, 64); err != nil { + return errors.Wrapf(err, "parse block number from %q failed", part) + } + d.ObjectLatestVersion = make(map[string]uint64) + for len(raw) > 0 { + part, raw, _ = strings.Cut(raw, "#") + objectID, ver, _ := strings.Cut(part, ":") + var version uint64 + if version, err = strconv.ParseUint(ver, 16, 64); err != nil { + return errors.Wrapf(err, "parse version for objectID %s from %q failed", objectID, ver) + } + d.ObjectLatestVersion[objectID] = version + } + return nil +} + +type HandlerAgentInterval struct { + controller.BaseHandlerAgent + + Client sui.Client `json:"-"` // used to check address is a ERC20 address + + IntervalConfig data.IntervalConfig + Filter chainsui.ObjectChangeFilter + NeedSelf bool + UnwrapDynamicObject bool +} + +const ( + MaxObjectDictLen = 100000 + MaxQueryObjectChangeRangeSize = 10000 +) + +func (a HandlerAgentInterval) PushObjectLatestVersion( + ctx context.Context, + blockNumber uint64, + dict *ObjectDict, +) (ObjectDict, error) { + _, logger := log.FromContext(ctx, "handler", a.HandlerID.String(), "filter", utils.MustJSONMarshal(a.Filter)) + if dict != nil && dict.BlockNumber == blockNumber { // may be retried, so dict.BlockNumber may be equal to blockNumber + return *dict, nil + } + var from uint64 + result := ObjectDict{ + BlockNumber: blockNumber, + ObjectLatestVersion: make(map[string]uint64), + } + if dict != nil { + from = dict.BlockNumber + 1 + result.ObjectLatestVersion = utils.CopyMap(dict.ObjectLatestVersion) + } + logger.Infof("will push object latest version dict in [%d,%d]", from, blockNumber) + for from <= blockNumber { + startAt := time.Now() + end := min(blockNumber, from+MaxQueryObjectChangeRangeSize-1) + changes, err := a.Client.GetObjectChanges(ctx, from, end, a.Filter) + if err != nil { + return ObjectDict{}, err + } + beforeSize := len(result.ObjectLatestVersion) + var deleteCount int + var updateCount int + var createCount int + for _, bn := range utils.GetOrderedMapKeys(changes) { + for _, oc := range changes[bn] { + objectID, objectVersion := oc.ObjectID.String(), oc.Version.Uint64() + if oc.Type.IsDeleted() { + logger.Debugf("pushing and deleted object %s/%d", objectID, objectVersion) + delete(result.ObjectLatestVersion, objectID) + deleteCount++ + } else if preVersion, has := result.ObjectLatestVersion[objectID]; has { + logger.Debugf("pushing and updated object %s/%d=>%d", objectID, preVersion, objectVersion) + result.ObjectLatestVersion[objectID] = objectVersion + updateCount++ + } else { + logger.Debugf("pushing and created object %s/%d", objectID, objectVersion) + result.ObjectLatestVersion[objectID] = objectVersion + createCount++ + } + } + } + logger.With("used", time.Since(startAt).String()). + Infof("pushed object latest version dict in [%d,%d], size %d => %d, created %d and updated %d and deleted %d", + from, end, beforeSize, len(result.ObjectLatestVersion), createCount, updateCount, deleteCount) + from = end + 1 + } + if size := len(result.ObjectLatestVersion); size > MaxObjectDictLen { + err := errors.Errorf("object latest version dict size for handler %s with filter %s is too big: %d > %d", + a.HandlerID.String(), utils.MustJSONMarshal(a.Filter), size, MaxObjectDictLen) + return ObjectDict{}, fetcher.Permanent(err) + } + return result, nil +} + +type objectDetails struct { + Version string `json:"version"` + Digest string `json:"digest"` + Content json.RawMessage `json:"content"` + Type string `json:"type"` +} + +type object struct { + Version uint64 + Digest string + Content json.RawMessage +} + +var ignoreNotExistObject = envconf.LoadBool("SENTIO_SUI_INTERVAL_HANDLER_IGNORE_NON_EXISTENT_OBJECTS", true) + +var dynamicType = types.TypeTagFromStringMust( + "0x2::dynamic_field::Field<0x2::dynamic_object_field::Wrapper, 0x2::object::ID>") + +func (a HandlerAgentInterval) ObjMgrKey() string { + return utils.MustJSONMarshal(a.Filter) +} + +func (a HandlerAgentInterval) BuildBindingDataList( + ctx context.Context, + bd *BlockData, +) (result []standard.BindingDataInner, err error) { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return + } + + dict := bd.objMgr.Get(a.ObjMgrKey()) + if dict == nil { + return + } + + _, logger := log.FromContext(ctx) + var opts = types.SuiObjectDataOptions{ + ShowContent: true, + ShowType: true, + } + var requests []types.SuiGetPastObjectRequest + for objectID, version := range dict.ObjectLatestVersion { + requests = append(requests, types.SuiGetPastObjectRequest{ + ObjectID: types.StrToObjectIDMust(objectID), + Version: types.Uint64ToNumber(version), + }) + } + contents := make(map[string]object) + for len(requests) > 0 { + var resp []types.SuiPastObjectResponse + if resp, err = a.Client.TryMultiGetPastObjects(ctx, requests, opts); err != nil { + return + } + var wrappedObjectIDList []string + for i, obj := range resp { + req := requests[i] + switch obj.Status { + case types.SuiPastObjectStatusVersionFound: + var details objectDetails + if err = json.Unmarshal(obj.Details, &details); err != nil { + return nil, errors.Wrapf(err, "object %s version %d unmarshal details failed", + req.ObjectID, req.Version.Uint64()) + } + objectType, _ := types.TypeTagFromString(details.Type) + if a.UnwrapDynamicObject && objectType != nil && dynamicType.Include(*objectType) { + var moveObj struct { + Fields struct { + Value string `json:"value"` + } `json:"fields"` + } + if err = json.Unmarshal(details.Content, &moveObj); err != nil { + return nil, errors.Wrapf(err, "object %s version %d unmarshal move object failed", + req.ObjectID, req.Version.Uint64()) + } + wrappedObjectIDList = append(wrappedObjectIDList, moveObj.Fields.Value) + } else { + contents[req.ObjectID.String()] = object{ + Version: req.Version.Uint64(), + Digest: details.Digest, + Content: details.Content, + } + } + case types.SuiPastObjectStatusObjectDeleted: + default: + message := fmt.Sprintf("object %s version %d has unexpected status %s", + req.ObjectID, req.Version.Uint64(), obj.Status) + if !ignoreNotExistObject { + return nil, errors.Errorf(message) + } + logger.Warnf("%s, will be ignored", message) + } + } + requests = requests[:0] + if len(wrappedObjectIDList) > 0 { + var stat []chainsui.ObjectStat + if stat, err = a.Client.GetObjectsStat(ctx, 0, bd.GetBlockNumber(), wrappedObjectIDList); err != nil { + return nil, err + } + for i, objectID := range wrappedObjectIDList { + if stat[i].Count > 0 { + requests = append(requests, types.SuiGetPastObjectRequest{ + ObjectID: types.StrToObjectIDMust(objectID), + Version: types.Uint64ToNumber(stat[i].MaxObjectVersion), + }) + } else { + logger.Warnf("Object %s has no history", objectID) + } + } + } + } + + if a.Filter.TypePattern != nil { + for objectID, obj := range contents { + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT, + Data: &protos.Data{ + Value: &protos.Data_SuiObject_{ + SuiObject: &protos.Data_SuiObject{ + RawSelf: utils.WrapPointer(string(obj.Content)), + ObjectId: objectID, + ObjectVersion: obj.Version, + ObjectDigest: obj.Digest, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: len(obj.Content), + }) + } + } + if a.Filter.OwnerFilter != nil { + var dataSize int + var owned []string + var self *object + for objectID, obj := range contents { + if utils.IndexOf(a.Filter.OwnerFilter.OwnerID, objectID) < 0 { + owned = append(owned, string(obj.Content)) + dataSize += len(obj.Content) + } else { + self = utils.WrapPointer(contents[objectID]) + } + } + var selfContent *string + var selfVersion uint64 + var selfDigest string + if self != nil { + selfContent = utils.WrapPointer(string(self.Content)) + selfVersion = self.Version + selfDigest = self.Digest + dataSize += len(self.Content) + } else if a.NeedSelf { + // need self but self not exist + return + } + result = append(result, standard.BindingDataInner{ + HandlerType: protos.HandlerType_SUI_OBJECT, + TxIndex: math.MaxInt, + Data: &protos.Data{ + Value: &protos.Data_SuiObject_{ + SuiObject: &protos.Data_SuiObject{ + RawSelf: selfContent, + ObjectId: a.Filter.OwnerFilter.OwnerID[0], + ObjectVersion: selfVersion, + ObjectDigest: selfDigest, + RawObjects: owned, + Timestamp: timestamppb.New(bd.GetBlockTime()), + Slot: bd.GetBlockNumber(), + }, + }, + }, + DataSize: dataSize, + }) + } + return +} + +func (a HandlerAgentInterval) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "IntervalConfig": a.IntervalConfig, + "Filter": a.Filter, + "NeedSelf": a.NeedSelf, + "UnwrapDynamicObject": a.UnwrapDynamicObject, + } +} diff --git a/driver/controller/standard/sui/handler_interval_test.go b/driver/controller/standard/sui/handler_interval_test.go new file mode 100644 index 0000000..c4892fe --- /dev/null +++ b/driver/controller/standard/sui/handler_interval_test.go @@ -0,0 +1,50 @@ +package sui + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_loadDump(t *testing.T) { + m0 := ObjectDictSetManager{} + assert.NoError(t, m0.load("")) + + m0.Put("x1", ObjectDict{ + BlockNumber: 100, + ObjectLatestVersion: map[string]uint64{ + "0x1": 100, + }, + }) + + m0.Put("x2", ObjectDict{ + BlockNumber: 101, + ObjectLatestVersion: map[string]uint64{ + "0x21": 999, + "0x22": 0, + }, + }) + + x3 := ObjectDict{ + BlockNumber: 100000000, + ObjectLatestVersion: map[string]uint64{}, + } + for i := 0; i < 100000; i++ { + id := fmt.Sprintf("0x%016x%016x%016x%016x", rand.Uint64(), rand.Uint64(), rand.Uint64(), rand.Uint64()) + ver := rand.Uint64() + x3.ObjectLatestVersion[id] = ver + } + m0.Put("x3", x3) + + d0 := m0.GetData() + t.Logf("data0: %d", len(d0)) + + var m1 ObjectDictSetManager + assert.NoError(t, m1.load(d0)) + + assert.Equal(t, m0.data, m1.data) + assert.Equal(t, m0.cachedBlockNumber, m1.cachedBlockNumber) + assert.Equal(t, m0.cachedData, m1.cachedData) +} diff --git a/driver/controller/standard/task.go b/driver/controller/standard/task.go new file mode 100644 index 0000000..ae66e73 --- /dev/null +++ b/driver/controller/standard/task.go @@ -0,0 +1,658 @@ +package standard + +import ( + "context" + "fmt" + "time" + + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/timer" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" + "sentioxyz/sentio-core/driver/timeseries" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type partitionWithIndex struct { + partition string + index uint64 +} +type waiter struct { + ready *concurrency.ResourceWaiter[uint64] + finish *concurrency.ResourceWaiter[partitionWithIndex] +} + +type task struct { + bindingData + + sp streamPool + waiter *waiter + + metricConfigs map[timeseries.MetaType]map[string]*protos.MetricConfig + webhookChannels map[string]string + chainID string + processor *models.Processor + + stream protos.ProcessorV3_ProcessBindingsStreamClient + index controller.TaskIndex + partition string + + timer *timer.Timer + + logger *log.SentioLogger +} + +func (b *task) errLogger() *log.SentioLogger { + return b.logger.With("binding", b.data.String()) +} + +func (b *task) Init(ctx context.Context, index controller.TaskIndex, progressbar controller.ProgressBar) { + b.index = index + if enableBindingDataPartition { + b.waiter.ready.NewResource(b.index.Global) + } else { + b.waiter.finish.NewResource(partitionWithIndex{index: b.index.Global}) + } + b.timer = timer.NewTimer() + _, b.logger = log.FromContext(ctx, + "block", controller.GetBlockSummary(b), + "latest", controller.GetBlockSummary(progressbar.LatestBlock), + "index", index, + "handler", b.handlerID.String()) +} + +func (b *task) GetHandlerID() controller.HandlerID { + return b.handlerID +} + +func (b *task) title() string { + return fmt.Sprintf("#%d binding data %d/%d for handler %s in block %s", + b.index.Global, b.index.InBlock, b.index.TotalInBlock, b.handlerID, controller.GetBlockSummary(b)) +} + +func (b *task) Summary() string { + return b.title() +} + +func (b *task) Exec( + ctx context.Context, + checkpointCtrl controller.CheckpointController, +) (extErr *controller.ExternalError) { + b.logger.Debug("task started") + start := b.timer.Start("ALL") + hmi := controller.TaskInfo{ + Processor: b.processor, + ChainID: b.chainID, + Handler: b.handlerID.Name, + Category: b.handlerID.Type, + DataSource: b.handlerID.DataSource, + } + var stat statistic + ctx = controller.N.BeforeEntityOperation(ctx, hmi) // to pass metric attrs to noticeController of entityController + defer func() { + used := start.End() + usedReport := b.timer.ReportDistribution("ALL", "*") + b.logger = b.logger.With("used", usedReport, "stat", stat) + if extErr == nil { + b.logger.Debugw("task succeed") + b.waiter.finish.ResourceReady(partitionWithIndex{partition: b.partition, index: b.index.Global}) + } else if errors.Is(extErr.Wrapped(), context.Canceled) { + b.logger.Warnfe(extErr, "task canceled") + } else { + b.errLogger().Errore(extErr, "task failed") + } + controller.N.TaskDone(ctx, hmi, extErr == nil, used) + for eventName, count := range stat.TimeSeries[timeseries.MetaTypeEvent] { + controller.N.DataEmitted(ctx, hmi, "event", "", eventName, int64(count)) + } + for counterName, count := range stat.TimeSeries[timeseries.MetaTypeCounter] { + controller.N.DataEmitted(ctx, hmi, "metric", "counter", counterName, int64(count)) + } + for gaugeName, count := range stat.TimeSeries[timeseries.MetaTypeGauge] { + controller.N.DataEmitted(ctx, hmi, "metric", "gauge", gaugeName, int64(count)) + } + for subtype, st := range stat.Entity { + for entityName, count := range st { + controller.N.DataEmitted(ctx, hmi, "entity", subtype, entityName, int64(count)) + } + } + }() + if extErr = b.getStream(ctx); extErr != nil { + return extErr + } + defer b.returnStream() + if extErr = b.sendBindingData(ctx); extErr != nil { + return extErr + } + if enableBindingDataPartition { + if extErr = b.recvPartition(ctx); extErr != nil { + return extErr + } + b.waiter.finish.NewResource(partitionWithIndex{partition: b.partition, index: b.index.Global}) + b.waiter.ready.ResourceReady(b.index.Global) + err := b.waiter.ready.Wait(ctx, func(u uint64) bool { + return u < b.index.Global + }) + if err != nil { + return controller.NewExternalError(controller.ErrCodeSystem, + errors.Errorf("waiting all previous tasks got partition failed: %v", err)) + } + if extErr = b.sendStartCommand(ctx); extErr != nil { + return extErr + } + } + stat, extErr = b.waitProcess(ctx, checkpointCtrl) + return extErr +} + +func (b *task) getStream(ctx context.Context) *controller.ExternalError { + select { + case b.stream = <-b.sp: + return nil + case <-ctx.Done(): + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("get stream for %s failed: %v", b.title(), ctx.Err())) + } +} + +func (b *task) returnStream() { + if b.stream != nil { + b.sp <- b.stream + } +} + +func (b *task) streamSend( + ctx context.Context, + req *protos.ProcessStreamRequest, + what, mark string, + timeout time.Duration, +) *controller.ExternalError { + start := b.timer.Start(mark) + err := timer.Wait(ctx, timeout, time.Minute, func() error { + return b.stream.Send(req) + }, func(used time.Duration) { + b.logger.Warnf("stream send %s already waited %s", what, used.String()) + }) + if err != nil { + if errors.Is(err, context.Canceled) { + b.logger.Warnfe(err, "stream send %s canceled", what) + } else { + b.errLogger().Errorfe(err, "stream send %s failed", what) + } + return controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Wrapf(err, "send %s for %s failed", what, b.title())) + } + used := start.End() + b.logger.With("used", used.String()).Debugf("stream sent %s", what) + return nil +} + +func (b *task) streamReceive( + ctx context.Context, + mark string, + timeout time.Duration, +) (*protos.ProcessStreamResponseV3, *controller.ExternalError) { + defer b.timer.Start(mark).End() + var resp *protos.ProcessStreamResponseV3 + err := timer.Wait(ctx, timeout, time.Minute, func() (recvErr error) { + resp, recvErr = b.stream.Recv() + return recvErr + }, func(used time.Duration) { + b.logger.Warnf("stream receive already waited %s", used.String()) + }) + if err != nil { + if errors.Is(err, context.Canceled) { + b.logger.Warnfe(err, "stream receive canceled") + } else { + b.errLogger().Errorfe(err, "stream receive failed") + } + return nil, controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Wrapf(err, "receive for %s failed", b.title())) + } + if uint64(resp.GetProcessId()) != b.index.Global { + return nil, controller.NewExternalError(controller.ErrCodeCallProcessorFailed, + errors.Errorf("unexpected ProcessID #%d for %s", resp.GetProcessId(), b.title())) + } + return resp, nil +} + +func (b *task) sendBindingData(ctx context.Context) *controller.ExternalError { + return b.streamSend(ctx, &protos.ProcessStreamRequest{ + ProcessId: int32(b.index.Global), + Value: &protos.ProcessStreamRequest_Binding{Binding: b.data}, + }, "binding data", "SB", time.Minute*30) +} + +func (b *task) recvPartition(ctx context.Context) *controller.ExternalError { + resp, extErr := b.streamReceive(ctx, "RP", time.Minute*30) + if extErr != nil { + return extErr + } + for _, p := range resp.GetPartitions().GetPartitions() { + b.partition = p.GetUserValue() + } + return nil +} + +func (b *task) sendStartCommand(ctx context.Context) *controller.ExternalError { + return b.streamSend(ctx, &protos.ProcessStreamRequest{ + ProcessId: int32(b.index.Global), + Value: &protos.ProcessStreamRequest_Start{Start: true}, + }, "start command", "SS", time.Minute*30) +} + +type statistic struct { + Get int + List int + ListEntities int + Upsert int + UpsertEntities int + Update int + UpdateEntities int + Delete int + DeleteEntities int + + Entity map[string]map[string]int // Entity[op][name] + TimeSeries map[timeseries.MetaType]map[string]int + Export map[string]int +} + +func (b *task) waitProcess( + ctx context.Context, + checkpointCtrl controller.CheckpointController, +) (statistic, *controller.ExternalError) { + var stat statistic + stat.Entity = make(map[string]map[string]int) + stat.TimeSeries = make(map[timeseries.MetaType]map[string]int) + stat.Export = make(map[string]int) + for { + resp, extErr := b.streamReceive(ctx, "RR", time.Minute*30) + if extErr != nil { + return stat, extErr + } + if resp.GetResult() != nil { + if errMsg := resp.GetResult().GetStates().GetError(); errMsg != "" { + b.errLogger().Errorf("got error in final result: %s", errMsg) + return stat, controller.NewExternalError(controller.ErrCodeProcessFailed, + errors.Errorf("got error in %s: %s", b.title(), errMsg)) + } + // Event v2 and metric (gauge/counter) v2 data are no longer supported by + // the streaming driver (driver v3+). A processor that still emits them is + // running an unsupported SDK and must be rejected instead of silently + // dropping the data. + if len(resp.GetResult().GetEvents()) > 0 { + return stat, controller.NewExternalError(controller.ErrCodeProcessFailed, + errors.Errorf("event v2 data is no longer supported in %s, please upgrade the processor SDK", b.title())) + } + if len(resp.GetResult().GetGauges()) > 0 || len(resp.GetResult().GetCounters()) > 0 { + return stat, controller.NewExternalError(controller.ErrCodeProcessFailed, + errors.Errorf("metric v2 data is no longer supported in %s, please upgrade the processor SDK", b.title())) + } + if tsr := resp.GetResult().GetTimeseriesResult(); len(tsr) > 0 { + b.logger.Debugf("got %d time series data in final process result", len(tsr)) + if data, convertErr := b.ConvertTimeSeriesData(tsr); convertErr != nil { + b.errLogger().Errorfe(convertErr, "convert time series data failed") + return stat, convertErr.Wrapf("invalid time series data in %s", b.title()) + } else { + checkpointCtrl.InsertTimeSeriesData(b.GetBlockNumber(), b.index, data) + timeseries.Statistic(data, stat.TimeSeries) + } + } + if epr := resp.GetResult().GetExports(); len(epr) > 0 { + b.logger.Debugf("got %d export data in final process result", len(epr)) + data := b.ConvertExportData(epr) + checkpointCtrl.InsertWebhookData(b.GetBlockNumber(), b.index, data) + controller.StatisticWebhookMessages(data, stat.Export) + } + return stat, nil + } + + if req := resp.GetTplRequest(); req != nil { + b.logger.Debugf("got %d templates", len(req.GetTemplates())) + extErr = checkpointCtrl.NewTemplateInstance(ctx, b, ConvertTemplateInstance(req.GetTemplates(), req.GetRemove())) + if extErr != nil { + return stat, extErr.Wrapf("add template instance in %s failed", b.title()) + } + continue + } + if req := resp.GetTsRequest(); req != nil { + b.logger.Debugf("got %d time series data", len(req.GetData())) + if data, convertErr := b.ConvertTimeSeriesData(req.GetData()); convertErr != nil { + b.errLogger().Errorfe(convertErr, "convert time series data failed: %v", req.GetData()) + return stat, convertErr.Wrapf("invalid time series data in %s", b.title()) + } else { + checkpointCtrl.InsertTimeSeriesData(b.GetBlockNumber(), b.index, data) + timeseries.Statistic(data, stat.TimeSeries) + } + continue + } + + // must be an db request + dbReq := resp.GetDbRequest() + dbResp := protos.DBResponse{OpId: dbReq.GetOpId()} + reqLogger := b.logger.With("opid", dbReq.GetOpId()) + reqErrLogger := func() *log.SentioLogger { + return reqLogger.With("binding", b.data.String(), "dbreq", dbReq.String()) + } + + // wait resource + if dbReq.GetGet() != nil || dbReq.GetList() != nil { + wr := b.timer.Start("WR") + err := b.waiter.finish.Wait(ctx, func(p partitionWithIndex) bool { + return p.partition == b.partition && p.index < b.index.Global + }) + if err != nil { + return stat, controller.NewExternalError(controller.ErrCodeSystem, + errors.Wrapf(err, "wait previous task finish for %s failed", b.title())) + } + waitUsed := wr.End() + reqLogger = reqLogger.With("waitUsed", waitUsed.String()) + } + + // do the db operation + var what string + start := b.timer.Start("DE") + if dbReq.GetGet() != nil { + entity, id := dbReq.GetGet().GetEntity(), dbReq.GetGet().GetId() + reqLogger = reqLogger.With("dbop", "get", "entity", entity, "id", id) + entityType := checkpointCtrl.GetEntityOrInterfaceType(entity) + if entityType == nil { + reqErrLogger().Errorf("get unknown entity") + return stat, controller.NewExternalError(controller.ErrCodeGetUnknownEntity, + errors.Errorf("get unknown entity %q for %s", entity, b.title())) + } + box, getErr := checkpointCtrl.GetEntity(ctx, entityType, id, b.GetBlockNumber()) + if getErr != nil { + reqErrLogger().Errorfe(getErr, "get entity failed") + return stat, getErr.Wrapf("get entity %s/%s in %s failed", entity, id, b.title()) + } + if box != nil && box.Data != nil { + data, convertErr := box.ToRichStruct(entityType) + if convertErr != nil { + reqErrLogger().Errorfe(getErr, "convert entity to RichStruct failed") + return stat, controller.NewExternalError(controller.ErrCodeInvalidEntityData, + errors.Wrapf(convertErr, "convert entity %s/%s in %s failed", entity, id, b.title())) + } + dbResp.Value = &protos.DBResponse_EntityList{ + EntityList: &protos.EntityList{ + Entities: []*protos.Entity{{ + Entity: box.Entity, + GenBlockNumber: box.GenBlockNumber, + GenBlockTime: timestamppb.New(box.GenBlockTime), + GenBlockChain: b.chainID, + Data: data, + }}, + }, + } + } else { + // not created or deleted, return an empty list + dbResp.Value = &protos.DBResponse_EntityList{ + EntityList: &protos.EntityList{}, + } + } + stat.Get++ + what = fmt.Sprintf("entity get response #%d", stat.Get) + } + if dbReq.GetList() != nil { + const defaultPageSize = 10000 + entity := dbReq.GetList().GetEntity() + pageSize := int(dbReq.GetList().GetPageSize()) + if pageSize <= 0 { + pageSize = defaultPageSize + } + reqLogger = reqLogger.With( + "dbop", "list", + "entity", entity, + "cursor", dbReq.GetList().GetCursor(), + "pageSize", pageSize, + "filters", dbReq.GetList().GetFilters()) + entityType := checkpointCtrl.GetEntityType(entity) + if entityType == nil { + reqErrLogger().Errorf("list unknown entity") + return stat, controller.NewExternalError(controller.ErrCodeListUnknownEntity, + errors.Errorf("list unknown entity %q for %s", entity, b.title())) + } + filters := make([]persistent.EntityFilter, len(dbReq.GetList().GetFilters())) + for fi, ft := range dbReq.GetList().GetFilters() { + field := entityType.GetFieldByName(ft.GetField()) + if field == nil { + reqErrLogger().Errorf("field %s.%s in entity list filter is not exist", entityType.Name, ft.GetField()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Errorf("field %s.%s in entity list filter is not exist for %s", + entityType.Name, ft.GetField(), b.title())) + } + if entityType.GetForeignKeyFieldByName(ft.GetField()).IsReverseField() { + reqErrLogger().Errorf("field %s.%s in entity list filter is a reverse foreign key", + entityType.Name, ft.GetField()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Errorf("field %s.%s in entity list filter is a reverse foreign key for %s", + entityType.Name, ft.GetField(), b.title())) + } + fieldTitle := fmt.Sprintf("%s.%s %s", entityType.Name, ft.GetField(), field.Type.String()) + filters[fi] = persistent.EntityFilter{Field: field} + switch ft.GetOp() { + case protos.DBRequest_EQ: + filters[fi].Op = persistent.EntityFilterOpEq + case protos.DBRequest_NE: + filters[fi].Op = persistent.EntityFilterOpNe + case protos.DBRequest_GT: + filters[fi].Op = persistent.EntityFilterOpGt + case protos.DBRequest_GE: + filters[fi].Op = persistent.EntityFilterOpGe + case protos.DBRequest_LT: + filters[fi].Op = persistent.EntityFilterOpLt + case protos.DBRequest_LE: + filters[fi].Op = persistent.EntityFilterOpLe + case protos.DBRequest_IN: + filters[fi].Op = persistent.EntityFilterOpIn + case protos.DBRequest_NOT_IN: + filters[fi].Op = persistent.EntityFilterOpNotIn + case protos.DBRequest_LIKE: + filters[fi].Op = persistent.EntityFilterOpLike + case protos.DBRequest_NOT_LIKE: + filters[fi].Op = persistent.EntityFilterOpNotLike + case protos.DBRequest_HAS_ALL: + filters[fi].Op = persistent.EntityFilterOpHasAll + case protos.DBRequest_HAS_ANY: + filters[fi].Op = persistent.EntityFilterOpHasAny + default: + reqErrLogger().Errorf("unknown list filter op %s", ft.GetOp()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Errorf("unknown list filter operator %v for %s", ft.GetOp(), b.title())) + } + filterValueType := field.Type + if ft.GetOp() == protos.DBRequest_HAS_ALL || ft.GetOp() == protos.DBRequest_HAS_ANY { + fieldType := schema.BreakType(field.Type) + if fieldType.CountListLayer() != 1 { + reqErrLogger().Errorf("field %s cannot use op %s", fieldTitle, ft.GetOp().String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Errorf("field %s cannot use op %s for %s", fieldTitle, ft.GetOp().String(), b.title())) + } + filterValueType = schema.BreakType(field.Type).SkipListLayer(1).Join() + } + for _, val := range ft.GetValue().GetValues() { + value, convertErr := persistent.FromRichValue(val, filterValueType) + if convertErr != nil { + reqErrLogger().Errorfe(convertErr, "convert filter value %v for field %s with op %s to go value failed", + val, fieldTitle, ft.GetOp().String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Wrapf(convertErr, "convert filter value %v for field %s with op %s to go value for %s failed", + val, fieldTitle, ft.GetOp().String(), b.title())) + } + filters[fi].Value = append(filters[fi].Value, value) + } + if initErr := filters[fi].Init(); initErr != nil { + reqErrLogger().Errorfe(initErr, "init filter for field %s with op %s", fieldTitle, ft.GetOp().String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, + errors.Wrapf(initErr, "init filter for field %s with op %s for %s failed", + fieldTitle, ft.GetOp().String(), b.title())) + } + } + boxes, nextCursor, listErr := checkpointCtrl.ListEntity( + ctx, entityType, filters, dbReq.GetList().GetCursor(), pageSize, b.GetBlockNumber()) + if listErr != nil { + reqErrLogger().Errorfe(listErr, "list entity failed") + return stat, listErr.Wrapf("list entity for %s failed", b.title()) + } + data := &protos.EntityList{Entities: make([]*protos.Entity, len(boxes))} + for k, box := range boxes { + one, convertErr := box.ToRichStruct(entityType) + if convertErr != nil { + reqErrLogger().With("box", box.String()).Errorfe(convertErr, "convert entity to RichStruct failed") + return stat, controller.NewExternalError(controller.ErrCodeInvalidEntityData, + errors.Wrapf(convertErr, "convert entity %s to RichStruct for %s failed ", box.String(), b.title())) + } + data.Entities[k] = &protos.Entity{ + Entity: box.Entity, + GenBlockNumber: box.GenBlockNumber, + GenBlockTime: timestamppb.New(box.GenBlockTime), + GenBlockChain: b.chainID, + Data: one, + } + } + dbResp.Value = &protos.DBResponse_EntityList{EntityList: data} + dbResp.NextCursor = nextCursor + stat.List++ + stat.ListEntities += len(boxes) + what = fmt.Sprintf("entity list response #%d/%d", stat.List, stat.ListEntities) + } + if dbReq.GetUpsert() != nil { + reqLogger = reqLogger.With("dbop", "upsert", "count", len(dbReq.GetUpsert().GetId())) + entities := dbReq.GetUpsert().GetEntity() + ids := dbReq.GetUpsert().GetId() + datas := dbReq.GetUpsert().GetEntityData() + if len(ids) != len(datas) || len(ids) != len(entities) { + reqErrLogger().Errorf("len(entity),length(id),length(data) = %d,%d,%d must be equal", + len(entities), len(ids), len(datas)) + return stat, controller.NewExternalError(controller.ErrCodeInvalidUpsertEntityRequest, + errors.Errorf("len(entity),length(id),length(data) = %d,%d,%d must be equal in db upsert request for %s", + len(entities), len(ids), len(datas), b.title())) + } + for k := range ids { + entity, id, data := entities[k], ids[k], datas[k] + summary := fmt.Sprintf("%d/%d %s/%s", k+1, len(ids), entity, id) + entityType := checkpointCtrl.GetEntityType(entity) + if entityType == nil { + reqErrLogger().Errorf("set unknown entity %s failed", summary) + return stat, controller.NewExternalError(controller.ErrCodeUpsertUnknownEntity, + errors.Errorf("set unknown entity %s for %s", summary, b.title())) + } + box := persistent.UncommittedEntityBox{EntityBox: persistent.EntityBox{ + ID: id, + GenBlockNumber: b.GetBlockNumber(), + GenBlockTime: b.GetBlockTime(), + GenBlockHash: b.GetBlockHash(), + }} + if convertErr := box.FromRichStruct(entityType, data); convertErr != nil { + reqErrLogger().Errorfe(convertErr, "convert entity %s/%s failed", summary, data.String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidUpsertEntityRequest, + errors.Wrapf(convertErr, "convert entity %s/%s failed for %s failed", summary, data.String(), b.title())) + } + if setErr := checkpointCtrl.SetEntity(ctx, entityType, box); setErr != nil { + reqErrLogger().Errorfe(setErr, "set entity %s %s failed", summary, box.String()) + return stat, setErr.Wrapf("set entity %s %s for %s failed", summary, box.String(), b.title()) + } + reqLogger.Debugf("set entity %s completed", summary) + utils.IncrK2Map(stat.Entity, "upsert", entity, 1) + } + stat.Upsert++ + stat.UpsertEntities += len(ids) + what = fmt.Sprintf("entity upsert response #%d/%d", stat.Upsert, stat.UpsertEntities) + } + if dbReq.GetUpdate() != nil { + reqLogger = reqLogger.With("dbop", "update", "count", len(dbReq.GetUpdate().GetId())) + entities := dbReq.GetUpdate().GetEntity() + ids := dbReq.GetUpdate().GetId() + datas := dbReq.GetUpdate().GetEntityData() + if len(ids) != len(datas) || len(ids) != len(entities) { + reqErrLogger().Errorf("len(entity),length(id),length(data) = %d,%d,%d must be equal", + len(entities), len(ids), len(datas)) + return stat, controller.NewExternalError(controller.ErrCodeInvalidUpdateEntityRequest, + errors.Errorf("len(entity),length(id),length(data) = %d,%d,%d must be equal in db update request for %s", + len(entities), len(ids), len(datas), b.title())) + } + for k := range ids { + entity, id, data := entities[k], ids[k], datas[k] + summary := fmt.Sprintf("%d/%d %s/%s", k+1, len(ids), entity, id) + entityType := checkpointCtrl.GetEntityType(entity) + if entityType == nil { + reqErrLogger().Errorf("set unknown entity %s failed", summary) + return stat, controller.NewExternalError(controller.ErrCodeUpdateUnknownEntity, + errors.Errorf("set unknown entity %s for %s", summary, b.title())) + } + box := persistent.UncommittedEntityBox{EntityBox: persistent.EntityBox{ + ID: id, + GenBlockNumber: b.GetBlockNumber(), + GenBlockTime: b.GetBlockTime(), + GenBlockHash: b.GetBlockHash(), + }} + if convertErr := box.FromEntityUpdateData(entityType, data); convertErr != nil { + reqErrLogger().Errorfe(convertErr, "convert entity %s/%s failed", summary, data.String()) + return stat, controller.NewExternalError(controller.ErrCodeInvalidUpdateEntityRequest, + errors.Wrapf(convertErr, "convert entity %s/%s failed for %s failed", summary, data.String(), b.title())) + } + if setErr := checkpointCtrl.SetEntity(ctx, entityType, box); setErr != nil { + reqErrLogger().Errorfe(setErr, "set entity %s %s failed", summary, box.String()) + return stat, setErr.Wrapf("set entity %s %s for %s failed", summary, box.String(), b.title()) + } + reqLogger.Debugf("set entity %s completed", summary) + utils.IncrK2Map(stat.Entity, "update", entity, 1) + } + stat.Update++ + stat.UpdateEntities += len(ids) + what = fmt.Sprintf("entity update response #%d/%d", stat.Update, stat.UpdateEntities) + } + if dbReq.GetDelete() != nil { + reqLogger = reqLogger.With("dbop", "delete", "count", len(dbReq.GetDelete().GetId())) + entities, ids := dbReq.GetDelete().GetEntity(), dbReq.GetDelete().GetId() + if len(ids) != len(entities) { + reqErrLogger().Errorf("len(entity),length(id) = %d,%d must be equal", len(entities), len(ids)) + return stat, controller.NewExternalError(controller.ErrCodeInvalidDeleteEntityRequest, + errors.Errorf("len(entity),length(id) = %d,%d must be equal in db delete request for %s", + len(entities), len(ids), b.title())) + } + for k := range ids { + entity, id := entities[k], ids[k] + summary := fmt.Sprintf("%d/%d %s/%s", k+1, len(ids), entity, id) + entityType := checkpointCtrl.GetEntityType(entity) + if entityType == nil { + reqErrLogger().Errorf("delete unknown entity %s failed", summary) + return stat, controller.NewExternalError(controller.ErrCodeDeleteUnknownEntity, + errors.Errorf("delete unknown entity %s for %s", summary, b.title())) + } + box := persistent.UncommittedEntityBox{EntityBox: persistent.EntityBox{ + ID: id, + GenBlockNumber: b.GetBlockNumber(), + GenBlockTime: b.GetBlockTime(), + GenBlockHash: b.GetBlockHash(), + }} + if delErr := checkpointCtrl.SetEntity(ctx, entityType, box); delErr != nil { + reqErrLogger().Errorfe(delErr, "delete entity %s failed", summary) + return stat, delErr.Wrapf("delete entity %s for %s failed", summary, b.title()) + } + reqLogger.Debugf("delete entity %s completed", summary) + utils.IncrK2Map(stat.Entity, "delete", entity, 1) + } + stat.Delete++ + stat.DeleteEntities += len(ids) + what = fmt.Sprintf("entity delete response #%d/%d", stat.Delete, stat.DeleteEntities) + } + reqLogger.With("used", start.End().String()).Debugf("%s is ready", what) + + // send db result + req := &protos.ProcessStreamRequest{ + ProcessId: resp.GetProcessId(), + Value: &protos.ProcessStreamRequest_DbResult{DbResult: &dbResp}, + } + if sendErr := b.streamSend(ctx, req, what, "SE", time.Minute*30); sendErr != nil { + return stat, sendErr + } + } +} diff --git a/driver/controller/startup/BUILD.bazel b/driver/controller/startup/BUILD.bazel new file mode 100644 index 0000000..20494da --- /dev/null +++ b/driver/controller/startup/BUILD.bazel @@ -0,0 +1,89 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "startup", + srcs = [ + "checkpoint.go", + "entity.go", + "quota.go", + "standard.go", + "startup.go", + "subgraph.go", + "timeseries.go", + "utils.go", + "webhook.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/startup", + visibility = ["//visibility:public"], + deps = [ + "//chain/evm", + "//chain/sui/types", + "//common/chains", + "//common/chx", + "//common/clickhousemanager", + "//common/compress", + "//common/concurrency", + "//common/envconf", + "//common/errgroup", + "//common/gonanoid", + "//common/log", + "//common/protojson", + "//common/set", + "//common/sparsify", + "//common/tracker", + "//common/utils", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/aptos", + "//driver/controller/data/evm", + "//driver/controller/data/fuel", + "//driver/controller/data/sol", + "//driver/controller/data/sui", + "//driver/controller/standard", + "//driver/controller/standard/aptos", + "//driver/controller/standard/evm", + "//driver/controller/standard/fuel", + "//driver/controller/standard/sol", + "//driver/controller/standard/sui", + "//driver/controller/standard/sui/grpc", + "//driver/controller/subgraph", + "//driver/entity/clickhouse", + "//driver/entity/persistent", + "//driver/entity/schema", + "//driver/exitcode", + "//driver/subgraph/manifest", + "//driver/timeseries", + "//driver/timeseries/clickhouse", + "//processor/protos", + "//service/common/errors", + "//service/common/models", + "//service/common/rpc", + "//service/database_registry/protos", + "//service/processor/models", + "//service/processor/protos", + "//service/usage/protos", + "//service/webhook/protos", + "@com_github_clickhouse_clickhouse_go_v2//:clickhouse-go", + "@com_github_ipfs_go_ipfs_api//:go-ipfs-api", + "@com_github_pkg_errors//:errors", + "@com_google_cloud_go_pubsub//:pubsub", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//encoding/gzip", + "@org_golang_google_grpc//status", + "@org_golang_google_protobuf//types/known/timestamppb", + ], +) + +go_test( + name = "startup_test", + srcs = ["startup_test.go"], + embed = [":startup"], + deps = [ + "//common/log", + "//service/common/errors", + "//service/processor/models", + "@com_github_stretchr_testify//assert", + ], +) diff --git a/driver/controller/startup/checkpoint.go b/driver/controller/startup/checkpoint.go new file mode 100644 index 0000000..d73b5b6 --- /dev/null +++ b/driver/controller/startup/checkpoint.go @@ -0,0 +1,191 @@ +package startup + +import ( + "context" + "encoding/json" + "math" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/compress" + "sentioxyz/sentio-core/driver/controller" + sentioerror "sentioxyz/sentio-core/service/common/errors" + "sentioxyz/sentio-core/service/processor/models" + "sentioxyz/sentio-core/service/processor/protos" +) + +type checkpointStore struct { + processor *models.Processor + chainID string + cli protos.ProcessorServiceClient + chainState *models.ChainState + latestCheckpoint *controller.Checkpoint +} + +func newCheckpointStore( + processor *models.Processor, + chainID string, + cli protos.ProcessorServiceClient, + chainState *models.ChainState, +) *checkpointStore { + return &checkpointStore{ + processor: processor, + chainID: chainID, + cli: cli, + chainState: chainState, + } +} + +func LoadCheckpoints(cs *models.ChainState) (checkpoints []controller.Checkpoint, err error) { + if len(cs.IndexerState) == 0 { + return nil, nil + } + if err = compress.Load(cs.IndexerState, &checkpoints); err != nil { + return nil, errors.Wrap(err, "unmarshal checkpoints failed") + } + return checkpoints, nil +} + +func LoadTemplates(cs *models.ChainState) (templates map[uint64][]controller.TemplateInstance, err error) { + if len(cs.Templates) == 0 { + return make(map[uint64][]controller.TemplateInstance), nil + } + if err = compress.Load([]byte(cs.Templates), &templates); err != nil { + return nil, errors.Wrapf(err, "unmarshal templates for chain %s failed", cs.ChainID) + } + return templates, nil +} + +func SetCheckpoints(cs *models.ChainState, checkpoints []controller.Checkpoint) error { + // set indexer_state + b, err := compress.Dump(checkpoints) + if err != nil { + return errors.Wrapf(err, "dump checkpoints failed") + } + if len(checkpoints) > 2 && len(b) > maxIndexerStateSize { + return errors.Wrapf(controller.ErrCheckpointsTooBig, + "length of IndexerState in %d checkpoints is %d, over the limit %d", + len(checkpoints), len(b), maxIndexerStateSize) + } + cs.IndexerState = b + // set progress and state + cs.ProcessedBlockNumber = -1 + cs.ProcessedBlockHash = "" + cs.ProcessedTimestampMicros = 0 + cs.InitialStartBlockNumber = 0 + cs.EstimatedLatestBlockNumber = math.MaxInt64 + cs.LastBlockNumber = 0 + cs.State = int32(protos.ChainState_Status_CATCHING_UP) + if len(checkpoints) > 0 { + last := checkpoints[len(checkpoints)-1] + cs.ProcessedBlockNumber = int64(last.BlockNumber) + cs.ProcessedBlockHash = last.BlockHash + cs.ProcessedTimestampMicros = last.BlockTime.UnixMicro() + cs.InitialStartBlockNumber = int64(last.FullBlockRange.StartBlock) + cs.EstimatedLatestBlockNumber = int64(last.CurrentLastBlockNumber()) + if last.FullBlockRange.EndBlock != nil { + cs.LastBlockNumber = int64(*last.FullBlockRange.EndBlock) + } + if last.InWatching() || last.AllDone() { + cs.State = int32(protos.ChainState_Status_PROCESSING_LATEST) + } + } + // remove error + cs.ErrorRecord = sentioerror.ErrorRecord{} + return nil +} + +func SetTemplates(cs *models.ChainState, templates map[uint64][]controller.TemplateInstance) error { + b, err := compress.Dump(templates) + if err != nil { + return errors.Wrapf(err, "dump templates failed") + } + cs.Templates = string(b) + return nil +} + +func SetHandlerStat(cs *models.ChainState, agentStat map[string]int) error { + b, err := json.Marshal(agentStat) + if err != nil { + return errors.Wrapf(err, "dump agent stat failed") + } + cs.HandlerStat = b + return nil +} + +func (s *checkpointStore) Load(ctx context.Context) ( + checkpoints []controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + err error, +) { + if checkpoints, err = LoadCheckpoints(s.chainState); err != nil { + return nil, nil, err + } + if templates, err = LoadTemplates(s.chainState); err != nil { + return nil, nil, err + } + if len(checkpoints) > 0 { + s.latestCheckpoint = &checkpoints[len(checkpoints)-1] + } + return +} + +const maxIndexerStateSize = 50 * 1024 * 1024 // 50MB + +func (s *checkpointStore) Save( + ctx context.Context, + checkpoints []controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + agentStat map[string]int, +) error { + var latest *controller.Checkpoint + if len(checkpoints) > 0 { + latest = &checkpoints[len(checkpoints)-1] + } + reorg := (latest == nil && s.latestCheckpoint != nil) || + (latest != nil && s.latestCheckpoint != nil && latest.BlockNumber < s.latestCheckpoint.BlockNumber) + var reorgBlocks uint64 + var reduceToBlock int64 + if reorg { + if latest == nil { + reorgBlocks = s.latestCheckpoint.BlockNumber - s.latestCheckpoint.FullBlockRange.StartBlock + 1 + reduceToBlock = -1 + } else { + reorgBlocks = s.latestCheckpoint.BlockNumber - latest.BlockNumber + reduceToBlock = int64(latest.BlockNumber) + } + } + + cs := *s.chainState + if err := SetCheckpoints(&cs, checkpoints); err != nil { + return err + } + if err := SetTemplates(&cs, templates); err != nil { + return err + } + if err := SetHandlerStat(&cs, agentStat); err != nil { + return err + } + + if err := updateChainState(ctx, s.cli, cs); err != nil { + return err + } + s.chainState = &cs + s.latestCheckpoint = latest + + if reorg { + controller.N.ReorgDone(ctx, s.processor, s.chainID, reorgBlocks, reduceToBlock) + } + return nil +} + +func (s *checkpointStore) SaveError(ctx context.Context, extErr *controller.ExternalError) error { + cs := *s.chainState + cs.ErrorRecord = newErrorRecord(extErr) + cs.State = int32(protos.ChainState_Status_ERROR) + if err := updateChainState(ctx, s.cli, cs); err != nil { + return err + } + s.chainState = &cs + return nil +} diff --git a/driver/controller/startup/entity.go b/driver/controller/startup/entity.go new file mode 100644 index 0000000..2e72fbb --- /dev/null +++ b/driver/controller/startup/entity.go @@ -0,0 +1,155 @@ +package startup + +import ( + "context" + "errors" + "time" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/entity/clickhouse" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" +) + +type entityController struct { + *persistent.Controller +} + +func newEntityController( + store *clickhouse.Store, + chainID string, + storeCacheSize int, + storeFullCacheSize int, + monitor persistent.MetricsMonitor, +) *entityController { + return &entityController{ + Controller: persistent.NewController( + clickhouse.NewChainStore(store, chainID, storeCacheSize, storeFullCacheSize), + monitor, + ), + } +} + +func (c entityController) Reset(ctx context.Context, checkpoint *controller.Checkpoint) *controller.ExternalError { + var blockNumber int64 = -1 + if checkpoint != nil { + blockNumber = int64(checkpoint.BlockNumber) + } + if err := c.Controller.Reorg(ctx, blockNumber); err != nil { + return controller.NewExternalError(controller.ErrCodeCleanEntityDataFailed, err) + } + return nil +} + +var maxUncommitedEntityChanges = envconf.LoadUInt64("SENTIO_MAX_UNCOMMITED_ENTITY_CHANGES", 1000000, + envconf.WithMin(10000), envconf.WithMax(1000000)) + +func (c entityController) CachedTooMuch(blockNumber uint64) bool { + return uint64(c.Controller.CountUncommittedChanges(blockNumber)) > maxUncommitedEntityChanges +} + +func (c entityController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (map[string]int, map[string]int, *controller.ExternalError) { + created, updated, err := c.Controller.Commit(ctx, blockNumber, blockTime) + if err != nil { + if errors.Is(err, persistent.ErrUpdateImmutable) { + return created, updated, controller.NewExternalError(controller.ErrCodeUpdateImmutableEntity, err) + } + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return created, updated, controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return created, updated, controller.NewExternalError(controller.ErrCodeSaveEntityDataFailed, err) + } + return created, updated, nil +} + +func (c entityController) GetEntity( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (*persistent.EntityBox, *controller.ExternalError) { + box, err := c.Controller.GetEntity(ctx, typ, id, blockNumber) + if err != nil { + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return box, controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return box, controller.NewExternalError(controller.ErrCodeGetEntityFromDBFailed, err) + } + return box, nil +} + +func (c entityController) GetEntityInBlock( + ctx context.Context, + typ schema.EntityOrInterface, + id string, + blockNumber uint64, +) (*persistent.EntityBox, *controller.ExternalError) { + box, err := c.Controller.GetEntityInBlock(ctx, typ, id, blockNumber) + if err != nil { + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return box, controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return box, controller.NewExternalError(controller.ErrCodeGetEntityFromDBFailed, err) + } + return box, nil +} + +func (c entityController) ListEntity( + ctx context.Context, + entityType *schema.Entity, + filters []persistent.EntityFilter, + cursor string, + limit int, + blockNumber uint64, +) ([]*persistent.EntityBox, *string, *controller.ExternalError) { + boxes, next, err := c.Controller.ListEntity(ctx, entityType, filters, cursor, limit, blockNumber) + if err != nil { + if errors.Is(err, persistent.ErrInvalidListFilter) { + return boxes, next, controller.NewExternalError(controller.ErrCodeInvalidListEntityFilter, err) + } + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return boxes, next, controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return boxes, next, controller.NewExternalError(controller.ErrCodeListEntityFromDBFailed, err) + } + return boxes, next, nil +} + +func (c entityController) ListRelated( + ctx context.Context, + entityType *schema.Entity, + id string, + fieldName string, + blockNumber uint64, +) ([]*persistent.EntityBox, schema.EntityOrInterface, *controller.ExternalError) { + boxes, target, err := c.Controller.ListRelated(ctx, entityType, id, fieldName, blockNumber) + if err != nil { + if errors.Is(err, persistent.ErrInvalidField) { + return boxes, target, controller.NewExternalError(controller.ErrCodeListRelatedEntityWithInvalidField, err) + } + return boxes, target, controller.NewExternalError(controller.ErrCodeListEntityFromDBFailed, err) + } + return boxes, target, nil +} + +func (c entityController) SetEntity( + ctx context.Context, + entityType *schema.Entity, + box persistent.UncommittedEntityBox, +) *controller.ExternalError { + if err := c.Controller.SetEntity(ctx, entityType, box); err != nil { + if errors.Is(err, persistent.ErrUpdateImmutable) { + return controller.NewExternalError(controller.ErrCodeUpdateImmutableEntity, err) + } + if errors.Is(err, persistent.ErrInvalidFieldValue) { + return controller.NewExternalError(controller.ErrCodeInvalidEntityFieldValue, err) + } + return controller.NewExternalError(controller.ErrCodeSaveEntityDataFailed, err) + } + return nil +} diff --git a/driver/controller/startup/quota.go b/driver/controller/startup/quota.go new file mode 100644 index 0000000..3adc184 --- /dev/null +++ b/driver/controller/startup/quota.go @@ -0,0 +1,128 @@ +package startup + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + "google.golang.org/protobuf/types/known/timestamppb" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/timeseries" + "sentioxyz/sentio-core/service/processor/models" + "sentioxyz/sentio-core/service/usage/protos" +) + +type quotaService struct { + chainID string + processor *models.Processor + cli protos.UsageServiceClient +} + +func newQuotaService(chainID string, processor *models.Processor, cli protos.UsageServiceClient) *quotaService { + return "aService{chainID: chainID, processor: processor, cli: cli} +} + +func (s *quotaService) CheckOverQuota(ctx context.Context) (*controller.OverQuota, error) { + req := protos.CheckOverLimitRequest{ + ProjectId: s.processor.ProjectID, + ProcessorId: &s.processor.ID, + Now: timestamppb.Now(), + Sku: "metric", // We can use any SKU here, since we only care about the overall quota. + } + resp, err := s.cli.CheckOverLimit(ctx, &req) + if err != nil { + return nil, errors.Wrapf(err, "check over quota failed") + } + if len(resp.GetOver()) == 0 { + return nil, nil + } + return &controller.OverQuota{ + Msg: strings.Join(resp.Over, "\n"), + Detail: strings.Join(resp.OverDetail, "\n"), + }, nil +} + +func (s *quotaService) SaveUsage(ctx context.Context, used controller.Usage, inWatching bool) error { + var version = fmt.Sprintf("%d", s.processor.Version) + type record struct { + sku string + count int + tags map[string]string + } + var records []record + // metric v3 + for _, metricType := range []timeseries.MetaType{timeseries.MetaTypeCounter, timeseries.MetaTypeGauge} { + for name, count := range used.TimeSeries[metricType] { + records = append(records, record{ + sku: "metricv3", + count: count, + tags: map[string]string{"name": name, "version": version, "type": string(metricType)}, + }) + controller.N.DataSaved(ctx, s.processor, s.chainID, "metricV3", string(metricType), name, int64(count)) + } + } + // event v3 + for name, count := range used.TimeSeries[timeseries.MetaTypeEvent] { + records = append(records, record{ + sku: "eventv3", + count: count, + tags: map[string]string{"name": name, "version": version}, + }) + controller.N.DataSaved(ctx, s.processor, s.chainID, "eventV3", "", name, int64(count)) + } + // webhook + for name, count := range used.Export { + records = append(records, record{ + sku: "webhook", + count: count, + tags: map[string]string{"name": name, "version": version}, + }) + } + // entity + for name, count := range used.EntityCreated { + records = append(records, record{ + sku: "entity_created", + count: count, + tags: map[string]string{"name": name, "version": version}, + }) + controller.N.DataSaved(ctx, s.processor, s.chainID, "entity", "created", name, int64(count)) + } + for name, count := range utils.MergeMapSum(used.EntityCreated, used.EntityUpdated) { + records = append(records, record{ + sku: "entity", + count: count, + tags: map[string]string{"name": name, "version": version}, + }) + } + for name, count := range used.EntityUpdated { + controller.N.DataSaved(ctx, s.processor, s.chainID, "entity", "updated", name, int64(count)) + } + + // build request and async save + var req protos.AsyncSaveRequest + for _, r := range records { + if r.count == 0 { + continue + } + req.Dialogues = append(req.Dialogues, &protos.Dialogue{ + RequestTime: timestamppb.Now(), + Succeed: true, + Units: uint64(r.count), + Tags: &protos.Tags{ + ProjectId: s.processor.ProjectID, + ProcessorId: &s.processor.ID, + Sku: utils.Select(inWatching, r.sku, r.sku+"_backfill"), + CustomTags: r.tags, + }, + }) + } + if _, err := s.cli.AsyncSave(ctx, &req); err != nil { + _, logger := log.FromContext(ctx, "records", records) + logger.Warnfe(err, "save usage failed") + } + return nil +} diff --git a/driver/controller/startup/standard.go b/driver/controller/startup/standard.go new file mode 100644 index 0000000..243ace0 --- /dev/null +++ b/driver/controller/startup/standard.go @@ -0,0 +1,368 @@ +package startup + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "time" + + evmchain "sentioxyz/sentio-core/chain/evm" + suitypes "sentioxyz/sentio-core/chain/sui/types" + "sentioxyz/sentio-core/common/chains" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/protojson" + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + aptosdata "sentioxyz/sentio-core/driver/controller/data/aptos" + evmdata "sentioxyz/sentio-core/driver/controller/data/evm" + fueldata "sentioxyz/sentio-core/driver/controller/data/fuel" + soldata "sentioxyz/sentio-core/driver/controller/data/sol" + suidata "sentioxyz/sentio-core/driver/controller/data/sui" + "sentioxyz/sentio-core/driver/controller/standard" + "sentioxyz/sentio-core/driver/controller/standard/aptos" + "sentioxyz/sentio-core/driver/controller/standard/evm" + "sentioxyz/sentio-core/driver/controller/standard/fuel" + "sentioxyz/sentio-core/driver/controller/standard/sol" + "sentioxyz/sentio-core/driver/controller/standard/sui" + suigrpc "sentioxyz/sentio-core/driver/controller/standard/sui/grpc" + "sentioxyz/sentio-core/driver/exitcode" + "sentioxyz/sentio-core/driver/subgraph/manifest" + "sentioxyz/sentio-core/processor/protos" + "sentioxyz/sentio-core/service/common/rpc" + protossvc "sentioxyz/sentio-core/service/processor/protos" + + "github.com/pkg/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type standardStartupController struct { + baseStartupController + + processorClients []protos.ProcessorV3Client + initResult *protos.InitResponse +} + +func (c *standardStartupController) buildProcessorUrlList() ([]string, error) { + processorUrlList := []string{c.config.ProcessorUrl} + if c.processor.NumWorkers <= 1 { + return processorUrlList, nil + } + host, port, has := strings.Cut(c.config.ProcessorUrl, ":") + var basePort uint64 = 80 + if has { + basePort, _ = strconv.ParseUint(port, 10, 64) + } + for i := uint64(1); i < uint64(c.processor.NumWorkers); i++ { + processorUrlList = append(processorUrlList, fmt.Sprintf("%s:%d", host, basePort+i)) + } + return processorUrlList, nil +} + +func (c *standardStartupController) buildMainControllers(ctx context.Context) ( + ctrls map[string]*controller.MainController, + exitCode exitcode.Code, + err error, +) { + _, logger := log.FromContext(ctx) + ctrls = make(map[string]*controller.MainController) + + // connect to webhook service and create webhook subscription + if err = c.createWebhookSubscription(ctx); err != nil { + return ctrls, 0, errors.Wrapf(err, "create webhook subscription failed") + } + + // create gcp pub sub topic + if err = c.createPubSubTopic(ctx); err != nil { + return ctrls, 0, errors.Wrapf(err, "create pubsub topic failed") + } + + // connect to clickhouse + if err = c.connectClickhouse(ctx); err != nil { + return ctrls, 0, errors.Wrapf(err, "connect to clickhouse failed") + } + + // build time series store + if err = c.buildTimeSeriesStore(ctx); err != nil { + return ctrls, exitcode.AlwaysRetry, errors.Wrapf(err, "build time series store failed") + } + + // load all templates + var templates []*protos.TemplateInstance + for _, cs := range c.processor.ChainStates { + if chainTemplates, err := LoadTemplates(cs); err != nil { + return ctrls, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "invaid templates")) + } else { + templates = append(templates, standard.ConvertTemplateInstanceBack(cs.ChainID, chainTemplates)...) + } + } + + // connect to all processor and init them + processorUrlList, buildErr := c.buildProcessorUrlList() + if buildErr != nil { + return ctrls, exitcode.NeverRetry, buildErr + } + c.processorClients = make([]protos.ProcessorV3Client, len(processorUrlList)) + var initResultText string + for i, processorUrl := range processorUrlList { + // connect to processor + conn, connErr := rpc.DialInsecure(processorUrl) + if connErr != nil { + return ctrls, 0, errors.Wrapf(connErr, "connect to processor #%d %s failed", i, processorUrl) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + c.processorClients[i] = protos.NewProcessorV3Client(conn) + logger.Infof("connected to processor #%d %s", i, processorUrl) + + // call start function + for { + _, startErr := c.processorClients[i].Start(ctx, &protos.StartRequest{TemplateInstances: templates}) + if startErr == nil { + break + } + if status.Code(startErr) == codes.InvalidArgument { + return ctrls, exitcode.AlwaysRetry, errors.Wrapf(startErr, "call start for processor failed") + } + logger.Warnfe(startErr, "call start for processor #%d failed, will retry after %s", i, initRetryInterval) + if startErr = utils.Sleep(ctx, initRetryInterval); startErr != nil { + return ctrls, exitcode.AlwaysRetry, errors.Wrapf(startErr, "call start for processor failed") + } + } + logger.Infof("called start for processor #%d", i) + + // get init result + resp, getConfigErr := c.processorClients[i].GetConfig(ctx, &protos.ProcessConfigRequest{}) + if getConfigErr != nil { + return ctrls, exitcode.AlwaysRetry, errors.Wrapf(getConfigErr, "get processor config failed") + } + chainIDSet := set.New[string]() + for _, cc := range resp.GetContractConfigs() { + chainIDSet.Add(cc.GetContract().GetChainId()) + } + for _, ac := range resp.GetAccountConfigs() { + chainIDSet.Add(ac.GetChainId()) + } + chainIDList := chainIDSet.DumpValues() + sort.Strings(chainIDList) + initResult := &protos.InitResponse{ + ChainIds: chainIDList, + DbSchema: resp.GetDbSchema(), + Config: resp.GetConfig(), + ExecutionConfig: resp.GetExecutionConfig(), + MetricConfigs: resp.GetMetricConfigs(), + ExportConfigs: resp.GetExportConfigs(), + EventLogConfigs: resp.GetEventLogConfigs(), + } + + // check init result + if i == 0 { + c.initResult, initResultText = initResult, string(protojson.MustJSONMarshal(initResult)) + } else if another := string(protojson.MustJSONMarshal(initResult)); another != initResultText { + logger.With("this", another, "before", initResultText).Errorf("configs from processor #%d was different", i) + return ctrls, exitcode.AlwaysRetry, errors.Errorf("configs from different processor has diff") + } + logger.Infof("got config from processor #%d succeed", i) + } + logger.Infow("init processors succeed", "initResult", initResultText) + + // check chain + if len(c.initResult.GetChainIds()) == 0 { + return ctrls, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("no chain in processor")) + } + for _, chainID := range c.initResult.GetChainIds() { + if _, has := c.chainConfigs[chainID]; !has { + return ctrls, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("chain %s is not supported", chainID)) + } + } + + // build entity store + if schemeCnt := c.initResult.GetDbSchema().GetGqlSchema(); len(schemeCnt) > 0 { + if extErr := c.buildEntityStore(ctx, schemeCnt); extErr != nil { + if extErr.IsUserError() { + return ctrls, exitcode.NeverRetry, extErr + } + return ctrls, exitcode.AlwaysRetry, extErr + } + req := &protossvc.SetProcessorEntitySchemaRequest{ProcessorId: c.config.ProcessorID, Schema: schemeCnt} + if _, err = c.processorClient.SetProcessorEntitySchema(ctx, req); err != nil { + return ctrls, exitcode.NeverRetry, errors.Wrapf(err, "update processor entity schema failed") + } + } + + // build main controller for each chain + for _, chainID := range c.initResult.GetChainIds() { + ctrls[chainID], exitCode, err = c.buildMainController(ctx, chainID) + if err != nil { + err = errors.Wrapf(err, "build main controller for chain %s failed", chainID) + return + } + logger.Infof("main controller of chain %s is ready", chainID) + } + return +} + +func (c *standardStartupController) buildMainController( + ctx context.Context, + chainID string, +) (ctrl *controller.MainController, exitCode exitcode.Code, err error) { + chainConfig := c.chainConfigs[chainID] + if chainConfig.IsCustomizedEndpoint { + if err = evmchain.CheckArchiveNode(context.Background(), chainConfig.Endpoint); err != nil { + return nil, exitcode.AlwaysRetry, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Wrapf(err, "invalid customized endpoint %q for chain %s", chainConfig.Endpoint, chainID)) + } + } + chainType, hasChain := chains.GetChainType(chains.ChainID(chainID)) + if !hasChain { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeUnexpectedProcessorConfig, + errors.Errorf("unknown chain id %s", chainID)) + } + // prepare client and handler controller + var checkLink bool + var cli controller.Client + var handlerCtrl controller.HandlerController + switch { + case chainID == manifest.CustomizedChainID || chains.IsEVMChains(chainID): + checkLink = true + evmCli, newClientErr := evmdata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + chainConfig.ProcessingDelayBlocks, + controller.SubscribeMinWatchInterval, + time.Second*3, + ) + if newClientErr != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build evm client failed") + } + handlerCtrl = evm.NewHandlerController(c.processor, c.initResult, chainConfig, evmCli, c.processorClients) + cli = evmCli + case chains.IsAptosChain(chainID): + aptosCli, newClientErr := aptosdata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + controller.SubscribeMinWatchInterval, + ) + if newClientErr != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build aptos client failed") + } + handlerCtrl = aptos.NewHandlerController(c.processor, c.initResult, chainConfig, aptosCli, c.processorClients) + cli = aptosCli + case chains.IsSuiChain(chainID): + suiCli, newClientErr := suidata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + controller.SubscribeMinWatchInterval, + ) + if newClientErr != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build sui client failed") + } + // grpc-format data is supported only for the sui variation (not iota) at DriverVersion >= 2. + if suitypes.VariationFromChainID(chains.SuiChainID(chainID)) == suitypes.VariationSUI && + c.processor.DriverVersion >= 2 { + handlerCtrl = suigrpc.NewHandlerController(c.processor, c.initResult, chainConfig, suiCli, c.processorClients) + } else { + handlerCtrl = sui.NewHandlerController(c.processor, c.initResult, chainConfig, suiCli, c.processorClients) + } + cli = suiCli + case chains.IsFuelChain(chainID): + fuelCli, newClientErr := fueldata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + controller.SubscribeMinWatchInterval, + ) + if newClientErr != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build fuel client failed") + } + handlerCtrl = fuel.NewHandlerController(c.processor, c.initResult, chainConfig, fuelCli, c.processorClients) + cli = fuelCli + case chains.IsSolanaChain(chainID): + solCli, newClientErr := soldata.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + controller.SubscribeMinWatchInterval, + c.processor.DriverVersion, + ) + if newClientErr != nil { + // A retryable NewClient error (e.g. the super-node probe kept hitting transient HTTP/timeout + // errors) means the endpoint may simply be temporarily unavailable; restart the pod and try + // again instead of failing permanently. + if data.IsNewClientRetryable(newClientErr) { + return nil, exitcode.AlwaysRetry, errors.Wrapf(newClientErr, "build sol client failed") + } + return nil, exitcode.NeverRetry, errors.Wrapf(newClientErr, "build sol client failed") + } + handlerCtrl = sol.NewHandlerController(c.processor, c.initResult, chainConfig, solCli, c.processorClients) + cli = solCli + default: + // chainID is OK but the chainType is not supported, so is a system error + return nil, exitcode.NeverRetry, errors.Errorf("chain type %s is not supported", chainType) + } + // block builder + blockBuilder := controller.NewBlockBuilder(handlerCtrl, cli, checkLink) + // webhook controller + var webhookCtrl controller.WebhookController = controller.EmptyWebhookController{} + if c.pubSubTopic != nil { + webhookCtrl = newWebhookController(c.processor, c.pubSubTopic) + } + // time series controller + var timeSeriesCtrl controller.TimeSeriesController = controller.EmptyTimeSeriesController{} + if c.timeSeriesStore != nil { + timeSeriesCtrl = newTimeSeriesController(chainID, c.timeSeriesStore) + } + // entity controller + var entityCtrl controller.EntityController = controller.EmptyEntityController{} + if c.entityStore != nil { + entityCtrl = newEntityController( + c.entityStore, + chainID, + c.config.EntityStoreCacheSize, + c.config.EntityStoreFullCacheSize, + c.config.EntityMetricsMonitor) + } + // checkpoint store + var store controller.CheckpointStore + if store, err = c.getCheckpointStore(ctx, chainID); err != nil { + return nil, exitcode.AlwaysRetry, controller.NewExternalError(controller.ErrCodeSaveCheckpointFailed, err) + } + // checkpoint controller + var checkpointCtrl controller.CheckpointController + checkpointCtrl, err = controller.NewCheckpointController( + ctx, + chainID, + controller.SaveCheckpointDelay, + controller.SaveCheckpointInterval, + controller.MaxKeepCheckpointCount, + store, + c.getQuotaService(chainID), + timeSeriesCtrl, + entityCtrl, + webhookCtrl, + c.buildCommitCtx, + ) + if err != nil { + return nil, exitcode.AlwaysRetry, errors.Wrapf(err, "build checkpoint controller failed") + } + // main controller + seqMode := c.initResult.GetExecutionConfig().GetSequential() + ctrl = controller.NewMainController(blockBuilder, checkpointCtrl, seqMode, c.processor, chainID) + return ctrl, exitcode.AlwaysRetry, nil +} diff --git a/driver/controller/startup/startup.go b/driver/controller/startup/startup.go new file mode 100644 index 0000000..f0b721e --- /dev/null +++ b/driver/controller/startup/startup.go @@ -0,0 +1,632 @@ +package startup + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "time" + + "cloud.google.com/go/pubsub" + "github.com/ClickHouse/clickhouse-go/v2" + shell "github.com/ipfs/go-ipfs-api" + "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/encoding/gzip" + + "sentioxyz/sentio-core/common/chx" + ckhmanager "sentioxyz/sentio-core/common/clickhousemanager" + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/gonanoid" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/tracker" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + entitychs "sentioxyz/sentio-core/driver/entity/clickhouse" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/entity/schema" + "sentioxyz/sentio-core/driver/exitcode" + "sentioxyz/sentio-core/driver/timeseries" + timeserieschs "sentioxyz/sentio-core/driver/timeseries/clickhouse" + sentioerror "sentioxyz/sentio-core/service/common/errors" + commonmodels "sentioxyz/sentio-core/service/common/models" + "sentioxyz/sentio-core/service/common/rpc" + protosregistry "sentioxyz/sentio-core/service/database_registry/protos" + "sentioxyz/sentio-core/service/processor/models" + protossvc "sentioxyz/sentio-core/service/processor/protos" + protousage "sentioxyz/sentio-core/service/usage/protos" + protoswebhook "sentioxyz/sentio-core/service/webhook/protos" +) + +type baseStartupController struct { + config Config + + processorClient protossvc.ProcessorServiceClient + + usageClient protousage.UsageServiceClient + registryClient protosregistry.DatabaseRegistryServiceClient + + processor *models.Processor + chainConfigs map[string]*config.ChainConfig + + pubSubTopic *pubsub.Topic + + // timeseries / entity clickhouse controllers, built by the injected + // ClickhouseConnector once the processor is loaded (nil => store disabled). + tsController *chx.Controller + entityController *chx.Controller + + ipfsShell *shell.Shell + + timeSeriesStore timeseries.Store + entityStore *entitychs.Store + + release []func() +} + +func (c *baseStartupController) releaseAll() { + for i := len(c.release) - 1; i >= 0; i-- { + c.release[i]() + } +} + +func (c *baseStartupController) connectToProcessorService(ctx context.Context) error { + _, logger := log.FromContext(ctx) + conn, err := rpc.DialAuto(c.config.ProcessorService, rpc.RetryDialOption) + if err != nil { + return errors.Wrapf(err, "dial to processor service %s failed", c.config.ProcessorService) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + + c.processorClient = protossvc.NewProcessorServiceClient(conn) + logger.Infof("connected to processor service %s", c.config.ProcessorService) + return nil +} + +func (c *baseStartupController) connectToUsageService(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.UsageService == "" { + logger.Warnf("no usage service so will not connect to usage service") + return nil + } + conn, err := rpc.DialAuto(c.config.UsageService, rpc.RetryDialOption) + if err != nil { + return errors.Wrapf(err, "dial to usage service %s failed", c.config.UsageService) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + c.usageClient = protousage.NewUsageServiceClient(conn) + logger.Infof("connected to usage service %s", c.config.UsageService) + return nil +} + +// connectToDBRegistryService dials the gRPC endpoint that hosts +// DatabaseRegistryService and keeps the client stub on baseStartupController. +// The address is logically independent from the billing/usage service address. +// When empty (cloud deployments), this is a no-op and registration is gated +// away later by newEntityProbe and newTimeSeriesProbe based on the processor's TablePattern. +func (c *baseStartupController) connectToDBRegistryService(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.DBRegistryService == "" { + logger.Warnf("no db registry service configured so will not connect to it") + return nil + } + conn, err := rpc.DialAuto(c.config.DBRegistryService, rpc.RetryDialOption) + if err != nil { + return errors.Wrapf(err, "dial to db registry service %s failed", c.config.DBRegistryService) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + c.registryClient = protosregistry.NewDatabaseRegistryServiceClient(conn) + logger.Infof("connected to db registry service %s", c.config.DBRegistryService) + return nil +} + +func (c *baseStartupController) loadChainsConfig(ctx context.Context) (err error) { + _, logger := log.FromContext(ctx) + c.chainConfigs, err = config.LoadChainsConfig( + c.config.ChainConfigFile, config.PatchChainsConfigEnv, c.processor.NetworkOverrides) + if err == nil { + logger.Info("loaded chain config") + } + return err +} + +func (c *baseStartupController) getProcessor(ctx context.Context) error { + _, logger := log.FromContext(ctx) + req := &protossvc.GetProcessorRequest{ProcessorId: c.config.ProcessorID} + response, err := c.processorClient.GetProcessorWithProject(ctx, req) + if err != nil { + return err + } + var p models.Processor + if err = p.FromPB(response.Processor); err != nil { + return err + } + p.Project = &commonmodels.Project{} + p.Project.FromPB(response.Project) + c.processor = &p + logger.Infof("got processor from processor service") + return nil +} + +func (c *baseStartupController) createWebhookSubscription(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.WebhookService == "" { + logger.Warn("no webhook service so will not create webhook subscription") + return nil + } + conn, err := rpc.DialAuto(c.config.WebhookService, rpc.RetryDialOption) + if err != nil { + return errors.Wrapf(err, "dial to webhook service %s failed", c.config.WebhookService) + } + c.release = append(c.release, func() { + _ = conn.Close() + }) + logger.Infof("connected to webhook service %s", c.config.WebhookService) + + req := &protoswebhook.CreateSubscriptionRequest{ProcessorId: c.config.ProcessorID} + if _, err = protoswebhook.NewWebhookServiceClient(conn).CreateSubscription(ctx, req); err != nil { + return errors.Wrapf(err, "create subscription failed") + } + logger.Infof("created webhook subscription") + return nil +} + +func (c *baseStartupController) createPubSubTopic(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.WebhookTopic == "" || c.config.PubSubProject == "" { + logger.Warnf("no webhook topic or pubsub project so will not create pubsub topic") + return nil + } + cli, err := pubsub.NewClient(ctx, c.config.PubSubProject) + if err != nil { + return errors.Wrapf(err, "create gcp pubsub client failed") + } + c.release = append(c.release, func() { + _ = cli.Close() + }) + c.pubSubTopic = cli.Topic(c.config.WebhookTopic) + logger.Infof("created pubsub topic %s in gcp", c.config.WebhookTopic) + return nil +} + +// connectClickhouse asks the injected ClickhouseConnector to build the +// timeseries and entity chx.Controller for the loaded processor. All sharding / +// housegate wiring lives in the connector implementation in the driver binary. +func (c *baseStartupController) connectClickhouse(ctx context.Context) error { + _, logger := log.FromContext(ctx) + if c.config.ClickhouseConnector == nil { + logger.Warnf("no clickhouse connector so will not connect to clickhouse") + return nil + } + ts, entity, err := c.config.ClickhouseConnector.Connect(ctx, c.processor) + if err != nil { + return errors.Wrap(err, "connect to clickhouse failed") + } + c.tsController = ts + c.entityController = entity + return nil +} + +func (c *baseStartupController) newTimeSeriesProbe() (timeserieschs.Probe, error) { + if c.processor.TablePattern != models.TablePatternNetworkV1 { + return nil, nil + } + if c.registryClient == nil { + return nil, errors.Errorf( + "processor TablePattern=%s requires onchain database registration, "+ + "but no db registry service is configured (set -db-registry-service flag)", + c.processor.TablePattern, + ) + } + return &timeSeriesProbe{ + client: c.registryClient, + processorID: c.processor.ID, + processorReplica: c.config.ProcessorReplica, + }, nil +} + +func (c *baseStartupController) buildTimeSeriesStore(ctx context.Context) error { + // build time series clickhouse store, each chain will use it to build time series controller + _, logger := log.FromContext(ctx) + + if c.tsController == nil { + logger.Warnf("no clickhouse connection so will not create time series store") + return nil + } + ctrl := *c.tsController + + probe, err := c.newTimeSeriesProbe() + if err != nil { + return err + } + + c.timeSeriesStore = timeserieschs.NewStore(ctrl, timeserieschs.Option{}, probe) + if err := c.timeSeriesStore.Init(ctx); err != nil { + return errors.Wrapf(err, "init time series store failed") + } + logger.Info("time series store is ready") + return nil +} + +func (c *baseStartupController) newEntityProbe() (entitychs.Probe, error) { + if c.processor.TablePattern != models.TablePatternNetworkV1 { + return nil, nil + } + if c.registryClient == nil { + return nil, errors.Errorf( + "processor TablePattern=%s requires onchain database registration, "+ + "but no db registry service is configured (set -db-registry-service flag)", + c.processor.TablePattern, + ) + } + return &entityProbe{ + client: c.registryClient, + processorID: c.processor.ID, + processorReplica: c.config.ProcessorReplica, + }, nil +} + +func (c *baseStartupController) buildEntityStore(ctx context.Context, schemaText string) *controller.ExternalError { + _, logger := log.FromContext(ctx) + + if c.entityController == nil { + return controller.NewExternalError(controller.ErrCodeSystem, + errors.Errorf("need clickhouse connection but no clickhouse connector configured")) + } + ctrl := *c.entityController + + entityFea := entitychs.BuildFeatures(c.processor.EntitySchemaVersion) + entitySchema, buildErr := schema.ParseAndVerifySchema(schemaText, entityFea.BuildVerifyOptions()...) + if buildErr != nil { + return controller.NewExternalError(controller.ErrCodeInvalidEntitySchema, buildErr) + } + + probe, err := c.newEntityProbe() + if err != nil { + return controller.NewExternalError(controller.ErrCodeSystem, err) + } + + c.entityStore = entitychs.NewStore(ctrl, entityFea, entitySchema, entitychs.DefaultCreateTableOption, probe) + if initErr := c.entityStore.InitEntitySchema(ctx); initErr != nil { + return controller.NewExternalError(controller.ErrCodeInitEntityFailed, initErr) + } + logger.Infow("entity store is ready", "schema", schemaText, "feature", entityFea) + return nil +} + +func (c *baseStartupController) buildIpfsShell(ctx context.Context) { + if c.config.IpfsNodeAddr == "" { + return + } + _, logger := log.FromContext(ctx) + c.ipfsShell = shell.NewShell(c.config.IpfsNodeAddr) + c.ipfsShell.SetTimeout(5 * time.Second) + logger.Infow("ipfs shell is ready") +} + +func (c *baseStartupController) getQuotaService(chainID string) controller.QuotaService { + if c.usageClient == nil { + return controller.EmptyQuotaService{} + } + return newQuotaService(chainID, c.processor, c.usageClient) +} + +func (c *baseStartupController) buildCommitCtx(ctx context.Context, chainID string, cur controller.Checkpoint) context.Context { + sign, _ := json.Marshal(map[string]any{ + "processor_id": c.config.ProcessorID, + "processor_replica": c.config.ProcessorReplica, + "chain_id": chainID, + "watching": cur.InWatching(), + "current_block_number": strconv.FormatUint(cur.BlockNumber, 10), + "current_block_time": cur.BlockTime.Format(time.RFC3339Nano), + }) + return ckhmanager.ContextMergeSettings(ctx, clickhouse.Settings{"log_comment": string(sign)}) +} + +func newErrorRecord(err error) (er sentioerror.ErrorRecord) { + er.ID, _ = gonanoid.GenerateLongID() + er.CreatedAt = time.Now() + er.Namespace = sentioerror.DRIVER + er.Message = err.Error() + er.Code = controller.ErrCodeSystem + var extErr *controller.ExternalError + if errors.As(err, &extErr) { + er.Code = int32(extErr.Code()) + if extErr.IsUserError() { + er.Namespace = sentioerror.PROCESSOR + } + } + return +} + +func updateChainState(ctx context.Context, cli protossvc.ProcessorServiceClient, cs models.ChainState) error { + var name string + if cs.ChainID == "meta" { + name = "meta chain state" + } else { + name = fmt.Sprintf("chain state of chain %s", cs.ChainID) + } + csPb, err := cs.ToPB() + if err != nil { + return errors.Wrapf(err, "convert %s to pb failed", name) + } + req := protossvc.UpdateChainProcessorStatusRequest{ + Id: cs.ProcessorID, + ChainState: csPb, + } + // ChainState may be very big, so need to use compressor + if _, err = cli.UpdateChainProcessorStatus(ctx, &req, grpc.UseCompressor(gzip.Name)); err != nil { + _, logger := log.FromContext(ctx, + "IndexerState", utils.StringSummaryV2(string(cs.IndexerState)), + "MeterState", utils.StringSummaryV2(string(cs.MeterState)), + "HandlerStat", utils.StringSummaryV2(string(cs.HandlerStat)), + "Templates", utils.StringSummaryV2(cs.Templates)) + logger.Warnfe(err, "update %s failed", name) + return errors.Wrapf(err, "update %s failed", name) + } + return nil +} + +func (c *baseStartupController) findChainState(chainID string) (*models.ChainState, bool) { + for _, chainState := range c.processor.ChainStates { + if chainState.ChainID == chainID { + return chainState, true + } + } + return &models.ChainState{ + ID: fmt.Sprintf("%s_%s", c.config.ProcessorID, chainID), + ChainID: chainID, + ProcessorID: c.config.ProcessorID, + ProcessedBlockNumber: -1, + ProcessedVersion: c.processor.Version, + State: int32(protossvc.ChainState_Status_CATCHING_UP), + }, false +} + +func (c *baseStartupController) getCheckpointStore( + ctx context.Context, + chainID string, +) (controller.CheckpointStore, error) { + cs, has := c.findChainState(chainID) + store := newCheckpointStore(c.processor, chainID, c.processorClient, cs) + if has { + return store, nil + } + // no chain state for the chain, create the initial one + if err := store.Save(ctx, nil, nil, nil); err != nil { + return nil, errors.Wrapf(err, "save initial chain state failed") + } + return store, nil +} + +func (c *baseStartupController) updateMetaState(ctx context.Context, metaErr error) error { + cs := models.ChainState{ + ID: fmt.Sprintf("%s_meta", c.processor.ID), + ChainID: "meta", + ProcessorID: c.processor.ID, + ProcessedBlockNumber: -1, + ProcessedVersion: -1, + State: int32(protossvc.ChainState_Status_PROCESSING_LATEST), + } + if metaErr != nil { + cs.State = int32(protossvc.ChainState_Status_ERROR) + cs.ErrorRecord = newErrorRecord(metaErr) + } + return updateChainState(ctx, c.processorClient, cs) +} + +const ( + initRetryInterval = time.Second * 5 +) + +func main(ctx context.Context, config Config) (exitCode exitcode.Code, err error) { + _, logger := log.FromContext(ctx) + logger.Infow("startup now", "startupConfig", config) + + base := baseStartupController{config: config} + defer base.releaseAll() + + // 1. connect to processor service + if err = base.connectToProcessorService(ctx); err != nil { + logger.Errorfe(err, "startup failed") + return 0, nil + } + + // 2. get processor from processor service + if err = base.getProcessor(ctx); err != nil { + return 0, errors.Wrapf(err, "get processor failed") + } + + // check if using all streaming mode + if base.processor.DriverVersion < 1 { + logger.Warnf("driver version is %d < 1, will not use all stream mode", base.processor.DriverVersion) + return -1, nil + } + + // record meta error + defer func() { + if err != nil { + logger.Errorf("startup failed: %+v", err) + if updateErr := base.updateMetaState(ctx, err); updateErr != nil { + logger.Errorfe(updateErr, "update meta chain state failed") + } + } + }() + + // 3. connect to usage service + if err = base.connectToUsageService(ctx); err != nil { + return 0, err + } + + // 3b. connect to db registry service (optional, only required for + // network_v1 processors). The check that the client stub exists when + // it is actually needed happens in newEntityProbe and newTimeSeriesProbe. + if err = base.connectToDBRegistryService(ctx); err != nil { + return 0, err + } + + // 4. load chain configs + if err = base.loadChainsConfig(ctx); err != nil { + return exitcode.AlwaysRetry, errors.Wrapf(err, "load chains config failed") + } + + // build main controllers + ctrls := make(map[string]*controller.MainController) + switch base.processor.Project.Type { + case commonmodels.ProjectTypeSentio: + std := standardStartupController{baseStartupController: base} + if ctrls, exitCode, err = std.buildMainControllers(ctx); err != nil { + return exitCode, err + } + case commonmodels.ProjectTypeSubgraph: + ss := subgraphStartupController{baseStartupController: base} + if ctrls, exitCode, err = ss.buildMainControllers(ctx); err != nil { + return exitCode, err + } + default: + return exitcode.NeverRetry, errors.Errorf("project type %s is not supported", base.processor.Project.Type) + } + + // update meta chain state + if updateErr := base.updateMetaState(ctx, nil); updateErr != nil { + logger.Errorfe(updateErr, "update meta chain state failed") + return 0, nil + } + logger.Info("startup succeed") + + // start all main controllers + g, gctx := errgroup.WithContext(ctx) + for chainID_, ctrl_ := range ctrls { + chainID, ctrl := chainID_, ctrl_ + tracker.AddOrReplaceTrackedObject("MainController::"+chainID, ctrl) + g.Go(func() error { + mainCtx, _ := log.FromContext(gctx, "chain_id", chainID) + return ctrl.Main(mainCtx) + }) + } + if err = g.Wait(); err != nil { + var extErr *controller.ExternalError + if errors.As(err, &extErr) { + switch { + case extErr.IsBillingError(): + return exitcode.OverQuota, extErr + case extErr.IsUserError(): + return exitcode.NeverRetry, extErr + } + } + return exitcode.AlwaysRetry, err + } + logger.Info("all chains are done") + <-ctx.Done() + return exitcode.AlwaysRetry, nil +} + +// ClickhouseConnector builds the timeseries and entity chx.Controller used by a +// processor's stores. Its implementation lives in the driver binary and +// encapsulates the sharding / housegate wiring, so the controller depends only +// on chx, which is in sentio-core. +// +// A nil returned controller means the corresponding store is not available +// (e.g. no clickhouse configured). +type ClickhouseConnector interface { + Connect(ctx context.Context, processor *models.Processor) (timeseries, entity *chx.Controller, err error) +} + +type Config struct { + ProcessorID string + ProcessorReplica int + ProcessorService string + UsageService string + DBRegistryService string + WebhookService string + WebhookTopic string + ProcessorUrl string + ChainConfigFile string + IpfsNodeAddr string + EntityStoreCacheSize int + EntityStoreFullCacheSize int + SubgraphTotalMemSize uint + SubgraphDebugTrace bool + // PubSubProject is the GCP project used to create the webhook pubsub topic; + // empty disables pubsub topic creation. Provided by the driver binary. + PubSubProject string + + // ClickhouseConnector builds the timeseries/entity chx.Controller for a + // processor. Its implementation lives in the driver binary. + ClickhouseConnector ClickhouseConnector + // EntityMetricsMonitor is the metric monitor for the entity store; it carries + // the otel instrument and is built by the driver binary so the controller + // does not import the binary's metrics package. + EntityMetricsMonitor persistent.MetricsMonitor +} + +func Main(config Config) { + const retryInterval = time.Second * 30 + ctx, logger := log.FromContext(concurrency.NewSignalContext(context.Background()), "processorID", config.ProcessorID) + for { + exitCode, _ := main(ctx, config) + if exitCode < 0 { + logger.Warnf("do not use all streaming mode") + return + } + if exitCode > 0 { + logger.Infof("EXIT WITH CODE %d", exitCode) + os.Exit(int(exitCode)) + } + + logger.Infof("startup failed, will retry after %s", retryInterval.String()) + select { + case <-time.After(retryInterval): + case <-ctx.Done(): + os.Exit(0) + } + } +} + +type timeSeriesProbe struct { + client protosregistry.DatabaseRegistryServiceClient + processorID string + processorReplica int +} + +func (p *timeSeriesProbe) PreCreateTable(ctx context.Context, tv chx.TableOrView) error { + metaType, _, err := timeseries.CutTableName(tv.GetName()) + if err != nil { + return err + } + _, err = p.client.EnsureTable(ctx, &protosregistry.EnsureTableRequest{ + ProcessorId: p.processorID, + ReplicaIndex: uint32(p.processorReplica), + TableId: tv.GetName(), + TableType: string(metaType), + }) + return err +} + +type entityProbe struct { + client protosregistry.DatabaseRegistryServiceClient + processorID string + processorReplica int +} + +func (p *entityProbe) PreCreateTable(ctx context.Context, tv chx.TableOrView) error { + if _, ok := tv.(chx.Table); !ok { + return nil + } + _, err := p.client.EnsureTable(ctx, &protosregistry.EnsureTableRequest{ + ProcessorId: p.processorID, + ReplicaIndex: uint32(p.processorReplica), + TableId: tv.GetName(), + TableType: "entity", + }) + return err +} diff --git a/driver/controller/startup/startup_test.go b/driver/controller/startup/startup_test.go new file mode 100644 index 0000000..61551a6 --- /dev/null +++ b/driver/controller/startup/startup_test.go @@ -0,0 +1,43 @@ +package startup + +import ( + "testing" + + "sentioxyz/sentio-core/common/log" + sentioerror "sentioxyz/sentio-core/service/common/errors" + "sentioxyz/sentio-core/service/processor/models" + + "github.com/stretchr/testify/assert" +) + +func Test_errorRecord(t *testing.T) { + var er sentioerror.ErrorRecord + pb := er.ToPB() + log.Infof("pb.createdAt: %s", pb.GetCreatedAt().AsTime().String()) + var ner sentioerror.ErrorRecord + ner.FromPB(pb) + log.Infof("er.createdAt: %s", ner.CreatedAt.String()) + assert.True(t, ner.CreatedAt.IsZero()) +} + +func Test_buildProcessorUrls(t *testing.T) { + var s standardStartupController + + s.config.ProcessorUrl = "aaa" + s.processor = &models.Processor{NumWorkers: 1} + urls, err := s.buildProcessorUrlList() + assert.NoError(t, err) + assert.Equal(t, []string{"aaa"}, urls) + + s.config.ProcessorUrl = "aaa" + s.processor.NumWorkers = 3 + urls, err = s.buildProcessorUrlList() + assert.NoError(t, err) + assert.Equal(t, []string{"aaa", "aaa:81", "aaa:82"}, urls) + + s.config.ProcessorUrl = "aaa.bbb:9999" + s.processor.NumWorkers = 3 + urls, err = s.buildProcessorUrlList() + assert.NoError(t, err) + assert.Equal(t, []string{"aaa.bbb:9999", "aaa.bbb:10000", "aaa.bbb:10001"}, urls) +} diff --git a/driver/controller/startup/subgraph.go b/driver/controller/startup/subgraph.go new file mode 100644 index 0000000..b4e8d9c --- /dev/null +++ b/driver/controller/startup/subgraph.go @@ -0,0 +1,133 @@ +package startup + +import ( + "context" + "time" + + evmchain "sentioxyz/sentio-core/chain/evm" + "sentioxyz/sentio-core/common/chains" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/subgraph" + "sentioxyz/sentio-core/driver/exitcode" + "sentioxyz/sentio-core/driver/subgraph/manifest" + + "github.com/pkg/errors" +) + +type subgraphStartupController struct { + baseStartupController +} + +func (c *subgraphStartupController) buildMainControllers(ctx context.Context) ( + ctrls map[string]*controller.MainController, + exitCode exitcode.Code, + err error, +) { + ctrls = make(map[string]*controller.MainController) + // connect to clickhouse + if err = c.connectClickhouse(ctx); err != nil { + return ctrls, 0, errors.Wrapf(err, "connect to clickhouse failed") + } + // build ipfs client + c.buildIpfsShell(ctx) + // manifest + var mf *manifest.Manifest + if mf, err = manifest.LoadFromIpfs(c.ipfsShell, c.processor.IpfsHash, true); err != nil { + if errors.Is(err, manifest.ErrInvalidManifest) || errors.Is(err, manifest.ErrInvalidCustomizedEndpoint) { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, err) + } + return nil, exitcode.AlwaysRetry, errors.Wrapf(err, "load subgraph manifest failed") + } + // chainID and chainConfig and client + chainID, endpoint, _ := manifest.GetChainID(mf.GetNetwork(), false) + var client evm.Client + var chainConfig *config.ChainConfig + if chainID == manifest.CustomizedChainID { + chainConfig = config.NewCustomizedChainConfig(manifest.CustomizedChainID, endpoint) + } else if chains.IsEVMChains(chainID) { + var has bool + chainConfig, has = c.chainConfigs[chainID] + if !has { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("unsupported evm chain id %q for the network %q", chainID, mf.GetNetwork())) + } + if chainConfig.IsCustomizedEndpoint { + // chainConfig.Endpoint is not in the manifest file, it is in the NetworkOverrides, + // so need to check archive node here + if err = evmchain.CheckArchiveNode(ctx, chainConfig.Endpoint); err != nil { + return nil, exitcode.AlwaysRetry, errors.Wrapf(err, + "invalid customized endpoint %q for chain %s", chainConfig.Endpoint, chainID) + } + } + } else { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("unknown network %q in the manifest", mf.GetNetwork())) + } + client, err = evm.NewClient( + ctx, + chainConfig.Endpoint, + int(controller.ClientMaxConcurrency), + chainConfig.StartBlockOverride, + chainConfig.ProcessingDelayBlocks, + controller.SubscribeMinWatchInterval, + time.Second*3, + ) + if err != nil { + return nil, exitcode.NeverRetry, errors.Wrapf(err, "build evm client failed") + } + // handler controller + var handlerCtrl *subgraph.HandlerController + handlerCtrl, err = subgraph.NewHandlerController( + ctx, + c.processor, + chainConfig, + client, + c.ipfsShell, + mf, + uint32(c.config.SubgraphTotalMemSize), + c.config.SubgraphDebugTrace, + ) + if err != nil { + return nil, exitcode.NeverRetry, controller.NewExternalError(controller.ErrCodeWasmInitFailed, err) + } + // block builder + blockBuilder := controller.NewBlockBuilder(handlerCtrl, client, true) + // c.entityStore + if extErr := c.buildEntityStore(ctx, mf.Schema.File.GetContent()); extErr != nil { + if extErr.IsUserError() { + return nil, exitcode.NeverRetry, extErr + } + return nil, exitcode.AlwaysRetry, extErr + } + // entity controller + entityCtrl := newEntityController(c.entityStore, chainID, c.config.EntityStoreCacheSize, + c.config.EntityStoreFullCacheSize, c.config.EntityMetricsMonitor) + // checkpoint store + var store controller.CheckpointStore + if store, err = c.getCheckpointStore(ctx, chainID); err != nil { + return nil, exitcode.AlwaysRetry, controller.NewExternalError(controller.ErrCodeSaveCheckpointFailed, err) + } + // checkpoint controller + var checkpointCtrl controller.CheckpointController + checkpointCtrl, err = controller.NewCheckpointController( + ctx, + chainID, + controller.SaveCheckpointDelay, + controller.SaveCheckpointInterval, + controller.MaxKeepCheckpointCount, + store, + c.getQuotaService(chainID), + controller.EmptyTimeSeriesController{}, + entityCtrl, + controller.EmptyWebhookController{}, + c.buildCommitCtx, + ) + if err != nil { + return nil, exitcode.AlwaysRetry, errors.Wrapf(err, "build checkpoint controller failed") + } + // main controller + ctrls[chainID] = controller.NewMainController(blockBuilder, checkpointCtrl, false, c.processor, chainID) + return ctrls, exitcode.AlwaysRetry, nil +} diff --git a/driver/controller/startup/timeseries.go b/driver/controller/startup/timeseries.go new file mode 100644 index 0000000..8dc781b --- /dev/null +++ b/driver/controller/startup/timeseries.go @@ -0,0 +1,138 @@ +package startup + +import ( + "context" + "math" + "sync" + "time" + + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/timeseries" +) + +type timeSeriesController struct { + chainID string + store timeseries.Store + + mu sync.Mutex + cached map[uint64]map[uint64][]timeseries.Dataset // map[][] + committed *uint64 +} + +func newTimeSeriesController(chainID string, store timeseries.Store) *timeSeriesController { + return &timeSeriesController{ + chainID: chainID, + store: store, + cached: make(map[uint64]map[uint64][]timeseries.Dataset), + } +} + +func (c *timeSeriesController) Reset(ctx context.Context, checkpoint *controller.Checkpoint) *controller.ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + if checkpoint == nil { + c.cached = make(map[uint64]map[uint64][]timeseries.Dataset) + } else { + utils.MapDelete(c.cached, func(bn uint64) bool { + return bn > checkpoint.BlockNumber + }) + } + var blockNumber int64 = -1 + if checkpoint != nil { + blockNumber = int64(checkpoint.BlockNumber) + } + if err := c.store.DeleteData(ctx, c.chainID, blockNumber); err != nil { + return controller.NewExternalError(controller.ErrCodeCleanTimeSeriesDataFailed, err) + } + return nil +} + +var maxUncommitedTimeSeries = envconf.LoadUInt64("SENTIO_MAX_UNCOMMITED_TIME_SERIES_DATA", 1000000, + envconf.WithMin(10000), envconf.WithMax(1000000)) + +func (c *timeSeriesController) getCachedSize(blockNumber uint64) (total uint64) { + for bn, blockData := range c.cached { + if bn > blockNumber { + continue + } + for _, dss := range blockData { + for _, ds := range dss { + total += uint64(len(ds.Rows)) + } + } + } + return total +} + +func (c *timeSeriesController) CachedTooMuch(blockNumber uint64) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.getCachedSize(blockNumber) > maxUncommitedTimeSeries +} + +func (c *timeSeriesController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (stat map[timeseries.MetaType]map[string]int, extErr *controller.ExternalError) { + // collect data to commit + var data []timeseries.Dataset + c.mu.Lock() + for _, bn := range utils.GetOrderedMapKeys(c.cached) { + if bn <= blockNumber { + data = append(data, utils.MergeArr(utils.GetMapValuesOrderByKey(c.cached[bn])...)...) + } + } + c.mu.Unlock() + + // actually save dataset + if err := c.store.AppendData(ctx, data, c.chainID, blockTime); err != nil { + if errors.Is(err, timeseries.ErrInvalidMetaDiff) { + return nil, controller.NewExternalError(controller.ErrCodeTimeSeriesDataSchemaChanged, err) + } + if errors.Is(err, timeseries.ErrInvalidMeta) { + return nil, controller.NewExternalError(controller.ErrCodeInvalidTimeSeriesData, err) + } + return nil, controller.NewExternalError(controller.ErrCodeSaveTimeSeriesDataFailed, + errors.Wrapf(err, "failed to commit timeseries data: %s", timeseries.GetDatasetsSummary(data))) + } + stat = make(map[timeseries.MetaType]map[string]int) + for _, ds := range data { + utils.IncrK2Map(stat, ds.Type, ds.Name, len(ds.Rows)) + } + + // save succeed, clean c.cached + c.mu.Lock() + defer c.mu.Unlock() + utils.MapDelete(c.cached, func(bn uint64) bool { + return bn <= blockNumber + }) + c.committed = &blockNumber + return +} + +func (c *timeSeriesController) Insert(blockNumber uint64, taskIndex controller.TaskIndex, data []timeseries.Dataset) { + c.mu.Lock() + defer c.mu.Unlock() + org, _ := utils.GetFromK2Map(c.cached, blockNumber, taskIndex.Global) + utils.PutIntoK2Map(c.cached, blockNumber, taskIndex.Global, append(org, data...)) +} + +func (c *timeSeriesController) Snapshot() any { + c.mu.Lock() + defer c.mu.Unlock() + return map[string]any{ + "committed": c.committed, + "uncommitedTotal": c.getCachedSize(math.MaxUint64), + "uncommited": cacheSnapshot(c.cached, func(dss []timeseries.Dataset) (s int) { + for _, ds := range dss { + s += len(ds.Rows) + } + return s + }), + } +} diff --git a/driver/controller/startup/utils.go b/driver/controller/startup/utils.go new file mode 100644 index 0000000..e1e3103 --- /dev/null +++ b/driver/controller/startup/utils.go @@ -0,0 +1,48 @@ +package startup + +import ( + "sentioxyz/sentio-core/common/sparsify" + "sentioxyz/sentio-core/common/utils" +) + +func cacheSnapshot[T any](cache map[uint64]map[uint64]T, getSize func(T) int) any { + if len(cache) == 0 { + return nil + } + type segment struct { + Start uint64 + End uint64 + BlockCount int + DataCount int + } + ss := make([]segment, 0, len(cache)) + for _, bn := range utils.GetOrderedMapKeys(cache) { + s := segment{Start: bn, End: bn, BlockCount: 1} + for _, d := range cache[bn] { + s.DataCount += getSize(d) + } + ss = append(ss, s) + } + remove := sparsify.Remove(ss, func(s segment) uint64 { + return s.Start + }) + merge := func(ss []segment) (r segment) { + r = ss[0] + for i := 1; i < len(ss); i++ { + r.Start = min(ss[i].Start, r.Start) + r.End = max(ss[i].End, r.End) + r.BlockCount += ss[i].BlockCount + r.DataCount += ss[i].DataCount + } + return r + } + var result []segment + var s int + for p := 1; p < len(ss); p++ { + if !remove[p] { + result = append(result, merge(ss[s:p])) + s = p + } + } + return append(result, merge(ss[s:])) +} diff --git a/driver/controller/startup/webhook.go b/driver/controller/startup/webhook.go new file mode 100644 index 0000000..58a2844 --- /dev/null +++ b/driver/controller/startup/webhook.go @@ -0,0 +1,176 @@ +package startup + +import ( + "context" + "encoding/json" + "math" + "sync" + "time" + + "cloud.google.com/go/pubsub" + "github.com/pkg/errors" + + "sentioxyz/sentio-core/common/envconf" + "sentioxyz/sentio-core/common/errgroup" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/service/processor/models" +) + +type webhookController struct { + processor *models.Processor + topic *pubsub.Topic + + mu sync.Mutex + cached map[uint64]map[uint64][]controller.WebhookMessage // map[][] + committed *uint64 +} + +func newWebhookController(processor *models.Processor, topic *pubsub.Topic) *webhookController { + return &webhookController{ + processor: processor, + topic: topic, + cached: make(map[uint64]map[uint64][]controller.WebhookMessage), + } +} + +func (c *webhookController) Reset(ctx context.Context, checkpoint *controller.Checkpoint) *controller.ExternalError { + c.mu.Lock() + defer c.mu.Unlock() + if checkpoint == nil { + c.cached = make(map[uint64]map[uint64][]controller.WebhookMessage) + } else { + utils.MapDelete(c.cached, func(bn uint64) bool { + return bn > checkpoint.BlockNumber + }) + } + // sent msg cannot be canceled + return nil +} + +type SingleWebhookMessage struct { + ExportName string `json:"export_name"` + EventID uint64 `json:"event_id"` + TimestampMicros uint64 `json:"timestamp_micros"` + Version uint64 `json:"version"` + Data string `json:"data"` +} + +const MaxMessagesPerBlock = 10000000 + +var maxUncommitedWebhookMessages = envconf.LoadUInt64("SENTIO_MAX_UNCOMMITED_WEBHOOK_MESSAGES", 1000000, + envconf.WithMin(10000), envconf.WithMax(1000000)) + +func (c *webhookController) getCachedSize(blockNumber uint64) (total uint64) { + for bn, blockMsgs := range c.cached { + if bn > blockNumber { + continue + } + for _, msgs := range blockMsgs { + total += uint64(len(msgs)) + } + } + return total +} + +func (c *webhookController) CachedTooMuch(blockNumber uint64) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.getCachedSize(blockNumber) > maxUncommitedWebhookMessages +} + +func (c *webhookController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (stat map[string]int, extErr *controller.ExternalError) { + // build SingleWebhookMessage from c.cached into dict + stat = make(map[string]int) + dict := make(map[string][]SingleWebhookMessage) + c.mu.Lock() + for _, bn := range utils.GetOrderedMapKeys(c.cached) { + if bn > blockNumber { + continue + } + blockMsgs := c.cached[bn] + if count := utils.CountMap(blockMsgs); count >= MaxMessagesPerBlock { + c.mu.Unlock() + return nil, controller.NewExternalError(controller.ErrCodeTooManyWebhookMsgEntity, + errors.Errorf("too many messages in block %d, %d > %d", bn, count, MaxMessagesPerBlock)) + } + var blockSeq uint64 + for _, messages := range utils.GetMapValuesOrderByKey(blockMsgs) { + for _, msg := range messages { + blockSeq++ + dict[msg.Channel] = append(dict[msg.Channel], SingleWebhookMessage{ + ExportName: msg.Name, + EventID: bn*MaxMessagesPerBlock + blockSeq, + TimestampMicros: uint64(msg.BlockTime.UnixMicro()), + Version: uint64(c.processor.Version), + Data: msg.Payload, + }) + stat[msg.Name] += 1 + } + } + } + c.mu.Unlock() + + // actually send SingleWebhookMessage + g, gctx := errgroup.WithContext(ctx) + for channel, messages := range dict { + data, err := json.Marshal(messages) + if err != nil { + panic(errors.Wrapf(err, "json marshal message for channel %s failed", channel)) + } + pubSubMsg := &pubsub.Message{ + Data: data, + Attributes: map[string]string{ + "channel_name": channel, + "project_id": c.processor.ProjectID, + "processor_id": c.processor.ID, + }, + } + g.Go(func() error { + _, pubErr := c.topic.Publish(gctx, pubSubMsg).Get(gctx) + if pubErr != nil { + return errors.Wrapf(pubErr, "publish message for channel %s failed", channel) + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, controller.NewExternalError(controller.ErrCodeSendWebhookDataFailed, err) + } + + // send succeed, clean c.cached + c.mu.Lock() + defer c.mu.Unlock() + utils.MapDelete(c.cached, func(bn uint64) bool { + return bn <= blockNumber + }) + c.committed = &blockNumber + return +} + +func (c *webhookController) Insert( + blockNumber uint64, + taskIndex controller.TaskIndex, + messages []controller.WebhookMessage, +) { + c.mu.Lock() + defer c.mu.Unlock() + org, _ := utils.GetFromK2Map(c.cached, blockNumber, taskIndex.Global) + utils.PutIntoK2Map(c.cached, blockNumber, taskIndex.Global, append(org, messages...)) +} + +func (c *webhookController) Snapshot() any { + c.mu.Lock() + defer c.mu.Unlock() + return map[string]any{ + "committed": c.committed, + "uncommitedTotal": c.getCachedSize(math.MaxUint64), + "uncommited": cacheSnapshot(c.cached, func(msgs []controller.WebhookMessage) (s int) { + return len(msgs) + }), + } +} diff --git a/driver/controller/subgraph/BUILD.bazel b/driver/controller/subgraph/BUILD.bazel new file mode 100644 index 0000000..3c729a1 --- /dev/null +++ b/driver/controller/subgraph/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "subgraph", + srcs = [ + "binding_data.go", + "block_data.go", + "handler.go", + "handler_block.go", + "handler_call.go", + "handler_event.go", + "instance.go", + "task.go", + ], + importpath = "sentioxyz/sentio-core/driver/controller/subgraph", + visibility = ["//visibility:public"], + deps = [ + "//chain/evm", + "//common/concurrency", + "//common/log", + "//common/set", + "//common/timer", + "//common/utils", + "//common/wasm", + "//driver/controller", + "//driver/controller/config", + "//driver/controller/data", + "//driver/controller/data/evm", + "//driver/controller/fetcher", + "//driver/entity/persistent", + "//driver/subgraph/abiutil", + "//driver/subgraph/common", + "//driver/subgraph/ethereum", + "//driver/subgraph/manifest", + "//service/processor/models", + "@com_github_ethereum_go_ethereum//accounts/abi", + "@com_github_ethereum_go_ethereum//common", + "@com_github_ethereum_go_ethereum//core/types", + "@com_github_ethereum_go_ethereum//crypto", + "@com_github_ipfs_go_ipfs_api//:go-ipfs-api", + "@com_github_mr_tron_base58//base58", + "@com_github_pkg_errors//:errors", + ], +) diff --git a/driver/controller/subgraph/binding_data.go b/driver/controller/subgraph/binding_data.go new file mode 100644 index 0000000..4226e89 --- /dev/null +++ b/driver/controller/subgraph/binding_data.go @@ -0,0 +1,28 @@ +package subgraph + +import ( + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/subgraph/manifest" +) + +type taskData struct { + callHandlerParam any + + dataSource *manifest.DataSource + handlerID controller.HandlerID + txIndex int + logIndex int + + size int +} + +func (t taskData) Cmp(a taskData) int { + if r := utils.Cmp(t.txIndex, a.txIndex); r != 0 { + return r + } + if r := utils.Cmp(t.handlerID.ID, a.handlerID.ID); r != 0 { + return r + } + return utils.Cmp(t.logIndex, a.logIndex) +} diff --git a/driver/controller/subgraph/block_data.go b/driver/controller/subgraph/block_data.go new file mode 100644 index 0000000..42395ce --- /dev/null +++ b/driver/controller/subgraph/block_data.go @@ -0,0 +1,325 @@ +package subgraph + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/accounts/abi" + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/pkg/errors" + + "sentioxyz/sentio-core/chain/evm" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/common/wasm" + "sentioxyz/sentio-core/driver/controller" + evmExtend "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/subgraph/abiutil" + "sentioxyz/sentio-core/driver/subgraph/common" + "sentioxyz/sentio-core/driver/subgraph/ethereum" +) + +type BlockData struct { + evmExtend.BlockHeader + + mainData evmExtend.BlockMainData + extendData evmExtend.BlockExtendData + + cachedBlock *ethereum.Block + cachedTxn map[string]*ethereum.Transaction + cachedTxnReceipt map[string]*ethereum.TransactionReceipt + + taskList []controller.Task + taskTotalSize int + dataSource string + + checkpointData map[string]string +} + +func (d *BlockData) GetTaskList() []controller.Task { + return d.taskList +} + +func (d *BlockData) CheckpointData() map[string]string { + return d.checkpointData +} + +func (d *BlockData) DataSource() string { + return d.dataSource +} + +func (d *BlockData) Size() int { + return d.taskTotalSize +} + +func (d *BlockData) buildBlock() (b *ethereum.Block, err error) { + if d.cachedBlock == nil { + var payload evm.ExtendedHeader + if err = json.Unmarshal(d.BlockHeader.Raw, &payload); err != nil { + return nil, errors.Wrapf(err, "unmarshal header for block %d failed: %s", + d.GetBlockNumber(), string(d.BlockHeader.Raw)) + } + defer func() { + if panicErr := recover(); panicErr != nil { + var is bool + if err, is = panicErr.(error); !is { + err = errors.Errorf("build ethereum block %d from raw data (%s) failed: %v", + d.GetBlockNumber(), string(d.BlockHeader.Raw), panicErr) + } else { + err = errors.Wrapf(err, "build ethereum block %d from raw data (%s) failed", + d.GetBlockNumber(), string(d.BlockHeader.Raw)) + } + } + }() + d.cachedBlock = ðereum.Block{ + Hash: wasm.MustBuildByteArrayFromHex(d.BlockHash), + ParentHash: wasm.MustBuildByteArrayFromHex(d.ParentBlockHash), + UnclesHash: wasm.MustBuildByteArrayFromHex(payload.UncleHash.Hex()), + StateRoot: wasm.MustBuildByteArrayFromHex(payload.Root.Hex()), + TransactionsRoot: wasm.MustBuildByteArrayFromHex(payload.TxHash.Hex()), + ReceiptsRoot: wasm.MustBuildByteArrayFromHex(payload.ReceiptHash.Hex()), + Number: common.MustBuildBigInt(d.BlockNumber), + GasUsed: common.MustBuildBigInt(payload.GasUsed), + GasLimit: common.MustBuildBigInt(payload.GasLimit), + Timestamp: common.MustBuildBigInt(payload.Time), + Difficulty: common.MustBuildBigInt(payload.Difficulty), + TotalDifficulty: common.MustBuildBigInt(payload.TotalDifficulty.ToInt()), + BaseFeePerGas: common.MustBuildBigInt(payload.BaseFee), + } + if payload.Author != "" { + d.cachedBlock.Author = common.MustBuildAddressFromString(payload.Author) + } + if payload.Size != nil { + d.cachedBlock.Size = common.MustBuildBigInt(uint64(*payload.Size)) + } + } + return d.cachedBlock, nil +} + +func (d *BlockData) transactionSucceed(txHash string) (bool, error) { + if receipt, has := d.extendData.Receipts[txHash]; !has { + return false, errors.Errorf("unreachable, transaction receipt %s not loaded, already loaded %v in block %d", + txHash, utils.GetMapKeys(d.extendData.Receipts), d.BlockNumber) + } else { + return receipt.Status > 0, nil + } +} + +func (d *BlockData) buildTransaction(txHash string) (tx *ethereum.Transaction, err error) { + if d.cachedTxn == nil { + d.cachedTxn = make(map[string]*ethereum.Transaction) + } + var has bool + if tx, has = d.cachedTxn[txHash]; !has { + var payload evm.RPCTransaction + if payload, has = d.extendData.Transactions[txHash]; !has { + return nil, errors.Errorf("unreachable, transaction %s not loaded, already loaded %v in block %d", + txHash, utils.GetMapKeys(d.extendData.Transactions), d.BlockNumber) + } + defer func() { + if panicErr := recover(); panicErr != nil { + var is bool + if err, is = panicErr.(error); !is { + err = errors.Errorf("build ethereum transaction %s from raw data (%s) failed: %v", + txHash, utils.MustJSONMarshal(payload), panicErr) + } else { + err = errors.Wrapf(err, "build ethereum transaction %s from raw data (%s) failed", + txHash, utils.MustJSONMarshal(payload)) + } + } + }() + tx = ðereum.Transaction{ + Hash: wasm.MustBuildByteArrayFromHex(txHash), + Index: common.MustBuildBigInt(uint64(payload.TransactionIndex)), + From: common.MustBuildAddressFromString(payload.From.String()), + Value: common.MustBuildBigInt(payload.Value.ToInt()), + GasLimit: common.MustBuildBigInt(uint64(payload.Gas)), + GasPrice: common.MustBuildBigInt(payload.GasPrice.ToInt()), + Input: &wasm.ByteArray{Data: payload.Input}, + Nonce: common.MustBuildBigInt(uint64(payload.Nonce)), + } + if payload.To != nil { + tx.To = common.MustBuildAddressFromString(payload.To.String()) + } + d.cachedTxn[txHash] = tx + } + return +} + +func (d *BlockData) buildReceipt(txHash string) (receipt *ethereum.TransactionReceipt, err error) { + if d.cachedTxnReceipt == nil { + d.cachedTxnReceipt = make(map[string]*ethereum.TransactionReceipt) + } + var has bool + if receipt, has = d.cachedTxnReceipt[txHash]; !has { + var payload evm.ExtendedReceipt + if payload, has = d.extendData.Receipts[txHash]; !has { + return nil, errors.Errorf("unreachable, transaction receipt %s not loaded, already loaded %v in block %d", + txHash, utils.GetMapKeys(d.extendData.Receipts), d.BlockNumber) + } + defer func() { + if panicErr := recover(); panicErr != nil { + var is bool + if err, is = panicErr.(error); !is { + err = errors.Errorf("build ethereum transaction receipt %s from raw data (%s) failed: %v", + txHash, utils.MustJSONMarshal(payload), panicErr) + } else { + err = errors.Wrapf(err, "build ethereum transaction receipt %s from raw data (%s) failed", + txHash, utils.MustJSONMarshal(payload)) + } + } + }() + receipt = ðereum.TransactionReceipt{ + TransactionHash: wasm.MustBuildByteArrayFromHex(txHash), + TransactionIndex: common.MustBuildBigInt(uint64(payload.TransactionIndex)), + BlockHash: &wasm.ByteArray{Data: payload.BlockHash.Bytes()}, + BlockNumber: common.MustBuildBigInt(payload.BlockNumber.ToInt()), + CumulativeGasUsed: common.MustBuildBigInt(uint64(payload.CumulativeGasUsed)), + GasUsed: common.MustBuildBigInt(uint64(payload.GasUsed)), + ContractAddress: nil, + Logs: nil, + Status: common.MustBuildBigInt(uint64(payload.Status)), + Root: &wasm.ByteArray{Data: payload.Root}, + LogsBloom: &wasm.ByteArray{Data: payload.Bloom.Bytes()}, + } + if payload.ContractAddress != nil { + receipt.ContractAddress = common.MustBuildAddressFromString(payload.ContractAddress.String()) + } + receipt.Logs = &wasm.ObjectArray[*ethereum.Log]{Data: make([]*ethereum.Log, len(payload.Logs))} + for i, rawLog := range payload.Logs { + receipt.Logs.Data[i] = ðereum.Log{ + Address: common.MustBuildAddressFromString(rawLog.Address.String()), + Topics: &wasm.ObjectArray[*wasm.ByteArray]{ + Data: utils.MapSliceNoError(rawLog.Topics, func(topic ethcommon.Hash) *wasm.ByteArray { + return &wasm.ByteArray{Data: topic.Bytes()} + }), + }, + Data: &wasm.ByteArray{Data: rawLog.Data}, + BlockHash: &wasm.ByteArray{Data: rawLog.BlockHash.Bytes()}, + BlockNumber: common.MustBuildBigInt(rawLog.BlockNumber), + TransactionHash: &wasm.ByteArray{Data: rawLog.TxHash.Bytes()}, + TransactionIndex: common.MustBuildBigInt(rawLog.TxIndex), + LogIndex: common.MustBuildBigInt(rawLog.Index), + TransactionLogIndex: common.MustBuildBigInt(i), + LogType: nil, // TODO unknown field + Removed: &common.Wrapped[wasm.Bool]{Inner: wasm.Bool(rawLog.Removed)}, + } + } + d.cachedTxnReceipt[txHash] = receipt + } + return +} + +func (d *BlockData) buildCall(trace evmExtend.Trace, funcABI *abi.Method) (*ethereum.Call, int, error) { + var size int + var payload evm.ParityTrace + err := json.Unmarshal(trace.Raw, &payload) + if err != nil { + return nil, size, errors.Wrapf(err, "unmarshal call trace failed: %s", string(trace.Raw)) + } + size += len(trace.Raw) + ca := ðereum.Call{} + ca.Block, err = d.buildBlock() + if err != nil { + return ca, size, err + } + size += len(d.BlockHeader.Raw) + ca.Transaction, err = d.buildTransaction(trace.TransactionHash) + if err != nil { + return ca, size, err + } + size += 1000 + ca.To, err = common.BuildAddressFromString(payload.Action.To) + if err != nil { + return ca, size, err + } + ca.From, err = common.BuildAddressFromString(payload.Action.From.Hex()) + if err != nil { + return ca, size, err + } + fn := abiutil.GetMethodSig(funcABI, true) + // first 4 bytes is method ID + ca.InputValues, err = ethereum.UnpackParams("input data of function "+fn, payload.Action.Input[4:], funcABI.Inputs) + if err != nil { + return ca, size, err + } + ca.OutputValues, err = ethereum.UnpackParams("output data of function "+fn, payload.Result.Output, funcABI.Outputs) + return ca, size, err +} + +func (d *BlockData) buildEvent(event types.Log, eventABI *abi.Event) (ev *ethereum.Event, size int, err error) { + txHash := event.TxHash.String() + receipt, has := d.extendData.Receipts[txHash] + if !has { + return nil, size, errors.Errorf("unreachable, receipt %s for the log %s not loaded, already loaded %v in block %d", + txHash, utils.MustJSONMarshal(event), utils.GetMapKeys(d.extendData.Receipts), d.BlockNumber) + } + var txLogIndex = -1 + for i, transactionLog := range receipt.Logs { + if transactionLog.Index == event.Index { + txLogIndex = i + break + } + } + if txLogIndex < 0 { + return nil, size, errors.Errorf("unreachable, log with logIndex %d not found in all transaction receipt logs %s", + event.Index, utils.MustJSONMarshal(receipt.Logs)) + } + defer func() { + if panicErr := recover(); panicErr != nil { + var is bool + if err, is = panicErr.(error); !is { + err = errors.Errorf("build ethereum event in block %d from raw data (%s) failed: %v", + d.GetBlockNumber(), utils.MustJSONMarshal(event), panicErr) + } else { + err = errors.Wrapf(err, "build ethereum event in block %d from raw data (%s) failed", + d.GetBlockNumber(), utils.MustJSONMarshal(event)) + } + } + }() + ev = ðereum.Event{ + Address: common.MustBuildAddressFromString(event.Address.String()), + LogIndex: common.MustBuildBigInt(uint64(event.Index)), + TransactionLogIndex: common.MustBuildBigInt(txLogIndex), + LogType: nil, // TODO unknown field + } + size += 1000 // event self + if ev.Block, err = d.buildBlock(); err != nil { + return ev, size, err + } + size += len(d.BlockHeader.Raw) + if ev.Transaction, err = d.buildTransaction(txHash); err != nil { + return ev, size, err + } + size += 1000 // tx part + if ev.Receipt, err = d.buildReceipt(txHash); err != nil { + return ev, size, err + } + size += 1000 + 1000*len(d.extendData.Receipts[txHash].Logs) // receipt main part + logs + + // arguments of the log + arguments := make(map[string]any) + + // unpack non-indexed arguments + if err = eventABI.Inputs.NonIndexed().UnpackIntoMap(arguments, event.Data); err != nil { + return nil, size, errors.Wrapf(ethereum.ErrABINotMatch, "unpack event data of raw log failed: %v", err) + } + + // parse indexed arguments + var indexedInputs abi.Arguments + for _, input := range eventABI.Inputs { + if input.Indexed { + indexedInputs = append(indexedInputs, input) + } + } + if err = abi.ParseTopicsIntoMap(arguments, indexedInputs, event.Topics[1:]); err != nil { + // TODO if the type of the indexed argument is tuple, here will got an error, it is unexpected + return nil, size, errors.Wrapf(ethereum.ErrABINotMatch, "parse topics of raw log failed: %v", err) + } + + // convert args to ev.Parameters + ev.Parameters = ðereum.EventParams{} + for _, input := range eventABI.Inputs { + ev.Parameters.Data = append(ev.Parameters.Data, ethereum.MustConvertEventParam(arguments[input.Name], input)) + } + return ev, size, nil +} diff --git a/driver/controller/subgraph/handler.go b/driver/controller/subgraph/handler.go new file mode 100644 index 0000000..efe19e8 --- /dev/null +++ b/driver/controller/subgraph/handler.go @@ -0,0 +1,478 @@ +package subgraph + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "sentioxyz/sentio-core/common/concurrency" + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/config" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/subgraph/manifest" + "sentioxyz/sentio-core/service/processor/models" + + shell "github.com/ipfs/go-ipfs-api" + "github.com/pkg/errors" +) + +type HandlerAgent interface { + controller.HandlerAgent + + GetExtendRequirements(context.Context, *BlockData) (evm.BlockExtendRequirement, error) + BuildTaskDataList(context.Context, *BlockData) ([]taskData, error) +} + +const ( + HandlerTypeEvent = "event" + HandlerTypeBlock = "block" + HandlerTypeCall = "call" +) + +type HandlerController struct { + processor *models.Processor + chainConfig *config.ChainConfig + client evm.Client + ipfsShell *shell.Shell + manifest *manifest.Manifest + memHardLimit uint32 + debugTrace bool + + instance *instance // TODO support multi instances + + agents []HandlerAgent // built from Manifest + + addressStart map[string]uint64 + addressStartData string + + waiter *concurrency.ResourceWaiter[uint64] +} + +func NewHandlerController( + ctx context.Context, + processor *models.Processor, + chainConfig *config.ChainConfig, + client evm.Client, + ipfsShell *shell.Shell, + manifest *manifest.Manifest, + memHardLimit uint32, + debugTrace bool, +) (ctrl *HandlerController, err error) { + ctrl = &HandlerController{ + processor: processor, + chainConfig: chainConfig, + client: client, + ipfsShell: ipfsShell, + manifest: manifest, + memHardLimit: memHardLimit, + debugTrace: debugTrace, + } + ctrl.instance, err = ctrl.newInstance(ctx) + return ctrl, err +} + +func (c *HandlerController) chainID() string { + return c.chainConfig.ChainID +} + +func (c *HandlerController) Prologue( + ctx context.Context, + checkpoint *controller.Checkpoint, + templates map[uint64][]controller.TemplateInstance, + first uint64, + latest controller.BlockHeader, +) *controller.ExternalError { + // reset instance + if err := c.instance.Reset(ctx); err != nil { + return controller.NewExternalError(controller.ErrCodeResetWasmInstanceFailed, err) + } + // load contract start + if extErr := c.LoadAddressStart(checkpoint); extErr != nil { + return extErr + } + // build agents + if extErr := c.buildAgents(ctx, first, latest.GetBlockNumber(), templates); extErr != nil { + return extErr + } + // build checkpoint data for contract start + c.AddressStartReady() + // reset waiter + c.waiter = concurrency.NewResourceWaiter[uint64]() + return nil +} + +func (c *HandlerController) buildAgents( + ctx context.Context, + first, latest uint64, + templates map[uint64][]controller.TemplateInstance, +) (extErr *controller.ExternalError) { + _, logger := log.FromContext(ctx) + // collect data sources + dataSources := c.manifest.DataSources + for _, tpls := range utils.GetMapValuesOrderByKey(templates) { + for _, tpl := range tpls { + if total := len(c.manifest.Templates); tpl.TemplateID < 0 || int(tpl.TemplateID) >= total { + logger.Warnf("template id %d out of range [0,%d), will be ignored", tpl.TemplateID, total) + continue + } + dataSources = append(dataSources, c.manifest.Templates[tpl.TemplateID].NewDataSource( + tpl.Address, + manifest.BuildBigIntFromUint(tpl.StartBlock), + tpl.TemplateName, + )) + } + } + // build agents + c.agents = nil + for dataSourceID, ds := range dataSources { + blockRange := controller.BlockRange{StartBlock: max(ds.Source.GetStartBlock(), first)} + if ds.Source.EndBlock != nil { + blockRange.EndBlock = utils.WrapPointer(ds.Source.GetEndBlock()) + } + contractRange := blockRange + var err error + contractRange.StartBlock, err = c.GetAddressStart(ctx, ds.Source.Address, blockRange.StartBlock, latest) + if err != nil { + return controller.NewExternalError(controller.ErrCodeGetContractStartBlockFailed, err) + } + for i, eventHandler := range ds.Mapping.EventHandlers { + if ds.Source.Address == "" { + return controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("data source #%d %s has event handler but no contract address", dataSourceID, ds.Name)) + } + agent := HandlerAgentEvent{ + BaseHandlerAgent: controller.BaseHandlerAgent{ + HandlerID: controller.HandlerID{ + DataSource: ds.Name, + DataSourceID: dataSourceID, + Type: HandlerTypeEvent, + Name: eventHandler.Handler, + ID: int32(i), + }, + Range: contractRange, + }, + DataSource: ds, + Filter: evm.LogFilter{ + Topics: [][]string{{eventHandler.Topic0}}, + Address: []string{strings.ToLower(ds.Source.Address)}, + }, + } + c.agents = append(c.agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + for i, callHandler := range ds.Mapping.CallHandlers { + if ds.Source.Address == "" { + return controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("data source #%d %s has call handler but no contract address", dataSourceID, ds.Name)) + } + agent := HandlerAgentCall{ + BaseHandlerAgent: controller.BaseHandlerAgent{ + HandlerID: controller.HandlerID{ + DataSource: ds.Name, + DataSourceID: dataSourceID, + Type: HandlerTypeCall, + Name: callHandler.Handler, + ID: int32(i), + }, + Range: contractRange, + }, + DataSource: ds, + Filter: evm.TraceFilter{ + Signature: []string{callHandler.Signature}, + Address: []string{strings.ToLower(ds.Source.Address)}, + }, + } + c.agents = append(c.agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + for i, blockHandler := range ds.Mapping.BlockHandlers { + agent := HandlerAgentBlock{ + BaseHandlerAgent: controller.BaseHandlerAgent{ + HandlerID: controller.HandlerID{ + DataSource: ds.Name, + DataSourceID: dataSourceID, + Type: HandlerTypeBlock, + Name: blockHandler.Handler, + ID: int32(i), + }, + Range: blockRange, + }, + DataSource: ds, + } + switch kind := blockHandler.Filter.GetKind(); kind { + case "once": + agent.Once = true + case "polling": + interval := blockHandler.Filter.GetEvery() + if interval <= 0 { + return controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("every should greater than 0 in data source %s #%d block handler", ds.Name, i)) + } + agent.IntervalConfig = data.IntervalConfig{ + BlockInterval: &data.BlockInterval{Backfill: uint64(interval), Watching: uint64(interval)}, + } + default: + return controller.NewExternalError(controller.ErrCodeInvalidSubgraphManifest, + errors.Errorf("filter kind %q is not supported in data source %s #%d block handler", kind, ds.Name, i)) + } + c.agents = append(c.agents, agent) + logger.Infow("has new agent", "agent", agent.Snapshot()) + } + } + return nil +} + +func (c *HandlerController) GetBlockRange() controller.BlockRange { + return controller.GetHandleAgentsBlockRange(c.agents) +} + +func (c *HandlerController) GetAgentStat() map[string]int { + stat := make(map[string]int) + for _, ag := range c.agents { + stat[fmt.Sprintf("%T", ag)] += 1 + } + return stat +} + +func (c *HandlerController) buildReportRequirements( + currentBlockNumber uint64, +) []data.IntervalRequirement { + endBlock := c.GetBlockRange().EndBlock + // In the backfill phase, at least one non-empty BlockData is generated for every DAY. + // In the watching phase, at least one non-empty BlockData is generated for every MINUTE. + reqs := []data.IntervalRequirement{{ + BlockRange: controller.BlockRange{StartBlock: currentBlockNumber, EndBlock: endBlock}, + IntervalConfig: data.IntervalConfig{TimeInterval: &data.TimeInterval{ + Backfill: time.Hour * 24, + Watching: time.Minute, + }}, + }} + if endBlock != nil { + reqs = append(reqs, data.IntervalRequirement{ + BlockRange: controller.BlockRange{StartBlock: *endBlock, EndBlock: endBlock}, + IntervalConfig: data.IntervalConfig{BlockInterval: &data.BlockInterval{ + Backfill: 1, + Watching: 1, + }}, + }) + } + return reqs +} + +func (c *HandlerController) BuildBlockDataFetcher( + firstBlockNumber uint64, + currentBlockNumber uint64, + latest controller.BlockHeader, +) controller.Fetcher[controller.BlockData] { + req := c.getDataRequirement() + req.Interval = append(req.Interval, c.buildReportRequirements(currentBlockNumber)...) + fetchNamePrefix := fmt.Sprintf("EVM::%s::", c.chainID()) + return fetcher.TransferFetcher( + fetchNamePrefix+"BlockDataFetcher", + evm.BuildBlockMainDataFetcher(fetchNamePrefix, req, firstBlockNumber, currentBlockNumber, latest, c.client), + latest, + controller.ProcessConcurrency, + 256*1024*1024, + 100, + time.Second*10, + 20, + time.Second, + func(ctx context.Context, blockNumber uint64, from evm.BlockMainData) (controller.BlockData, bool, error) { + if from.IsEmpty() { + return nil, false, nil + } + var err error + result := BlockData{mainData: from, checkpointData: make(map[string]string)} + // always need header + if result.BlockHeader, err = c.client.GetHeader(ctx, blockNumber); err != nil { + return nil, false, err + } + // check block hash of main data with the header got above + for _, l := range from.Logs { + if l.BlockHash.String() != result.GetBlockHash() { + return nil, false, fetcher.Permanent(errors.Errorf("invalid block hash of the log %s, expected is %s", + l.BlockHash.String(), controller.GetBlockSummary(result.BlockHeader))) + } + } + for _, t := range from.Traces { + if t.BlockHash != result.GetBlockHash() { + return nil, false, fetcher.Permanent(errors.Errorf("invalid block hash of the trace %s, expected is %s", + t.BlockHash, controller.GetBlockSummary(result.BlockHeader))) + } + } + // take the main data and ask the handler controller what extend data is needed + var r evm.BlockExtendRequirement + if r, err = c.getBlockExtendRequirements(ctx, &result); err != nil { + return nil, false, err + } + // actually get the extended data + if result.extendData, err = c.client.GetBlock(ctx, blockNumber, r); err != nil { + return nil, false, err + } + // build binding data + if result.taskList, result.taskTotalSize, err = c.BuildTaskList(ctx, &result); err != nil { + _, logger := log.FromContext(ctx, + "header", utils.MustJSONMarshal(result.BlockHeader), + "mainData", utils.MustJSONMarshal(result.mainData), + "extendData", utils.MustJSONMarshal(result.extendData)) + logger.Warnfe(err, "build task list failed") + return nil, false, err + } + c.DumpAddressStart(result.checkpointData) + return &result, true, nil + }, + ) +} + +func (c *HandlerController) getDataRequirement() (dr evm.DataRequirement) { + for _, agent := range c.agents { + switch ag := agent.(type) { + case HandlerAgentBlock: + if !ag.Once { + dr.Interval = append(dr.Interval, data.IntervalRequirement{ + IntervalConfig: ag.IntervalConfig, + BlockRange: ag.Range, + }) + } else { + dr.Exact = append(dr.Exact, ag.Range.StartBlock) + } + case HandlerAgentCall: + dr.Trace = append(dr.Trace, evm.TraceRequirement{ + TraceFilter: ag.Filter, + BlockRange: ag.Range, + }) + case HandlerAgentEvent: + dr.Log = append(dr.Log, evm.LogRequirement{ + LogFilter: ag.Filter, + BlockRange: ag.Range, + }) + } + } + return dr +} + +func (c *HandlerController) getBlockExtendRequirements( + ctx context.Context, + blockData *BlockData, +) (req evm.BlockExtendRequirement, err error) { + var ar evm.BlockExtendRequirement + for _, agent := range c.agents { + if ar, err = agent.GetExtendRequirements(ctx, blockData); err != nil { + return + } + req.Merge(ar) + } + return +} + +func (c *HandlerController) BuildTaskList( + ctx context.Context, + bd *BlockData, +) ([]controller.Task, int, error) { + var taskDatas []taskData + var taskTotalSize int + for _, agent := range c.agents { // 15308711065 + if agent.GetRange().Contains(bd.GetBlockNumber()) { + tds, err := agent.BuildTaskDataList(ctx, bd) + if err != nil { + return nil, 0, err + } + taskDatas = append(taskDatas, tds...) + for _, td := range tds { + taskTotalSize += td.size + } + } + } + // The purpose of using stable sorting is to ensure that tasks of the same handler remain in the order + // in which they were generated. + sort.SliceStable(taskDatas, func(i, j int) bool { + return taskDatas[i].Cmp(taskDatas[j]) < 0 + }) + var r []controller.Task + for _, td := range taskDatas { + r = append(r, &task{ + handlerCtrl: c, + BlockHeader: bd.BlockHeader, + taskData: td, + }) + } + return r, taskTotalSize, nil +} + +const checkpointDataKeyAddressStart = "AddressStart" + +func (c *HandlerController) LoadAddressStart(checkpoint *controller.Checkpoint) *controller.ExternalError { + c.addressStart = make(map[string]uint64) + if checkpoint == nil || checkpoint.Data == nil { + return nil + } + raw, has := checkpoint.Data[checkpointDataKeyAddressStart] + if !has { + return nil + } + err := json.Unmarshal([]byte(raw), &c.addressStart) + if err != nil { + return controller.NewExternalError(controller.ErrCodeInvalidCheckpointData, + errors.Wrapf(err, "load address start failed")) + } + return nil +} + +func (c *HandlerController) GetAddressStart( + ctx context.Context, + address string, + start uint64, + latest uint64, +) (uint64, error) { + if c.chainConfig.SkipStartBlockValidation || controller.SkipStartBlockValidation || address == "" { + return start, nil + } + var has bool + if start, has = c.addressStart[address]; has { + return start, nil + } + var err error + start, has, err = c.client.GetContractStartBlock(ctx, address, start, latest) + if err != nil { + return 0, errors.Wrapf(err, "get start block for contract %s failed", address) + } + if !has { + start = latest + 1 + } + c.addressStart[address] = start + return start, nil +} + +func (c *HandlerController) AddressStartReady() { + b, _ := json.Marshal(c.addressStart) + c.addressStartData = string(b) +} + +func (c *HandlerController) DumpAddressStart(checkpointData map[string]string) { + checkpointData[checkpointDataKeyAddressStart] = c.addressStartData +} + +func (c *HandlerController) Epilogue() { +} + +func (c *HandlerController) Snapshot() any { + return map[string]any{ + "chainConfig": c.chainConfig, + "manifest": c.manifest, + "memHardLimit": c.memHardLimit, + "debugTrace": c.debugTrace, + "agents": utils.MapSliceNoError(c.agents, func(a HandlerAgent) any { + return a.Snapshot() + }), + "addressStart": c.addressStart, + "agentStat": c.GetAgentStat(), + "wasmInstance": c.instance.Snapshot(), + } +} diff --git a/driver/controller/subgraph/handler_block.go b/driver/controller/subgraph/handler_block.go new file mode 100644 index 0000000..b11b39f --- /dev/null +++ b/driver/controller/subgraph/handler_block.go @@ -0,0 +1,61 @@ +package subgraph + +import ( + "context" + "math" + + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/subgraph/manifest" +) + +type HandlerAgentBlock struct { + controller.BaseHandlerAgent + DataSource *manifest.DataSource + + IntervalConfig data.IntervalConfig + Once bool +} + +func (a HandlerAgentBlock) GetExtendRequirements(context.Context, *BlockData) (evm.BlockExtendRequirement, error) { + return evm.BlockExtendRequirement{}, nil +} + +func (a HandlerAgentBlock) BuildTaskDataList(_ context.Context, bd *BlockData) ([]taskData, error) { + if a.Once { + if bd.GetBlockNumber() != a.Range.StartBlock { + return nil, nil + } + } else { + if !data.ContainsInterval(bd.mainData.Intervals, a.IntervalConfig) { + return nil, nil + } + } + block, err := bd.buildBlock() + if err != nil { + return nil, fetcher.Permanent(err) + } + return []taskData{{ + callHandlerParam: block, + dataSource: a.DataSource, + handlerID: a.HandlerID, + txIndex: utils.Select(a.Once, -1, math.MaxInt), + size: len(bd.BlockHeader.Raw), + }}, nil +} + +func (a HandlerAgentBlock) Snapshot() any { + sn := map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + } + if a.Once { + sn["Once"] = true + } else { + sn["IntervalConfig"] = a.IntervalConfig + } + return sn +} diff --git a/driver/controller/subgraph/handler_call.go b/driver/controller/subgraph/handler_call.go new file mode 100644 index 0000000..c0832e9 --- /dev/null +++ b/driver/controller/subgraph/handler_call.go @@ -0,0 +1,58 @@ +package subgraph + +import ( + "context" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/subgraph/manifest" +) + +type HandlerAgentCall struct { + controller.BaseHandlerAgent + DataSource *manifest.DataSource + + Filter evm.TraceFilter +} + +func (a HandlerAgentCall) GetExtendRequirements(_ context.Context, bd *BlockData) (evm.BlockExtendRequirement, error) { + txHashSet := set.New[string]() + for _, trace := range bd.mainData.Traces { + if a.Filter.Check(trace) { + txHashSet.Add(trace.TransactionHash) + } + } + return evm.BlockExtendRequirement{SpecialTransactions: txHashSet.DumpValues()}, nil +} + +func (a HandlerAgentCall) BuildTaskDataList(ctx context.Context, bd *BlockData) ([]taskData, error) { + funcAbi := a.DataSource.Mapping.CallHandlers[a.HandlerID.ID].GetABI() + var r []taskData + for _, trace := range bd.mainData.Traces { + if !a.Filter.Check(trace) { + continue + } + call, size, err := bd.buildCall(trace, funcAbi) + if err != nil { + return nil, fetcher.Permanent(err) + } + r = append(r, taskData{ + callHandlerParam: call, + dataSource: a.DataSource, + handlerID: a.HandlerID, + txIndex: int(trace.TransactionIndex), + size: size, + }) + } + return r, nil +} + +func (a HandlerAgentCall) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + } +} diff --git a/driver/controller/subgraph/handler_event.go b/driver/controller/subgraph/handler_event.go new file mode 100644 index 0000000..0c3f70e --- /dev/null +++ b/driver/controller/subgraph/handler_event.go @@ -0,0 +1,71 @@ +package subgraph + +import ( + "context" + + "sentioxyz/sentio-core/common/set" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/controller/data/evm" + "sentioxyz/sentio-core/driver/controller/fetcher" + "sentioxyz/sentio-core/driver/subgraph/manifest" +) + +type HandlerAgentEvent struct { + controller.BaseHandlerAgent + DataSource *manifest.DataSource + + Filter evm.LogFilter +} + +func (a HandlerAgentEvent) GetExtendRequirements(_ context.Context, bd *BlockData) (evm.BlockExtendRequirement, error) { + txHashSet := set.New[string]() + checker := a.Filter.BuildChecker(nil, nil) + for _, log := range bd.mainData.Logs { + if ok, _ := checker(log); ok { + txHashSet.Add(log.TxHash.String()) + } + } + txHashes := txHashSet.DumpValues() + return evm.BlockExtendRequirement{ + SpecialTransactions: txHashes, + SpecialTransactionReceipts: txHashes, + SpecialTransactionReceiptLogs: txHashes, + }, nil +} + +func (a HandlerAgentEvent) BuildTaskDataList(_ context.Context, bd *BlockData) ([]taskData, error) { + eventAbi := a.DataSource.Mapping.EventHandlers[a.HandlerID.ID].GetABI() + var r []taskData + checker := a.Filter.BuildChecker(nil, nil) + for _, log := range bd.mainData.Logs { + if ok, _ := checker(log); !ok { + continue + } + if succeed, err := bd.transactionSucceed(log.TxHash.String()); err != nil { + return nil, fetcher.Permanent(err) + } else if !succeed { + continue // tx failed + } + event, size, err := bd.buildEvent(log, eventAbi) + if err != nil { + return nil, fetcher.Permanent(err) + } + r = append(r, taskData{ + callHandlerParam: event, + dataSource: a.DataSource, + handlerID: a.HandlerID, + txIndex: int(log.TxIndex), + logIndex: int(log.Index), + size: size, + }) + } + return r, nil +} + +func (a HandlerAgentEvent) Snapshot() any { + return map[string]any{ + "HandlerID": a.HandlerID, + "Range": a.Range.String(), + "Filter": a.Filter, + } +} diff --git a/driver/controller/subgraph/instance.go b/driver/controller/subgraph/instance.go new file mode 100644 index 0000000..24de9e9 --- /dev/null +++ b/driver/controller/subgraph/instance.go @@ -0,0 +1,787 @@ +package subgraph + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strconv" + "time" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/common/wasm" + "sentioxyz/sentio-core/driver/controller" + "sentioxyz/sentio-core/driver/entity/persistent" + "sentioxyz/sentio-core/driver/subgraph/common" + "sentioxyz/sentio-core/driver/subgraph/ethereum" + "sentioxyz/sentio-core/driver/subgraph/manifest" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/mr-tron/base58/base58" + "github.com/pkg/errors" +) + +func (c *HandlerController) newInstance(ctx context.Context) (*instance, error) { + inst := &instance{ + mods: make(map[string]*wasm.Instance[CtxData]), + handlerCtrl: c, + } + + hashBelong := make(map[string][]string) + + _ = c.manifest.TravelDataSourcesAndTemplates(func(ds *manifest.DataSource, name string) error { + hash := ds.Mapping.File.GetIpfsHash() + m, has := inst.mods[hash] + if !has { + m = wasm.NewInstance[CtxData]( + fmt.Sprintf("%s/%s", name, hash), + []byte(ds.Mapping.File.GetContent()), + c.memHardLimit, + ) + inst.importFunctions(m) + if c.debugTrace { + m.SetDebugLevel(wasm.DebugLevelTrace) + } + } + switch ds.Kind { + case "ethereum/contract", "ethereum": + for _, evh := range ds.Mapping.EventHandlers { + m.MustExportFunction(evh.Handler, (func(*ethereum.Event))(nil)) + } + for _, cah := range ds.Mapping.CallHandlers { + m.MustExportFunction(cah.Handler, (func(*ethereum.Call))(nil)) + } + for _, blh := range ds.Mapping.BlockHandlers { + m.MustExportFunction(blh.Handler, (func(*ethereum.Block))(nil)) + } + case "file/ipfs": + m.MustExportFunction(ds.Mapping.Handler, (func(*wasm.ByteArray))(nil)) + } + inst.mods[hash] = m + hashBelong[hash] = append(hashBelong[hash], name) + return nil + }) + + // init modules + _, logger := log.FromContext(ctx) + for hash, m := range inst.mods { + if err := m.Init(logger); err != nil { + return nil, errors.Wrapf(err, "init wasm instance with ipfs hash %q for %v failed", hash, hashBelong[hash]) + } + } + + return inst, nil +} + +type CtxData struct { + task *task + checkpointCtrl controller.CheckpointController + + // This is the data source of the calling handler. + // Normally this should be equal to task.dataSource, when the entry handler of the task calling a file template + // handler, this will be a dynamic data source created from the file template + dataSource *manifest.DataSource +} + +func (c CtxData) String() string { + return "" +} + +type instance struct { + mods map[string]*wasm.Instance[CtxData] + handlerCtrl *HandlerController +} + +const ( + LogLevelCritical = 0 + LogLevelError = 1 + LogLevelWarning = 2 + LogLevelInfo = 3 + LogLevelDebug = 4 +) + +func (inst *instance) importFunctions(m *wasm.Instance[CtxData]) { + // TODO more functions + + // /** + // * Special function for ENS name lookups, not meant for general purpose use. + // * This function will only be useful if the graph-node instance has additional + // * data loaded ** + // */ + //export declare namespace ens { + // function nameByHash(hash: string): string | null; + // } + + m.MustImportFunction("env", "abort", + func(ctx *wasm.CallContext[CtxData], msg *wasm.String, filename *wasm.String, lineNum wasm.I32, colNum wasm.I32) { + errMsg := "" + if msg != nil && msg.String() != "" { + errMsg = msg.String() + } + abortErr := fmt.Errorf("abort at %s:%d:%d while calling %s, %s", + filename, lineNum, colNum, ctx.DumpCallStack(), errMsg) + ctx.Logger().UserVisible().Error(abortErr.Error()) + panic(abortErr) + }). + MustImportFunction("wasi_snapshot_preview1", "fd_write", + func(ctx *wasm.CallContext[CtxData], _, _, _, _ wasm.I32) wasm.U16 { + ctx.Logger().UserVisible().Warnf("Calling %s => wasi_snapshot_preview1.fd_write was ignored. "+ + "If you want to print log, please use log in @graphprotocol/graph-ts", ctx.DumpCallStack()) + return 0 + }). + //export declare namespace typeConversion { + // function bytesToString(bytes: Uint8Array): string + // function bytesToHex(bytes: Uint8Array): string + // function bigIntToString(bigInt: Uint8Array): string + // function bigIntToHex(bigInt: Uint8Array): string + // function stringToH160(s: string): Bytes + // function bytesToBase58(n: Uint8Array): string + //} + MustImportFunction("conversion", "typeConversion.bytesToString", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *wasm.String { + if arg == nil { + return wasm.BuildString("") + } + return wasm.BuildStringFromBytes(arg.Data) + }). + MustImportFunction("conversion", "typeConversion.bytesToHex", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *wasm.String { + return wasm.BuildString(arg.ToHex()) + }). + MustImportFunction("conversion", "typeConversion.bigIntToString", + func(_ *wasm.CallContext[CtxData], arg *common.BigInt) *wasm.String { + if arg == nil { + return wasm.BuildString("") + } + return wasm.BuildString(arg.String()) + }). + MustImportFunction("conversion", "typeConversion.bigIntToHex", + func(_ *wasm.CallContext[CtxData], arg *common.BigInt) *wasm.String { + if arg == nil { + return wasm.BuildString("") + } + return wasm.BuildString(arg.ToHex()) + }). + MustImportFunction("conversion", "typeConversion.stringToH160", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) *wasm.ByteArray { + arr, buildErr := wasm.BuildByteArrayFromHex(arg.String()) + if buildErr != nil { + panic(buildErr) + } + return arr + }). + MustImportFunction("conversion", "typeConversion.bytesToBase58", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *wasm.String { + if arg == nil { + return wasm.BuildString(base58.Encode(nil)) + } + return wasm.BuildString(base58.Encode(arg.Data)) + }). + //export declare namespace bigInt { + // function plus(x: BigInt, y: BigInt): BigInt + // function minus(x: BigInt, y: BigInt): BigInt + // function times(x: BigInt, y: BigInt): BigInt + // function dividedBy(x: BigInt, y: BigInt): BigInt + // function dividedByDecimal(x: BigInt, y: BigDecimal): BigDecimal + // function mod(x: BigInt, y: BigInt): BigInt + // function pow(x: BigInt, exp: u8): BigInt + // function fromString(s: string): BigInt + // function bitOr(x: BigInt, y: BigInt): BigInt + // function bitAnd(x: BigInt, y: BigInt): BigInt + // function leftShift(x: BigInt, bits: u8): BigInt + // function rightShift(x: BigInt, bits: u8): BigInt + //} + MustImportFunction("numbers", "bigInt.plus", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.Plus(y) + }). + MustImportFunction("numbers", "bigInt.minus", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.Minus(y) + }). + MustImportFunction("numbers", "bigInt.times", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.Times(y) + }). + MustImportFunction("numbers", "bigInt.dividedBy", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.DividedBy(y) + }). + MustImportFunction("numbers", "bigInt.dividedByDecimal", + func(_ *wasm.CallContext[CtxData], x *common.BigInt, y *common.BigDecimal) *common.BigDecimal { + return common.BuildBigDecimalFromBigInt(x, 0).DividedBy(y) + }). + MustImportFunction("numbers", "bigInt.mod", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.Mod(y) + }). + MustImportFunction("numbers", "bigInt.pow", + func(_ *wasm.CallContext[CtxData], x *common.BigInt, exp wasm.U8) *common.BigInt { + return x.Pow(exp) + }). + MustImportFunction("numbers", "bigInt.fromString", + func(_ *wasm.CallContext[CtxData], x *wasm.String) *common.BigInt { + return common.MustBuildBigInt(x.String()) + }). + MustImportFunction("numbers", "bigInt.bitOr", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.BitOr(y) + }). + MustImportFunction("numbers", "bigInt.bitAnd", + func(_ *wasm.CallContext[CtxData], x, y *common.BigInt) *common.BigInt { + return x.BitAnd(y) + }). + MustImportFunction("numbers", "bigInt.leftShift", + func(_ *wasm.CallContext[CtxData], x *common.BigInt, y wasm.U8) *common.BigInt { + return x.LeftShift(y) + }). + MustImportFunction("numbers", "bigInt.rightShift", + func(_ *wasm.CallContext[CtxData], x *common.BigInt, y wasm.U8) *common.BigInt { + return x.RightShift(y) + }). + //export declare namespace bigDecimal { + // function plus(x: BigDecimal, y: BigDecimal): BigDecimal + // function minus(x: BigDecimal, y: BigDecimal): BigDecimal + // function times(x: BigDecimal, y: BigDecimal): BigDecimal + // function dividedBy(x: BigDecimal, y: BigDecimal): BigDecimal + // function equals(x: BigDecimal, y: BigDecimal): boolean + // function toString(bigDecimal: BigDecimal): string + // function fromString(s: string): BigDecimal + //} + MustImportFunction("numbers", "bigDecimal.plus", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) *common.BigDecimal { + return x.Plus(y) + }). + MustImportFunction("numbers", "bigDecimal.minus", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) *common.BigDecimal { + return x.Minus(y) + }). + MustImportFunction("numbers", "bigDecimal.times", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) *common.BigDecimal { + return x.Times(y) + }). + MustImportFunction("numbers", "bigDecimal.dividedBy", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) *common.BigDecimal { + return x.DividedBy(y) + }). + MustImportFunction("numbers", "bigDecimal.equals", + func(_ *wasm.CallContext[CtxData], x, y *common.BigDecimal) wasm.Bool { + return wasm.Bool(x.Equals(y)) + }). + MustImportFunction("numbers", "bigDecimal.toString", + func(_ *wasm.CallContext[CtxData], arg *common.BigDecimal) *wasm.String { + return wasm.BuildString(arg.String()) + }). + MustImportFunction("numbers", "bigDecimal.fromString", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) *common.BigDecimal { + return common.MustBuildBigDecimalFromString(arg.String()) + }). + //export declare namespace json { + // function fromBytes(data: Bytes): JSONValue; + // function try_fromBytes(data: Bytes): Result; + // function toI64(decimal: string): i64; + // function toU64(decimal: string): u64; + // function toF64(decimal: string): f64; + // function toBigInt(decimal: string): BigInt; + //} + MustImportFunction("json", "json.fromBytes", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *common.JSONValue { + val := &common.JSONValue{} + if err := val.FromBytes(arg.Data); err != nil { + panic(err) + } + return val + }). + MustImportFunction("json", "json.try_fromBytes", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *common.Result[*common.JSONValue, wasm.Bool] { + result := &common.Result[*common.JSONValue, wasm.Bool]{} + var val common.JSONValue + if err := val.FromBytes(arg.Data); err != nil { + result.Error = &common.Wrapped[wasm.Bool]{Inner: true} + } else { + result.Value = &common.Wrapped[*common.JSONValue]{Inner: &val} + } + return result + }). + MustImportFunction("json", "json.toI64", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) wasm.I64 { + v, err := strconv.ParseInt(arg.String(), 0, 64) + if err != nil { + panic(err) + } + return wasm.I64(v) + }). + MustImportFunction("json", "json.toU64", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) wasm.U64 { + v, err := strconv.ParseUint(arg.String(), 0, 64) + if err != nil { + panic(err) + } + return wasm.U64(v) + }). + MustImportFunction("json", "json.toF64", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) wasm.F64 { + v, err := strconv.ParseFloat(arg.String(), 64) + if err != nil { + panic(err) + } + return wasm.F64(v) + }). + MustImportFunction("json", "json.toBigInt", + func(_ *wasm.CallContext[CtxData], arg *wasm.String) *common.BigInt { + return common.MustBuildBigInt(arg.String()) + }). + //export declare namespace crypto { + // function keccak256(input: ByteArray): ByteArray + //} + MustImportFunction("index", "crypto.keccak256", + func(_ *wasm.CallContext[CtxData], arg *wasm.ByteArray) *wasm.ByteArray { + return &wasm.ByteArray{Data: crypto.Keccak256(arg.Data)} + }). + //export declare namespace ethereum { + // function call(call: SmartContractCall): Array | null + // function encode(token: Value): Bytes | null + // function decode(types: String, data: Bytes): Value | null + // } + MustImportFunction("ethereum", "ethereum.call", inst.EthCall). + MustImportFunction("ethereum", "ethereum.encode", inst.EthEncode). + MustImportFunction("ethereum", "ethereum.decode", inst.EthDecode). + //export declare namespace dataSource { + // function create(name: string, params: Array): void + // function createWithContext( + // name: string, + // params: Array, + // context: DataSourceContext, + // ): void + // + // // Properties of the data source that fired the event. + // function address(): Address + // function network(): string + // function context(): DataSourceContext + // } + MustImportFunction("datasource", "dataSource.create", inst.CreateDataSource). + MustImportFunction("datasource", "dataSource.createWithContext", inst.CreateDataSourceWithCtx). + MustImportFunction("datasource", "dataSource.network", inst.GetNetwork). + MustImportFunction("datasource", "dataSource.address", + func(ctx *wasm.CallContext[CtxData]) *common.Address { + return common.MustBuildAddressFromString(ctx.TopParams().Data.dataSource.Source.Address) + }). + MustImportFunction("datasource", "dataSource.context", + func(ctx *wasm.CallContext[CtxData]) *common.Entity { + ctxText := ctx.TopParams().Data.dataSource.Context + if ctxText == "" { + return &common.Entity{Properties: &wasm.ObjectArray[*common.EntityProperty]{}} // empty context + } + var ctxEntity common.Entity + if err := json.Unmarshal([]byte(ctxText), &ctxEntity); err != nil { + panic(err) + } + return &ctxEntity + }). + //export declare namespace ipfs { + // function cat(hash: string): Bytes | null + // function map(hash: string, callback: string, userData: Value, flags: string[]): void + //} + MustImportFunction("index", "ipfs.cat", + func(ctx *wasm.CallContext[CtxData], hash *wasm.String) *wasm.ByteArray { + r, err := inst.handlerCtrl.ipfsShell.Cat(hash.String()) + if err != nil { + ctx.Logger().UserVisible().Warnf("ipfs cat %q failed: %v", hash.String(), err) + panic(controller.NewExternalError(controller.ErrCodeSubgraphIpfsCatFailed, + errors.Wrapf(err, "ipfs cat %q failed", hash.String()))) + } + cnt, readErr := io.ReadAll(r) + if readErr != nil { + ctx.Logger().UserVisible().Warnf("ipfs cat %q failed: read failed: %v", hash.String(), readErr) + panic(controller.NewExternalError(controller.ErrCodeSubgraphIpfsCatFailed, + errors.Wrapf(err, "ipfs cat %q failed", hash.String()))) + } + return &wasm.ByteArray{Data: cnt} + }). + MustImportFunction("index", "ipfs.map", + func( + _ *wasm.CallContext[CtxData], + hash *wasm.String, + callback *wasm.String, + userData *common.Value, + flags *wasm.ObjectArray[*wasm.String], + ) { + panic(errors.Errorf("ipfs.map is not supported")) + }). + //export declare namespace store { + // function get(entity: string, id: string): Entity | null + // function get_in_block(entity: string, id: string): Entity | null; + // function loadRelated(entity: string, id: string, field: string): Array; + // function set(entity: string, id: string, data: Entity): void + // function remove(entity: string, id: string): void + // } + MustImportFunction("index", "store.set", inst.SetEntity). + MustImportFunction("index", "store.get", inst.GetEntity). + MustImportFunction("index", "store.get_in_block", inst.GetEntityInBlock). + MustImportFunction("index", "store.loadRelated", inst.GetRelatedEntities). + MustImportFunction("index", "store.remove", inst.RemoveEntity). + //export declare namespace log { + // // Host export for logging, providing basic logging functionality + // export function log(level: Level, msg: string): void + //} + MustImportFunction("index", "log.log", + func(ctx *wasm.CallContext[CtxData], level wasm.I32, msg *wasm.String) { + logger := ctx.Logger().UserVisible() + switch level { + case LogLevelCritical: + logger.Fatal(msg.String()) + case LogLevelError: + logger.Error(msg.String()) + case LogLevelWarning: + logger.Warn(msg.String()) + case LogLevelInfo: + logger.Info(msg.String()) + case LogLevelDebug: + logger.Debug(msg.String()) + default: + panic(errors.Errorf("invalid log level %d with msg %q", level, msg.String())) + } + }) +} + +func (inst *instance) getEntity( + ctx *wasm.CallContext[CtxData], + argName *wasm.String, + argID *wasm.String, + inBlock bool, +) (entity *common.Entity) { + tk := ctx.TopParams().Data.task + ckc := ctx.TopParams().Data.checkpointCtrl + name, id := argName.String(), argID.String() + entityType := ckc.GetEntityOrInterfaceType(name) + if entityType == nil { + panic(controller.NewExternalError(controller.ErrCodeGetUnknownEntity, errors.Errorf("get unknown entity %q", name))) + } + var box *persistent.EntityBox + var extErr *controller.ExternalError + ctxEx := controller.N.BeforeEntityOperation(ctx, tk.taskInfo()) + if inBlock { + box, extErr = ckc.GetEntityInBlock(ctxEx, entityType, id, tk.GetBlockNumber()) + } else { + box, extErr = ckc.GetEntity(ctxEx, entityType, id, tk.GetBlockNumber()) + } + if extErr != nil { + panic(extErr.Wrapf("get entity %s/%s failed", name, id)) + } + if box != nil && box.Data != nil { + entity = &common.Entity{} + entity.FromGoType(box.Data, entityType) + } + return entity +} + +func (inst *instance) GetEntity( + ctx *wasm.CallContext[CtxData], + argName *wasm.String, + argID *wasm.String, +) (entity *common.Entity) { + return inst.getEntity(ctx, argName, argID, false) +} + +func (inst *instance) GetEntityInBlock( + ctx *wasm.CallContext[CtxData], + argName *wasm.String, + argID *wasm.String, +) (entity *common.Entity) { + return inst.getEntity(ctx, argName, argID, true) +} + +func (inst *instance) GetRelatedEntities( + ctx *wasm.CallContext[CtxData], + argName, argID, argField *wasm.String, +) *wasm.ObjectArray[*common.Entity] { + tk := ctx.TopParams().Data.task + ckc := ctx.TopParams().Data.checkpointCtrl + name, id, field := argName.String(), argID.String(), argField.String() + entityType := ckc.GetEntityType(name) + if entityType == nil { + panic(controller.NewExternalError(controller.ErrCodeListUnknownEntity, + errors.Errorf("list related with unknown entity %q", name))) + } + ctxEx := controller.N.BeforeEntityOperation(ctx, tk.taskInfo()) + boxes, target, extErr := ckc.ListRelated(ctxEx, entityType, id, field, tk.GetBlockNumber()) + if extErr != nil { + panic(extErr) + } + arr := &wasm.ObjectArray[*common.Entity]{Data: make([]*common.Entity, len(boxes))} + for i, box := range boxes { + if box != nil && box.Data != nil { + arr.Data[i] = &common.Entity{} + arr.Data[i].FromGoType(box.Data, target) + } + } + return arr +} + +func (inst *instance) SetEntity( + ctx *wasm.CallContext[CtxData], + argName, argID *wasm.String, + entity *common.Entity, +) { + tk := ctx.TopParams().Data.task + ckc := ctx.TopParams().Data.checkpointCtrl + name, id := argName.String(), argID.String() + entityType := ckc.GetEntityType(name) + if entityType == nil { + panic(controller.NewExternalError(controller.ErrCodeListUnknownEntity, + errors.Errorf("set unknown entity %q", name))) + } + + box := persistent.UncommittedEntityBox{EntityBox: persistent.EntityBox{ + ID: id, + GenBlockNumber: tk.GetBlockNumber(), + GenBlockTime: tk.GetBlockTime(), + GenBlockHash: tk.GetBlockHash(), + }} + if entity != nil { + box.Data = entity.ToGoType() + box.FillLostFields(make(map[string]any), entityType) + } + ctxEx := controller.N.BeforeEntityOperation(ctx, tk.taskInfo()) + if extErr := ckc.SetEntity(ctxEx, entityType, box); extErr != nil { + panic(extErr.Wrapf("set entity %s/%s %s failed", name, id, box.String())) + } + subtype := "upsert" + if entity == nil { + subtype = "delete" + } + controller.N.DataEmitted(ctxEx, tk.taskInfo(), "entity", subtype, name, 1) +} + +func (inst *instance) RemoveEntity(ctx *wasm.CallContext[CtxData], name *wasm.String, id *wasm.String) { + inst.SetEntity(ctx, name, id, nil) +} + +func (inst *instance) GetNetwork(ctx *wasm.CallContext[CtxData]) *wasm.String { + return wasm.BuildString(inst.handlerCtrl.chainID()) +} + +func (inst *instance) EthCall( + ctx *wasm.CallContext[CtxData], + call *ethereum.SmartContractCall, +) *wasm.ObjectArray[*ethereum.Value] { + start := time.Now() + top := ctx.TopParams() + tk, ds := top.Data.task, top.Data.dataSource + contractName, methodSig := call.ContractName.String(), call.FunctionSignature.String() + contractABI := ds.GetABIByName(contractName) + if contractABI == nil { + panic(controller.NewExternalError(controller.ErrCodeSubgraphEthCallWithInvalidParam, + errors.Errorf("contract %s is not found in data source %s", contractName, ds.Name))) + } + methodABI := contractABI.FindMethodBySig(methodSig) + if methodABI == nil { + panic(controller.NewExternalError(controller.ErrCodeSubgraphEthCallWithInvalidParam, + errors.Errorf("method with signature %q is not found in contract %s in data source %s", + contractName, methodSig, ds.Name))) + } + logger := ctx.Logger().With("ethCall", map[string]any{ + "contractName": call.ContractName.String(), + "contractAddr": call.ContractAddress.String(), + "functionName": call.FunctionName.String(), + "functionSignature": call.FunctionSignature.String(), + }) + ret, err := ethereum.EthCall( + ctx, + logger, + inst.handlerCtrl.client, + call.ContractAddress.Data, + methodABI, + call.FunctionParams, + tk.GetBlockNumber()) + + controller.N.SubgraphRPCDone(ctx, tk.taskInfoForCall(), err == nil, time.Since(start)) + + if err != nil { + if errors.Is(err, ethereum.ErrEthCallDataFormatErr) { + panic(controller.NewExternalError(controller.ErrCodeSubgraphEthCallWithInvalidParam, + errors.Wrapf(err, "calling %s.%s in data source %s failed", contractName, methodSig, ds.Name))) + } + panic(controller.NewExternalError(controller.ErrCodeSubgraphEthCallFailed, + errors.Wrapf(err, "calling %s.%s in data source %s failed", contractName, methodSig, ds.Name))) + } + return ret +} + +func (inst *instance) EthEncode(ctx *wasm.CallContext[CtxData], value *ethereum.Value) *wasm.ByteArray { + logger := ctx.Logger().With("value", value.String()) + b, err := ethereum.Encode(value) + if err != nil { + logger.Warne(err, "ethereum.encode failed") + return nil + } + result := &wasm.ByteArray{Data: b} + logger.Debugw("ethereum.encode succeed", "result", result.String()) + return result +} + +func (inst *instance) EthDecode(ctx *wasm.CallContext[CtxData], types *wasm.String, data *wasm.ByteArray) *ethereum.Value { + logger := ctx.Logger().With("types", types.String(), "data", data.String()) + val, err := ethereum.Decode(types.String(), data.Data) + if err != nil { + logger.Warne(err, "ethereum.decode failed") + return nil + } + logger.Debugw("ethereum.decode succeed", "value", val.String()) + return val +} + +func (inst *instance) CreateDataSource(ctx *wasm.CallContext[CtxData], tplName *wasm.String, params *wasm.ObjectArray[*wasm.String]) { + inst.CreateDataSourceWithCtx(ctx, tplName, params, nil) +} + +func (inst *instance) CreateDataSourceWithCtx( + ctx *wasm.CallContext[CtxData], + tplName *wasm.String, + params *wasm.ObjectArray[*wasm.String], + ctxEntity *common.Entity, +) { + top := ctx.TopParams() + tk := top.Data.task + // find template + templateID, tpl := tk.handlerCtrl.manifest.FindTemplateByName(tplName.String()) + if templateID < 0 { + panic(controller.NewExternalError(controller.ErrCodeCreateTemplateFailed, errors.Errorf( + "create data source by template with name %q failed: template not found", tplName.String()))) + } + createFailedText := fmt.Sprintf("create data source by template #%d/%s failed", templateID, tplName.String()) + // need one param as contract address or file hash + if len(params.Data) == 0 { + panic(controller.NewExternalError(controller.ErrCodeCreateTemplateFailed, + errors.Errorf("%s: params is empty", createFailedText))) + } + // build context text + var ctxStr string + if ctxEntity != nil { + ctxBytes, err := json.Marshal(ctxEntity) + if err != nil { + panic(controller.NewExternalError(controller.ErrCodeCreateTemplateFailed, + errors.Wrapf(err, "%s: marshal data source ctx %s failed", createFailedText, ctxEntity.String()))) + } + ctxStr = string(ctxBytes) + } + + // file template, ipfs cat file and call file handler + if tpl.Kind == "file/ipfs" { + handlerFullName := fmt.Sprintf("%s/%s", tplName.String(), tpl.Mapping.Handler) + file := params.Data[0].String() + r, catErr := tk.handlerCtrl.ipfsShell.Cat(file) + if catErr != nil { + panic(controller.NewExternalError(controller.ErrCodeSubgraphIpfsCatFailed, + errors.Wrapf(catErr, "%s: ipfs cat %q failed", createFailedText, file))) + } + cnt, readErr := io.ReadAll(r) + if readErr != nil { + panic(controller.NewExternalError(controller.ErrCodeSubgraphIpfsCatFailed, + errors.Wrapf(catErr, "%s: ipfs cat %q failed", createFailedText, file))) + } + logger := ctx.Logger().UserVisible() + logger.Infof("will call file handler %s with ipfs file hash %q", handlerFullName, file) + err := inst.CallHandler( + ctx, + wasm.CallParams[CtxData]{ + ExportFuncName: tpl.Mapping.Handler, + Logger: top.Logger, + Data: CtxData{ + dataSource: tpl.NewDataSource("", manifest.BuildBigIntFromUint(tk.GetBlockNumber()), ctxStr), + task: tk, + checkpointCtrl: top.Data.checkpointCtrl, + }, + }, + &wasm.ByteArray{Data: cnt}) + if err != nil { + logger.Errorfe(err, "call file handler %s failed", handlerFullName) + panic(err) + } + return + } + + // contract template, save it + newTpl := controller.TemplateInstance{ + TemplateID: int32(templateID), + Address: params.Data[0].String(), + TemplateName: ctxStr, + BlockRange: controller.BlockRange{StartBlock: tk.GetBlockNumber()}, + } + extErr := top.Data.checkpointCtrl.NewTemplateInstance(ctx, tk, []controller.TemplateInstance{newTpl}) + if extErr != nil { + panic(extErr) + } +} + +func (inst *instance) Reset(ctx context.Context) error { + _, logger := log.FromContext(ctx) + for _, mod := range inst.mods { + if err := mod.Reset(logger); err != nil { + return err + } + } + return nil +} + +const ( + callTimeUsedWarningLimit = time.Second + callHandlerMaxDeep = 10 +) + +func (inst *instance) CallHandler( + ctx *wasm.CallContext[CtxData], + params wasm.CallParams[CtxData], + args ...any, +) (extErr *controller.ExternalError) { + tk, ds := params.Data.task, params.Data.dataSource + hash := ds.Mapping.File.GetIpfsHash() + mod, has := inst.mods[hash] + if !has { + return controller.NewExternalError(controller.ErrCodeSystem, + errors.Errorf("mod not found for data source %q with ipfs hash %q", ds.Name, hash)) + } + + if deep := ctx.CallStackDeep(); deep >= callHandlerMaxDeep { + return controller.NewExternalError(controller.ErrCodeWasmStackOverFlow, + errors.Errorf("call stack deep %d over flow: %s => %s::%s", + deep, ctx.DumpCallStack(), mod.Name(), params.ExportFuncName)) + } + + // Actually call the event handler. + _, report, err := mod.CallExportFunction(ctx, params, args...) + + // process call error + var errCallingImportFunc *wasm.ErrCallingImportFunc + if err != nil && errors.As(err, &errCallingImportFunc) { + if !errors.As(errCallingImportFunc.Err, &extErr) { + extErr = controller.NewExternalError(controller.ErrCodeCallWasmExportFunctionFailed, errCallingImportFunc.Err) + } + } else if errors.Is(err, wasm.ErrPrepareCallExportFunc) || errors.Is(err, wasm.ErrPanic) { + extErr = controller.NewExternalError(controller.ErrCodeCallWasmExportFunctionFailed, err) + } else if err != nil { + extErr = controller.NewExternalError(controller.ErrCodeWasmError, err) + } + + // print logs + if err != nil { + tk.errLogger().With("report", report).Errorfe(err, "called handler") + } else if report.TimeUsed > callTimeUsedWarningLimit { + tk.logger.With("report", report).Warnf("called handler") + } else { + tk.logger.With("report", report).Infof("called handler") + } + + // report metrics + controller.N.SubgraphTaskDone(ctx, tk.taskInfoForCall(), err == nil, + report.TimeUsed, report.ImportFuncCallUsed, report.MemoryUsed) + + return extErr +} + +func (inst *instance) Snapshot() any { + return utils.MapMapNoError(inst.mods, func(mod *wasm.Instance[CtxData]) any { + return mod.Snapshot() + }) +} diff --git a/driver/controller/subgraph/task.go b/driver/controller/subgraph/task.go new file mode 100644 index 0000000..a72cc3d --- /dev/null +++ b/driver/controller/subgraph/task.go @@ -0,0 +1,93 @@ +package subgraph + +import ( + "context" + "fmt" + + "sentioxyz/sentio-core/common/log" + "sentioxyz/sentio-core/common/timer" + "sentioxyz/sentio-core/common/utils" + "sentioxyz/sentio-core/common/wasm" + "sentioxyz/sentio-core/driver/controller" + + "github.com/pkg/errors" +) + +type task struct { + controller.BlockHeader + taskData + + handlerCtrl *HandlerController + instance *instance + + index controller.TaskIndex + timer *timer.Timer + logger *log.SentioLogger +} + +func (t *task) taskInfo() controller.TaskInfo { + return controller.TaskInfo{ + Processor: t.handlerCtrl.processor, + ChainID: t.handlerCtrl.chainID(), + Handler: t.handlerID.Name, + Category: t.handlerID.Type, + DataSource: t.handlerID.DataSource, + } +} + +func (t *task) GetHandlerID() controller.HandlerID { + return t.handlerID +} + +func (t *task) Init(ctx context.Context, index controller.TaskIndex, progressbar controller.ProgressBar) { + t.handlerCtrl.waiter.NewResource(index.Global) + t.instance = t.handlerCtrl.instance // select one + t.index = index + t.timer = timer.NewTimer() + _, t.logger = log.FromContext(ctx, + "block", controller.GetBlockSummary(t), + "latest", controller.GetBlockSummary(progressbar.LatestBlock), + "index", index, + "handler", t.handlerID.String()) +} + +func (t *task) errLogger() *log.SentioLogger { + return t.logger.With("callHandlerArg", utils.MustJSONMarshal(t.callHandlerParam)) +} + +// taskInfoForCall is like taskInfo but uses the resolved data +// source name (set when the handler is actually invoked). +func (t *task) taskInfoForCall() controller.TaskInfo { + h := t.taskInfo() + h.DataSource = t.dataSource.Name + return h +} + +func (t *task) Summary() string { + return fmt.Sprintf("#%d binding data %d/%d for handler %s in block %s", + t.index.Global, t.index.InBlock, t.index.TotalInBlock, t.handlerID, controller.GetBlockSummary(t)) +} + +func (t *task) Exec(ctx context.Context, checkpointCtrl controller.CheckpointController) *controller.ExternalError { + err := t.handlerCtrl.waiter.Wait(ctx, func(u uint64) bool { + return u < t.index.Global + }) + if err != nil { + return controller.NewExternalError(controller.ErrCodeSystem, + errors.Errorf("waiting all previous tasks finish failed: %v", err)) + } + extErr := t.instance.CallHandler( + wasm.NewCallContext[CtxData](ctx), + wasm.CallParams[CtxData]{ + ExportFuncName: t.handlerID.Name, + Logger: t.logger, + Data: CtxData{dataSource: t.dataSource, task: t, checkpointCtrl: checkpointCtrl}, + }, + t.callHandlerParam, + ) + if extErr != nil { + return extErr.Wrapf("process %s failed", t.Summary()) + } + t.handlerCtrl.waiter.ResourceReady(t.index.Global) + return nil +} diff --git a/driver/controller/timeseries.go b/driver/controller/timeseries.go new file mode 100644 index 0000000..1a15338 --- /dev/null +++ b/driver/controller/timeseries.go @@ -0,0 +1,50 @@ +package controller + +import ( + "context" + "time" + + "sentioxyz/sentio-core/driver/timeseries" +) + +type TimeSeriesController interface { + Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError + CachedTooMuch(blockNumber uint64) bool + Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, + ) (stat map[timeseries.MetaType]map[string]int, err *ExternalError) + + Insert(blockNumber uint64, taskIndex TaskIndex, data []timeseries.Dataset) + + Snapshot() any +} + +type EmptyTimeSeriesController struct{} + +func (c EmptyTimeSeriesController) Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError { + return nil +} + +func (c EmptyTimeSeriesController) CachedTooMuch(blockNumber uint64) bool { + return false +} + +func (c EmptyTimeSeriesController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (map[timeseries.MetaType]map[string]int, *ExternalError) { + return nil, nil +} + +func (c EmptyTimeSeriesController) Insert(blockNumber uint64, taskIndex TaskIndex, data []timeseries.Dataset) { + if len(data) > 0 { + panic("do not support time series data") + } +} + +func (c EmptyTimeSeriesController) Snapshot() any { + return nil +} diff --git a/driver/controller/webhook.go b/driver/controller/webhook.go new file mode 100644 index 0000000..8670c31 --- /dev/null +++ b/driver/controller/webhook.go @@ -0,0 +1,55 @@ +package controller + +import ( + "context" + "time" +) + +type WebhookController interface { + Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError + CachedTooMuch(blockNumber uint64) bool + Commit(ctx context.Context, blockNumber uint64, blockTime time.Time) (stat map[string]int, err *ExternalError) + Insert(blockNumber uint64, taskIndex TaskIndex, messages []WebhookMessage) + Snapshot() any +} + +type WebhookMessage struct { + Name string + Channel string + BlockTime time.Time + Payload string +} + +func StatisticWebhookMessages(msgs []WebhookMessage, stat map[string]int) { + for _, msg := range msgs { + stat[msg.Name] += 1 + } +} + +type EmptyWebhookController struct{} + +func (c EmptyWebhookController) Reset(ctx context.Context, checkpoint *Checkpoint) *ExternalError { + return nil +} + +func (c EmptyWebhookController) CachedTooMuch(blockNumber uint64) bool { + return false +} + +func (c EmptyWebhookController) Commit( + ctx context.Context, + blockNumber uint64, + blockTime time.Time, +) (map[string]int, *ExternalError) { + return nil, nil +} + +func (c EmptyWebhookController) Insert(blockNumber uint64, taskIndex TaskIndex, messages []WebhookMessage) { + if len(messages) > 0 { + panic("do not support webhook data") + } +} + +func (c EmptyWebhookController) Snapshot() any { + return nil +} diff --git a/driver/exitcode/BUILD.bazel b/driver/exitcode/BUILD.bazel new file mode 100644 index 0000000..63efdb5 --- /dev/null +++ b/driver/exitcode/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "exitcode", + srcs = ["exitcode.go"], + importpath = "sentioxyz/sentio-core/driver/exitcode", + visibility = ["//visibility:public"], +) diff --git a/driver/exitcode/exitcode.go b/driver/exitcode/exitcode.go new file mode 100644 index 0000000..7d874db --- /dev/null +++ b/driver/exitcode/exitcode.go @@ -0,0 +1,17 @@ +package exitcode + +// Code is the process exit code the streaming driver uses to tell its +// supervisor how to retry. The concrete ErrXXX sentinel errors and the Halt +// helper that map onto these codes are used by driver v2 and stay in the sentio +// repository. +type Code int + +const ( + AlwaysRetry Code = 1 + LimitedRetry Code = 10 + NeverRetry Code = 11 + OverQuota Code = 12 + RetryAfterOneHour Code = 20 + RetryNextDay Code = 21 + RetryNextMonth Code = 22 +) diff --git a/go.mod b/go.mod index 0d100da..6f45af4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.3 require ( cloud.google.com/go/bigquery v1.67.0 + cloud.google.com/go/pubsub v1.49.0 github.com/BurntSushi/toml v1.5.0 github.com/ClickHouse/clickhouse-go/v2 v2.42.0 github.com/DmitriyVTitov/size v1.5.0 @@ -151,6 +152,7 @@ require ( github.com/zeebo/xxh3 v1.0.2 // indirect go.etcd.io/bbolt v1.3.5 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.opencensus.io v0.24.0 // indirect go.uber.org/ratelimit v0.3.1 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect @@ -256,7 +258,7 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.51.0 // indirect - golang.org/x/sync v0.20.0 // indirect + golang.org/x/sync v0.20.0 golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect diff --git a/go.sum b/go.sum index 8b7ad0f..1e2da30 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.120.0 h1:wc6bgG9DHyKqF5/vQvX1CiZrtHnxJjBlKUyF9nP6meA= cloud.google.com/go v0.120.0/go.mod h1:/beW32s8/pGRuj4IILWQNd4uuebeT4dkOhKmkfit64Q= cloud.google.com/go/auth v0.16.1 h1:XrXauHMd30LhQYVRHLGvJiYeczweKQXZxsTbV9TiguU= @@ -14,10 +15,14 @@ cloud.google.com/go/datacatalog v1.26.0 h1:eFgygb3DTufTWWUB8ARk+dSuXz+aefNJXTlkW cloud.google.com/go/datacatalog v1.26.0/go.mod h1:bLN2HLBAwB3kLTFT5ZKLHVPj/weNz6bR0c7nYp0LE14= cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/kms v1.21.2 h1:c/PRUSMNQ8zXrc1sdAUnsenWWaNXN+PzTXfXOcSFdoE= +cloud.google.com/go/kms v1.21.2/go.mod h1:8wkMtHV/9Z8mLXEXr1GK7xPSBdi6knuLXIhqjuWcI6w= cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/pubsub v1.49.0 h1:5054IkbslnrMCgA2MAEPcsN3Ky+AyMpEZcii/DoySPo= +cloud.google.com/go/pubsub v1.49.0/go.mod h1:K1FswTWP+C1tI/nfi3HQecoVeFvL4HUOB1tdaNXKhUY= cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= @@ -104,12 +109,15 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= @@ -184,9 +192,13 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/ethereum/c-kzg-4844/v2 v2.1.6 h1:xQymkKCT5E2Jiaoqf3v4wsNgjZLY0lRSkZn27fRjSls= @@ -257,10 +269,20 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= @@ -270,7 +292,13 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -282,6 +310,7 @@ github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9 github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= @@ -530,6 +559,7 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= @@ -682,11 +712,15 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.einride.tech/aip v0.68.1 h1:16/AfSxcQISGN5z9C5lM+0mLYXihrHbQ1onvYTr93aQ= +go.einride.tech/aip v0.68.1/go.mod h1:XaFtaj4HuA3Zwk9xoBtTWgNubZ0ZZXv9BZJCkuKuWbg= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/detectors/gcp v1.42.0 h1:kpt2PEJuOuqYkPcktfJqWWDjTEd/FNgrxcniL7kQrXQ= @@ -756,8 +790,12 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -765,12 +803,16 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= @@ -779,9 +821,11 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -791,6 +835,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -831,7 +876,10 @@ golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -849,14 +897,33 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.233.0 h1:iGZfjXAJiUFSSaekVB7LzXl6tRfEKhUN7FkZN++07tI= google.golang.org/api v0.233.0/go.mod h1:TCIVLLlcwunlMpZIhIp7Ltk77W+vUSdUKAAIlbxY44c= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 h1:1tXaIXCracvtsRxSBsYDiSBN0cuJvM7QYW+MrpIRY78= google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2/go.mod h1:49MsLSx0oWMOZqcpB3uL8ZOkAh1+TndpJ8ONoCBWiZk= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= @@ -895,6 +962,8 @@ gorm.io/gorm v1.31.0 h1:0VlycGreVhK7RF/Bwt51Fk8v0xLiiiFdbGDPIZQ7mJY= gorm.io/gorm v1.31.0/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=